From b6e3ab33699b909c8fa76afdb92beaa5ef6c537e Mon Sep 17 00:00:00 2001 From: Jarry Shaw Date: Sun, 25 Oct 2026 00:54:37 -0400 Subject: [PATCH] docs(foundation): fifth whole-repo pass over foundation, toolkit, dumpkit, interface and utilities (#1103) - engines: the name= class-keyword collision holds below 3.11, not on 3.10 alone; _pcap_backend does import pcapkit.corekit.infoclass; PyShark 3.13 is inferred, not measured; upstream file:line anchors become names; unsupported_reason() is consulted by engine selection, not run() alone. - reassembly: TCP pseudo-code matches FIN cutting the hole list and the sentinel hole; IP departures name the fixed timeout; IPv6 and DF frames are not submitted for every frame; examples import from the submodules. - traceflow/toolkit: labels come from the opening packet; the link type of a PCAP-NG packet is its interface's, not a global header's. - registry: meth is a method name or a (parser, constructor) pair, not a callable; register_dumper accepts any dictdumper Dumper subclass. - compat, config, dumpkit, utilities: stale version range, unresolvable DictDumper reference, "dumper class itself", missing config/datetime. Prose only; AST guard clean; docs, conventions and slice tests pass. --- .../pcapkit/foundation/engines/3rdparty.rst | 8 +- .../pcapkit/foundation/engines/index.rst | 7 +- .../foundation/reassembly/ip/index.rst | 4 + .../pcapkit/foundation/reassembly/ip/ipv4.rst | 4 +- .../pcapkit/foundation/reassembly/ip/ipv6.rst | 8 +- .../pcapkit/foundation/reassembly/tcp.rst | 32 +++++--- pcapkit/dumpkit/common.py | 2 +- pcapkit/dumpkit/null.py | 2 +- pcapkit/dumpkit/pcap.py | 2 +- pcapkit/foundation/engines/_pcap_backend.py | 3 +- pcapkit/foundation/engines/engine.py | 10 +-- pcapkit/foundation/engines/pcap_ct.py | 13 ++-- pcapkit/foundation/engines/pcapng.py | 2 +- pcapkit/foundation/engines/pypcap.py | 2 +- pcapkit/foundation/engines/pyshark.py | 12 +-- pcapkit/foundation/extraction.py | 13 ++-- pcapkit/foundation/reassembly/data/data.py | 11 +-- pcapkit/foundation/reassembly/ip.py | 11 +-- pcapkit/foundation/reassembly/ipv4.py | 2 +- pcapkit/foundation/reassembly/ipv6.py | 2 +- pcapkit/foundation/reassembly/tcp.py | 6 +- pcapkit/foundation/registry/protocols.py | 76 +++++++++++++++---- pcapkit/foundation/traceflow/data/tcp.py | 9 ++- pcapkit/foundation/traceflow/tcp.py | 9 ++- pcapkit/foundation/traceflow/traceflow.py | 4 +- pcapkit/toolkit/dpkt.py | 5 +- pcapkit/toolkit/pcapng.py | 2 +- pcapkit/toolkit/scapy.py | 2 +- pcapkit/utilities/__init__.py | 6 +- pcapkit/utilities/compat.py | 6 +- pcapkit/utilities/config.py | 4 +- 31 files changed, 179 insertions(+), 100 deletions(-) diff --git a/docs/source/pcapkit/foundation/engines/3rdparty.rst b/docs/source/pcapkit/foundation/engines/3rdparty.rst index 0c8e96beba..fb6dbcdc78 100644 --- a/docs/source/pcapkit/foundation/engines/3rdparty.rst +++ b/docs/source/pcapkit/foundation/engines/3rdparty.rst @@ -115,7 +115,8 @@ support, as is used by :class:`pcapkit.foundation.extraction.Extractor`. 3.10 and 3.11 return a loop silently, 3.12 returns one with a :exc:`DeprecationWarning`, and **3.14 raises** ``RuntimeError: There is no current event loop in thread 'MainThread'``. Python - 3.13 works, being on the deprecated-but-functional side of that change. + 3.13 was not measured; it is expected to work, being on the + deprecated-but-functional side of that change. **Wireshark's** :program:`tshark` **binary.** |pyshark|_ shells out to it and parses nothing itself. It need not be on :envvar:`PATH`: |pyshark|_ consults @@ -464,8 +465,9 @@ and ``import pcap`` resolves to whichever the import system finds first. they actually got rather than assume, and this module is the one place that answers it -- deliberately shared, since the engines must agree and a copy of the detection in each would be a chance to disagree. It is *only* detection, -and it imports nothing from :mod:`pcapkit`, so it cannot introduce an import -cycle. +and it imports nothing from :mod:`pcapkit` but :mod:`pcapkit.corekit.infoclass`, +which imports nothing from :mod:`pcapkit.foundation`, so it cannot introduce an +import cycle. .. autodata:: pcapkit.foundation.engines._pcap_backend.PYPCAP diff --git a/docs/source/pcapkit/foundation/engines/index.rst b/docs/source/pcapkit/foundation/engines/index.rst index eebbcc6735..019595784c 100644 --- a/docs/source/pcapkit/foundation/engines/index.rst +++ b/docs/source/pcapkit/foundation/engines/index.rst @@ -171,9 +171,10 @@ A constraint that an import cannot reveal is enforced by overriding one warning naming the actual cause and a fall back to the built-in parser. :class:`~pcapkit.foundation.engines.pyshark.PyShark` checks its Python ceiling and for ``tshark``, :class:`~pcapkit.foundation.engines.pypcapfile.PyPCAPFile` its Python ceiling, -:class:`~pcapkit.foundation.engines.pcap_ct.PCAP_CT` for a system ``libpcap``, and -:class:`~pcapkit.foundation.engines.pypcap.PyPCAP` and -:class:`~pcapkit.foundation.engines.pcap_ct.PCAP_CT` which distribution owns |pcap|_. +and :class:`~pcapkit.foundation.engines.pypcap.PyPCAP` and +:class:`~pcapkit.foundation.engines.pcap_ct.PCAP_CT` which distribution owns |pcap|_ and +whether it loads at all -- for :class:`~pcapkit.foundation.engines.pcap_ct.PCAP_CT`, whether +a system ``libpcap`` is present (:issue:`2132`). :class:`~pcapkit.foundation.engines.pypcap.PyPCAP` deliberately has no Python ceiling check, even though upstream cannot be installed on 3.12 or newer: what matters is which distribution is present, and a version check would refuse a working build. diff --git a/docs/source/pcapkit/foundation/reassembly/ip/index.rst b/docs/source/pcapkit/foundation/reassembly/ip/index.rst index 5231712ccd..989e854734 100644 --- a/docs/source/pcapkit/foundation/reassembly/ip/index.rst +++ b/docs/source/pcapkit/foundation/reassembly/ip/index.rst @@ -96,6 +96,10 @@ reported as a separate incomplete datagram; this is intended (:issue:`1507`). Departures from the procedure above: +* The timer is not set from ``TLB`` and ``TTL``. A buffer is abandoned a fixed + timeout after its first-arriving fragment, on the capture's clock -- 60 + seconds by default, per :rfc:`1122#section-3.3.2` for IPv4 and + :rfc:`8200#section-4.5` for IPv6. * Only the final fragment's partial last ``RCVBT`` block is set. A non-final fragment carries a multiple of 8 octets, so one ending mid-block was cut or is malformed, and setting its last block would complete the datagram with diff --git a/docs/source/pcapkit/foundation/reassembly/ip/ipv4.rst b/docs/source/pcapkit/foundation/reassembly/ip/ipv4.rst index 0d9245b5f7..b4f776dd26 100644 --- a/docs/source/pcapkit/foundation/reassembly/ip/ipv4.rst +++ b/docs/source/pcapkit/foundation/reassembly/ip/ipv4.rst @@ -96,8 +96,8 @@ Terminology .. note:: ``packet`` is analysed on the first read, not when the datagram is - submitted. A datagram is submitted for *every* frame -- an unfragmented - one included, since nothing upstream filters it out -- and the analysis + submitted. A datagram is submitted for *every* frame whose **DF** flag + is clear -- an unfragmented one included -- and the analysis is a second full parse of the payload, so running it eagerly would charge every caller for a result most never read. Reading the attribute, or any mapping view of it (``datagram['packet']``, ``to_dict()``, ``items()``, diff --git a/docs/source/pcapkit/foundation/reassembly/ip/ipv6.rst b/docs/source/pcapkit/foundation/reassembly/ip/ipv6.rst index d89cbf281f..c105fd3168 100644 --- a/docs/source/pcapkit/foundation/reassembly/ip/ipv6.rst +++ b/docs/source/pcapkit/foundation/reassembly/ip/ipv6.rst @@ -140,11 +140,9 @@ Terminology .. note:: ``packet`` is analysed on the first read, not when the datagram is - submitted. A datagram is submitted for *every* frame -- an unfragmented - one included, since nothing upstream filters it out -- and the analysis - is a second full parse of the payload, so running it eagerly would - charge every caller for a result most never read. Reading the attribute, or any - mapping view of it (``datagram['packet']``, ``to_dict()``, ``items()``, + submitted. The analysis is a second full parse of the payload, so + running it eagerly would charge every caller for a result most never + read. Reading the attribute, or any mapping view of it (``datagram['packet']``, ``to_dict()``, ``items()``, ``repr()``), runs it and keeps the result; see :class:`~pcapkit.foundation.reassembly.data.data.Deferred`. diff --git a/docs/source/pcapkit/foundation/reassembly/tcp.rst b/docs/source/pcapkit/foundation/reassembly/tcp.rst index ccb8859b5b..d1a0875221 100644 --- a/docs/source/pcapkit/foundation/reassembly/tcp.rst +++ b/docs/source/pcapkit/foundation/reassembly/tcp.rst @@ -83,6 +83,11 @@ Algorithm [from octet fragment.first to octet fragment.last]; HDL <- [one hole from PSN + fragment.len to infinity, i.e. HOLE_OPEN_BOUND]; + IF (SYN is false) { + /* the start of the stream is unknown: this hole is the one + before the capture */ + insert one hole from -HOLE_OPEN_BOUND to PSN - 1 into HDL; + } } ELSE { put data from fragment into data buffer with BUFID [from octet fragment.first to octet fragment.last]; @@ -94,7 +99,13 @@ Algorithm } } - IF (FIN is true or RST is true) { + /* a FIN fixes the end of the stream; data sent before it may still + arrive after it */ + IF (FIN is true) { + cut every hole in HDL back to end below the FIN; + } + + IF (RST is true or HDL holds no hole but the one before the capture) { submit datagram to next step; free all reassembly resources for this BUFID; BREAK. @@ -116,13 +127,14 @@ Algorithm new_hole.last <- fragment.first - 1; } - IF (fragment.last < hole.last AND FIN is false AND RST is false) { + IF (fragment.last < hole.last) { create new entry "new_hole" in HDL; new_hole.first <- fragment.last + 1; new_hole.last <- hole.last; } - BREAK. + /* one segment may span several holes */ + CONTINUE. } give up until (no entry from HDL) } @@ -141,16 +153,18 @@ here is the process: new hole descriptor ``new_hole`` with ``new_hole.first`` equal to ``hole.first``, and ``new_hole.last`` equal to ``fragment.first`` minus one (``-1``). -6. If ``fragment.last`` is less than ``hole.last`` and neither ``FIN`` - nor ``RST`` is set -- TCP has no *more fragments* flag, so the - termination flags take its place -- then create a new hole +6. If ``fragment.last`` is less than ``hole.last``, then create a new hole descriptor ``new_hole``, with ``new_hole.first`` equal to ``fragment.last`` plus one (``+1``) and ``new_hole.last`` equal to ``hole.last``. 7. Go to step one. -8. If the hole descriptor list is now empty, the datagram is now - complete. Pass it on to the higher level protocol processor - for further handling. Otherwise, return. +8. If the hole descriptor list holds no hole but the one below a capture + that missed the handshake, the stream is complete. Pass it on to the + higher level protocol processor for further handling. Otherwise, return. + +TCP has no *more fragments* flag, so the hole above the last octet received +stays open until a FIN cuts it back to end at the sequence number the FIN +occupies. A RST submits the stream as it stands, holes and all. Terminology =========== diff --git a/pcapkit/dumpkit/common.py b/pcapkit/dumpkit/common.py index f7ac4b5879..a318e700bb 100644 --- a/pcapkit/dumpkit/common.py +++ b/pcapkit/dumpkit/common.py @@ -421,7 +421,7 @@ def excluded(info: 'Info | MultiDict[Any, Any]') -> 'Collection[Any]': The :attr:`~pcapkit.corekit.infoclass.Info.__excluded__` of a field model, or of the field model an export was made from -- metadata such as ``__next_type__``, which only a rebuild reads - (:issue:`2003`). None for an option list, whose keys are its + (:issue:`2003`). Empty for an option list, whose keys are its options. """ diff --git a/pcapkit/dumpkit/null.py b/pcapkit/dumpkit/null.py index 2ec84f3dd3..13129435ff 100644 --- a/pcapkit/dumpkit/null.py +++ b/pcapkit/dumpkit/null.py @@ -49,7 +49,7 @@ def __call__(self, value: 'dict[str, Any]', name: 'Optional[str]' = None) -> 'No name: name of current content block Returns: - The dumper class itself (to support chain calling). + The dumper itself (to support chain calling). """ return self diff --git a/pcapkit/dumpkit/pcap.py b/pcapkit/dumpkit/pcap.py index b7b5c84dc4..745b1be447 100644 --- a/pcapkit/dumpkit/pcap.py +++ b/pcapkit/dumpkit/pcap.py @@ -148,7 +148,7 @@ def __call__(self, value: 'Data_Frame', name: 'Optional[str]' = None) -> 'PCAPIO name: name of current content block Returns: - The dumper class itself (to support chain calling). + The dumper itself (to support chain calling). """ with open(self._file, 'ab') as file: diff --git a/pcapkit/foundation/engines/_pcap_backend.py b/pcapkit/foundation/engines/_pcap_backend.py index 7c6db14686..6333adeee4 100644 --- a/pcapkit/foundation/engines/_pcap_backend.py +++ b/pcapkit/foundation/engines/_pcap_backend.py @@ -24,7 +24,8 @@ the one place that answers that, deliberately shared: the engines must agree on the answer, and a copy of the detection in each would be a chance to disagree. It is *only* detection -- nothing here is engine behaviour -- and it imports -nothing from :mod:`pcapkit`, so it cannot introduce an import cycle. +nothing from :mod:`pcapkit` but :mod:`pcapkit.corekit.infoclass`, which imports +nothing from :mod:`pcapkit.foundation`, so it cannot introduce an import cycle. """ import contextvars diff --git a/pcapkit/foundation/engines/engine.py b/pcapkit/foundation/engines/engine.py index 52fd9d3876..473236dc43 100644 --- a/pcapkit/foundation/engines/engine.py +++ b/pcapkit/foundation/engines/engine.py @@ -290,15 +290,15 @@ def __init_subclass__(cls, /, engine: 'Optional[str]' = None, *args: 'Any', **kw Note: The keyword is ``engine`` rather than ``name`` because ``name`` - cannot be passed as a class keyword at all on Python 3.10: + cannot be passed as a class keyword at all below Python 3.11: ``abc.ABCMeta.__new__`` takes ``mcls``, ``name``, ``bases`` and ``namespace`` as positional-*or-keyword* parameters - before 3.11, so a class keyword by any of those names collides + there, so a class keyword by any of those names collides with one of them and the class statement raises :exc:`TypeError` from the metaclass before this method is reached. ``engine`` is outside that set, so the documented registration path works on every - supported version. Measured on 3.10.21, 3.11.15 and 3.14.7; those - names are the whole of the ``abc.ABCMeta.__new__`` collision + supported version. Measured on 3.8.20, 3.9.25, 3.10.21, 3.11.15 and + 3.14.7; those names are the whole of the ``abc.ABCMeta.__new__`` collision surface. Separately, and for an unrelated reason that holds on every version, ``metaclass`` is not usable as a class keyword either: a ``class`` statement consumes it to choose the metaclass, so it never @@ -316,7 +316,7 @@ def __init_subclass__(cls, /, engine: 'Optional[str]' = None, *args: 'Any', **kw # ``class MyEngine(Engine, engnie='x')`` skip registration altogether -- # no exception, no warning. So reject it. # - # One typo this cannot catch is ``name=``, and only on Python 3.10: it is + # One typo this cannot catch is ``name=``, and only below Python 3.11: it is # one of the names that collide with ``ABCMeta.__new__``, so it fails # in the metaclass before reaching here. It is still loud, just as a # ``TypeError`` rather than an ``UnsupportedCall``. diff --git a/pcapkit/foundation/engines/pcap_ct.py b/pcapkit/foundation/engines/pcap_ct.py index 39318efac8..de19474bad 100644 --- a/pcapkit/foundation/engines/pcap_ct.py +++ b/pcapkit/foundation/engines/pcap_ct.py @@ -251,10 +251,11 @@ def __init__(self, extractor: 'Extractor') -> 'None': Raises: UnsupportedCall: If the environment cannot support this engine, for either of the reasons :meth:`unsupported_reason` describes. That - hook is only consulted by :meth:`Extractor.run - `, which degrades to - the default engine with a warning; constructing the engine - directly bypasses it, so this is the backstop for that path. + hook is only consulted by + :class:`~pcapkit.foundation.extraction.Extractor`'s engine + selection, which degrades to the default engine with a warning; + constructing the engine directly bypasses it, so this is the + backstop for that path. """ # NOTE: the probe comes *before* ``import pcap``, and the order is @@ -464,8 +465,8 @@ def close(self) -> 'None': a second call, where upstream ``pypcap`` 1.3.0 segfaults on one. The :attr:`_closed ` flag is kept regardless: it is not this engine's business to depend on a pre-release's undocumented - forgiveness, and the flag also covers the case of an engine that was - never opened. + forgiveness. An engine that was never opened has no + :attr:`_handle ` to close, and is skipped too. """ if self._closed or self._handle is None: diff --git a/pcapkit/foundation/engines/pcapng.py b/pcapkit/foundation/engines/pcapng.py index f0369498f1..6102f17e7e 100644 --- a/pcapkit/foundation/engines/pcapng.py +++ b/pcapkit/foundation/engines/pcapng.py @@ -344,7 +344,7 @@ def _block_name(self, kind: 'str') -> 'str': with every Section Header Block, so numbering a block by their length gives the first IDB of every section the same name. The output dumpers key by name, so a repeated name overwrites the earlier block (#1256). The number is therefore counted across the whole file instead; - for a single-section file it is the same number as before. + for a single-section file the two counts agree. Args: kind: The kind of the block, e.g. ``'Interface Description'``. diff --git a/pcapkit/foundation/engines/pypcap.py b/pcapkit/foundation/engines/pypcap.py index ba766d6de0..b4b9482ccb 100644 --- a/pcapkit/foundation/engines/pypcap.py +++ b/pcapkit/foundation/engines/pypcap.py @@ -208,7 +208,7 @@ def __init__(self, extractor: 'Extractor') -> 'None': than upstream `PyPCAP`_, or is installed but cannot load. These are the conditions :meth:`unsupported_reason` reports, kept here as well because that hook is only consulted by - :meth:`Extractor.run `; + :class:`~pcapkit.foundation.extraction.Extractor`'s engine selection; constructing the engine directly bypasses it, and running `pcap-ct`_ under the :class:`~pcapkit.foundation.engines.pypcap.PyPCAP` name would misreport what did the work. Via :class:`~pcapkit.foundation.extraction.Extractor` diff --git a/pcapkit/foundation/engines/pyshark.py b/pcapkit/foundation/engines/pyshark.py index 08b4185203..a58d28242f 100644 --- a/pcapkit/foundation/engines/pyshark.py +++ b/pcapkit/foundation/engines/pyshark.py @@ -64,10 +64,10 @@ def _is_packet(packet: 'PySharkPacket') -> 'bool': records only. The marker is ``frame.encap_type`` on the record's :func:`_frame_layer`. - Wireshark adds it for ``REC_TYPE_PACKET`` records only: - :file:`epan/dissectors/packet-frame.c` lines 1001-1002 in 4.2.2, and 828-890 - in 4.6.9. The summary line is not used, because it differs between those - versions. A record with no ``frame`` layer is not a packet either. + Wireshark adds it for ``REC_TYPE_PACKET`` records only, in ``dissect_frame`` + of :file:`epan/dissectors/packet-frame.c`, in 4.2.2 and 4.6.9 alike. The + summary line is not used, because it differs between those versions. A + record with no ``frame`` layer is not a packet either. """ frame = _frame_layer(packet) @@ -119,8 +119,8 @@ def unsupported_reason(cls) -> 'Optional[str]': error rather than degrading to the default engine with a warning. **The interpreter.** |pyshark|_ 0.6 does - ``asyncio.get_event_loop_policy().get_event_loop()`` at - :file:`pyshark/capture/capture.py:183`, in a fresh interpreter with no + ``asyncio.get_event_loop_policy().get_event_loop()`` in + ``Capture._setup_eventloop``, in a fresh interpreter with no running loop. Measured: 3.10 and 3.11 return a loop silently, 3.12 returns one with a :exc:`DeprecationWarning`, and 3.14 raises ``RuntimeError: There is no current event loop in thread diff --git a/pcapkit/foundation/extraction.py b/pcapkit/foundation/extraction.py index 7d3be41272..2a7e09c8f0 100644 --- a/pcapkit/foundation/extraction.py +++ b/pcapkit/foundation/extraction.py @@ -515,7 +515,9 @@ class it names, restores the shipped entry rather than storing ext: file extension Raises: - RegistryError: If ``dumper`` is not a class, or not a :class:`~pcapkit.dumpkit.common.Dumper` subclass. + RegistryError: If ``dumper`` is not a class, or not a + :class:`dictdumper.dumper.Dumper` subclass, such as + :class:`~pcapkit.dumpkit.common.Dumper`. Warns: RegistryWarning: If a different dumper is already registered for @@ -1670,10 +1672,11 @@ def __del__(self) -> 'None': :attr:`~object.__dict__` rather than attribute access. That matters for more than tidiness: ``_flag_s`` is assigned early in :meth:`__init__` and ``_ifile`` only much later, so a constructor - that fails in between -- an unreadable path, an unknown format -- - leaves an instance whose flag says "mine" and which has no stream at - all. The close itself is guarded too, that being what a close during - interpreter shutdown can fail at. + that fails in between -- an unreadable path, or a negative + ``reasm_timeout`` with ``reassembly=True`` and ``ip``, ``ipv4``, + ``ipv6`` or ``tcp`` set -- leaves an instance whose flag says "mine" + and which has no stream at all. The close itself is guarded too, that + being what a close during interpreter shutdown can fail at. """ if not self._owns_input(): diff --git a/pcapkit/foundation/reassembly/data/data.py b/pcapkit/foundation/reassembly/data/data.py index 59009c9bf8..b2c7020dd0 100644 --- a/pcapkit/foundation/reassembly/data/data.py +++ b/pcapkit/foundation/reassembly/data/data.py @@ -71,8 +71,9 @@ class Completion(EnumLookup, StrEnum): COMPLETE = auto() #: Fragments were still outstanding when the buffer was flushed -- at the end - #: of the capture, or when the session was torn down (a TCP FIN/RST, or an - #: IPv4 datagram whose identifier was reused by an unfragmented packet). + #: of the capture, or when the session was torn down (a TCP RST, a TCP SYN on + #: a 4-tuple already buffered, or an IPv4 datagram whose identifier was + #: reused by an unfragmented packet). #: The missing octets may simply not have been captured. PARTIAL = auto() @@ -111,9 +112,9 @@ class Deferred: :class:`~pcapkit.foundation.reassembly.data.data.ReassemblyData` rather than in either protocol's data module. - IP reassembly is the case that made it necessary. It submits a datagram for - *every* frame -- not only the fragmented ones, since a frame that is not - fragmented in any sense still reaches + IP reassembly is the case that made it necessary. IPv4 reassembly submits a + datagram for *every* frame whose **DF** flag is clear -- not only the + fragmented ones, since an unfragmented frame still reaches :meth:`IP.reassembly ` and is submitted there as a trivially complete datagram -- so an eager parse would re-parse captures holding no fragments at all: :file:`http.pcap` has 1117 diff --git a/pcapkit/foundation/reassembly/ip.py b/pcapkit/foundation/reassembly/ip.py index 432a5a99e3..eb89c9eefe 100644 --- a/pcapkit/foundation/reassembly/ip.py +++ b/pcapkit/foundation/reassembly/ip.py @@ -511,11 +511,12 @@ def submit(self, buf: 'Buffer[_AT]', *, bufid: 'tuple[_AT, _AT, int, TransType]' index=tuple(index), header=header, payload=payload, - # NOTE: ``analyze`` is a second full parse of the payload, and a - # datagram is submitted for every frame rather than only for the - # fragmented ones, so running it here charges every caller for a - # result most of them never read. ``Deferred`` postpones it to the - # first read of ``Datagram.packet``. + # NOTE: ``analyze`` is a second full parse of the payload, and an + # IPv4 datagram is submitted for every frame with **DF** clear + # rather than only for the fragmented ones, so running it here + # charges every caller for a result most of them never read. + # ``Deferred`` postpones it to the first read of + # ``Datagram.packet``. packet=Deferred(self.protocol.analyze, bufid[3], payload), conflict=conflict, ) diff --git a/pcapkit/foundation/reassembly/ipv4.py b/pcapkit/foundation/reassembly/ipv4.py index aca3189187..3d11aa74a8 100644 --- a/pcapkit/foundation/reassembly/ipv4.py +++ b/pcapkit/foundation/reassembly/ipv4.py @@ -34,7 +34,7 @@ class IPv4(IP): :data:`None` selects :attr:`__timeout__` Example: - >>> from pcapkit.foundation.reassembly import IPv4 + >>> from pcapkit.foundation.reassembly.ipv4 import IPv4 # Initialise instance: >>> ipv4_reassembly = IPv4() # Call reassembly: diff --git a/pcapkit/foundation/reassembly/ipv6.py b/pcapkit/foundation/reassembly/ipv6.py index 6152d314e7..a5ea494759 100644 --- a/pcapkit/foundation/reassembly/ipv6.py +++ b/pcapkit/foundation/reassembly/ipv6.py @@ -79,7 +79,7 @@ class IPv6(IP): :data:`None` selects :attr:`__timeout__` Example: - >>> from pcapkit.foundation.reassembly import IPv6 + >>> from pcapkit.foundation.reassembly.ipv6 import IPv6 # Initialise instance: >>> ipv6_reassembly = IPv6() # Call reassembly: diff --git a/pcapkit/foundation/reassembly/tcp.py b/pcapkit/foundation/reassembly/tcp.py index 64533a0138..7adf3f71c4 100644 --- a/pcapkit/foundation/reassembly/tcp.py +++ b/pcapkit/foundation/reassembly/tcp.py @@ -39,7 +39,7 @@ class TCP(ReassemblyBase[Packet, Datagram, BufferID, Buffer]): expiry Example: - >>> from pcapkit.foundation.reassembly import TCP + >>> from pcapkit.foundation.reassembly.tcp import TCP # Initialise instance: >>> tcp_reassembly = TCP() # Call reassembly: @@ -390,7 +390,7 @@ def _reassemble_append(self, info: 'Packet', fragment: 'Fragment', PSN: 'int') - Covers both an ordinary (or zero-length) forward gap and a tail-side overlap. Mutates ``fragment`` in place: its ``gap`` list, its ``conflict`` list, and finally its ``raw``/``len`` via - :meth:`~pcapkit.foundation.reassembly.data.data.Info.__update__`. + :meth:`~pcapkit.corekit.infoclass.Info.__update__`. Arguments: info: :term:`info ` dict of the arriving segment @@ -445,7 +445,7 @@ def _reassemble_prepend(self, info: 'Packet', fragment: 'Fragment', PSN: 'int') which may also extend a new tail past the buffered end in the same call. Mutates ``fragment`` in place: its ``isn``, its ``gap`` list, its ``conflict`` list, and finally its ``raw``/``len`` via - :meth:`~pcapkit.foundation.reassembly.data.data.Info.__update__`. + :meth:`~pcapkit.corekit.infoclass.Info.__update__`. Arguments: info: :term:`info ` dict of the arriving segment diff --git a/pcapkit/foundation/registry/protocols.py b/pcapkit/foundation/registry/protocols.py index 10e0b337fc..b3389b2154 100644 --- a/pcapkit/foundation/registry/protocols.py +++ b/pcapkit/foundation/registry/protocols.py @@ -754,7 +754,10 @@ def register_ipv4_option(code: 'IPv4_OptionNumber', meth: 'str | tuple[IPv4_Opti Args: code: :class:`IPv4 ` option code as in :class:`~pcapkit.const.ipv4.option_number.OptionNumber`. - meth: Method name or callable to parse and/or construct the option. + meth: Method name, or a ``(parser, constructor)`` pair of callables, to + parse and construct the option. A name names the + ``_read_opt_*`` and ``_make_opt_*`` methods, and has to name + an existing ``_read_opt_*`` one. schema: :class:`~pcapkit.protocols.schema.schema.Schema` class for the option. It should be a subclass of :class:`pcapkit.protocols.schema.internet.ipv4.Option`. @@ -779,7 +782,10 @@ def register_hip_parameter(code: 'HIP_Parameter', meth: 'str | tuple[HIP_Paramet Args: code: :class:`~pcapkit.protocols.internet.hip.HIP` parameter code as in :class:`~pcapkit.const.hip.parameter.Parameter`. - meth: Method name or callable to parse and/or construct the parameter. + meth: Method name, or a ``(parser, constructor)`` pair of callables, to + parse and construct the parameter. A name names the + ``_read_param_*`` and ``_make_param_*`` methods, and has to name + an existing ``_read_param_*`` one. schema: :class:`~pcapkit.protocols.schema.schema.Schema` class for the parameter. It should be a subclass of :class:`pcapkit.protocols.schema.internet.hip.Parameter`. @@ -804,7 +810,10 @@ def register_hopopt_option(code: 'IPv6_Option', meth: 'str | tuple[HOPOPT_Option Args: code: :class:`~pcapkit.protocols.internet.hopopt.HOPOPT` option code as in :class:`~pcapkit.const.ipv6.option.Option`. - meth: Method name or callable to parse and/or construct the option. + meth: Method name, or a ``(parser, constructor)`` pair of callables, to + parse and construct the option. A name names the + ``_read_opt_*`` and ``_make_opt_*`` methods, and has to name + an existing ``_read_opt_*`` one. schema: :class:`~pcapkit.protocols.schema.schema.Schema` class for the option. It should be a subclass of :class:`pcapkit.protocols.schema.internet.hopopt.Option`. @@ -829,7 +838,10 @@ def register_ipv6_opts_option(code: 'IPv6_Option', meth: 'str | tuple[IPv6_Opts_ Args: code: :class:`IPv6-Opts ` option code as in :class:`~pcapkit.const.ipv6.option.Option`. - meth: Method name or callable to parse and/or construct the option. + meth: Method name, or a ``(parser, constructor)`` pair of callables, to + parse and construct the option. A name names the + ``_read_opt_*`` and ``_make_opt_*`` methods, and has to name + an existing ``_read_opt_*`` one. schema: :class:`~pcapkit.protocols.schema.schema.Schema` class for the option. It should be a subclass of :class:`pcapkit.protocols.schema.internet.ipv6_opts.Option`. @@ -854,7 +866,10 @@ def register_ipv6_route_routing(code: 'IPv6_Routing', meth: 'str | tuple[IPv6_Ro Args: code: :class:`IPv6-Route ` data type code as in :class:`~pcapkit.const.ipv6.routing.Routing`. - meth: Method name or callable to parse and/or construct the data. + meth: Method name, or a ``(parser, constructor)`` pair of callables, to + parse and construct the data. A name names the + ``_read_data_type_*`` and ``_make_data_type_*`` methods, and has to name + an existing ``_read_data_type_*`` one. schema: :class:`~pcapkit.protocols.schema.schema.Schema` class for the routing data. It should be a subclass of :class:`pcapkit.protocols.schema.internet.ipv6_route.RoutingType`. @@ -879,7 +894,10 @@ def register_mh_message(code: 'MH_Packet', meth: 'str | tuple[MH_PacketParser, M Args: code: :class:`~pcapkit.protocols.internet.mh.MH` message type code as in :class:`~pcapkit.const.mh.packet.Packet`. - meth: Method name or callable to parse and/or construct the data. + meth: Method name, or a ``(parser, constructor)`` pair of callables, to + parse and construct the data. A name names the + ``_read_msg_*`` and ``_make_msg_*`` methods, and has to name + an existing ``_read_msg_*`` one. schema: :class:`~pcapkit.protocols.schema.schema.Schema` class for the message type. It should be a subclass of :class:`pcapkit.protocols.schema.internet.mh.Packet`. @@ -904,7 +922,10 @@ def register_mh_option(code: 'MH_Option', meth: 'str | tuple[MH_OptionParser, MH Args: code: :class:`~pcapkit.protocols.internet.mh.MH` option code as in :class:`~pcapkit.const.mh.option.Option`. - meth: Method name or callable to parse and/or construct the data. + meth: Method name, or a ``(parser, constructor)`` pair of callables, to + parse and construct the data. A name names the + ``_read_opt_*`` and ``_make_opt_*`` methods, and has to name + an existing ``_read_opt_*`` one. schema: :class:`~pcapkit.protocols.schema.schema.Schema` class for the option. It should be a subclass of :class:`pcapkit.protocols.schema.internet.mh.Option`. @@ -929,7 +950,10 @@ def register_mh_extension(code: 'MH_CGAExtension', meth: 'str | tuple[MH_Extensi Args: code: :class:`~pcapkit.protocols.internet.mh.MH` CGA extension code as in :class:`~pcapkit.const.mh.cga_extension.CGAExtension`. - meth: Method name or callable to parse and/or construct the data. + meth: Method name, or a ``(parser, constructor)`` pair of callables, to + parse and construct the data. A name names the + ``_read_ext_*`` and ``_make_ext_*`` methods, and has to name + an existing ``_read_ext_*`` one. schema: :class:`~pcapkit.protocols.schema.schema.Schema` class for the CGA extension. It should be a subclass of :class:`pcapkit.protocols.schema.internet.mh.CGAExtension`. @@ -1194,7 +1218,10 @@ def register_tcp_option(code: 'TCP_Option', meth: 'str | tuple[TCP_OptionParser, Args: code: :class:`~pcapkit.protocols.transport.tcp.TCP` option code as in :class:`~pcapkit.const.tcp.option.Option`. - meth: Method name or callable to parse and/or construct the option. + meth: Method name, or a ``(parser, constructor)`` pair of callables, to + parse and construct the option. A name names the + ``_read_mode_*`` and ``_make_mode_*`` methods, and has to name + an existing ``_read_mode_*`` one. schema: :class:`~pcapkit.protocols.schema.schema.Schema` class for the option. It should be a subclass of :class:`pcapkit.protocols.schema.transport.tcp.Option`. @@ -1219,7 +1246,10 @@ def register_tcp_mp_option(code: 'TCP_MPTCPOption', meth: 'str | tuple[TCP_MPOpt Args: code: Multipath :class:`~pcapkit.protocols.transport.tcp.TCP` option code as in :class:`~pcapkit.const.tcp.mp_tcp_option.MPTCPOption`. - meth: Method name or callable to parse and/or construct the option. + meth: Method name, or a ``(parser, constructor)`` pair of callables, to + parse and construct the option. A name names the + ``_read_mptcp_*`` and ``_make_mptcp_*`` methods, and has to name + an existing ``_read_mptcp_*`` one. schema: :class:`~pcapkit.protocols.schema.schema.Schema` class for the option. It should be a subclass of :class:`pcapkit.protocols.schema.transport.tcp.MPTCP`. @@ -1332,7 +1362,10 @@ def register_http_frame(code: 'HTTP_Frame', meth: 'str | tuple[HTTP_FrameParser, Args: code: :class:`HTTP/2 ` frame type code as in :class:`~pcapkit.const.http.frame.Frame`. - meth: Method name or callable to parse and/or construct the frame. + meth: Method name, or a ``(parser, constructor)`` pair of callables, to + parse and construct the frame. A name names the + ``_read_http_*`` and ``_make_http_*`` methods, and has to name + an existing ``_read_http_*`` one. schema: :class:`~pcapkit.protocols.schema.schema.Schema` class for the frame. It should be a subclass of :class:`pcapkit.protocols.schema.application.httpv2.FrameType`. @@ -1362,7 +1395,10 @@ def register_pcapng_block(code: 'PCAPNG_BlockType', meth: 'str | tuple[PCAPNG_Bl Args: code: :class:`PCAPNG ` block type code as in :class:`~pcapkit.const.pcapng.block_type.BlockType`. - meth: Method name or callable to parse and/or construct the block. + meth: Method name, or a ``(parser, constructor)`` pair of callables, to + parse and construct the block. A name names the + ``_read_block_*`` and ``_make_block_*`` methods, and has to name + an existing ``_read_block_*`` one. schema: :class:`~pcapkit.protocols.schema.schema.Schema` class for the block. It should be a subclass of :class:`pcapkit.protocols.schema.misc.pcapng.BlockType`. @@ -1387,7 +1423,10 @@ def register_pcapng_option(code: 'PCAPNG_OptionType', meth: 'str | tuple[PCAPNG_ Args: code: :class:`PCAPNG ` option type code as in :class:`~pcapkit.const.pcapng.option_type.OptionType`. - meth: Method name or callable to parse and/or construct the option. + meth: Method name, or a ``(parser, constructor)`` pair of callables, to + parse and construct the option. A name names the + ``_read_option_*`` and ``_make_option_*`` methods, and has to name + an existing ``_read_option_*`` one. schema: :class:`~pcapkit.protocols.schema.schema.Schema` class for the option. It should be a subclass of :class:`pcapkit.protocols.schema.misc.pcapng.Option`. @@ -1412,8 +1451,10 @@ def register_pcapng_record(code: 'PCAPNG_RecordType', meth: 'str | tuple[PCAPNG_ Args: code: :class:`PCAPNG ` name resolution record type code as in :class:`~pcapkit.const.pcapng.record_type.RecordType`. - meth: Method name or callable to parse and/or construct the name - resolution record. + meth: Method name, or a ``(parser, constructor)`` pair of callables, to + parse and construct the name resolution record. A name names the + ``_read_record_*`` and ``_make_record_*`` methods, and has to name + an existing ``_read_record_*`` one. schema: :class:`~pcapkit.protocols.schema.schema.Schema` class for the name resolution record. It should be a subclass of :class:`pcapkit.protocols.schema.misc.pcapng.NameResolutionRecord`. @@ -1438,7 +1479,10 @@ def register_pcapng_secrets(code: 'PCAPNG_SecretsType', meth: 'str | tuple[PCAPN Args: code: :class:`PCAPNG ` decryption secrets type code as in :class:`~pcapkit.const.pcapng.secrets_type.SecretsType`. - meth: Method name or callable to parse and/or construct the decryption secrets. + meth: Method name, or a ``(parser, constructor)`` pair of callables, to + parse and construct the decryption secrets. A name names the + ``_read_secrets_*`` and ``_make_secrets_*`` methods, and has to name + an existing ``_read_secrets_*`` one. schema: :class:`~pcapkit.protocols.schema.schema.Schema` class for the decryption secrets. It should be a subclass of :class:`pcapkit.protocols.schema.misc.pcapng.DSBSecrets`. diff --git a/pcapkit/foundation/traceflow/data/tcp.py b/pcapkit/foundation/traceflow/data/tcp.py index 4ef253bf3f..6c34815a5d 100644 --- a/pcapkit/foundation/traceflow/data/tcp.py +++ b/pcapkit/foundation/traceflow/data/tcp.py @@ -49,7 +49,8 @@ class Packet(Info, Generic[_AT]): """ - #: Data link type from global header. + #: Data link type: the PCAP global header's, or that of the PCAP-NG + #: interface the packet was captured on. protocol: 'Enum_LinkType' #: Frame number. index: 'int' @@ -109,7 +110,8 @@ class Buffer(Info, Generic[_AT]): #: authoritative ordering; :attr:`forward` and :attr:`reverse` are #: subsequences of it. index: 'list[int]' - #: Flow label generated from ``BUFID``. + #: Flow label, built from the packet that opened the flow, cf. + #: :meth:`TCP.trace `. label: 'str' #: ``(address, port)`` of the endpoint whose packet opened this flow. It #: defines what "forward" means for the flow, and it is the endpoint the @@ -169,7 +171,8 @@ class Index(DeferredPacket, Info): fpout: 'Optional[str]' #: Tuple of frame index, **both directions**, in capture order. index: 'tuple[int, ...]' - #: Flow label generated from ``BUFID``. + #: Flow label, built from the packet that opened the flow, cf. + #: :meth:`TCP.trace `. label: 'str' #: Frame index of the packets travelling in the direction that opened the #: flow, in capture order. That endpoint is the one the :attr:`label` names diff --git a/pcapkit/foundation/traceflow/tcp.py b/pcapkit/foundation/traceflow/tcp.py index d40d0b7243..3d78846cec 100644 --- a/pcapkit/foundation/traceflow/tcp.py +++ b/pcapkit/foundation/traceflow/tcp.py @@ -116,10 +116,11 @@ class TCP(TraceFlowBase[BufferID, Buffer[_AT], Index, Packet[_AT]], Generic[_AT] and are concatenated after all. Measured: two requests sent back to back on one acknowledgement number come back as a single datagram carrying ``b'req1req2'``, which - :meth:`test_pipelined_sends_share_a_datagram_because_the_ack_never_moved` - pins. So this narrows the concatenation to within one exchange rather than - eliminating it, and an application parser handed a pipelined datagram - still sees only the first message. + ``test_pipelined_sends_share_a_datagram_because_the_ack_never_moved`` in + :file:`tests/foundation/traceflow/test_tcp.py` pins. So this narrows the + concatenation to within one exchange rather than eliminating it, and an + application parser handed a pipelined datagram still sees only the first + message. A direction that carried one acknowledgement number throughout -- a single request and its reply, which is what most of the unit tests exercise -- diff --git a/pcapkit/foundation/traceflow/traceflow.py b/pcapkit/foundation/traceflow/traceflow.py index 94257ec307..7bcd259ac6 100644 --- a/pcapkit/foundation/traceflow/traceflow.py +++ b/pcapkit/foundation/traceflow/traceflow.py @@ -242,7 +242,9 @@ class it names, restores the shipped entry rather than storing ext: file extension Raises: - RegistryError: If ``dumper`` is not a class, or not a :class:`~pcapkit.dumpkit.common.Dumper` subclass. + RegistryError: If ``dumper`` is not a class, or not a + :class:`dictdumper.dumper.Dumper` subclass, such as + :class:`~pcapkit.dumpkit.common.Dumper`. Warns: RegistryWarning: If a different dumper is already registered for diff --git a/pcapkit/toolkit/dpkt.py b/pcapkit/toolkit/dpkt.py index ce3c233e4b..a146f5e6c3 100644 --- a/pcapkit/toolkit/dpkt.py +++ b/pcapkit/toolkit/dpkt.py @@ -830,7 +830,8 @@ def tcp_traceflow(packet: 'Packet', timestamp: 'float', *, :func:`ipv4_reassembly`. The PCAP frame record is built from the exact timestamp attached by :func:`attach_timestamp`, if any; see :func:`packet2frame`. - data_link: Data link layer protocol (from global header). + data_link: Data link layer protocol: the global header's for PCAP, and + that of the interface the packet was captured on for PCAP-NG. count: Packet index. If not provided, defaults to ``-1``. Returns: @@ -864,7 +865,7 @@ def tcp_traceflow(packet: 'Packet', timestamp: 'float', *, wire, _ = _layer2bytes(packet, tcp) data = TF_TCP_Packet( # type: ignore[type-var] - protocol=data_link, # data link type from global header + protocol=data_link, # data link type index=count, # frame number frame=packet2frame(packet, timestamp, data_link=data_link), # extracted packet syn=syn, # TCP synchronise (SYN) flag diff --git a/pcapkit/toolkit/pcapng.py b/pcapkit/toolkit/pcapng.py index cd991379c7..2e6eff13a1 100644 --- a/pcapkit/toolkit/pcapng.py +++ b/pcapkit/toolkit/pcapng.py @@ -211,7 +211,7 @@ def tcp_traceflow(frame: 'PCAPNG', *, nanosecond: 'bool' = False) -> 'TF_TCP_Pac if (segment := tcp_segment(frame)) is not None: frame_info = cast('Packet', frame.info) data = TF_TCP_Packet( # type: ignore[type-var] - protocol=frame.linktype, # data link type from global header + protocol=frame.linktype, # data link type of the block's interface index=frame_info.number, # frame number frame=block2frame(frame_info, nanosecond=nanosecond), # extracted frame info diff --git a/pcapkit/toolkit/scapy.py b/pcapkit/toolkit/scapy.py index a46a31d3ea..40a592cf40 100644 --- a/pcapkit/toolkit/scapy.py +++ b/pcapkit/toolkit/scapy.py @@ -11,7 +11,7 @@ .. warning:: - This module requires installed `Scapy`_ engine. + This module requires `Scapy`_ to be installed. .. note:: diff --git a/pcapkit/utilities/__init__.py b/pcapkit/utilities/__init__.py index 2c377a6f20..08ff69fe81 100644 --- a/pcapkit/utilities/__init__.py +++ b/pcapkit/utilities/__init__.py @@ -7,8 +7,10 @@ :mod:`pcapkit.utilities` holds the cross-cutting services :mod:`pcapkit` is built with: user-refined exceptions and -warnings, logging, a compatibility layer, and wrappers around -third-party libraries. It imports nothing from :mod:`pcapkit` +warnings, logging, the tunable limits of +:mod:`~pcapkit.utilities.config`, a compatibility layer, +timestamp conversion, and wrappers around third-party +libraries. It imports nothing from :mod:`pcapkit` outside itself (:issue:`1712`). .. versionremoved:: 1.5.0 diff --git a/pcapkit/utilities/compat.py b/pcapkit/utilities/compat.py index 454af876f3..f501c73111 100644 --- a/pcapkit/utilities/compat.py +++ b/pcapkit/utilities/compat.py @@ -105,9 +105,9 @@ def unquote(string: 'str | bytes', encoding: 'str' = 'utf-8', errors: 'str' = 'r # leaving no runtime trace at all. That dunder is *read* -- # :meth:`pcapkit.corekit.infoclass.Info.__init_subclass__` and # :meth:`pcapkit.protocols.schema.schema.Schema.__init_subclass__` refuse to -# derive from a class carrying it in its own ``__dict__`` -- so on 3.10, which is -# in the CI matrix, keying the guard on 3.8 would leave the runtime check reading -# an attribute nothing had set: a silent no-op on one supported interpreter and +# derive from a class carrying it in its own ``__dict__`` -- so on 3.8 to 3.10, +# all in the CI matrix, keying the guard on 3.8 would leave the runtime check +# reading an attribute nothing had set: a silent no-op on those interpreters and # an enforced rule on the rest. ``typing_extensions`` is a declared dependency # for exactly ``python_version < '3.11'`` and its ``final`` sets the attribute on # every version, so asking for it below 3.11 costs nothing and closes the gap. diff --git a/pcapkit/utilities/config.py b/pcapkit/utilities/config.py index b81d3376cf..3b7af9dc73 100644 --- a/pcapkit/utilities/config.py +++ b/pcapkit/utilities/config.py @@ -217,8 +217,8 @@ def read_flag(name: 'str', default: 'bool' = False) -> 'bool': IPV6_TCP_WARNED_LIMIT = read_limit('IPV6_TCP_WARNED_LIMIT', 32, risk=( 'each capture remembered pins one of its packets in memory')) -#: int: Most objects -#: :meth:`DictDumper._append_fallback ` +#: int: Most objects the ``_append_fallback`` method of a dumper +#: :func:`~pcapkit.dumpkit.common.make_dumper` builds #: expands for one top-level object, counting an object once per time it is met, #: :envvar:`PCAPKIT_FALLBACK_LIMIT`; ``-1`` expands without bound. #: