fix(protocols): dissect LINKTYPE_RAW by version nibble; add the BSD loopback layer for LINKTYPE_NULL/LOOP (#1502, #1574)

docs/source/ext.rst

@@ -30,6 +30,8 @@ The protocol classes :mod:`pcapkit` ships:
 +                                                                  +                +-----------------------+-------------------------------------------------------------+
 |                                                                  | L2TP Family    | :class:`pcapkit.protocols.link.l2tpv2.L2TPv2`                                       |
 +                                                                  +----------------+-----------------------+-------------------------------------------------------------+
+|                                                                  | :class:`pcapkit.protocols.link.loopback.Loopback`                                                    |
++                                                                  +----------------+-----------------------+-------------------------------------------------------------+
 |                                                                  |                | :class:`pcapkit.protocols.link.vlan.VLAN`                                           |
 +                                                                  +                +-----------------------+-------------------------------------------------------------+
 |                                                                  | VLAN Family    | :class:`pcapkit.protocols.link.c_tag.C_Tag`                                         |

docs/source/pcapkit/protocols/index.rst

@@ -27,7 +27,7 @@ from given information. The class hierarchy of :mod:`pcapkit.protocols`:
        A{{ProtocolMeta}} -.->|metaclass| B(ProtocolBase)
 
        subgraph link [Link Layer]
-           Link --> Ethernet & L2TP & VLAN & ARP
+           Link --> Ethernet & L2TP & Loopback & VLAN & ARP
 
            subgraph arp [ARP Family]
                ARP --> InARP
@@ -108,6 +108,7 @@ from given information. The class hierarchy of :mod:`pcapkit.protocols`:
        click Ethernet "/pcapkit/protocols/link/ethernet.html#pcapkit.protocols.link.ethernet.Ethernet"
        click L2TP "/pcapkit/protocols/link/l2tp.html#pcapkit.protocols.link.l2tp.L2TP"
        click L2TPv2 "/pcapkit/protocols/link/l2tpv2.html#pcapkit.protocols.link.l2tpv2.L2TPv2"
+       click Loopback "/pcapkit/protocols/link/loopback.html#pcapkit.protocols.link.loopback.Loopback"
        click VLAN "/pcapkit/protocols/link/vlan.html#pcapkit.protocols.link.vlan.VLAN"
        click C_Tag "/pcapkit/protocols/link/c_tag.html#pcapkit.protocols.link.c_tag.C_Tag"
        click S_Tag "/pcapkit/protocols/link/s_tag.html#pcapkit.protocols.link.s_tag.S_Tag"

docs/source/pcapkit/protocols/link/index.rst

@@ -16,6 +16,7 @@ link layer, with detailed implementation and methods.
    arp
    l2tp
    l2tpv2
+   loopback
    vlan
    c_tag
    s_tag

docs/source/pcapkit/protocols/link/loopback.rst

@@ -0,0 +1,65 @@
+BSD Loopback Encapsulation
+==========================
+
+.. module:: pcapkit.protocols.link.loopback
+
+:mod:`pcapkit.protocols.link.loopback` contains
+:class:`~pcapkit.protocols.link.loopback.Loopback`
+only, which implements extractor for the BSD loopback
+encapsulation [*]_ of ``LINKTYPE_NULL`` and ``LINKTYPE_LOOP``,
+whose structure is described as below:
+
+.. table::
+
+   ====== ===== =================== ===================================
+   Octets Bits  Name                Description
+   ====== ===== =================== ===================================
+   0          0 ``loopback.family`` Address Family (Internet Layer)
+   ====== ===== =================== ===================================
+
+The address family is in the byte order of the capturing host for
+``LINKTYPE_NULL``, which no capture format records, so it is inferred from the
+octets; for ``LINKTYPE_LOOP`` it is always in network byte order. Either way it
+is kept as ``loopback.byteorder``.
+
+.. autoclass:: pcapkit.protocols.link.loopback.Loopback
+   :no-members:
+   :show-inheritance:
+
+   .. autoproperty:: name
+   .. autoproperty:: length
+   .. autoproperty:: protocol
+
+   .. automethod:: read
+   .. automethod:: make
+
+   .. automethod:: _make_data
+
+   .. automethod:: __index__
+
+   .. autoattribute:: __proto__
+      :no-value:
+
+.. autofunction:: pcapkit.protocols.link.loopback.family_byteorder
+
+Header Schemas
+--------------
+
+.. module:: pcapkit.protocols.schema.link.loopback
+
+.. autoclass:: pcapkit.protocols.schema.link.loopback.Loopback
+   :members:
+   :show-inheritance:
+
+Data Models
+-----------
+
+.. module:: pcapkit.protocols.data.link.loopback
+
+.. autoclass:: pcapkit.protocols.data.link.loopback.Loopback
+   :members:
+   :show-inheritance:
+
+.. rubric:: Footnotes
+
+.. [*] https://www.tcpdump.org/linktypes/LINKTYPE_NULL.html

docs/source/pcapkit/protocols/misc/pcap.rst

@@ -114,6 +114,7 @@ whose structure is described as below:
    .. automethod:: make
 
    .. automethod:: _make_data
+   .. automethod:: _dispatch_linktype
 
    .. autoattribute:: __proto__
       :no-value:

docs/source/pcapkit/protocols/misc/pcapng.rst

@@ -38,6 +38,7 @@ which implements extractor for PCAP-NG file format [*]_.
    .. automethod:: index
 
    .. automethod:: _decode_next_layer
+   .. automethod:: _dispatch_linktype
 
    .. automethod:: _get_payload
    .. automethod:: _make_data

examples/generators/dispatch.py

@@ -1,5 +1,5 @@
 # -*- coding: utf-8 -*-
-"""Enumerate the seven ``__proto__`` dispatch registries, and probe every entry.
+"""Enumerate the eight ``__proto__`` dispatch registries, and probe every entry.
 
 :file:`pcapkit/foundation/registry/protocols.py` documents the registries that
 decide *which* :class:`~pcapkit.protocols.protocol.Protocol` subclass parses the
@@ -274,6 +274,12 @@ def _ethernet(ethertype: 'int', payload: 'bytes') -> 'bytes':
     return bytes(Ethernet(type=ethertype, payload=payload))
 
 
+def _loopback(family: 'int', payload: 'bytes', byteorder: 'str' = 'little') -> 'bytes':
+    """A BSD loopback header carrying ``payload``."""
+    from pcapkit.protocols.link.loopback import Loopback
+    return bytes(Loopback(family=family, byteorder=byteorder, payload=payload))
+
+
 ###############################################################################
 # Link -- EtherType.
 ###############################################################################
@@ -508,7 +514,8 @@ def _linktype_payload(code: 'Any') -> 'tuple[bytes, bool]':
     Returns:
         The record's octets, and whether they still need an Ethernet wrapper
         (true for :attr:`~pcapkit.const.reg.linktype.LinkType.ETHERNET`, false
-        for the two raw-IP link types, which carry no link layer at all).
+        for the two raw-IP link types, which carry no link layer at all, and
+        for the two loopback link types, whose header is built here).
 
     """
     from pcapkit.const.reg.linktype import LinkType
@@ -519,6 +526,10 @@ def _linktype_payload(code: 'Any') -> 'tuple[bytes, bool]':
         return _ip4(6, _tcp(9999)), False
     if code == LinkType.IPV6:
         return _ip6(6, _tcp(9999)), False
+    if code == LinkType.NULL:  # in the capturing host's byte order (#1574)
+        return _loopback(2, _ip4(6, _tcp(9999))), False
+    if code == LinkType.LOOP:  # in network byte order (#1574)
+        return _loopback(2, _ip4(6, _tcp(9999)), byteorder='big'), False
     raise LookupError(f'linktype: no payload builder for {code!r}')  # pragma: no cover
 
 
@@ -539,6 +550,21 @@ def _pcapng_build(code: 'Any') -> 'Any':
     return _extract_first(_pcapng_bytes(code, record), '.pcapng')
 
 
+###############################################################################
+# Loopback -- address family.
+###############################################################################
+
+
+def _loopback_registry() -> 'Any':
+    from pcapkit.protocols.link.loopback import Loopback
+    return Loopback.__proto__
+
+
+def _loopback_build(family: 'int') -> 'Any':
+    payload = _ip4(6, _tcp(9999)) if family == 2 else _ip6(6, _tcp(9999))
+    return _extract_first(_pcap_bytes((_loopback(family, payload),), linktype=0), '.pcap')
+
+
 ###############################################################################
 # The families, the cases, and the probe.
 ###############################################################################
@@ -571,7 +597,7 @@ def _internet_enum() -> 'Any':
 
 
 #: Every family this module exercises, one per ``__proto__`` table named in
-#: GitHub issue #496.
+#: GitHub issue #496, and the loopback table #1574 added.
 FAMILIES = (
     Family('link', _link_registry, _link_build, _link_enum),
     Family('internet', _internet_registry, _internet_build, _internet_enum),
@@ -580,6 +606,7 @@ def _internet_enum() -> 'Any':
     Family('sctp', _sctp_registry, _sctp_build),
     Family('pcap-frame', _pcap_frame_registry, _pcap_frame_build),
     Family('pcapng-frame', _pcapng_registry, _pcapng_build),
+    Family('loopback', _loopback_registry, _loopback_build),
 )
 
 #: The families keyed by label, for a caller -- :func:`probe` -- that wants
@@ -665,14 +692,24 @@ def _internet_enum() -> 'Any':
         ('pcapkit.protocols.application.ngap', 'NGAP'),
 
     # -- Frame.__proto__ (pcap LinkType) ---------------------------------------
+    'pcap-frame/NULL': ('pcapkit.protocols.link', 'Loopback'),
     'pcap-frame/ETHERNET': ('pcapkit.protocols.link', 'Ethernet'),
+    'pcap-frame/LOOP': ('pcapkit.protocols.link', 'Loopback'),
     'pcap-frame/IPV4': ('pcapkit.protocols.internet', 'IPv4'),
     'pcap-frame/IPV6': ('pcapkit.protocols.internet', 'IPv6'),
 
     # -- PCAPNG.__proto__ (LinkType) --------------------------------------------
+    'pcapng-frame/NULL': ('pcapkit.protocols.link', 'Loopback'),
     'pcapng-frame/ETHERNET': ('pcapkit.protocols.link', 'Ethernet'),
+    'pcapng-frame/LOOP': ('pcapkit.protocols.link', 'Loopback'),
     'pcapng-frame/IPV4': ('pcapkit.protocols.internet', 'IPv4'),
     'pcapng-frame/IPV6': ('pcapkit.protocols.internet', 'IPv6'),
+
+    # -- Loopback.__proto__ (address family) ------------------------------------
+    'loopback/2': ('pcapkit.protocols.internet.ipv4', 'IPv4'),
+    'loopback/24': ('pcapkit.protocols.internet.ipv6', 'IPv6'),
+    'loopback/28': ('pcapkit.protocols.internet.ipv6', 'IPv6'),
+    'loopback/30': ('pcapkit.protocols.internet.ipv6', 'IPv6'),
 }
 
 

pcapkit/__init__.py

@@ -113,7 +113,7 @@
     'Raw',                                                  # Raw Packet
 
     'ARP', 'C_Tag', 'Ethernet', 'InARP', 'L2TP',            # Link Layer
-    'L2TPv2', 'S_Tag', 'VLAN',
+    'L2TPv2', 'Loopback', 'S_Tag', 'VLAN',
 
     'AH', 'ESP', 'IP', 'IPsec', 'IPv4', 'IPv6', 'IPX',      # Internet Layer
     'HIP', 'HOPOPT', 'IPv6_Frag', 'IPv6_Ext', 'IPv6_Opts', 'IPv6_Route', 'MH',

pcapkit/all.py

@@ -120,7 +120,7 @@
     'NoPayload',                                            # No Payload
     'Raw',                                                  # Raw Packet
     'ARP', 'C_Tag', 'Ethernet', 'InARP', 'L2TP', 'L2TPv2',
-    'S_Tag', 'VLAN',                                        # Link Layer
+    'Loopback', 'S_Tag', 'VLAN',                            # Link Layer
     'AH', 'ESP', 'IP', 'IPsec', 'IPv4', 'IPv6', 'IPX',      # Internet Layer
     'HIP', 'HOPOPT', 'IPv6_Ext', 'IPv6_Frag', 'IPv6_Opts', 'IPv6_Route', 'MH',
                                                             # IPv6 Extension Header

pcapkit/foundation/engines/dpkt.py

@@ -11,6 +11,7 @@
 
 """
 import decimal
+import functools
 from typing import TYPE_CHECKING, cast
 
 from pcapkit.const.pcapng.block_type import BlockType as Enum_BlockType
@@ -298,6 +299,58 @@ def __iter__(self) -> 'Iterator[tuple[float, bytes]]':
     return _Reader
 
 
+@functools.lru_cache(maxsize=None)
+def _loopback_protocol(base: 'Type[DPKTPacket]', ip: 'Type[DPKTPacket]', ip6: 'Type[DPKTPacket]',
+                       linktype: 'int') -> 'Type[DPKTPacket]':
+    """A :class:`dpkt.loopback.Loopback` that reads as the default engine's.
+
+    It reads the address family in the byte order
+    :func:`~pcapkit.protocols.link.loopback.family_byteorder` gives for
+    ``linktype`` -- network byte order for ``LINKTYPE_LOOP``, inferred for
+    ``LINKTYPE_NULL`` -- and dispatches on the same families as
+    :class:`~pcapkit.protocols.link.loopback.Loopback`. Unlike `DPKT`_'s own, it
+    names an IPv4 or IPv6 payload ``ip`` or ``ip6``, as
+    :class:`dpkt.ethernet.Ethernet` does and as :mod:`pcapkit.toolkit.dpkt`
+    looks for it, and keeps a payload of any other family as raw octets rather
+    than parsing it as Ethernet (:issue:`1574`).
+
+    Args:
+        base: :class:`dpkt.loopback.Loopback` itself.
+        ip: :class:`dpkt.ip.IP`, the payload of family 2.
+        ip6: :class:`dpkt.ip6.IP6`, the payload of families 24, 28 and 30.
+        linktype: ``LINKTYPE_NULL`` or ``LINKTYPE_LOOP``.
+
+    Returns:
+        The packet type, built once per argument tuple rather than once per
+        packet. Every class it is built from is part of that key, so a reloaded
+        `DPKT`_ module gets a class of its own instead of one holding stale
+        classes.
+
+    .. _DPKT: https://dpkt.readthedocs.io
+
+    """
+    import dpkt  # isort:skip
+
+    from pcapkit.protocols.link.loopback import family_byteorder  # isort:skip
+
+    layers = {2: ip, 24: ip6, 28: ip6, 30: ip6}
+
+    class Loopback(base):
+        """BSD loopback encapsulation."""
+
+        def unpack(ext, buf: 'bytes') -> 'None':
+            dpkt.dpkt.Packet.unpack(ext, buf)
+            family = buf[:ext.__hdr_len__]
+            ext.family = int.from_bytes(family, family_byteorder(family, linktype))
+            layer = layers.get(ext.family)
+            if layer is not None:
+                payload = layer(buf[ext.__hdr_len__:])
+                ext.data = payload
+                setattr(ext, layer.__name__.lower(), payload)
+
+    return Loopback
+
+
 class DPKT(EngineBase['DPKTPacket']):
     """DPKT engine support.
 
@@ -434,7 +487,7 @@ def read_frame(self) -> 'DPKTPacket':
         # interface this packet was captured on, so ask only after reading it
         timestamp, pkt = cast('tuple[float, bytes]', next(reader))
         linktype = Enum_LinkType.get(reader.datalink())
-        protocol = self._get_protocol(linktype)
+        protocol = self._get_protocol(linktype, pkt)
         try:
             packet = protocol(pkt)  # type: DPKTPacket
         except Exception as exc:  # pylint: disable=broad-except
@@ -511,25 +564,36 @@ def read_frame(self) -> 'DPKTPacket':
     # Utilities.
     ##########################################################################
 
-    def _get_protocol(self, linktype: 'Optional[Enum_LinkType]' = None) -> 'Type[DPKTPacket]':
+    def _get_protocol(self, linktype: 'Optional[Enum_LinkType]' = None,
+                      data: 'bytes' = b'') -> 'Type[DPKTPacket]':
         """Returns the protocol for parsing the current packet.
 
         Args:
             linktype: Link type code.
+            data: Octets of the current packet. ``LINKTYPE_RAW`` (101) carries
+                a bare IPv4 or IPv6 datagram, and only the version nibble of
+                its first octet tells which (:issue:`1502`).
+
+        Notes:
+            ``LINKTYPE_NULL`` (0) and ``LINKTYPE_LOOP`` (108) are parsed with
+            :func:`_loopback_protocol` (:issue:`1574`).
 
         """
         dpkt = self._expkg
         reader = self._extmp
 
         if linktype is None:
             linktype = Enum_LinkType.get(reader.datalink())
+        version = data[0] >> 4 if data else None
 
         if linktype == Enum_LinkType.ETHERNET:
             pkg = dpkt.ethernet.Ethernet
-        elif linktype.value == Enum_LinkType.IPV4:
+        elif linktype.value == Enum_LinkType.IPV4 or (linktype.value == Enum_LinkType.RAW and version == 4):
             pkg = dpkt.ip.IP
-        elif linktype.value == Enum_LinkType.IPV6:
+        elif linktype.value == Enum_LinkType.IPV6 or (linktype.value == Enum_LinkType.RAW and version == 6):
             pkg = dpkt.ip6.IP6
+        elif linktype.value in (Enum_LinkType.NULL, Enum_LinkType.LOOP):
+            pkg = _loopback_protocol(dpkt.loopback.Loopback, dpkt.ip.IP, dpkt.ip6.IP6, linktype.value)
         else:
             warn('unrecognised link layer protocol; all analysis functions ignored',
                  DPKTWarning, stacklevel=stacklevel())

pcapkit/protocols/__init__.py

@@ -50,7 +50,7 @@
     'Raw',
 
     # Link Layer
-    'ARP', 'C_Tag', 'Ethernet', 'InARP', 'L2TP', 'L2TPv2', 'S_Tag', 'VLAN',
+    'ARP', 'C_Tag', 'Ethernet', 'InARP', 'L2TP', 'L2TPv2', 'Loopback', 'S_Tag', 'VLAN',
 
     # Internet Layer
     'AH', 'ESP', 'IP', 'IPsec', 'IPv4', 'IPv6', 'IPX',

pcapkit/protocols/data/__init__.py

@@ -61,6 +61,9 @@
     # Ethernet Protocol
     'Ethernet',
 
+    # BSD Loopback Encapsulation
+    'Loopback',
+
     # 802.1Q/802.1ad VLAN Tag Types
     'VLAN', 'VLAN_TCI',
 

pcapkit/protocols/data/link/__init__.py

@@ -9,6 +9,9 @@
 # Ethernet Protocol
 from pcapkit.protocols.data.link.ethernet import Ethernet
 
+# BSD Loopback Encapsulation
+from pcapkit.protocols.data.link.loopback import Loopback
+
 # 802.1Q/802.1ad VLAN Tag Types
 from pcapkit.protocols.data.link.vlan import TCI as VLAN_TCI
 from pcapkit.protocols.data.link.vlan import VLAN
@@ -20,6 +23,9 @@
     # Ethernet Protocol
     'Ethernet',
 
+    # BSD Loopback Encapsulation
+    'Loopback',
+
     # 802.1Q/802.1ad VLAN Tag Types
     'VLAN', 'VLAN_TCI',
 ]