From b1c6cc2481a939735454d45e4d6a7a94b1306d48 Mon Sep 17 00:00:00 2001 From: Jarry Shaw Date: Fri, 9 Oct 2026 18:38:16 -0400 Subject: [PATCH] fix(toolkit): accept tshark's True/False TCP flags in the pyshark trace (#1514) - Flags: tcp_traceflow parsed tcp.flags_syn/fin/reset with bool(int(...)), but tshark 4.x reports them as 'True'/'False', so the first traced TCP frame raised ValueError. _parse_flag accepts both spellings and raises ProtocolError otherwise. - Timestamp: frame.time_epoch was handed over as text; it is now float(Decimal(...)), the default engine's float. - Sequence numbers: tcp.seq/tcp.ack are tshark's relative numbers; read tcp.seq_raw/tcp.ack_raw, falling back where they are absent. - Frame layer: packet2dict, encap_type and time_epoch read frame_info, which is pkt_comment on a commented packet; look the frame layer up by name. - Index: take the engine's frame count, as dpkt and scapy do, not tshark's frame number. - Engine agreement: delete the #1514 Gap row; compare pyshark trace inputs without header/payload and pyshark flows by label and index only, since pyshark has no frame bytes (ruling on #1507). Unit tests cover each fix; tests/toolkit, the engine module, tests/project, the docstring contract and the tier guard pass. Closes #1514 --- pcapkit/foundation/engines/pyshark.py | 2 +- pcapkit/toolkit/pyshark.py | 163 ++++++++- .../engines/test_engine_agreement_runtime.py | 40 ++- tests/toolkit/test_pyshark_unit.py | 313 +++++++++++++++++- 4 files changed, 492 insertions(+), 26 deletions(-) diff --git a/pcapkit/foundation/engines/pyshark.py b/pcapkit/foundation/engines/pyshark.py index 7628cce4b0..c2093b9cb7 100644 --- a/pcapkit/foundation/engines/pyshark.py +++ b/pcapkit/foundation/engines/pyshark.py @@ -261,7 +261,7 @@ def read_frame(self) -> 'PySharkPacket': # trace flows if ext._flag_t: if ext._tcp: - data_tf_tcp = tcp_traceflow(packet) + data_tf_tcp = tcp_traceflow(packet, count=ext._frnum) if data_tf_tcp is not None: ext._trace.tcp(data_tf_tcp) diff --git a/pcapkit/toolkit/pyshark.py b/pcapkit/toolkit/pyshark.py index d9e2324d69..b4173afa81 100644 --- a/pcapkit/toolkit/pyshark.py +++ b/pcapkit/toolkit/pyshark.py @@ -16,12 +16,13 @@ the `PyShark`_ engine. """ +import decimal import ipaddress from typing import TYPE_CHECKING, cast from pcapkit.const.reg.linktype import LinkType as Enum_LinkType from pcapkit.foundation.traceflow.data.tcp import Packet as TF_TCP_Packet -from pcapkit.utilities.exceptions import MissingKeyError +from pcapkit.utilities.exceptions import MissingKeyError, ProtocolError if TYPE_CHECKING: from typing import Any @@ -316,6 +317,107 @@ 'xra': Enum_LinkType.DOCSIS31_XRA31, # docsis31_xra31 } # type: dict[str, Enum_LinkType] +#: PyShark's text for a boolean field -> its value. tshark 4.x spells one ``'True'``/``'False'`` +#: (measured for ``tcp.flags_syn``, ``tcp.flags_fin`` and ``tcp.flags_reset`` with PyShark 0.6 +#: on tshark 4.6.9); ``'1'``/``'0'`` is the spelling this module first parsed (#1514). +_BOOLEAN_FIELD = {'1': True, '0': False, 'True': True, 'False': False} # type: dict[str, bool] + + +def _parse_flag(value: 'Any', name: 'str') -> 'bool': + """Parse a boolean PyShark field, such as ``tcp.flags_syn``. + + Args: + value: The field as PyShark reports it, a ``LayerFieldsContainer`` + (a :obj:`str` subclass). + name: The field's name, for the error message. + + Returns: + The flag's value. + + Raises: + ProtocolError: If ``value`` is not a :obj:`str` spelling ``'1'``, + ``'0'``, ``'True'`` or ``'False'``. + + """ + if isinstance(value, str): + flag = _BOOLEAN_FIELD.get(str(value)) + if flag is not None: + return flag + raise ProtocolError(f'invalid PyShark boolean field {name}: {value!r}') + + +def _parse_epoch(value: 'Any') -> 'float': + """Parse PyShark's ``frame.time_epoch``, such as ``'1500000000.000638000'``. + + Args: + value: The field as PyShark reports it, decimal text. + + Returns: + The timestamp, as the same :obj:`float` the default engine reports. + + Note: + The default engine computes the epoch as a :class:`~decimal.Decimal` + and hands the tracer ``float()`` of it (:mod:`pcapkit.toolkit.pcap`). + This takes the same route, so both are the double nearest the same + decimal value: identical, not merely close. + + Raises: + ProtocolError: If ``value`` is not decimal text. + + """ + try: + return float(decimal.Decimal(str(value))) + except decimal.InvalidOperation: + raise ProtocolError(f'invalid PyShark frame.time_epoch: {value!r}') from None + + +def _absolute(tcp: 'Any', name: 'str') -> 'int': + """Read the absolute TCP sequence or acknowledgement number. + + Args: + tcp: PyShark TCP layer. + name: ``'seq'`` or ``'ack'``. + + Returns: + ``tcp._raw``, the number on the wire, or ``tcp.`` where + tshark does not report the former. + + Note: + tshark's ``tcp.seq`` and ``tcp.ack`` are *relative* to the flow's first + number by default (``0`` where the wire says ``832175175``). + ``tcp.seq_raw`` and ``tcp.ack_raw`` exist from Wireshark 3.2: they are + absent from 3.0.0's ``epan/dissectors/packet-tcp.c`` and present in + 3.2.0's and 4.2.2's. On an older tshark the fallback is relative unless + its ``tcp.relative_sequence_numbers`` preference is off. + + """ + value = getattr(tcp, f'{name}_raw', None) + if value is None: + value = getattr(tcp, name) + return int(value) + + +def _frame_layer(packet: 'Packet') -> 'Any': + """Find the ``frame`` layer of a PyShark packet. + + Args: + packet: PyShark packet. + + Returns: + The layer named ``frame``, or :data:`None` if the record has none. + + Note: + PyShark takes the second PDML ```` as ``frame_info``. On a packet + with a comment that is ``pkt_comment``, and the frame layer is + ``layers[0]`` (measured on ``test.pcapng`` frames 1 and 6 with tshark + 4.6.9), so the layer is looked up by name, as the engine does (#1531). + + """ + for layer in (packet.frame_info, *packet.layers): + if layer.layer_name == 'frame': + return layer + return None + def packet2dict(packet: 'Packet') -> 'dict[str, Any]': """Convert PyShark packet into :obj:`dict`. @@ -324,16 +426,27 @@ def packet2dict(packet: 'Packet') -> 'dict[str, Any]': packet: PyShark packet. Returns: - A :obj:`dict` mapping of packet data. + A :obj:`dict` mapping of packet data: the ``frame`` layer's fields at + the top level, and each later layer nested in the one before it. A + packet comment, which PyShark reports as ``frame_info`` in place of the + frame layer, is kept under its own ``PKT_COMMENT`` key. A record with + no ``frame`` layer keeps PyShark's ``frame_info`` at the top level. """ dict_ = {} # type: dict[str, Any] - frame = packet.frame_info + frame = _frame_layer(packet) + if frame is None: + frame = packet.frame_info for field in frame.field_names: dict_[field] = getattr(frame, field) + if packet.frame_info is not frame: + note = packet.frame_info + dict_[note.layer_name.upper()] = {field: getattr(note, field) for field in note.field_names} tempdict = dict_ for layer in packet.layers: + if layer is frame: + continue tempdict[layer.layer_name.upper()] = {} tempdict = tempdict[layer.layer_name.upper()] for field in layer.field_names: @@ -342,11 +455,14 @@ def packet2dict(packet: 'Packet') -> 'dict[str, Any]': return dict_ -def tcp_traceflow(packet: 'Packet') -> 'TF_TCP_Packet | None': +def tcp_traceflow(packet: 'Packet', *, count: 'int' = -1) -> 'TF_TCP_Packet | None': """Trace packet flow for TCP. Args: packet: PyShark packet. + count: Packet index. If not provided, default to ``-1``. The engine + passes its own frame count, not tshark's ``frame.number``, which + also numbers records that are not packets (#1515). Returns: Data for TCP flow tracing. @@ -356,6 +472,12 @@ def tcp_traceflow(packet: 'Packet') -> 'TF_TCP_Packet | None': * If the ``packet`` can be reassembled, then the :obj:`dict` mapping of data for TCP flow tracing (:term:`trace.tcp.packet`) will be returned; otherwise, returns :data:`None`. + Raises: + MissingKeyError: If the frame's link type cannot be resolved. + ProtocolError: If the packet has no ``frame`` layer, a TCP flag field is + not a boolean tshark spelling, or ``frame.time_epoch`` is not decimal + text. + See Also: :class:`pcapkit.foundation.traceflow.tcp.TCP` @@ -395,9 +517,16 @@ def tcp_traceflow(packet: 'Packet') -> 'TF_TCP_Packet | None': # :exc:`~pcapkit.utilities.exceptions.MissingKeyError` -- this # package's own house exception for a lookup miss -- rather than # letting it escape this public function. - encap_type = getattr(packet.frame_info, 'encap_type', None) + # + # The frame layer is found by name, not taken as ``frame_info``, which + # is ``pkt_comment`` on a commented packet; the root layer is then the + # first layer that is not the frame layer. + frame = _frame_layer(packet) + if frame is None: + raise ProtocolError(f'PyShark packet {packet.number} has no frame layer') + encap_type = getattr(frame, 'encap_type', None) if encap_type is None: - name = packet.layers[0].layer_name + name = [layer for layer in packet.layers if layer is not frame][0].layer_name try: protocol = FILTER_NAME_TO_LINKTYPE[name.lower()] except KeyError: @@ -408,20 +537,28 @@ def tcp_traceflow(packet: 'Packet') -> 'TF_TCP_Packet | None': except (KeyError, ValueError): raise MissingKeyError('frame.encap_type=%s' % encap_type) from None + # A Simple Packet Block carries no timestamp, so tshark reports no + # ``frame.time_epoch`` for it; the PDML ``geninfo`` timestamp PyShark + # keeps as ``sniff_timestamp`` is then ``'0.000000000'``, which is what + # the default engine reports (measured on ``test.pcapng`` frame 4). + epoch = getattr(frame, 'time_epoch', None) + if epoch is None: + epoch = packet.sniff_timestamp + data = TF_TCP_Packet( # type: ignore[type-var] protocol=protocol, # data link type - index=int(packet.number), # frame number + index=count, # frame number frame=packet2dict(packet), # extracted packet - syn=bool(int(tcp.flags_syn)), # TCP synchronise (SYN) flag - fin=bool(int(tcp.flags_fin)), # TCP finish (FIN) flag - rst=bool(int(tcp.flags_reset)), # TCP reset (RST) flag + syn=_parse_flag(tcp.flags_syn, 'tcp.flags_syn'), # TCP synchronise (SYN) flag + fin=_parse_flag(tcp.flags_fin, 'tcp.flags_fin'), # TCP finish (FIN) flag + rst=_parse_flag(tcp.flags_reset, 'tcp.flags_reset'), # TCP reset (RST) flag src=ipaddress.ip_address(ip.src), # source IP dst=ipaddress.ip_address(ip.dst), # destination IP srcport=int(tcp.srcport), # TCP source port dstport=int(tcp.dstport), # TCP destination port - timestamp=packet.frame_info.time_epoch, # timestamp - seq=int(tcp.seq), # TCP sequence number - ack=int(tcp.ack), # TCP acknowledgement number + timestamp=_parse_epoch(epoch), # timestamp + seq=_absolute(tcp, 'seq'), # TCP sequence number + ack=_absolute(tcp, 'ack'), # TCP acknowledgement number # NOTE: PyShark reports dissected *fields*, not the octets behind # them, so there is no header or payload to hand over -- which is # the same reason this module carries no ``tcp_reassembly`` at all. diff --git a/tests/foundation/engines/test_engine_agreement_runtime.py b/tests/foundation/engines/test_engine_agreement_runtime.py index 2acb389412..c542466174 100644 --- a/tests/foundation/engines/test_engine_agreement_runtime.py +++ b/tests/foundation/engines/test_engine_agreement_runtime.py @@ -19,12 +19,14 @@ hands the reassembler, keyed by frame number -- every field; ``trace`` every :class:`~pcapkit.foundation.traceflow.data.tcp.Packet` handed to the - flow tracer, keyed by frame number -- every field but ``frame``, which is - the engine's own frame object; + flow tracer, keyed by frame number -- every field except ``frame``, which + is the engine's own frame object, and except ``header`` and ``payload`` for + an engine in :data:`NO_TCP_OCTETS`; ``datagrams`` every reassembled datagram, every field but the lazily parsed ``packet``; ``flows`` - every traced flow's label and index, and the octets of its PCAP file. + every traced flow's label and index, and the octets of its PCAP file unless + the engine is in :data:`JSON_TRACE`. The inputs are captured by wrapping :meth:`ReassemblyBase.__call__ ` and @@ -80,6 +82,20 @@ 'pyshark': 'pyshark reports dissected fields, not the frame octets', 'pypcapfile': 'the pypcapfile engine replaces the frame octets with decoded layers', } +#: Engines whose flow-tracing input carries no TCP octets, and why. Their ``trace`` +#: comparison leaves out ``header`` and ``payload`` and compares every other field. +NO_TCP_OCTETS = { + 'pyshark': 'pyshark reports dissected fields, not the octets behind them, so tcp_traceflow ' + "hands the tracer empty ones and Extractor refuses trace_analyse for it " + '(pcapkit/foundation/extraction.py:1446-1455; ruling on #1507)', +} +#: Engines whose flow files are JSON whatever ``trace_format`` asks for, and why. Their +#: ``flows`` comparison checks each flow's label and frame indices, not the file's octets. +JSON_TRACE = { + 'pyshark': 'pyshark cannot supply the frame bytes a PCAP trace needs, so Extractor writes ' + "JSON with a FormatWarning (pcapkit/foundation/extraction.py:1440-1444; ruling " + 'on #1507)', +} #: Captures with at least one TCP flow, measured on ``00d232890``. TRACED = ('http.pcap', 'http6.cap', 'in.pcap', 'many_interfaces.pcapng', 'options-tcp.pcap', 'options-transport.pcap', 'stream.pcap', 'tcp.pcap', 'test.pcap') @@ -205,6 +221,12 @@ def _run(engine: 'str', capture: 'str') -> '_Run': return _RUNS[key] +def _without_tcp_octets(inputs: 'dict[int, Any]') -> 'dict[int, Any]': + """``trace`` inputs without ``header`` and ``payload``, per :data:`NO_TCP_OCTETS`.""" + return {number: [{key: value for key, value in packet.items() if key not in ('header', 'payload')} + for packet in packets] for number, packets in inputs.items()} + + def _first_difference(got: 'dict[int, Any]', want: 'dict[int, Any]') -> 'Optional[str]': for number in sorted(set(got) | set(want)): if number not in got: @@ -251,7 +273,10 @@ def compare(engine: 'str', capture: 'str', aspect: 'str') -> 'Outcome': return Outcome('RECORDS', f'{len(mine)} {aspect} != {len(theirs)}') return Outcome('OK') if aspect in run.inputs: - diff = _first_difference(run.inputs[aspect], base.inputs[aspect]) + mine, theirs = run.inputs[aspect], base.inputs[aspect] + if aspect == 'trace' and engine in NO_TCP_OCTETS: + mine, theirs = _without_tcp_octets(mine), _without_tcp_octets(theirs) + diff = _first_difference(mine, theirs) return Outcome('INPUTS', diff) if diff else Outcome('OK') if aspect == 'datagrams': kinds = [kind for kind in ('ipv4', 'ipv6', 'tcp') if run.datagrams[kind] is not None] @@ -273,6 +298,8 @@ def compare(engine: 'str', capture: 'str', aspect: 'str') -> 'Outcome': if [f[:2] for f in run.flows] != [f[:2] for f in base.flows]: return Outcome('FLOWS', f'flows {[f[:2] for f in run.flows]!r:.200} != ' f'{[f[:2] for f in base.flows]!r:.200}') + if engine in JSON_TRACE: + return Outcome('OK') for (label, _, mine), (_, _, theirs) in zip(run.flows, base.flows): if mine != theirs: return Outcome('FLOWS', f'{label}: file of {len(mine)} octets != {len(theirs)} ' @@ -313,11 +340,6 @@ class TestEngineAgreement(harness.RoundTripBase): tuple(f'{capture}/pypcapfile/{aspect}' for capture in ('http6.cap', 'stream.pcap', 'tcp.pcap', 'test.pcap') for aspect in ('tcp', 'trace', 'datagrams', 'flows'))), - Gap(1514, "the pyshark engine raises ValueError on the first TCP frame it traces: " - 'tcp_traceflow reads the flags with bool(int(tcp.flags_syn)) (pcapkit/toolkit/' - "pyshark.py:415-417), and tshark 4.x reports them as 'True' and 'False'", - 'ERROR', "ValueError: invalid literal for int() with base 10: ", - tuple(f'{capture}/pyshark/*' for capture in TRACED)), Gap(1515, 'the pyshark engine counts a Systemd Journal Export Block as a frame: tshark ' 'numbers it as one, and read_frame takes every tshark frame ' '(pcapkit/foundation/engines/pyshark.py:245-248)', diff --git a/tests/toolkit/test_pyshark_unit.py b/tests/toolkit/test_pyshark_unit.py index fc7b82ea5d..8c40dd4094 100644 --- a/tests/toolkit/test_pyshark_unit.py +++ b/tests/toolkit/test_pyshark_unit.py @@ -20,9 +20,13 @@ def __init__(self, layer_name: str, **fields: object) -> None: class FakePySharkPacket: def __init__(self, *, ipv6: bool = False, tcp: bool = True, ip: bool = True, ether: bool = True, - link_layer: str = 'eth', encap_type: 'str | None' = None) -> None: + link_layer: str = 'eth', encap_type: 'str | None' = None, + time_epoch: 'object' = 70.5, comment: 'str | None' = None) -> None: self.number = '12' - frame_fields = {'time_epoch': 70.5, 'cap_len': '54'} # type: dict[str, object] + frame_fields = {'cap_len': '54'} # type: dict[str, object] + if time_epoch is not None: + # absent for a Simple Packet Block, which carries no timestamp + frame_fields['time_epoch'] = time_epoch if encap_type is not None: # Real PyShark hands every PDML field over as a # ``LayerFieldsContainer``, which subclasses ``str`` -- so @@ -80,6 +84,14 @@ def __init__(self, *, ipv6: bool = False, tcp: bool = True, # What is under test is the unknown-name path, which ``'ip'`` # exercises regardless of how a capture would reach it. self.layers = self.layers[1:] + if comment is not None: + # PyShark takes the second PDML ```` as ``frame_info``. On a + # commented packet that is ``pkt_comment``, and the frame layer + # becomes ``layers[0]`` -- measured on ``test.pcapng`` frames 1 and + # 6 with tshark 4.6.9: ``frame_info`` ``pkt_comment``, ``layers`` + # ``['frame', 'eth', 'ip', 'udp', 'dhcp']``. + self.layers.insert(0, self.frame_info) + self.frame_info = FakeLayer('pkt_comment', frame_comment=comment) def __contains__(self, name: str) -> bool: return name in self._contains @@ -413,7 +425,7 @@ def test_tcp_traceflow_end_to_end_for_an_ethernet_capture(self) -> None: from pcapkit.toolkit import pyshark as toolkit packet = FakePySharkPacket() - data = toolkit.tcp_traceflow(packet) + data = toolkit.tcp_traceflow(packet, count=12) self.assertIsNotNone(data) self.assertEqual(data.protocol, LinkType.ETHERNET) @@ -432,5 +444,300 @@ def test_tcp_traceflow_end_to_end_for_an_ethernet_capture(self) -> None: self.assertEqual(data.payload, bytearray()) +class LayerField(str): + """Stands in for PyShark's ``LayerFieldsContainer``, a :obj:`str` subclass.""" + + +@unittest.skipUnless(HAS_RUNTIME, 'runtime dependencies not installed') +class PySharkFlagTests(unittest.TestCase): + """#1514: tshark 4.x reports a boolean field as ``'True'``/``'False'``, which + ``bool(int(...))`` refused with :exc:`ValueError` on the first traced TCP frame. + + """ + + def setUp(self) -> None: + reimport_once_per_class(self) + + def test_parse_flag_accepts_both_spellings(self) -> None: + from pcapkit.toolkit import pyshark as toolkit + + for text, expected in (('1', True), ('0', False), ('True', True), ('False', False)): + for value in (text, LayerField(text)): + with self.subTest(value=value, type=type(value).__name__): + self.assertIs(toolkit._parse_flag(value, 'tcp.flags_syn'), expected) + + def test_parse_flag_rejects_anything_else(self) -> None: + from pcapkit.toolkit import pyshark as toolkit + from pcapkit.utilities.exceptions import ProtocolError + + for value in ('true', 'false', 'Set', '2', '', ' 1', 1, True, None): + with self.subTest(value=value): + with self.assertRaises(ProtocolError) as ctx: + toolkit._parse_flag(value, 'tcp.flags_fin') + self.assertEqual(str(ctx.exception), + f'invalid PyShark boolean field tcp.flags_fin: {value!r}') + + def test_tcp_traceflow_reads_tshark_4_flags(self) -> None: + from pcapkit.toolkit import pyshark as toolkit + + packet = FakePySharkPacket() + for flags in ((True, False, True), (False, True, False)): + syn, fin, rst = (LayerField(str(flag)) for flag in flags) + packet.tcp.flags_syn, packet.tcp.flags_fin, packet.tcp.flags_reset = syn, fin, rst + with self.subTest(flags=flags): + data = toolkit.tcp_traceflow(packet) + self.assertIsNotNone(data) + self.assertEqual((data.syn, data.fin, data.rst), flags) + + def test_tcp_traceflow_raises_for_an_unparsable_flag(self) -> None: + from pcapkit.toolkit import pyshark as toolkit + from pcapkit.utilities.exceptions import ProtocolError + + packet = FakePySharkPacket() + packet.tcp.flags_reset = LayerField('maybe') + with self.assertRaises(ProtocolError) as ctx: + toolkit.tcp_traceflow(packet) + self.assertIn('tcp.flags_reset', str(ctx.exception)) + + +@unittest.skipUnless(HAS_RUNTIME, 'runtime dependencies not installed') +class PySharkTraceFieldTests(unittest.TestCase): + """The ``timestamp``, ``seq`` and ``ack`` ``tcp_traceflow`` hands the tracer + must equal the default engine's. #1514's ``ValueError`` hid that they did not. + + """ + + def setUp(self) -> None: + reimport_once_per_class(self) + + def test_timestamp_is_the_default_engines_float(self) -> None: + """tshark prints ``frame.time_epoch`` to the nanosecond. The default + engine computes ``ts_sec + Decimal(ts_usec) / 10**6`` (or ``10**9``) + and hands the tracer ``float()`` of it + (``pcapkit/protocols/misc/pcap/frame.py``), so the float must be + identical, not just close. + + """ + import decimal + + from pcapkit.toolkit import pyshark as toolkit + + cases = [ + ('1500000000.000638000', 1500000000, 638, 1_000_000), + ('1511106549.913720000', 1511106549, 913720, 1_000_000), + ('1500000000.123456789', 1500000000, 123456789, 1_000_000_000), + ('1500000000.000000000', 1500000000, 0, 1_000_000), + ] + for text, seconds, fraction, resolution in cases: + with self.subTest(time_epoch=text): + packet = FakePySharkPacket() + packet.frame_info.time_epoch = LayerField(text) + data = toolkit.tcp_traceflow(packet) + self.assertIs(type(data.timestamp), float) + expected = float(decimal.Decimal(seconds) + decimal.Decimal(fraction) / resolution) + self.assertEqual(data.timestamp.hex(), expected.hex()) + + def test_unparsable_timestamp_raises(self) -> None: + from pcapkit.toolkit import pyshark as toolkit + from pcapkit.utilities.exceptions import ProtocolError + + packet = FakePySharkPacket() + packet.frame_info.time_epoch = LayerField('Jul 14, 2017') + with self.assertRaises(ProtocolError) as ctx: + toolkit.tcp_traceflow(packet) + self.assertEqual(str(ctx.exception), "invalid PyShark frame.time_epoch: 'Jul 14, 2017'") + + def test_seq_and_ack_are_absolute(self) -> None: + """tshark's ``tcp.seq``/``tcp.ack`` are relative by default; the + absolute numbers are ``tcp.seq_raw``/``tcp.ack_raw``. Measured on + ``http.pcap`` frame 2 with tshark 4.6.9: ``seq`` 0, ``seq_raw`` + 2847915401, ``ack`` 1, ``ack_raw`` 832175176. + + """ + from pcapkit.toolkit import pyshark as toolkit + + packet = FakePySharkPacket() + packet.tcp.seq, packet.tcp.seq_raw = LayerField('0'), LayerField('2847915401') + packet.tcp.ack, packet.tcp.ack_raw = LayerField('1'), LayerField('832175176') + data = toolkit.tcp_traceflow(packet) + self.assertEqual((data.seq, data.ack), (2847915401, 832175176)) + + def test_seq_and_ack_fall_back_without_raw_fields(self) -> None: + """A tshark older than 3.2 reports no ``*_raw`` field.""" + from pcapkit.toolkit import pyshark as toolkit + + packet = FakePySharkPacket() + self.assertFalse(hasattr(packet.tcp, 'seq_raw')) + self.assertFalse(hasattr(packet.tcp, 'ack_raw')) + data = toolkit.tcp_traceflow(packet) + self.assertEqual((data.seq, data.ack), (101, 202)) + + def test_simple_packet_block_takes_the_geninfo_timestamp(self) -> None: + """A Simple Packet Block has no ``frame.time_epoch``. PyShark's + ``sniff_timestamp`` is then ``'0.000000000'`` (``test.pcapng`` frame + 4, tshark 4.6.9), and the default engine reports 0 too. + + """ + from pcapkit.toolkit import pyshark as toolkit + + packet = FakePySharkPacket(time_epoch=None) + packet.sniff_timestamp = LayerField('0.000000000') + data = toolkit.tcp_traceflow(packet) + self.assertIs(type(data.timestamp), float) + self.assertEqual(data.timestamp, 0.0) + + def test_index_is_the_engine_count_not_the_tshark_number(self) -> None: + """Like the dpkt and scapy adapters, the index is the ``count`` the + engine passes. tshark's ``frame.number`` also counts records that are + not packets, such as a systemd Journal Export Block (#1515). + + """ + from pcapkit.toolkit import pyshark as toolkit + + packet = FakePySharkPacket() + packet.number = '6' + self.assertEqual(toolkit.tcp_traceflow(packet, count=5).index, 5) + self.assertEqual(toolkit.tcp_traceflow(packet).index, -1) + + def test_journal_block_then_two_tcp_packets(self) -> None: + """The #1531 review's capture: a journal block, then two TCP packets. + tshark numbers the packets 2 and 3; the default engine, and the + PyShark engine once it skips the journal block (#1531), count them 1 + and 2, and the index follows that count. + + """ + from pcapkit.toolkit import pyshark as toolkit + + indices = [] + for number, count in (('2', 1), ('3', 2)): + packet = FakePySharkPacket() + packet.number = number + indices.append(toolkit.tcp_traceflow(packet, count=count).index) + self.assertEqual(indices, [1, 2]) + + def test_the_engine_passes_its_frame_count(self) -> None: + """:meth:`PyShark.read_frame` hands :func:`tcp_traceflow` its own frame + count. Without #1531 that count is still tshark's number, so the journal + block shifts it here (2 and 3); with #1531 it is 1 and 2. Either way the + traced index is the engine's count, never ``packet.number`` read again. + + """ + import types + + from pcapkit.foundation.engines.pyshark import PyShark + + journal = FakePySharkPacket(ip=False, tcp=False) + journal.number = '1' + journal.layers = [FakeLayer('systemd_journal', message='hello')] + journal.frame_info = FakeLayer('frame', time_epoch='1500000000.004000000') # no encap_type + first, second = FakePySharkPacket(encap_type='1'), FakePySharkPacket(encap_type='1') + first.number, second.number = '2', '3' + + traced = [] # type: list[int] + stream = iter([journal, first, second]) + extractor = types.SimpleNamespace( + _frnum=0, _vfunc=lambda e, f: None, _flag_q=True, _flag_d=False, _flag_t=True, + _tcp=True, _trace=types.SimpleNamespace(tcp=lambda data: traced.append(data.index))) + engine = PyShark.__new__(PyShark) # skip __init__, which imports pyshark + engine._extractor = extractor + engine._extmp = types.SimpleNamespace(next=lambda: next(stream)) + + counts = [] + while True: + try: + packet = engine.read_frame() + except StopIteration: + break + if packet is not journal: + counts.append(extractor._frnum) + self.assertEqual(traced, counts) + self.assertIn(traced, ([2, 3], [1, 2])) + + +@unittest.skipUnless(HAS_RUNTIME, 'runtime dependencies not installed') +class PySharkCommentedPacketTests(unittest.TestCase): + """On a packet with a comment, PyShark's ``frame_info`` is the + ``pkt_comment`` layer and the ``frame`` layer is ``layers[0]``, so the + frame fields are looked up on the layer named ``frame``. + + """ + + def setUp(self) -> None: + reimport_once_per_class(self) + + def test_encap_type_and_time_epoch_come_from_the_frame_layer(self) -> None: + from pcapkit.const.reg.linktype import LinkType + from pcapkit.toolkit import pyshark as toolkit + + packet = FakePySharkPacket(encap_type='174', time_epoch=LayerField('1500000000.005000000'), + comment='a packet comment') + self.assertEqual(packet.frame_info.layer_name, 'pkt_comment') + self.assertEqual([layer.layer_name for layer in packet.layers], ['frame', 'eth', 'ip', 'tcp']) + + data = toolkit.tcp_traceflow(packet, count=6) + self.assertEqual(data.protocol, LinkType.LOOP) # 174, not the 'eth' fallback + self.assertEqual(data.timestamp, 1500000000.005) + self.assertEqual(data.index, 6) + + def test_a_commented_tcp_packet_resolves_its_link_type(self) -> None: + """Reading ``encap_type`` from ``frame_info`` found none on the + ``pkt_comment`` layer, fell back to ``layers[0]``, which is ``frame``, + and raised ``MissingKeyError: 'frame'``. + + """ + from pcapkit.const.reg.linktype import LinkType + from pcapkit.toolkit import pyshark as toolkit + + packet = FakePySharkPacket(encap_type='1', comment='first packet of the second section') + data = toolkit.tcp_traceflow(packet) + self.assertEqual(data.protocol, LinkType.ETHERNET) + self.assertEqual(data.timestamp, 70.5) + + def test_packet2dict_puts_the_frame_layer_on_top(self) -> None: + """``frame_info`` put the comment's fields at the top level and nested + the frame fields under ``FRAME``. The frame layer's fields belong on + top, as for any other packet, and the comment keeps its own key. + + """ + from pcapkit.toolkit import pyshark as toolkit + + converted = toolkit.packet2dict(FakePySharkPacket(encap_type='1', comment='synthetic frame 0')) + self.assertEqual(converted['encap_type'], '1') + self.assertEqual(converted['time_epoch'], 70.5) + self.assertNotIn('frame_comment', converted) + self.assertNotIn('FRAME', converted) + self.assertEqual(converted['PKT_COMMENT'], {'frame_comment': 'synthetic frame 0'}) + self.assertEqual(converted['ETH']['IP']['TCP']['srcport'], '1234') + + def test_packet2dict_without_a_frame_layer_keeps_frame_info(self) -> None: + """A record with no ``frame`` layer still converts, from ``frame_info``.""" + from pcapkit.toolkit import pyshark as toolkit + + packet = FakePySharkPacket(tcp=False) + packet.frame_info = FakeLayer('syscall', num='1') + self.assertEqual(toolkit.packet2dict(packet)['num'], '1') + + def test_filter_name_fallback_skips_the_frame_layer(self) -> None: + """Without ``encap_type``, the root layer is the first one after + ``frame``, not ``layers[0]``, which is ``frame`` itself here. + + """ + from pcapkit.const.reg.linktype import LinkType + from pcapkit.toolkit import pyshark as toolkit + + data = toolkit.tcp_traceflow(FakePySharkPacket(comment='a packet comment')) + self.assertEqual(data.protocol, LinkType.ETHERNET) + + def test_a_packet_without_a_frame_layer_raises(self) -> None: + from pcapkit.toolkit import pyshark as toolkit + from pcapkit.utilities.exceptions import ProtocolError + + packet = FakePySharkPacket(comment='a packet comment') + packet.layers = packet.layers[1:] + with self.assertRaises(ProtocolError) as ctx: + toolkit.tcp_traceflow(packet) + self.assertEqual(str(ctx.exception), 'PyShark packet 12 has no frame layer') + + if __name__ == '__main__': unittest.main()