From 481d2b39094d72bf75606df45231456254d2796f Mon Sep 17 00:00:00 2001 From: Jarry Shaw Date: Tue, 20 Oct 2026 13:21:41 -0400 Subject: [PATCH 1/2] fix(pcapng): cut an unknown, custom or journal block the file truncates (#1984) - UnknownBlock.body, CustomBlock.data and SystemdJournalExportBlock.entry are each one BytesField sized from the Block Total Length. A length past the end of the file made that field declare the missing octets: within the zero-pad ceiling they were zero-filled, past it FieldValueError was raised inside the header schema, outside PCAPNG.read's _read_block_cut fallback, and the extraction lost every earlier frame. - The new held_body callback bounds the span by what is left of the stream, so the body is the octets the file holds and PCAPNG.read reports the truncation as it does for every other block type. It is a no-op on well-formed blocks and on the packing path. - Tests cover all three types above and below the ceiling, and check the earlier frames, the warning, the held body, and byte-exact rebuilds. Closes #1984 --- pcapkit/protocols/schema/misc/pcapng.py | 54 +++++++- ...est_pcapng_cut_block_past_eof_1984_unit.py | 131 ++++++++++++++++++ 2 files changed, 181 insertions(+), 4 deletions(-) create mode 100644 tests/protocols/misc/test_pcapng_cut_block_past_eof_1984_unit.py diff --git a/pcapkit/protocols/schema/misc/pcapng.py b/pcapkit/protocols/schema/misc/pcapng.py index ff61179147..2c7c018ad6 100644 --- a/pcapkit/protocols/schema/misc/pcapng.py +++ b/pcapkit/protocols/schema/misc/pcapng.py @@ -323,6 +323,52 @@ def callback(pkt: 'dict[str, Any]') -> 'int': return callback +def held_body(framing: 'int') -> 'Callable[[dict[str, Any]], int]': + """Bound a block body read as one field to the octets the stream holds. + + Args: + framing: Octets the block's fixed fields span, both copies of the Block + Total Length included. + + Returns: + A callback returning the block's Block Total Length less ``framing``, + never below zero, and never past what is left of the stream. + + The bodies of :class:`UnknownBlock`, :class:`CustomBlock` and + :class:`SystemdJournalExportBlock` are each one + :class:`~pcapkit.corekit.fields.strings.BytesField` spanning the whole + block, so a Block Total Length that runs past the end of the file is one + field declaring that many octets. Unbounded, that field is zero-filled + within :data:`~pcapkit.utilities.config.MAX_ZERO_PAD_LENGTH`, and past it + :meth:`FieldBase.unpack ` + raises :exc:`~pcapkit.utilities.exceptions.FieldValueError` from inside the + block's schema, where :meth:`PCAPNG.read + ` cannot keep the block as + captured, so the whole extraction ends (:issue:`1984`). The other block + types read their bodies field by field, so none of them declares that much + in one field. + + Bounded, the body is the octets the file actually holds, whatever the + block declares, and :meth:`PCAPNG.read + ` reports the truncation as it + does for every other block type. On a well-formed block the stream holds + the body and the trailing Block Total Length, so this is a no-op; on the + packing path :meth:`Schema.pack + ` leaves ``__length__`` at + ``-1`` for "unknown", and the declared span is kept. + + """ + floored = nonnegative(lambda pkt: pkt['length'] - framing) + + def callback(pkt: 'dict[str, Any]') -> 'int': + nominal = floored(pkt) + remaining = pkt.get('__length__') + if not isinstance(remaining, int) or remaining < 0: + return nominal + return min(nominal, remaining) + return callback + + def bounded_option(length: 'Callable[[dict[str, Any]], int]') -> 'Callable[[dict[str, Any]], int]': """Refuse an option or record payload that runs past its area. @@ -903,7 +949,7 @@ class UnknownBlock(BlockType): #: Block total length. length: 'int' = UInt32Field(callback=byteorder_callback) #: Block body (including padding). - body: 'bytes' = BytesField(length=nonnegative(lambda pkt: pkt['length'] - GENERIC_BLOCK_MINIMUM)) + body: 'bytes' = BytesField(length=held_body(GENERIC_BLOCK_MINIMUM)) #: Block total length. length2: 'int' = UInt32Field(callback=byteorder_callback) @@ -2089,7 +2135,7 @@ class SystemdJournalExportBlock(BlockType, code=Enum_BlockType.systemd_Journal_E #: Block total length. length: 'int' = UInt32Field(callback=byteorder_callback) #: Journal entry. - entry: 'bytes' = BytesField(length=nonnegative(lambda pkt: pkt['length'] - GENERIC_BLOCK_MINIMUM)) + entry: 'bytes' = BytesField(length=held_body(GENERIC_BLOCK_MINIMUM)) #: Block total length. length2: 'int' = UInt32Field(callback=byteorder_callback) @@ -2517,8 +2563,8 @@ class CustomBlock(BlockType, code=[Enum_BlockType.Custom_Block_that_rewriters_ca # NOTE: both custom block types span the same fixed fields, so the one # entry stands for both. #: Custom data (incl. padding and options). - data: 'bytes' = BytesField(length=nonnegative( - lambda pkt: pkt['length'] - BLOCK_MINIMUM[Enum_BlockType.Custom_Block_that_rewriters_can_copy_into_new_files])) + data: 'bytes' = BytesField(length=held_body( + BLOCK_MINIMUM[Enum_BlockType.Custom_Block_that_rewriters_can_copy_into_new_files])) #: Block total length. length2: 'int' = UInt32Field(callback=byteorder_callback) diff --git a/tests/protocols/misc/test_pcapng_cut_block_past_eof_1984_unit.py b/tests/protocols/misc/test_pcapng_cut_block_past_eof_1984_unit.py new file mode 100644 index 0000000000..e1d61c6d4f --- /dev/null +++ b/tests/protocols/misc/test_pcapng_cut_block_past_eof_1984_unit.py @@ -0,0 +1,131 @@ +# -*- coding: utf-8 -*- +"""GitHub issue #1984: an unknown, custom or journal block the file cuts short is kept. + +The bodies of an unknown block, a Custom Block and a systemd Journal Export +Block are each one field spanning the whole block. A Block Total Length past +the end of the file made that one field declare the missing octets, and past +the zero-pad ceiling the read raised +:exc:`~pcapkit.utilities.exceptions.FieldValueError` from inside the block's +schema, which ended the whole extraction and lost every earlier frame. The body +is now sized from the octets the file holds, at any declared length, so the +block is reported as truncated as every other block type is, and extraction +keeps the frames before it. + +:mod:`pcapkit` is imported inside each test, after +:func:`~tests._support.reimport_once_per_class`. + +""" + +import io +import struct +import unittest +import warnings + +from tests.protocols.misc.test_pcapng_roundtrip2_unit import PCAPNGTestCase + +FRAME = bytes(range(1, 61)) + +#: An unregistered block type, the systemd Journal Export Block, and a Custom Block. +UNKNOWN = 0x77 +JOURNAL = 0x9 +CUSTOM = 0xBAD + +#: Block Total Lengths past and within the zero-pad ceiling. +LENGTHS = (0x7FFF_FFF0, 0x10000) + +#: The octets the file holds of a cut block after its Block Total Length. +TAIL = bytes(range(1, 9)) + +#: The body field of each block type. +FIELD = {UNKNOWN: 'body', JOURNAL: 'entry', CUSTOM: 'data'} + + +def _block(btype: 'int', body: 'bytes') -> 'bytes': + """A little-endian block of ``btype`` around ``body``, padded to 32 bits.""" + body += bytes(-len(body) % 4) + total = 12 + len(body) + return struct.pack(' 'bytes': + """An Enhanced Packet Block on interface 0 holding :data:`FRAME`.""" + return _block(6, struct.pack(' 'bytes': + """A block of ``btype`` declaring ``length`` that the file ends inside.""" + return struct.pack(' 'tuple[list[bytes], list[str]]': + """Extract ``data``, returning its frames' packets and the warnings raised.""" + import pcapkit + + with warnings.catch_warnings(record=True) as caught: + warnings.simplefilter('always') + extraction = pcapkit.extract(fin=io.BytesIO(data), nofile=True, store=True, + engine='default') + return ([bytes(frame.info.packet) for frame in extraction.frame], + [str(item.message) for item in caught]) + + def test_earlier_frames_survive(self) -> None: + for btype in FIELD: + for length in LENGTHS: + with self.subTest(btype=hex(btype), length=hex(length)): + frames, messages = self._extract(SHB + IDB + _epb() + _cut(btype, length)) + self.assertEqual(frames, [FRAME]) + self.assertIn(f'PCAP-NG: [Block {btype}] block length {length} exceeds the ' + f'{len(TAIL) + 8} octet(s) left in the file; block truncated', + messages) + + def test_body_is_the_octets_held(self) -> None: + for btype, name in FIELD.items(): + for length in LENGTHS: + with self.subTest(btype=hex(btype), length=hex(length)): + octets = _cut(btype, length) + with warnings.catch_warnings(): + warnings.simplefilter('ignore') + parsed = self._parse(octets) + held = TAIL[4:] if btype == CUSTOM else TAIL + self.assertEqual(getattr(parsed.__header__.block, name), held) + self.assertEqual(parsed.info.length, length) + + def test_cut_block_rebuilds_as_captured(self) -> None: + for btype in FIELD: + for length in LENGTHS: + with self.subTest(btype=hex(btype), length=hex(length)): + with warnings.catch_warnings(): + warnings.simplefilter('ignore') + self.assertRebuilds(_cut(btype, length)) + + def test_well_formed_blocks_are_unchanged(self) -> None: + for btype, body in BODY.items(): + with self.subTest(btype=hex(btype)): + octets = _block(btype, body) + frames, messages = self._extract(SHB + IDB + _epb() + octets + _epb()) + self.assertEqual(frames, [FRAME, FRAME]) + self.assertFalse(any('block truncated' in text for text in messages), messages) + self.assertFalse(any('block length mismatch' in text for text in messages), messages) + parsed = self._parse(octets) + held = body[4:] if btype == CUSTOM else body + self.assertEqual(getattr(parsed.__header__.block, FIELD[btype]), + held + bytes(-len(held) % 4)) + self.assertRebuilds(octets) + + +if __name__ == '__main__': + unittest.main() From a7af33d8d3aaea7787ab1d42382e431f6c8f600d Mon Sep 17 00:00:00 2001 From: Jarry Shaw Date: Tue, 20 Oct 2026 13:28:26 -0400 Subject: [PATCH 2/2] test(pcapng): drop timed wording from the #1984 test (#1984) --- .../protocols/misc/test_pcapng_cut_block_past_eof_1984_unit.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/protocols/misc/test_pcapng_cut_block_past_eof_1984_unit.py b/tests/protocols/misc/test_pcapng_cut_block_past_eof_1984_unit.py index e1d61c6d4f..2cbbc29de3 100644 --- a/tests/protocols/misc/test_pcapng_cut_block_past_eof_1984_unit.py +++ b/tests/protocols/misc/test_pcapng_cut_block_past_eof_1984_unit.py @@ -7,7 +7,7 @@ the zero-pad ceiling the read raised :exc:`~pcapkit.utilities.exceptions.FieldValueError` from inside the block's schema, which ended the whole extraction and lost every earlier frame. The body -is now sized from the octets the file holds, at any declared length, so the +is sized from the octets the file holds, at any declared length, so the block is reported as truncated as every other block type is, and extraction keeps the frames before it.