From e8273d9d67ad9e097caca0097f4645edcc6745dc Mon Sep 17 00:00:00 2001 From: Jarry Shaw Date: Sat, 10 Oct 2026 01:50:12 -0400 Subject: [PATCH] fix(reassembly): only the final fragment's partial block counts, and an oversized datagram never completes (#1566, #1567) - #1566: both completion checks tested all(RCVBT[0:(TDL + 7) // 8]), a slice that stops at the table's end, so a datagram declaring more data than the 65,535-octet buffer completed short. One helper, IP._is_complete, now also requires TDL <= len(datagram). - #1566: strict mode dropped a datagram with no run received while lax mode reported it. Both now report it, strict with payload (). - #1567: a non-final fragment ending mid-block marked that block, so a snaplen-cut IPv6 fragment (tl comes from the captured payload) completed its datagram zero-filled. Only the final fragment's partial last block is marked now; this replaces #1564's Total Length 0 special case and also covers a malformed IPv4 one. A fragment the buffer's end clips stays counted as cut there, so 7 and 8 octets at offset 65,528 now agree. Sample-capture reassembly is unchanged (9,456 rows, default/dpkt/scapy). --- .../foundation/reassembly/ip/index.rst | 8 + .../pcapkit/foundation/reassembly/ip/ipv6.rst | 11 ++ pcapkit/foundation/reassembly/ip.py | 116 ++++++++------ tests/foundation/reassembly/test_ip.py | 14 +- ...est_ip_cut_non_final_fragment_1567_unit.py | 108 +++++++++++++ .../test_ip_last_fragment_offset_1565_unit.py | 12 +- .../test_ip_oversized_datagram_1566_unit.py | 149 ++++++++++++++++++ ...pv6_cut_non_final_fragment_1567_runtime.py | 133 ++++++++++++++++ 8 files changed, 499 insertions(+), 52 deletions(-) create mode 100644 tests/foundation/reassembly/test_ip_cut_non_final_fragment_1567_unit.py create mode 100644 tests/foundation/reassembly/test_ip_oversized_datagram_1566_unit.py create mode 100644 tests/foundation/reassembly/test_ipv6_cut_non_final_fragment_1567_runtime.py diff --git a/docs/source/pcapkit/foundation/reassembly/ip/index.rst b/docs/source/pcapkit/foundation/reassembly/ip/index.rst index 87e90ccb14..056fa9361d 100644 --- a/docs/source/pcapkit/foundation/reassembly/ip/index.rst +++ b/docs/source/pcapkit/foundation/reassembly/ip/index.rst @@ -93,3 +93,11 @@ Algorithm Because completion frees the buffer (:rfc:`791#section-3.2`), a fragment that arrives after its datagram has completed opens a new buffer, which is later reported as a separate incomplete datagram; this is intended (:issue:`1507`). + +Two departures from the procedure above: + +* Only the final fragment's partial last ``RCVBT`` block is set. A non-final + fragment carries a multiple of 8 octets, so one ending mid-block was cut or + is malformed, and setting its last block would complete the datagram with + octets nobody sent (:issue:`1567`). +* A datagram longer than the data buffer is never complete (:issue:`1566`). diff --git a/docs/source/pcapkit/foundation/reassembly/ip/ipv6.rst b/docs/source/pcapkit/foundation/reassembly/ip/ipv6.rst index c0f83e4e81..a4550b196e 100644 --- a/docs/source/pcapkit/foundation/reassembly/ip/ipv6.rst +++ b/docs/source/pcapkit/foundation/reassembly/ip/ipv6.rst @@ -63,6 +63,17 @@ Terminology back off. All four adapters (``pcap``, ``pcapng``, ``dpkt`` and ``scapy``) agree on the three fields. + .. note:: + + ``tl`` is derived from the captured payload, not from the Payload + Length field, so a fragment the snapshot length cut does not show it + in ``tl``. A non-final fragment must carry a multiple of 8 octets + (:rfc:`8200#section-4.5`), so reassembly does not count the partial + last block of one that ends mid-block as received: the datagram + stays incomplete, rather than completing with zeros where the cut + octets were (:issue:`1567`). A cut *final* fragment cannot be told + from a whole one. + .. note:: The reassembly key is the Fragment header's *Identification* diff --git a/pcapkit/foundation/reassembly/ip.py b/pcapkit/foundation/reassembly/ip.py index c858051997..7ad4493413 100644 --- a/pcapkit/foundation/reassembly/ip.py +++ b/pcapkit/foundation/reassembly/ip.py @@ -35,14 +35,16 @@ class IP(ReassemblyBase[Packet[_AT], Datagram[_AT], BufferID, Buffer[_AT]], Gene Args: strict: if :data:`True`, report a datagram that is not completely - reassembled as the tuple of its received runs; otherwise as one - contiguous payload, its holes zero-filled -- or, while the total - length is unknown, only the prefix up to the first hole. Receipt - is tracked in 8-octet blocks, so the partial last block of a - snaplen-truncated fragment is not counted as received: it is left - out of the runs and ends the prefix, but once the total length is - known the contiguous payload keeps its real octets, with only the - rest of the hole zero-filled + reassembled as the tuple of its received runs, empty if none was; + otherwise as one contiguous payload, its holes zero-filled -- or, + while the total length is unknown, only the prefix up to the first + hole. Receipt is tracked in 8-octet blocks, and only the final + fragment's partial last block counts as received -- a non-final + fragment's, or a snaplen-truncated or buffer-clipped fragment's, + does not: it is left out of the runs and ends the prefix, but once + the total length is known the contiguous payload keeps its real + octets, with only the rest of the hole zero-filled. A datagram + longer than the data buffer is never complete store: if store reassembled datagram in memory, i.e., :attr:`self._dtgram ` (if not, datagram will be discarded after callback) @@ -168,12 +170,12 @@ def reassembly(self, info: 'Packet[_AT]') -> 'None': # grows to hold such a datagram rather than clipping it, ``RCVBT`` in # step with it. # - # With nothing declared, a snaplen cut cannot be seen in the length. - # A fragment with **MF** set must carry a multiple of 8 octets, so its - # partial last block is a cut and is left unmarked below. Any other cut - # -- an unfragmented frame's, or a final fragment's -- cannot be told - # from a whole one, since the frame's original length is not part of - # the input, and the datagram comes out ``COMPLETE`` but short. + # With nothing declared, a snaplen cut cannot be seen in the length. A + # non-final fragment's cut still shows, as a partial last block, which + # is left unmarked below. Any other cut -- an unfragmented frame's, or + # a final fragment's -- cannot be told from a whole one, since the + # frame's original length is not part of the input, and the datagram + # comes out ``COMPLETE`` but short. if TL == 0: length = len(info.payload) extent = FO + length @@ -206,13 +208,23 @@ def reassembly(self, info: 'Packet[_AT]') -> 'None': buf.datagram[start:stop] = payload # Set RCVBT bits (in 8 octets) for the blocks this fragment actually - # filled. A fragment held in full marks its last block even when it is - # short, as the final fragment's is; a truncated one leaves its partial - # last block clear, since the octets after the cut are a real hole -- - # as does a non-final fragment with Total Length 0, whose partial last - # block can only be a cut. + # filled. Only the final fragment, held in full, marks a partial last + # block: the octets after its end are past the datagram's. Every other + # partial last block is left clear, as a hole -- + # + # * a truncated fragment's, since the octets after the cut are missing. + # That includes one clipped by the data buffer's end, whose octets + # past it cannot be held, so it counts as cut there (:issue:`1566`). + # * a non-final fragment's. :rfc:`791#section-3.2` and + # :rfc:`8200#section-4.5` make every fragment but the last carry a + # multiple of 8 octets, so one that ends mid-block was cut or is + # malformed, and nobody sent the rest of its block. Marking it would + # complete the datagram with zeros in their place. This is the only + # way a cut shows when the length is not declared: under Total + # Length 0, and on IPv6, whose adapters derive it from the captured + # payload (:issue:`1567`). start = FO // 8 - if held == length and not (TL == 0 and MF): + if held == length and not MF: stop = (FO + held + 7) // 8 else: stop = (FO + held) // 8 @@ -224,16 +236,33 @@ def reassembly(self, info: 'Packet[_AT]') -> 'None': # fragment with MF clear has arrived, and ``0`` is a valid length if not MF: buf.__update__(TDL=length + FO) - TDL = buf.TDL # when datagram is reassembled in whole - start = 0 - stop = (TDL + 7) // 8 - if TDL >= 0 and all(buf.RCVBT[start:stop]): + if self._is_complete(buf): self._dtgram.extend( self.submit(self._buffer.pop(BUFID), bufid=BUFID, checked=True) ) + @staticmethod + def _is_complete(buf: 'Buffer[_AT]') -> 'bool': + """Tell whether a buffer holds its whole datagram. + + Arguments: + buf: buffer to check + + Returns: + Whether the total data length is known, fits the data buffer, and + every 8-octet block up to it is marked received. + + A datagram longer than the data buffer cannot be held, so it is never + complete. Checking ``RCVBT`` alone does not show that: slicing it past + its end stops silently, so the blocks past the buffer would read as + received (:issue:`1566`). + + """ + TDL = buf.TDL + return 0 <= TDL <= len(buf.datagram) and all(buf.RCVBT[:(TDL + 7) // 8]) + @staticmethod def _detect_conflicts(rcvbt: 'bytearray', tdl: 'int', datagram: 'bytearray', payload: 'bytearray', start: 'int', stop: 'int') -> 'list[tuple[int, int]]': @@ -331,9 +360,7 @@ def submit(self, buf: 'Buffer[_AT]', *, bufid: 'tuple[_AT, _AT, int, TransType]' datagram = buf.datagram conflict = tuple(buf.conflict) - start = 0 - stop = (TDL + 7) // 8 - flag = checked or (TDL >= 0 and all(RCVBT[start:stop])) + flag = checked or self._is_complete(buf) ret = [] # type: list[Datagram[_AT]] # How completely this datagram came out, and why it stopped. Derived once, @@ -361,23 +388,24 @@ def submit(self, buf: 'Buffer[_AT]', *, bufid: 'tuple[_AT, _AT, int, TransType]' # the last run may reach the end of the bit table if byte: data.append(bytes(byte)) - # strip empty packets - if data or header: - packet = Datagram( - completed=completion, - id=DatagramID( - src=bufid[0], - dst=bufid[1], - id=bufid[2], - proto=bufid[3], - ), - index=tuple(index), - header=header, - payload=tuple(data), - packet=None, - conflict=conflict, - ) - ret.append(packet) + # Report it even with no run received, its payload ``()``, as + # ``strict=False`` reports it with an empty payload: ``index`` + # still says which frames were its fragments (:issue:`1566`). + packet = Datagram( + completed=completion, + id=DatagramID( + src=bufid[0], + dst=bufid[1], + id=bufid[2], + proto=bufid[3], + ), + index=tuple(index), + header=header, + payload=tuple(data), + packet=None, + conflict=conflict, + ) + ret.append(packet) # if datagram is reassembled in whole -- or if it is not, and ``strict`` # asked for one contiguous payload rather than the received runs else: diff --git a/tests/foundation/reassembly/test_ip.py b/tests/foundation/reassembly/test_ip.py index fd1a2b6a72..7bd1155d84 100644 --- a/tests/foundation/reassembly/test_ip.py +++ b/tests/foundation/reassembly/test_ip.py @@ -98,16 +98,18 @@ class TestIP(IP): self.assertEqual(pending._buffer[self._packet(num=0, fo=0, mf=True, payload=b'').bufid].header, b'updated') + # a buffer with no run received is still reported, as strict=False + # reports it (#1566) empty = TestIP() src = ip_address('192.0.2.1') dst = ip_address('198.51.100.2') - self.assertEqual( - empty.submit( - Buffer(-1, bytearray(b'\x00\x00'), [], b'', bytearray(b''), 1000.0, []), - bufid=(src, dst, 42, TransType.UDP), - ), - [], + datagram, = empty.submit( + Buffer(-1, bytearray(b'\x00\x00'), [], b'', bytearray(b''), 1000.0, []), + bufid=(src, dst, 42, TransType.UDP), ) + self.assertFalse(datagram.completed) + self.assertEqual(datagram.payload, ()) + self.assertIsNone(datagram.packet) @unittest.skipUnless(HAS_RUNTIME, 'runtime dependencies not installed') diff --git a/tests/foundation/reassembly/test_ip_cut_non_final_fragment_1567_unit.py b/tests/foundation/reassembly/test_ip_cut_non_final_fragment_1567_unit.py new file mode 100644 index 0000000000..aa33f79c95 --- /dev/null +++ b/tests/foundation/reassembly/test_ip_cut_non_final_fragment_1567_unit.py @@ -0,0 +1,108 @@ +# -*- coding: utf-8 -*- +"""A non-final IP fragment that ends mid-block leaves a hole, not zeros. + +GitHub issue #1567. The IPv6 adapters derive ``tl`` from the captured payload, +so a fragment the snapshot length cut declares exactly what was captured, and +the reassembler marked its partial last block received. A 3-fragment datagram +with its first fragment cut from 64 octets to 60 then came out ``COMPLETE``, +octets 60-63 zero-filled; cut to 52, strict mode's first run carried 4 +zero-filled octets. + +Every fragment but the last carries a multiple of 8 octets +(:rfc:`8200#section-4.5`, :rfc:`791#section-3.2`), so only the final fragment's +partial last block is now counted received. That covers an IPv4 fragment whose +Total Length declares a non-multiple of 8 on a non-final fragment as well. + +Every case builds its fragments in memory and reads no capture. Classes are +imported inside each test, after :func:`~tests._support.reimport_once_per_class`. + +""" + +from ipaddress import ip_address +from itertools import permutations +import unittest + +from tests._support import reimport_once_per_class + +#: The datagram, with no zero octet in it, so a hole shows. +DATA = bytes((i * 7 + 3) % 251 + 1 for i in range(200)) + +#: Its three fragments, as ``(fragment offset, more fragments, payload)``. +FRAGMENTS = ((0, True, DATA[0:64]), (64, True, DATA[64:128]), (128, False, DATA[128:200])) + + +class TestIPReassemblyCutNonFinalFragment(unittest.TestCase): + """Pin that a cut non-final fragment leaves the datagram incomplete.""" + + def setUp(self) -> None: + reimport_once_per_class(self) + + def _packet(self, *, num: int, fo: int, mf: bool, payload: bytes, tl: 'int | None' = None): + """A fragment whose ``tl`` is, as the IPv6 adapters pass it, its captured length.""" + from pcapkit.const.reg.transtype import TransType + from pcapkit.foundation.reassembly.data.ip import Packet + + bufid = (ip_address('2001:db8::1'), ip_address('2001:db8::2'), 9, TransType.UDP) + return Packet(bufid, num, fo, 40, mf, 40 + len(payload) if tl is None else tl, + b'H' * 40, bytearray(payload), 1000.0 + num) + + def _reassemble(self, cls, packets, strict: bool): + reasm = cls(strict=strict) + for packet in packets: + reasm(packet) + datagram, = reasm.datagram + return datagram + + def test_a_cut_first_ipv6_fragment_leaves_a_hole(self) -> None: + from pcapkit.foundation.reassembly.data.data import Completion + from pcapkit.foundation.reassembly.ipv6 import IPv6 + + for cut in (60, 52): + for strict in (True, False): + with self.subTest(cut=cut, strict=strict): + packets = [self._packet(num=num, fo=fo, mf=mf, payload=payload[:cut] if fo == 0 else payload) + for num, (fo, mf, payload) in enumerate(FRAGMENTS, 1)] + datagram = self._reassemble(IPv6, packets, strict) + self.assertIs(datagram.completed, Completion.PARTIAL) + self.assertEqual(datagram.index, (1, 2, 3)) + if strict: + # the cut fragment's whole blocks only + self.assertEqual(datagram.payload, (DATA[:cut // 8 * 8], DATA[64:])) + else: + # its real octets kept, the rest of the hole zero-filled + self.assertEqual(datagram.payload, DATA[:cut] + bytes(64 - cut) + DATA[64:]) + + def test_a_non_final_ipv4_fragment_ending_mid_block_leaves_a_hole(self) -> None: + # Total Length 100 declares 60 octets on a non-final fragment, all of + # them captured: malformed rather than cut, but nobody sent 60-63. + from pcapkit.foundation.reassembly.data.data import Completion + from pcapkit.foundation.reassembly.ipv4 import IPv4 + + for strict in (True, False): + with self.subTest(strict=strict): + packets = [self._packet(num=num, fo=fo, mf=mf, payload=payload[:60] if fo == 0 else payload) + for num, (fo, mf, payload) in enumerate(FRAGMENTS, 1)] + datagram = self._reassemble(IPv4, packets, strict) + self.assertIs(datagram.completed, Completion.PARTIAL) + self.assertEqual(datagram.payload, + (DATA[:56], DATA[64:]) if strict else DATA[:60] + bytes(4) + DATA[64:]) + + def test_an_uncut_datagram_completes_byte_exact_in_any_order(self) -> None: + from pcapkit.foundation.reassembly.data.data import Completion + from pcapkit.foundation.reassembly.ipv4 import IPv4 + from pcapkit.foundation.reassembly.ipv6 import IPv6 + + for cls in (IPv4, IPv6): + for order in permutations(range(3)): + for strict in (True, False): + with self.subTest(cls=cls.__name__, order=order, strict=strict): + packets = [self._packet(num=index + 1, fo=FRAGMENTS[index][0], mf=FRAGMENTS[index][1], + payload=FRAGMENTS[index][2]) for index in order] + datagram = self._reassemble(cls, packets, strict) + self.assertIs(datagram.completed, Completion.COMPLETE) + self.assertEqual(datagram.payload, DATA) + self.assertEqual(datagram.conflict, ()) + + +if __name__ == '__main__': + unittest.main() diff --git a/tests/foundation/reassembly/test_ip_last_fragment_offset_1565_unit.py b/tests/foundation/reassembly/test_ip_last_fragment_offset_1565_unit.py index da89cc82b7..f7b360ec9b 100644 --- a/tests/foundation/reassembly/test_ip_last_fragment_offset_1565_unit.py +++ b/tests/foundation/reassembly/test_ip_last_fragment_offset_1565_unit.py @@ -65,7 +65,13 @@ def test_a_lone_fragment_at_the_largest_offset_is_buffered(self) -> None: datagram, = reasm.datagram self.assertIs(datagram.completed, Completion.PARTIAL) self.assertEqual(datagram.index, (1,)) - if strict: + if strict and mf: + # a non-final fragment ending mid-block was cut + # or is malformed, so its block is not received + # (#1567) -- and the datagram is still reported + # (#1566) + self.assertEqual(datagram.payload, ()) + elif strict: run, = datagram.payload self.assertEqual(run[:size], tail) elif mf: @@ -112,7 +118,9 @@ def test_a_non_final_fragment_at_the_largest_offset_leaves_the_datagram_open(sel run, = datagram.payload else: run = datagram.payload - self.assertEqual(run[:LAST + size], BODY + tail) + # the tail ends mid-block on a non-final fragment, so + # its block is not received (#1567) + self.assertEqual(run, BODY) def test_a_tso_fragment_at_the_largest_offset_composes_with_the_buffer_growth(self) -> None: # Total Length 0 (#1555): 7 octets end exactly at the preallocated diff --git a/tests/foundation/reassembly/test_ip_oversized_datagram_1566_unit.py b/tests/foundation/reassembly/test_ip_oversized_datagram_1566_unit.py new file mode 100644 index 0000000000..4a47332896 --- /dev/null +++ b/tests/foundation/reassembly/test_ip_oversized_datagram_1566_unit.py @@ -0,0 +1,149 @@ +# -*- coding: utf-8 -*- +"""An IP datagram longer than the data buffer is never complete. + +GitHub issue #1566. The data buffer holds 65,535 octets and ``RCVBT`` 8,192 +blocks. Both completion checks tested ``all(RCVBT[0:(TDL + 7) // 8])``, and the +slice stops silently at the table's end, so a final fragment declaring more data +than fits completed its datagram short. A datagram now completes only if its +total data length fits the buffer. + +A fragment the buffer's end clips counts as cut there: its partial last block is +a hole. An 8-octet non-final fragment at the largest offset therefore received +no block, and strict mode then dropped its datagram, while a 7-octet one was +reported. #1567 leaves the 7-octet one's block clear too, and strict mode now +reports a datagram with no run received, as lax mode does. + +Every case builds its fragments in memory and reads no capture. Classes are +imported inside each test, after :func:`~tests._support.reimport_once_per_class`. + +""" + +from ipaddress import ip_address +import unittest + +from tests._support import reimport_once_per_class + +#: The largest Fragment Offset, 8191 eight-octet units, in octets. +LAST = 65528 + +#: The data buffer's length. +BUFFER = 65535 + +#: Datagram octets, with no zero octet in them, so a hole shows. +DATA = (bytes(range(1, 256)) * 258)[:65579] + + +class TestIPReassemblyOversizedDatagram(unittest.TestCase): + """Pin that a datagram longer than the buffer is never ``COMPLETE``.""" + + def setUp(self) -> None: + reimport_once_per_class(self) + + def _packet(self, *, num: int, fo: int, mf: bool, payload: bytes, tl: 'int | None' = None): + from pcapkit.const.reg.transtype import TransType + from pcapkit.foundation.reassembly.data.ip import Packet + + bufid = (ip_address('192.0.2.1'), ip_address('198.51.100.2'), 7, TransType.UDP) + return Packet(bufid, num, fo, 20, mf, 20 + len(payload) if tl is None else tl, + b'H' * 20, bytearray(payload), 1000.0 + num) + + def _classes(self): + from pcapkit.foundation.reassembly.ipv4 import IPv4 + from pcapkit.foundation.reassembly.ipv6 import IPv6 + + return IPv4, IPv6 + + def test_a_final_fragment_declaring_more_than_the_buffer_does_not_complete(self) -> None: + # The 7-octet final fragment marks the last block, so every block + # reads received once the long one declares a total data length past + # the buffer: 65579 is the issue's, and 65536 the shortest -- which a + # check of ``(TDL + 7) // 8`` against ``len(RCVBT)`` lets through. + from pcapkit.foundation.reassembly.data.data import Completion + + for cls in self._classes(): + for tdl in (65579, 65536): + for strict in (True, False): + with self.subTest(cls=cls.__name__, tdl=tdl, strict=strict): + reasm = cls(strict=strict) + reasm(self._packet(num=1, fo=LAST, mf=False, payload=DATA[LAST:BUFFER])) + reasm(self._packet(num=2, fo=0, mf=True, payload=DATA[:64])) + reasm(self._packet(num=3, fo=64, mf=False, payload=DATA[64:tdl])) + datagram, = reasm.datagram + self.assertIs(datagram.completed, Completion.PARTIAL) + self.assertEqual(datagram.index, (1, 2, 3)) + self.assertEqual(datagram.payload, + (DATA[:BUFFER],) if strict else DATA[:BUFFER]) + self.assertEqual(datagram.conflict, ()) + + def test_the_issues_repro_is_partial(self) -> None: + # Octets 0-65527, a malformed 7-octet non-final fragment at the largest + # offset, then a final fragment at 64 declaring Total Length 65535, + # of which 8 octets were captured. + from pcapkit.foundation.reassembly.data.data import Completion + + for cls in self._classes(): + for strict in (True, False): + with self.subTest(cls=cls.__name__, strict=strict): + reasm = cls(strict=strict) + for num, fo in enumerate(range(0, LAST, 4096), 1): + reasm(self._packet(num=num, fo=fo, mf=True, payload=DATA[fo:min(fo + 4096, LAST)])) + reasm(self._packet(num=20, fo=LAST, mf=True, payload=b'M' * 7)) + reasm(self._packet(num=21, fo=64, mf=False, payload=DATA[64:72], tl=65535)) + datagram, = reasm.datagram + self.assertIs(datagram.completed, Completion.PARTIAL) + + def test_a_fragment_the_buffer_clips_counts_like_a_cut_one(self) -> None: + # 8 octets at the largest offset are clipped to 7; 7 octets on a + # non-final fragment were cut. Neither receives its block, and both + # report the same datagram. + from pcapkit.foundation.reassembly.data.data import Completion + + for cls in self._classes(): + for strict in (True, False): + with self.subTest(cls=cls.__name__, strict=strict): + reported = [] + for size in (7, 8): + reasm = cls(strict=strict) + reasm(self._packet(num=1, fo=LAST, mf=True, payload=b'A' * size)) + datagram, = reasm.datagram + self.assertIs(datagram.completed, Completion.PARTIAL) + self.assertEqual(datagram.index, (1,)) + reported.append(datagram.payload) + self.assertEqual(reported, [()] * 2 if strict else [b''] * 2) + + def test_a_tso_fragment_past_the_buffer_grows_it_rather_than_being_clipped(self) -> None: + # Total Length 0 (#1555): 8 octets at the largest offset end one past + # the preallocated buffer, which grows to hold them, so their block is + # received and every block reads received with the length still unknown. + from pcapkit.foundation.reassembly.data.data import Completion + from pcapkit.foundation.reassembly.ipv4 import IPv4 + + for strict in (True, False): + with self.subTest(strict=strict): + reasm = IPv4(strict=strict) + reasm(self._packet(num=1, fo=0, mf=True, payload=DATA[:32768])) + reasm(self._packet(num=2, fo=32768, mf=True, payload=DATA[32768:LAST])) + reasm(self._packet(num=3, fo=LAST, mf=True, payload=DATA[LAST:LAST + 8], tl=0)) + datagram, = reasm.datagram + self.assertIs(datagram.completed, Completion.PARTIAL) + self.assertEqual(datagram.payload, + (DATA[:LAST + 8],) if strict else DATA[:LAST + 8]) + + def test_strict_mode_reports_a_datagram_with_no_run_received(self) -> None: + # a non-final fragment whose capture holds 5 of its 8 octets fills no + # whole block + from pcapkit.foundation.reassembly.data.data import Completion + + for cls in self._classes(): + for strict in (True, False): + with self.subTest(cls=cls.__name__, strict=strict): + reasm = cls(strict=strict) + reasm(self._packet(num=1, fo=8, mf=True, payload=b'A' * 5, tl=28)) + datagram, = reasm.datagram + self.assertIs(datagram.completed, Completion.PARTIAL) + self.assertEqual(datagram.index, (1,)) + self.assertEqual(datagram.payload, () if strict else b'') + + +if __name__ == '__main__': + unittest.main() diff --git a/tests/foundation/reassembly/test_ipv6_cut_non_final_fragment_1567_runtime.py b/tests/foundation/reassembly/test_ipv6_cut_non_final_fragment_1567_runtime.py new file mode 100644 index 0000000000..76764ef91e --- /dev/null +++ b/tests/foundation/reassembly/test_ipv6_cut_non_final_fragment_1567_runtime.py @@ -0,0 +1,133 @@ +# -*- coding: utf-8 -*- +"""A snaplen-cut first IPv6 fragment leaves a hole under every engine. + +GitHub issue #1567. A 3-fragment IPv6 datagram of 64, 64 and 72 octets, its +first fragment captured to 60 octets, came out ``COMPLETE`` with octets 60-63 +zero-filled under the default, dpkt and scapy engines, since every IPv6 adapter +passes ``tl`` as the captured length. The fix is in the reassembler they share, +so each engine is run over the same capture here and has to report the same +``PARTIAL`` datagram. + +The captures are built in memory, through the engines, so the module belongs to +the fixture-dependent tier. + +""" +from __future__ import annotations + +import importlib.util +import os +import struct +import tempfile +import unittest +import warnings +from typing import TYPE_CHECKING + +from tests._support import close_extractor, reimport_once_per_class + +if TYPE_CHECKING: + from typing import Any + +RUNTIME_DEPS = ('tbtrim', 'aenum', 'chardet', 'dictdumper') +HAS_RUNTIME = all(importlib.util.find_spec(name) is not None for name in RUNTIME_DEPS) +ENGINES = ('default',) + tuple(name for name in ('dpkt', 'scapy') if importlib.util.find_spec(name) is not None) + +#: The datagram, with no zero octet in it, so a hole shows. +DATA = bytes((i * 7 + 3) % 251 + 1 for i in range(200)) + +#: Its three fragments, as ``(fragment offset, more fragments, payload)``. +FRAGMENTS = ((0, 1, DATA[0:64]), (64, 1, DATA[64:128]), (128, 0, DATA[128:200])) + +#: Ethernet, IPv6 and Fragment header octets before a fragment's payload. +HEADERS = 14 + 40 + 8 + + +def _frames() -> 'list[bytes]': + """The three fragments as Ethernet frames, uncut.""" + frames = [] + for offset, more, payload in FRAGMENTS: + # Next Header UDP; the offset field is (offset // 8) << 3, i.e. the offset in octets + frag = struct.pack('!BBHI', 17, 0, offset | more, 9) + ipv6 = struct.pack('!IHBB16s16s', 0x60000000, len(frag) + len(payload), 44, 64, + bytes.fromhex('20010db8000000000000000000000001'), + bytes.fromhex('20010db8000000000000000000000002')) + frames.append(bytes.fromhex('020000000002 020000000001 86dd') + ipv6 + frag + payload) + return frames + + +def _pcap(cut: 'int | None') -> 'bytes': + """A libpcap capture of the fragments, the first captured to ``cut`` payload octets.""" + output = bytearray(struct.pack(' 'bytes': + """The same capture as PCAP-NG: one section, one Ethernet interface.""" + def block(block_type: 'int', body: 'bytes') -> 'bytes': + body += b'\x00' * (-len(body) % 4) + return struct.pack(' None: + reimport_once_per_class(self) + tmp = tempfile.TemporaryDirectory() # pylint: disable=consider-using-with + self.addCleanup(tmp.cleanup) + self.tmp = tmp.name + + def _datagrams(self, name: 'str', octets: 'bytes', engine: 'str', strict: 'bool') -> 'Any': + from pcapkit import extract + + path = os.path.join(self.tmp, name) + with open(path, 'wb') as file: + file.write(octets) + with warnings.catch_warnings(): + warnings.simplefilter('ignore') + extractor = extract(fin=path, nofile=True, engine=engine, ipv6=True, + reassembly=True, reasm_strict=strict) + self.addCleanup(close_extractor, extractor) + return extractor.reassembly.ipv6 + + def _cases(self): + for fmt, build in (('pcap', _pcap), ('pcapng', _pcapng)): + for engine in ENGINES if fmt == 'pcap' else ('default',): + for strict in (True, False): + yield fmt, build, engine, strict + + def test_a_cut_first_fragment_leaves_a_hole(self) -> None: + from pcapkit.foundation.reassembly.data.data import Completion + + for cut in (60, 52): + for fmt, build, engine, strict in self._cases(): + with self.subTest(cut=cut, fmt=fmt, engine=engine, strict=strict): + datagram, = self._datagrams(f'cut-{cut}.{fmt}', build(cut), engine, strict) + self.assertIs(datagram.completed, Completion.PARTIAL) + self.assertEqual(datagram.index, (1, 2, 3)) + self.assertEqual(datagram.payload, + (DATA[:cut // 8 * 8], DATA[64:]) if strict + else DATA[:cut] + bytes(64 - cut) + DATA[64:]) + + def test_an_uncut_capture_completes_byte_exact(self) -> None: + from pcapkit.foundation.reassembly.data.data import Completion + + for fmt, build, engine, strict in self._cases(): + with self.subTest(fmt=fmt, engine=engine, strict=strict): + datagram, = self._datagrams(f'whole.{fmt}', build(None), engine, strict) + self.assertIs(datagram.completed, Completion.COMPLETE) + self.assertEqual(datagram.payload, DATA) + + +if __name__ == '__main__': + unittest.main()