fix(protocols): handle declared lengths above sys.maxsize as overruns (#1975)

docs/source/pcapkit/corekit/io.rst

@@ -8,7 +8,8 @@ I/O Objects
 :class:`io.BufferedReader`, and the stream proxies:
 :class:`~pcapkit.corekit.io.PeekableStream`, which adds ``peek`` to a seekable
 stream, and :class:`~pcapkit.corekit.io.NamedStream`, which gives a stream a
-``name``.
+``name``; and :func:`~pcapkit.corekit.io.advance`, which moves a stream past
+more octets than one read or seek can take.
 
 .. autoclass:: pcapkit.corekit.io.SeekableReader
    :no-members:
@@ -59,3 +60,5 @@ stream, and :class:`~pcapkit.corekit.io.NamedStream`, which gives a stream a
 
    .. autoattribute:: name
    .. automethod:: read
+
+.. autofunction:: pcapkit.corekit.io.advance

pcapkit/corekit/io.py

@@ -9,10 +9,12 @@
 implementation to :class:`io.BufferedReader`, and the stream proxies
 :class:`~pcapkit.corekit.io.PeekableStream`, which adds ``peek`` to a
 seekable stream, and :class:`~pcapkit.corekit.io.NamedStream`, which gives
-a stream a ``name``.
+a stream a ``name``, as well as :func:`~pcapkit.corekit.io.advance`, which moves
+a stream past more octets than one read or seek can take.
 
 """
 import io
+import sys
 import tempfile
 from typing import TYPE_CHECKING, cast
 
@@ -27,7 +29,7 @@
 
     from typing_extensions import Buffer
 
-__all__ = ['SeekableReader', 'PeekableStream', 'NamedStream']
+__all__ = ['SeekableReader', 'PeekableStream', 'NamedStream', 'advance']
 
 
 class SeekableReader(io.BufferedReader):
@@ -202,11 +204,14 @@ def _fill(self, size: 'int', minimum: 'Optional[int]' = None, /, *,
         minimum = size if minimum is None else minimum
         read = getattr(self._stream, 'read1', self._stream.read)
 
-        limit = config.READ_CHUNK
+        # NOTE: ``-1`` sets no limit of its own, but one read still goes no further
+        # than :data:`sys.maxsize`, whatever the setting, past which it raises
+        # :exc:`OverflowError` rather than coming back short (#1975).
+        limit = sys.maxsize if config.READ_CHUNK == -1 else min(config.READ_CHUNK, sys.maxsize)
         chunks = []  # type: list[bytes]
         count = 0
         while True:
-            chunk = read(size - count if limit == -1 else min(size - count, limit))
+            chunk = read(min(size - count, limit))
             if not chunk:
                 break
             chunks.append(chunk)
@@ -826,6 +831,46 @@ def peek(self, size: 'int' = 0) -> 'bytes':
         return buf
 
 
+#: Octets :func:`advance` reads and discards at a time when
+#: :data:`~pcapkit.utilities.config.READ_CHUNK` sets no limit.
+_DISCARD_CHUNK = 1 << 20
+
+
+def advance(file: 'IO[bytes]', count: 'int', /) -> 'None':
+    """Move ``file`` forward by ``count`` octets, however many one read or seek can take.
+
+    Args:
+        file: Stream to move.
+        count: Octets to move past, counted from the position.
+
+    A frame reader asks for a record no further than one read can go,
+    :data:`sys.maxsize`, so a record the file really holds past that is read only
+    as far as the limit; the next record still starts where this one declares it
+    ends (:issue:`1975`). A seekable stream gets there by relative seeks, none
+    longer than :data:`sys.maxsize`, and may land past its end, as an absolute seek
+    would. A :class:`SeekableReader`, or a stream that cannot seek, reads the
+    octets and discards them, no more than
+    :data:`~pcapkit.utilities.config.READ_CHUNK` at a time, and stops at the end
+    of the stream: a :class:`SeekableReader` seek reads at most a buffer's worth
+    past what it holds, so a longer one would stop short of the target.
+
+    """
+    if isinstance(file, SeekableReader) or not file.seekable():
+        chunk = _DISCARD_CHUNK if config.READ_CHUNK == -1 else config.READ_CHUNK
+        chunk = min(chunk, sys.maxsize)
+        while count > 0:
+            buf = file.read(min(count, chunk))
+            if not buf:
+                break
+            count -= len(buf)
+        return
+
+    while count > 0:
+        step = min(count, sys.maxsize)
+        file.seek(step, io.SEEK_CUR)
+        count -= step
+
+
 class PeekableStream:
     """Give a seekable binary stream without ``peek`` one.
 

pcapkit/foundation/engines/pcap.py

@@ -9,6 +9,7 @@
 
 """
 import io
+import sys
 from typing import TYPE_CHECKING
 
 from pcapkit.corekit.io import SeekableReader
@@ -240,4 +241,7 @@ def _bound_read_ahead(self) -> 'None':
         size = RECORD_HEADER_LEN
         if len(head) == RECORD_HEADER_LEN:
             size += int.from_bytes(head[8:12], self._gbhdr.byteorder)
-        ifile._read_ahead_size = size
+        # NOTE: No buffer holds more than :data:`sys.maxsize` octets, so a record
+        # longer than that is read ahead only so far; the reader moves past the
+        # rest itself (#1975).
+        ifile._read_ahead_size = min(size, sys.maxsize)

pcapkit/foundation/engines/pcapng.py

@@ -10,6 +10,7 @@
 """
 import collections
 import os
+import sys
 from typing import TYPE_CHECKING, cast
 
 from pcapkit.const.pcapng.block_type import BlockType as Enum_BlockType
@@ -453,7 +454,10 @@ def _bound_read_ahead(self) -> 'None':
                 if block_type == Enum_BlockType.Decryption_Secrets_Block:
                     secrets = int.from_bytes(self._peek(16)[12:16], byteorder)
                     size = max(size, BLOCK_MINIMUM[block_type] + secrets + -secrets % 4)
-        ifile._read_ahead_size = size
+        # NOTE: No buffer holds more than :data:`sys.maxsize` octets, so a block
+        # longer than that is read ahead only so far; the reader moves past the
+        # rest itself (#1975).
+        ifile._read_ahead_size = min(size, sys.maxsize)
 
     def _check_packet_block_context(self) -> 'None':
         """Reject a packet block in a section that describes no interface.

pcapkit/protocols/misc/pcap/frame.py

@@ -39,6 +39,7 @@
 from typing import TYPE_CHECKING, cast, overload
 
 from pcapkit.const.reg.linktype import LinkType as Enum_LinkType
+from pcapkit.corekit.io import advance
 from pcapkit.corekit.module import ModuleDescriptor
 from pcapkit.protocols.data.misc.pcap.frame import Frame as Data_Frame
 from pcapkit.protocols.data.misc.pcap.frame import FrameInfo as Data_FrameInfo
@@ -342,7 +343,13 @@ def read(self, length: 'Optional[int]' = None, *, _read: 'bool' = True,
             self._file.seek(_seek_set, io.SEEK_SET)
 
             #: bytes: Raw frame data.
-            self._data = self._read_fileng(self.length + _ilen)
+            #
+            # NOTE: Asked for no further than one read can go, :data:`sys.maxsize`.
+            # Past it the read raises :exc:`OverflowError` instead of coming back
+            # short, so a record claiming more than that would never reach the
+            # overrun handling below (#1975).
+            bound = min(self.length + _ilen, sys.maxsize)
+            self._data = self._read_fileng(bound)
 
             # NOTE: A record whose ``incl_len`` runs past the end of the data is
             # kept as captured, as the PCAP-NG reader keeps a captured length
@@ -357,8 +364,20 @@ def read(self, length: 'Optional[int]' = None, *, _read: 'bool' = True,
                      f'which holds {held} octet(s); kept as captured',
                      ProtocolWarning, stacklevel=stacklevel())
 
-            # move forward to the beginning of the next frame
-            self._file.seek(seek_cur, io.SEEK_SET)
+            # NOTE: A read that came back whole at the bound stopped there, not
+            # at the end of the data: the record runs on past it, and the next
+            # one starts where this one declares it ends, so the stream is
+            # moved over the rest (#1975). A shorter read met the end of the
+            # data, where a seek past :data:`sys.maxsize` could not go either,
+            # so the next frame then starts where the octets read end.
+            if len(self._data) == bound < self.length + _ilen:
+                advance(self._file, self.length + _ilen - bound)
+            else:
+                if seek_cur > sys.maxsize:
+                    seek_cur = _seek_set + len(self._data)
+
+                # move forward to the beginning of the next frame
+                self._file.seek(seek_cur, io.SEEK_SET)
 
             #: io.BytesIO: Source data stream.
             self._file = io.BytesIO(self._data)

pcapkit/protocols/misc/pcapng.py

@@ -67,6 +67,7 @@
 from pcapkit.corekit.enum import EnumLookup
 from pcapkit.corekit.fields.strings import DecodedString
 from pcapkit.corekit.infoclass import OrderedMultiInfo
+from pcapkit.corekit.io import advance
 from pcapkit.corekit.module import ModuleDescriptor
 from pcapkit.corekit.multidict import OrderedMultiDict
 from pcapkit.corekit.version import VersionInfo
@@ -1350,7 +1351,12 @@ def read(self, length: 'Optional[int]' = None, *, _read: 'bool' = True,
             self._file.seek(_seek_set, io.SEEK_SET)
 
             #: bytes: Raw block data.
-            self._data = self._read_fileng(schema.block.length)
+            #
+            # NOTE: Asked for no further than one read can go, :data:`sys.maxsize`,
+            # past which the read raises :exc:`OverflowError` instead of coming
+            # back short as the truncation handling below expects (#1975).
+            bound = min(schema.block.length, sys.maxsize)
+            self._data = self._read_fileng(bound)
 
             # NOTE: Block Total Length is cross-checked against its own trailing
             # copy and never against the file, so on a capture cut short it runs
@@ -1383,8 +1389,15 @@ def read(self, length: 'Optional[int]' = None, *, _read: 'bool' = True,
                 # again in place of building the block (:issue:`1470`).
                 block.__update__([('__truncated_raw__', self._data)])
 
-            # move backward to the beginning of next block
-            self._file.seek(seek_cur, io.SEEK_SET)
+            # NOTE: A read that came back whole at the bound stopped there, not
+            # at the end of the file: the block runs on past it, and the next one
+            # starts where its Block Total Length says, so the stream is moved
+            # over the rest (#1975).
+            if read == bound < schema.block.length:
+                advance(self._file, schema.block.length - bound)
+            else:
+                # move backward to the beginning of next block
+                self._file.seek(seek_cur, io.SEEK_SET)
 
             #: io.BytesIO: Source packet stream.
             self._file = io.BytesIO(self._data)
@@ -1636,7 +1649,8 @@ def _read_block_cut(self, header: 'Schema_PCAPNG', offset: 'int',
 
         current = self._file.tell()
         self._file.seek(offset + 8, io.SEEK_SET)
-        body = self._read_fileng(header.block.length - GENERIC_BLOCK_MINIMUM)
+        # NOTE: bounded as :meth:`read` bounds the block's read (#1975).
+        body = self._read_fileng(min(header.block.length - GENERIC_BLOCK_MINIMUM, sys.maxsize))
         self._file.seek(current, io.SEEK_SET)
 
         return Data_UnknownBlock(

pcapkit/protocols/schema/misc/pcap/frame.py

@@ -2,6 +2,7 @@
 # mypy: disable-error-code=assignment
 """header schema for frame header of PCAP file format"""
 
+import sys
 from typing import TYPE_CHECKING
 
 from pcapkit.corekit.fields.byteorder import byteorder_callback
@@ -36,8 +37,10 @@ class Frame(Schema):
     incl_len: 'int' = UInt32Field(callback=byteorder_callback)
     #: Actual length of packet.
     orig_len: 'int' = UInt32Field(callback=byteorder_callback)
-    #: Payload.
-    packet: 'bytes' = PayloadField(length=lambda pkt: pkt['incl_len'])
+    #: Payload, read no further than the record header and it together can go
+    #: in one read, :data:`sys.maxsize`; the frame reader moves past the rest
+    #: (:issue:`1975`).
+    packet: 'bytes' = PayloadField(length=lambda pkt: min(pkt['incl_len'], sys.maxsize - RECORD_HEADER_LEN))
 
     if TYPE_CHECKING:
         def __init__(self, ts_sec: 'int', ts_usec: 'int', incl_len: 'int',

pcapkit/protocols/schema/schema.py

@@ -6,6 +6,7 @@
 import collections.abc
 import io
 import itertools
+import sys
 from typing import TYPE_CHECKING, Any, Generic, TypeVar, cast
 
 from pcapkit.corekit.decorators import prepare
@@ -1070,7 +1071,12 @@ def unpack(cls, data: 'bytes | IO[bytes]',
                 if length < 0:
                     length = max(cast('int', packet['__length__']), 0)
 
-                payload = data.read(length)
+                # NOTE: A declared length is asked for no further than one read
+                # can go, :data:`sys.maxsize`; past it the read raises
+                # :exc:`OverflowError` rather than coming back short, which is
+                # how every other overrun is reported. No stream holds more than
+                # that, so the bounded read is the same short read (#1975).
+                payload = data.read(min(length, sys.maxsize))
                 self.__buffer__[field.name] = payload
 
                 packet['__length__'] -= length
@@ -1102,7 +1108,8 @@ def unpack(cls, data: 'bytes | IO[bytes]',
 
             length = field.length
 
-            byte = data.read(length)
+            # NOTE: bounded as the payload read above is (#1975).
+            byte = data.read(min(length, sys.maxsize))
             self.__buffer__[field.name] = byte
             if not isinstance(field, ForwardMatchField):
                 self._note_short_read(field.name, length, byte)

tests/dumpkit/test_pcap_trace_truncated_record_1541_unit.py

@@ -21,7 +21,6 @@
 import io
 import os
 import struct
-import sys
 import tempfile
 import unittest
 import warnings
@@ -111,9 +110,6 @@ def test_a_pcap_record_cut_at_eof_is_traced_as_the_octets_it_has(self) -> None:
         """#1541: a record claiming more than the file holds writes what it holds."""
         for claim in CLAIMS:
             with self.subTest(claim=claim):
-                if 16 + claim > sys.maxsize:
-                    self.skipTest('the frame reader asks the file for the 16-octet header and the claim at once, '
-                                  'past sys.maxsize on GraalPy (#1963)')
                 self.check(self.trace(_cut_pcap(claim)), _frames()[-1])
 
     def test_a_pcapng_captured_length_past_its_block_is_traced_as_the_octets_it_has(self) -> None:

tests/protocols/misc/pcap/test_frame_incl_len_overrun_1541_unit.py

@@ -15,7 +15,6 @@
 
 import io
 import struct
-import sys
 import unittest
 import warnings
 
@@ -55,9 +54,6 @@ def test_overrun_warns_and_is_kept_as_captured(self) -> None:
         header = Header(GLOBAL).info
         for claim in (61, 1000, 0x7FFF_FFFF):
             with self.subTest(claim=claim):
-                if 16 + claim > sys.maxsize:
-                    self.skipTest('the frame reader asks the file for the 16-octet header and the claim at once, '
-                                  'past sys.maxsize on GraalPy (#1963)')
                 octets = _record(claim)
                 frame, messages = self.parse(octets)
                 self.assertIn(f'PCAP: [Frame 4] captured length {claim} runs past the data, '