diff --git a/pcapkit/protocols/application/http.py b/pcapkit/protocols/application/http.py index 602a4b5032..5f4b546f1d 100644 --- a/pcapkit/protocols/application/http.py +++ b/pcapkit/protocols/application/http.py @@ -18,7 +18,9 @@ import struct from typing import TYPE_CHECKING, Generic +from pcapkit.corekit.protochain import ProtoChain from pcapkit.protocols.application.application import Application +from pcapkit.protocols.misc.null import NoPayload from pcapkit.protocols.protocol import _PT, _ST from pcapkit.utilities.exceptions import ProtocolError @@ -167,22 +169,30 @@ def id(cls) -> 'tuple[Literal["HTTP"], Literal["HTTPv1"], Literal["HTTPv2"]]': return ('HTTP', 'HTTPv1', 'HTTPv2') def read(self, length: 'Optional[int]' = None, *, - version: 'Optional[Literal[1, 2]]' = None, **kwargs: 'Any') -> '_PT': + version: 'Optional[Literal[1, 2]]' = None, + preface: 'bool' = False, # pylint: disable=unused-argument + **kwargs: 'Any') -> '_PT': """Read (parse) packet data. Args: length: Length of packet data. version: Version of HTTP. + preface: Construction keyword of :meth:`make`; whether a preface is + parsed is decided by the octets alone. **kwargs: Arbitrary keyword arguments. Returns: - Parsed packet data. + Parsed packet data. When the payload opens with the HTTP/2 + connection preface, its 24 octets are recorded as ``preface``. """ if length is None: length = len(self) - if version is None: + # An explicit ``version=2`` -- what :meth:`_make_data` hands back for a + # rebuild -- may still open with the connection preface, and the + # identification in :meth:`_guess_version` is what skips it. + if version is None or (version == 2 and self._test_preface(length)): http = self._guess_version(length, **kwargs) else: if version == 1: @@ -207,21 +217,44 @@ def read(self, length: 'Optional[int]' = None, *, self._version = http.version self._length = http.length + self._preface_length self._http = http - return http.info + + info = http.info + if self._preface_length: + info.__update__({ + 'preface': _HTTP2_PREFACE, + }) + + # The versioned parser may have kept octets after its message as the + # next layer, e.g. the frames after an HTTP/2 frame. + next_ = getattr(http, '_next', None) + if next_ is not None and not isinstance(next_, NoPayload): + self._next = next_ + self._protos = ProtoChain(self.__class__, self.alias, basis=next_.protochain) + return info def make(self, - version: 'Literal[1, 2]' = 1, + version: 'Literal[1, 2]' = 1, *, + preface: 'bool' = False, **kwargs: 'Any') -> '_ST': """Make (construct) packet data. Args: version: Version of HTTP. + preface: Whether the HTTP/2 connection preface goes ahead of the + frame; :meth:`pack` writes it. **kwargs: Arbitrary keyword arguments. Returns: bytes: Constructed packet data. + Raises: + ProtocolError: If ``version`` is unknown, or if ``preface`` is + requested for a version other than HTTP/2. + """ + if preface and version != 2: + raise ProtocolError(f'HTTP/{version}: connection preface is HTTP/2 only') + if version == 1: from pcapkit.protocols.application.httpv1 import HTTP as protocol # isort: skip # pylint: disable=line-too-long,import-outside-toplevel elif version == 2: @@ -243,6 +276,22 @@ def make(self, # ``make`` override that did would need a different dispatch here. return protocol.__new__(protocol).make(**kwargs) # type: ignore[return-value] + def pack(self, **kwargs: 'Any') -> 'bytes': + """Pack (construct) packet data. + + Args: + **kwargs: Arbitrary keyword arguments. + + Returns: + Constructed packet data, with the HTTP/2 connection preface ahead + of the frame if ``preface`` is set. + + """ + data = super().pack(**kwargs) + if kwargs.get('preface', False): + return _HTTP2_PREFACE + data + return data + ########################################################################## # Utilities. ########################################################################## @@ -274,9 +323,22 @@ def _make_data(cls, data: '_PT') -> 'dict[str, Any]': # type: ignore[override] if isinstance(data, Data_HTTPv1): return {'version': 1, **HTTPv1._make_data(data)} if isinstance(data, Data_HTTPv2): - return {'version': 2, **HTTPv2._make_data(data)} + return {'version': 2, 'preface': bool(data.get('preface')), **HTTPv2._make_data(data)} raise ProtocolError(f"invalid HTTP data: {type(data).__name__}") + def _test_preface(self, length: 'int') -> 'bool': + """Test whether the first ``length`` octets open with the HTTP/2 connection preface. + + Args: + length: Length of packet data. + + Returns: + :data:`True` if they do. + + """ + preface_len = len(_HTTP2_PREFACE) + return length >= preface_len and self._data[:preface_len] == _HTTP2_PREFACE + def _guess_version(self, length: 'int', **kwargs: 'Any') -> 'HTTP': """Identify the HTTP version of the payload, and parse it with that version. @@ -344,7 +406,7 @@ def _guess_version(self, length: 'int', **kwargs: 'Any') -> 'HTTP': # caller may hand this method fewer octets than the buffer holds, and a # preface must not be claimed out of octets outside this payload. preface_len = len(_HTTP2_PREFACE) - if length >= preface_len and self._data[:preface_len] == _HTTP2_PREFACE: + if self._test_preface(length): from pcapkit.protocols.application.httpv2 import HTTP as HTTPv2 # isort: skip # pylint: disable=line-too-long,import-outside-toplevel # The preface is not a frame -- the frames begin after it diff --git a/pcapkit/protocols/application/httpv2.py b/pcapkit/protocols/application/httpv2.py index 172d133ad8..e0a4dee839 100644 --- a/pcapkit/protocols/application/httpv2.py +++ b/pcapkit/protocols/application/httpv2.py @@ -38,6 +38,7 @@ ContinuationFrameFlags as Data_ContinuationFrameFlags from pcapkit.protocols.data.application.httpv2 import DataFrame as Data_DataFrame from pcapkit.protocols.data.application.httpv2 import DataFrameFlags as Data_DataFrameFlags +from pcapkit.protocols.data.application.httpv2 import Flags as Data_Flags from pcapkit.protocols.data.application.httpv2 import GoawayFrame as Data_GoawayFrame from pcapkit.protocols.data.application.httpv2 import HeadersFrame as Data_HeadersFrame from pcapkit.protocols.data.application.httpv2 import HeadersFrameFlags as Data_HeadersFrameFlags @@ -48,6 +49,7 @@ from pcapkit.protocols.data.application.httpv2 import \ PushPromiseFrameFlags as Data_PushPromiseFrameFlags from pcapkit.protocols.data.application.httpv2 import RSTStreamFrame as Data_RSTStreamFrame +from pcapkit.protocols.data.application.httpv2 import Settings as Data_Settings from pcapkit.protocols.data.application.httpv2 import SettingsFrame as Data_SettingsFrame from pcapkit.protocols.data.application.httpv2 import SettingsFrameFlags as Data_SettingsFrameFlags from pcapkit.protocols.data.application.httpv2 import UnassignedFrame as Data_UnassignedFrame @@ -80,6 +82,8 @@ from mypy_extensions import DefaultArg, KwArg, NamedArg from typing_extensions import Literal + from pcapkit.protocols.protocol import ProtocolBase + Flags = Schema_FrameType.Flags FrameParser = Callable[[Schema_FrameType, NamedArg(Schema_HTTP, 'header')], Data_HTTP] @@ -308,7 +312,9 @@ def read(self, length: 'Optional[int]' = None, **kwargs: 'Any') -> 'Data_HTTP': meth = name[0] http = meth(schema.frame, header=schema) - return http + # The frame ends at its declared Length; whatever follows it in the + # buffer (e.g. the next frame) is the undissected remainder. + return self._decode_next_layer(http, -1, length - 9 - schema.length) def make(self, # type: ignore[override] type: 'Enum_Frame | StdlibEnum | AenumEnum | str | int' = Enum_Frame.DATA, @@ -316,8 +322,10 @@ def make(self, # type: ignore[override] type_namespace: 'Optional[dict[str, int] | dict[int, str] | Type[StdlibEnum] | Type[AenumEnum]]' = None, # pylint: disable=line-too-long type_reversed: 'bool' = False, flags: 'Flags' = 0, # type: ignore[assignment] + reserved: 'int' = 0, sid: 'int' = 0, frame: 'bytes | Data_HTTP | Schema_FrameType | dict[str, Any]' = b'', + payload: 'bytes | ProtocolBase | Schema' = b'', **kwargs: 'Any') -> 'Schema_HTTP': """Make (construct) packet data. @@ -326,9 +334,12 @@ def make(self, # type: ignore[override] type_default: Default frame type. type_namespace: Namespace of frame type. type_reversed: Whether to reverse the namespace. - flags: Flags of HTTP/2 frame. + flags: Flags of HTTP/2 frame, combined with those the frame + constructor derives from ``frame``. + reserved: Reserved bit of the frame header. sid: Stream ID of HTTP/2 frame. frame: Frame data of HTTP/2 frame. + payload: Octets after the frame, e.g. the next frame. **kwargs: Arbitrary keyword arguments. Returns: @@ -351,9 +362,10 @@ def make(self, # type: ignore[override] meth = name[1] if isinstance(frame, dict): - frame_val, flags = meth(**frame) + frame_val, frame_flags = meth(**frame) else: - frame_val, flags = meth(frame) + frame_val, frame_flags = meth(frame) + flags = cast('Flags', int(flags) | int(frame_flags)) elif isinstance(frame, Schema): frame_val = frame else: @@ -370,11 +382,34 @@ def make(self, # type: ignore[override] type=type_val, flags=flags_val, # type: ignore[arg-type] stream={ + 'reserved': reserved, 'sid': sid, }, frame=frame_val, + payload=payload, ) + def pack(self, **kwargs: 'Any') -> 'bytes': + """Pack (construct) packet data. + + Args: + **kwargs: Arbitrary keyword arguments. + + Returns: + Constructed packet data. + + Notes: + :meth:`make` takes the frame payload as raw octets or as a built + frame schema as well, and neither carries what the frame readers + need, e.g. ``__flags__``. So the header :meth:`make` returns is + dropped once packed, and :meth:`unpack` parses the packed octets + back with the frame type's own schema. + + """ + data = super().pack(**kwargs) + self.__header__ = None # type: ignore[assignment] + return data + @staticmethod def _make_http_length(frame: 'bytes | Schema_FrameType', flags: 'Flags') -> 'int': """Calculate HTTP/2 frame payload length for construction. @@ -390,12 +425,12 @@ def _make_http_length(frame: 'bytes | Schema_FrameType', flags: 'Flags') -> 'int if isinstance(frame, Schema_UnassignedFrame): return len(frame.data) if isinstance(frame, Schema_DataFrame): - pad_len = frame.pad_len if flags_int & int(Schema_DataFrame.Flags.PADDED) else 0 - return len(frame.data) + (pad_len + 1 if pad_len else 0) + padded = flags_int & int(Schema_DataFrame.Flags.PADDED) + return len(frame.data) + ((frame.pad_len or 0) + 1 if padded else 0) if isinstance(frame, Schema_HeadersFrame): - pad_len = frame.pad_len if flags_int & int(Schema_HeadersFrame.Flags.PADDED) else 0 + padded = flags_int & int(Schema_HeadersFrame.Flags.PADDED) priority_len = 5 if flags_int & int(Schema_HeadersFrame.Flags.PRIORITY) else 0 - return len(frame.fragment) + priority_len + (pad_len + 1 if pad_len else 0) + return len(frame.fragment) + priority_len + ((frame.pad_len or 0) + 1 if padded else 0) if isinstance(frame, Schema_PriorityFrame): return 5 if isinstance(frame, Schema_RSTStreamFrame): @@ -406,8 +441,8 @@ def _make_http_length(frame: 'bytes | Schema_FrameType', flags: 'Flags') -> 'int return len(settings) return len(frame.settings) * 6 if isinstance(frame, Schema_PushPromiseFrame): - pad_len = frame.pad_len if flags_int & int(Schema_PushPromiseFrame.Flags.PADDED) else 0 - return len(frame.fragment) + 4 + (pad_len + 1 if pad_len else 0) + padded = flags_int & int(Schema_PushPromiseFrame.Flags.PADDED) + return len(frame.fragment) + 4 + ((frame.pad_len or 0) + 1 if padded else 0) if isinstance(frame, Schema_PingFrame): return 8 if isinstance(frame, Schema_GoawayFrame): @@ -471,10 +506,29 @@ def _make_data(cls, data: 'Data_HTTP') -> 'dict[str, Any]': # type: ignore[over return { 'type': data.type, 'flags': data.flags.__value__ if data.flags is not None else 0, + 'reserved': data.reserved, 'sid': data.sid, 'frame': data, + 'payload': cls._make_payload(data), } + @staticmethod + def _read_http_flags(schema: 'Schema_FrameType') -> 'Data_Flags': + """Read the flags of an HTTP/2 frame type that defines none. + + Args: + schema: Parsed frame schema. + + Returns: + Flags with the raw flags octet as ``__value__``. + + """ + flags = Data_Flags() + flags.__update__({ + '__value__': schema.__flags__, + }) + return flags + def _read_http_none(self, schema: 'Schema_UnassignedFrame', *, header: 'Schema_HTTP') -> 'Data_UnassignedFrame': """Read HTTP packet with unassigned type. @@ -487,14 +541,15 @@ def _read_http_none(self, schema: 'Schema_UnassignedFrame', *, Parsed packet data. """ - if any(header.flags): + if any(header.flags.values()): #raise ProtocolError(f'HTTP/2: [Type {frame}] invalid format') warn(f'HTTP/2: [Type {header.type}] invalid format', ProtocolWarning) data = Data_UnassignedFrame( length=header.length, type=header.type, - flags=None, + flags=self._read_http_flags(schema), + reserved=header.stream['reserved'], sid=header.stream['sid'], data=schema.data, ) @@ -543,6 +598,7 @@ def _read_http_data(self, schema: 'Schema_DataFrame', *, type=header.type, flags=flag, pad_len=schema.pad_len if flag.PADDED else 0, + reserved=header.stream['reserved'], sid=header.stream['sid'], data=schema.data, ) @@ -597,6 +653,7 @@ def _read_http_headers(self, schema: 'Schema_HeadersFrame', *, type=header.type, flags=flag, pad_len=schema.pad_len if flag.PADDED else 0, + reserved=header.stream['reserved'], sid=header.stream['sid'], excl_dependency=bool(schema.stream_dep['exclusive']) if flag.PRIORITY else False, stream_dependency=schema.stream_dep['sid'] if flag.PRIORITY else 0, @@ -642,7 +699,8 @@ def _read_http_priority(self, schema: 'Schema_PriorityFrame', *, data = Data_PriorityFrame( length=header.length, type=header.type, - flags=None, + flags=self._read_http_flags(schema), + reserved=header.stream['reserved'], sid=header.stream['sid'], excl_dependency=bool(schema.stream['exclusive']), stream_dependency=schema.stream['sid'], @@ -685,7 +743,8 @@ def _read_http_rst_stream(self, schema: 'Schema_RSTStreamFrame', *, data = Data_RSTStreamFrame( length=header.length, type=header.type, - flags=None, + flags=self._read_http_flags(schema), + reserved=header.stream['reserved'], sid=header.stream['sid'], error=schema.error, ) @@ -736,14 +795,18 @@ def _read_http_settings(self, schema: 'Schema_SettingsFrame', *, if flag.ACK and header.length != 0: raise ProtocolError(f'HTTP/2: [Type {header.type}] invalid format') - sets = OrderedMultiDict() # type: OrderedMultiDict[Enum_Setting, int] + sets = Data_Settings() + # ``add`` rather than item assignment: an identifier may repeat, and + # the values apply in order (:rfc:`9113#section-6.5.3`). ``Settings`` + # keeps every entry, and answers an index with the last value. for setting in schema.settings: - sets[setting.id] = setting.value + sets.add(setting.id, setting.value) data = Data_SettingsFrame( length=header.length, type=header.type, flags=flag, + reserved=header.stream['reserved'], sid=header.stream['sid'], settings=sets, ) @@ -799,8 +862,10 @@ def _read_http_push_promise(self, schema: 'Schema_PushPromiseFrame', *, length=header.length, type=header.type, flags=flag, + reserved=header.stream['reserved'], sid=header.stream['sid'], pad_len=schema.pad_len if flag.PADDED else 0, + promised_reserved=schema.stream['reserved'], promised_sid=schema.stream['sid'], fragment=schema.fragment, ) @@ -852,6 +917,7 @@ def _read_http_ping(self, schema: 'Schema_PingFrame', *, length=header.length, type=header.type, flags=flag, + reserved=header.stream['reserved'], sid=header.stream['sid'], data=schema.data, ) @@ -890,8 +956,10 @@ def _read_http_goaway(self, schema: 'Schema_GoawayFrame', *, data = Data_GoawayFrame( length=header.length, type=header.type, - flags=None, + flags=self._read_http_flags(schema), + reserved=header.stream['reserved'], sid=header.stream['sid'], + last_reserved=schema.stream['reserved'], last_sid=schema.stream['sid'], error=schema.error, debug_data=schema.debug, @@ -933,8 +1001,10 @@ def _read_http_window_update(self, schema: 'Schema_WindowUpdateFrame', *, data = Data_WindowUpdateFrame( length=header.length, type=header.type, - flags=None, + flags=self._read_http_flags(schema), + reserved=header.stream['reserved'], sid=header.stream['sid'], + increment_reserved=schema.size['reserved'], increment=schema.size['incr'], ) return data @@ -976,6 +1046,7 @@ def _read_http_continuation(self, schema: 'Schema_ContinuationFrame', *, length=header.length, type=header.type, flags=flag, + reserved=header.stream['reserved'], sid=header.stream['sid'], fragment=schema.fragment, ) @@ -1004,6 +1075,7 @@ def _make_http_none(self, frame: 'Optional[Data_UnassignedFrame]' = None, *, def _make_http_data(self, frame: 'Optional[Data_DataFrame]' = None, *, end_stream: 'bool' = False, + padded: 'bool' = False, pad_len: 'int' = 0, data: 'bytes' = b'', **kwargs: 'Any') -> 'tuple[Schema_DataFrame, Flags]': @@ -1012,6 +1084,7 @@ def _make_http_data(self, frame: 'Optional[Data_DataFrame]' = None, *, Args: frame: Frame data model. end_stream: End of stream flag. + padded: Padded flag, implied by a non-zero ``pad_len``. pad_len: Padding length. data: Frame data. **kwargs: Arbitrary keyword arguments. @@ -1022,6 +1095,7 @@ def _make_http_data(self, frame: 'Optional[Data_DataFrame]' = None, *, """ if frame is not None: end_stream = frame.flags.END_STREAM + padded = frame.flags.PADDED pad_len = frame.pad_len data = frame.data @@ -1029,7 +1103,7 @@ def _make_http_data(self, frame: 'Optional[Data_DataFrame]' = None, *, flags = Schema_DataFrame.Flags(0) if end_stream: flags |= Schema_DataFrame.Flags.END_STREAM - if pad_len: + if padded or pad_len: flags |= Schema_DataFrame.Flags.PADDED return Schema_DataFrame( @@ -1040,6 +1114,7 @@ def _make_http_data(self, frame: 'Optional[Data_DataFrame]' = None, *, def _make_http_headers(self, frame: 'Optional[Data_HeadersFrame]' = None, *, end_stream: 'bool' = False, end_headers: 'bool' = False, + padded: 'bool' = False, pad_len: 'int' = 0, excl_dep: 'bool' = False, sid_dep: 'Optional[int]' = None, @@ -1052,6 +1127,7 @@ def _make_http_headers(self, frame: 'Optional[Data_HeadersFrame]' = None, *, frame: Frame data model. end_stream: End of stream flag. end_headers: End of headers flag. + padded: Padded flag, implied by a non-zero ``pad_len``. pad_len: Padding length. excl_dep: Exclusive dependency flag. sid_dep: Dependency stream identifier. @@ -1067,6 +1143,7 @@ def _make_http_headers(self, frame: 'Optional[Data_HeadersFrame]' = None, *, priority = frame.flags.PRIORITY end_headers = frame.flags.END_HEADERS end_stream = frame.flags.END_STREAM + padded = frame.flags.PADDED pad_len = frame.pad_len excl_dep = frame.excl_dependency @@ -1082,7 +1159,7 @@ def _make_http_headers(self, frame: 'Optional[Data_HeadersFrame]' = None, *, flags |= Schema_HeadersFrame.Flags.END_STREAM if end_headers: flags |= Schema_HeadersFrame.Flags.END_HEADERS - if pad_len: + if padded or pad_len: flags |= Schema_HeadersFrame.Flags.PADDED if priority: flags |= Schema_HeadersFrame.Flags.PRIORITY @@ -1212,7 +1289,9 @@ def _make_http_settings(self, frame: 'Optional[Data_SettingsFrame]' = None, *, def _make_http_push_promise(self, frame: 'Optional[Data_PushPromiseFrame]' = None, *, end_headers: 'bool' = False, + padded: 'bool' = False, pad_len: 'int' = 0, + promised_reserved: 'int' = 0, promised_sid: 'int' = 0, fragment: 'bytes' = b'', **kwargs: 'Any') -> 'tuple[Schema_PushPromiseFrame, Flags]': @@ -1221,7 +1300,9 @@ def _make_http_push_promise(self, frame: 'Optional[Data_PushPromiseFrame]' = Non Args: frame: Frame data model. end_headers: End of headers flag. + padded: Padded flag, implied by a non-zero ``pad_len``. pad_len: Padding length. + promised_reserved: Reserved bit of the promised stream identifier. promised_sid: Promised stream identifier. fragment: Header block fragment. **kwargs: Arbitrary keyword arguments. @@ -1232,19 +1313,22 @@ def _make_http_push_promise(self, frame: 'Optional[Data_PushPromiseFrame]' = Non """ if frame is not None: end_headers = frame.flags.END_HEADERS + padded = frame.flags.PADDED pad_len = frame.pad_len + promised_reserved = frame.promised_reserved promised_sid = frame.promised_sid fragment = frame.fragment flags = Schema_PushPromiseFrame.Flags(0) if end_headers: flags |= Schema_PushPromiseFrame.Flags.END_HEADERS - if pad_len: + if padded or pad_len: flags |= Schema_PushPromiseFrame.Flags.PADDED return Schema_PushPromiseFrame( pad_len=pad_len, stream={ + 'reserved': promised_reserved, 'sid': promised_sid, }, fragment=fragment, @@ -1252,23 +1336,29 @@ def _make_http_push_promise(self, frame: 'Optional[Data_PushPromiseFrame]' = Non def _make_http_ping(self, frame: 'Optional[Data_PingFrame]' = None, *, ack: 'bool' = False, - opaque_data: 'bytes' = b'', + opaque_data: 'bytes' = bytes(8), **kwargs: 'Any') -> 'tuple[Schema_PingFrame, Flags]': """Make HTTP/2 ``PING`` frame. Args: frame: Frame data model. ack: Acknowledge flag. - opaque_data: Opaque data. + opaque_data: Opaque data, exactly 8 octets. **kwargs: Arbitrary keyword arguments. Returns: Constructed frame schema and updated flags. + Raises: + ProtocolError: If ``opaque_data`` is not 8 octets long. + """ if frame is not None: ack = frame.flags.ACK opaque_data = frame.data + if len(opaque_data) != 8: + raise ProtocolError(f'HTTP/2: [Type {Enum_Frame.PING}] opaque data must be 8 octets, ' + f'not {len(opaque_data)}') flags = Schema_PingFrame.Flags(0) if ack: @@ -1279,6 +1369,7 @@ def _make_http_ping(self, frame: 'Optional[Data_PingFrame]' = None, *, ), flags def _make_http_goaway(self, frame: 'Optional[Data_GoawayFrame]' = None, *, + last_reserved: 'int' = 0, last_sid: 'int' = 0, error: 'Enum_ErrorCode | str | int | StdlibEnum | AenumEnum' = Enum_ErrorCode.HTTP_1_1_REQUIRED, error_default: 'Optional[int]' = None, @@ -1290,6 +1381,7 @@ def _make_http_goaway(self, frame: 'Optional[Data_GoawayFrame]' = None, *, Args: frame: Frame data model. + last_reserved: Reserved bit of the last stream identifier. last_sid: Last stream identifier. error: Error code. error_default: Default value of error code. @@ -1303,6 +1395,7 @@ def _make_http_goaway(self, frame: 'Optional[Data_GoawayFrame]' = None, *, """ if frame is not None: + last_reserved = frame.last_reserved last_sid = frame.last_sid error_val = frame.error debug = frame.debug_data @@ -1313,6 +1406,7 @@ def _make_http_goaway(self, frame: 'Optional[Data_GoawayFrame]' = None, *, return Schema_GoawayFrame( stream={ + 'reserved': last_reserved, 'sid': last_sid, }, error=error_val, @@ -1320,12 +1414,14 @@ def _make_http_goaway(self, frame: 'Optional[Data_GoawayFrame]' = None, *, ), cast('Flags', 0) def _make_http_window_update(self, frame: 'Optional[Data_WindowUpdateFrame]' = None, *, + incr_reserved: 'int' = 0, incr: 'int' = 0, **kwargs: 'Any') -> 'tuple[Schema_WindowUpdateFrame, Flags]': """Make HTTP/2 ``WINDOW_UPDATE`` frame. Args: frame: Frame data model. + incr_reserved: Reserved bit of the window size increment. incr: Window size increment. **kwargs: Arbitrary keyword arguments. @@ -1334,10 +1430,12 @@ def _make_http_window_update(self, frame: 'Optional[Data_WindowUpdateFrame]' = N """ if frame is not None: + incr_reserved = frame.increment_reserved incr = frame.increment return Schema_WindowUpdateFrame( size={ + 'reserved': incr_reserved, 'incr': incr, } ), cast('Flags', 0) diff --git a/pcapkit/protocols/data/application/httpv2.py b/pcapkit/protocols/data/application/httpv2.py index 3c4d821b0e..185945abcd 100644 --- a/pcapkit/protocols/data/application/httpv2.py +++ b/pcapkit/protocols/data/application/httpv2.py @@ -4,24 +4,22 @@ from typing import TYPE_CHECKING from pcapkit.corekit.infoclass import info_final +from pcapkit.corekit.multidict import OrderedMultiDict from pcapkit.protocols.data.data import Data from pcapkit.protocols.data.protocol import Protocol if TYPE_CHECKING: - from typing import Optional - - from typing_extensions import Literal + from typing import Iterator from pcapkit.const.http.error_code import ErrorCode from pcapkit.const.http.frame import Frame from pcapkit.const.http.setting import Setting - from pcapkit.corekit.multidict import OrderedMultiDict from pcapkit.protocols.schema.application.httpv2 import FrameType __all__ = [ 'HTTP', - 'Flags', + 'Flags', 'Settings', 'DataFrameFlags', 'HeadersFrameFlags', 'SettingsFrameFlags', 'PushPromiseFrameFlags', 'PingFrameFlags', 'ContinuationFrameFlags', @@ -35,7 +33,8 @@ class Flags(Data): """Data model for HTTP/2 flags.""" if TYPE_CHECKING: - #: Flags as in combination value. + #: Flags as in combination value, bits the frame type leaves + #: undefined included. __value__: 'FrameType.Flags' @@ -47,7 +46,9 @@ class HTTP(Protocol): #: Frame type. type: 'Frame' #: Flags. - flags: 'Optional[Flags]' + flags: 'Flags' + #: Reserved bit of the frame header. + reserved: 'int' #: Stream ID. sid: 'int' @@ -57,12 +58,12 @@ class UnassignedFrame(HTTP): """Data model for HTTP/2 unassigned frame.""" #: Flags. - flags: 'Literal[None]' + flags: 'Flags' #: Frame payload. data: 'bytes' if TYPE_CHECKING: - def __init__(self, length: 'int', type: 'Frame', flags: 'Literal[None]', sid: 'int', data: 'bytes') -> 'None': ... # pylint: disable=unused-argument,super-init-not-called,multiple-statements,line-too-long,redefined-builtin + def __init__(self, length: 'int', type: 'Frame', flags: 'Flags', reserved: 'int', sid: 'int', data: 'bytes') -> 'None': ... # pylint: disable=unused-argument,super-init-not-called,multiple-statements,line-too-long,redefined-builtin @info_final @@ -90,7 +91,7 @@ class DataFrame(HTTP): data: 'bytes' if TYPE_CHECKING: - def __init__(self, length: 'int', type: 'Frame', flags: 'DataFrameFlags', pad_len: 'int', sid: 'int', data: 'bytes') -> 'None': ... # pylint: disable=unused-argument,super-init-not-called,multiple-statements,line-too-long,redefined-builtin + def __init__(self, length: 'int', type: 'Frame', flags: 'DataFrameFlags', reserved: 'int', pad_len: 'int', sid: 'int', data: 'bytes') -> 'None': ... # pylint: disable=unused-argument,super-init-not-called,multiple-statements,line-too-long,redefined-builtin @info_final @@ -128,7 +129,7 @@ class HeadersFrame(HTTP): fragment: 'bytes' if TYPE_CHECKING: - def __init__(self, length: 'int', type: 'Frame', flags: 'HeadersFrameFlags', pad_len: 'int', sid: 'int', excl_dependency: 'bool', stream_dependency: 'int', weight: 'int', fragment: 'bytes') -> 'None': ... # pylint: disable=unused-argument,super-init-not-called,multiple-statements,line-too-long,redefined-builtin + def __init__(self, length: 'int', type: 'Frame', flags: 'HeadersFrameFlags', reserved: 'int', pad_len: 'int', sid: 'int', excl_dependency: 'bool', stream_dependency: 'int', weight: 'int', fragment: 'bytes') -> 'None': ... # pylint: disable=unused-argument,super-init-not-called,multiple-statements,line-too-long,redefined-builtin @info_final @@ -136,7 +137,7 @@ class PriorityFrame(HTTP): """Data model for HTTP/2 ``PRIORITY`` frame.""" #: Flags. - flags: 'Literal[None]' + flags: 'Flags' #: Exclusive dependency. excl_dependency: 'bool' #: Stream dependency. @@ -145,7 +146,7 @@ class PriorityFrame(HTTP): weight: 'int' if TYPE_CHECKING: - def __init__(self, length: 'int', type: 'Frame', flags: 'Literal[None]', sid: 'int', excl_dependency: 'bool', stream_dependency: 'int', weight: 'int') -> 'None': ... # pylint: disable=unused-argument,super-init-not-called,multiple-statements,line-too-long,redefined-builtin + def __init__(self, length: 'int', type: 'Frame', flags: 'Flags', reserved: 'int', sid: 'int', excl_dependency: 'bool', stream_dependency: 'int', weight: 'int') -> 'None': ... # pylint: disable=unused-argument,super-init-not-called,multiple-statements,line-too-long,redefined-builtin @info_final @@ -153,12 +154,36 @@ class RSTStreamFrame(HTTP): """Data model for HTTP/2 ``RST_STREAM`` frame.""" #: Flags. - flags: 'Literal[None]' + flags: 'Flags' #: Error code. error: 'ErrorCode' if TYPE_CHECKING: - def __init__(self, length: 'int', type: 'Frame', flags: 'Literal[None]', sid: 'int', error: 'int') -> 'None': ... # pylint: disable=unused-argument,super-init-not-called,multiple-statements,line-too-long,redefined-builtin + def __init__(self, length: 'int', type: 'Frame', flags: 'Flags', reserved: 'int', sid: 'int', error: 'int') -> 'None': ... # pylint: disable=unused-argument,super-init-not-called,multiple-statements,line-too-long,redefined-builtin + + +class Settings(OrderedMultiDict['Setting', int]): + """Settings of an HTTP/2 ``SETTINGS`` frame. + + An identifier may repeat, and the values apply in order, so the last one + wins (:rfc:`9113#section-6.5`). Every entry is kept in wire order, as + :meth:`getlist` and ``items(multi=True)`` show and the rebuild writes, but + indexing, :meth:`get`, :meth:`values`, ``items()`` and :meth:`to_dict` + answer with the *last* value of each identifier rather than the first. + + """ + + def __getitem__(self, key: 'Setting') -> 'int': + if key in self: + return self.getlist(key)[-1] + return super().__getitem__(key) + + def items(self, multi: 'bool' = False) -> 'Iterator[tuple[Setting, int]]': # type: ignore[override] + if multi: + yield from super().items(multi=True) + return + for key, _ in super().items(): + yield key, self[key] @info_final @@ -179,10 +204,10 @@ class SettingsFrame(HTTP): #: Flags. flags: 'SettingsFrameFlags' #: Settings. - settings: 'OrderedMultiDict[Setting, int]' + settings: 'Settings' if TYPE_CHECKING: - def __init__(self, length: 'int', type: 'Frame', flags: 'Optional[Flags]', sid: 'int', settings: 'OrderedMultiDict[Setting, int]') -> 'None': ... # pylint: disable=unused-argument,super-init-not-called,multiple-statements,line-too-long,redefined-builtin + def __init__(self, length: 'int', type: 'Frame', flags: 'Flags', reserved: 'int', sid: 'int', settings: 'Settings') -> 'None': ... # pylint: disable=unused-argument,super-init-not-called,multiple-statements,line-too-long,redefined-builtin @info_final @@ -206,13 +231,15 @@ class PushPromiseFrame(HTTP): flags: 'PushPromiseFrameFlags' #: Padded length. pad_len: 'int' + #: Reserved bit of the promised stream ID. + promised_reserved: 'int' #: Promised stream ID. promised_sid: 'int' #: Header block fragment. fragment: 'bytes' if TYPE_CHECKING: - def __init__(self, length: 'int', type: 'Frame', flags: 'Optional[Flags]', pad_len: 'int', sid: 'int', promised_sid: 'int', fragment: 'bytes') -> 'None': ... # pylint: disable=unused-argument,super-init-not-called,multiple-statements,line-too-long,redefined-builtin + def __init__(self, length: 'int', type: 'Frame', flags: 'Flags', reserved: 'int', pad_len: 'int', sid: 'int', promised_reserved: 'int', promised_sid: 'int', fragment: 'bytes') -> 'None': ... # pylint: disable=unused-argument,super-init-not-called,multiple-statements,line-too-long,redefined-builtin @info_final @@ -236,7 +263,7 @@ class PingFrame(HTTP): data: 'bytes' if TYPE_CHECKING: - def __init__(self, length: 'int', type: 'Frame', flags: 'Optional[Flags]', sid: 'int', data: 'bytes') -> 'None': ... # pylint: disable=unused-argument,super-init-not-called,multiple-statements,line-too-long,redefined-builtin + def __init__(self, length: 'int', type: 'Frame', flags: 'Flags', reserved: 'int', sid: 'int', data: 'bytes') -> 'None': ... # pylint: disable=unused-argument,super-init-not-called,multiple-statements,line-too-long,redefined-builtin @info_final @@ -244,7 +271,9 @@ class GoawayFrame(HTTP): """Data model for HTTP/2 ``GOAWAY`` frame.""" #: Flags. - flags: 'Literal[None]' + flags: 'Flags' + #: Reserved bit of the last stream ID. + last_reserved: 'int' #: Last stream ID. last_sid: 'int' #: Error code. @@ -253,7 +282,7 @@ class GoawayFrame(HTTP): debug_data: 'bytes' if TYPE_CHECKING: - def __init__(self, length: 'int', type: 'Frame', flags: 'Optional[Flags]', sid: 'int', last_sid: 'int', error: 'int', debug_data: 'bytes') -> 'None': ... # pylint: disable=unused-argument,super-init-not-called,multiple-statements,line-too-long,redefined-builtin + def __init__(self, length: 'int', type: 'Frame', flags: 'Flags', reserved: 'int', sid: 'int', last_reserved: 'int', last_sid: 'int', error: 'int', debug_data: 'bytes') -> 'None': ... # pylint: disable=unused-argument,super-init-not-called,multiple-statements,line-too-long,redefined-builtin @info_final @@ -261,12 +290,14 @@ class WindowUpdateFrame(HTTP): """Data model for HTTP/2 ``WINDOW_UPDATE`` frame.""" #: Flags. - flags: 'Literal[None]' + flags: 'Flags' + #: Reserved bit of the window size increment. + increment_reserved: 'int' #: Window size increment. increment: 'int' if TYPE_CHECKING: - def __init__(self, length: 'int', type: 'Frame', flags: 'Optional[Flags]', sid: 'int', increment: 'int') -> 'None': ... # pylint: disable=unused-argument,super-init-not-called,multiple-statements,line-too-long,redefined-builtin + def __init__(self, length: 'int', type: 'Frame', flags: 'Flags', reserved: 'int', sid: 'int', increment_reserved: 'int', increment: 'int') -> 'None': ... # pylint: disable=unused-argument,super-init-not-called,multiple-statements,line-too-long,redefined-builtin @info_final @@ -290,4 +321,4 @@ class ContinuationFrame(HTTP): fragment: 'bytes' if TYPE_CHECKING: - def __init__(self, length: 'int', type: 'Frame', flags: 'Optional[Flags]', sid: 'int', fragment: 'bytes') -> 'None': ... # pylint: disable=unused-argument,super-init-not-called,multiple-statements,line-too-long,redefined-builtin + def __init__(self, length: 'int', type: 'Frame', flags: 'Flags', reserved: 'int', sid: 'int', fragment: 'bytes') -> 'None': ... # pylint: disable=unused-argument,super-init-not-called,multiple-statements,line-too-long,redefined-builtin diff --git a/pcapkit/protocols/schema/application/httpv2.py b/pcapkit/protocols/schema/application/httpv2.py index 468d478f81..18d1f04ad1 100644 --- a/pcapkit/protocols/schema/application/httpv2.py +++ b/pcapkit/protocols/schema/application/httpv2.py @@ -11,7 +11,7 @@ from pcapkit.const.http.setting import Setting as Enum_Setting from pcapkit.corekit.enum import EnumLookup from pcapkit.corekit.fields.collections import ListField -from pcapkit.corekit.fields.misc import ConditionalField, SchemaField, SwitchField +from pcapkit.corekit.fields.misc import ConditionalField, PayloadField, SchemaField, SwitchField from pcapkit.corekit.fields.numbers import EnumField, NumberField, UInt8Field, UInt32Field from pcapkit.corekit.fields.strings import BitField, BytesField, PaddingField from pcapkit.protocols.schema.schema import EnumSchema, Schema, schema_final @@ -30,6 +30,7 @@ from typing import Any, Optional from pcapkit.corekit.fields.field import FieldBase as Field + from pcapkit.protocols.protocol import ProtocolBase if SPHINX_TYPE_CHECKING: # pragma: no cover from typing_extensions import TypedDict @@ -49,6 +50,8 @@ class FrameFlags(TypedDict): class StreamID(TypedDict): """Stream identifier.""" + #: Reserved bit. + reserved: int #: Stream identifier. sid: int @@ -63,6 +66,8 @@ class StreamDependency(TypedDict): class WindowSize(TypedDict): """Window size increment.""" + #: Reserved bit. + reserved: int #: Window size increment. incr: int @@ -81,7 +86,12 @@ def http_frame_selector(pkt: 'dict[str, Any]') -> 'Field': """ type = cast('Enum_Frame', pkt['type']) schema = FrameType.registry[type] - return SchemaField(length=pkt['__length__'], schema=schema) + # The frame payload is bounded by the declared Length, so anything after + # it -- typically the next frame -- is left for :attr:`HTTP.payload`. It is + # bounded by the buffer as well, so that a Length the buffer cannot back + # reaches :meth:`HTTP.read `, + # which rejects it with :exc:`~pcapkit.utilities.exceptions.ProtocolError`. + return SchemaField(length=min(pkt['length'], pkt['__length__']), schema=schema) @schema_final @@ -105,15 +115,18 @@ class HTTP(Schema): }) #: Stream identifier. stream: 'StreamID' = BitField(length=4, namespace={ + 'reserved': (0, 1), 'sid': (1, 31), }) #: Frame payload. frame: 'FrameType' = SwitchField( selector=http_frame_selector, ) + #: Octets after the frame, e.g. the next frame. + payload: 'bytes' = PayloadField() if TYPE_CHECKING: - def __init__(self, length: 'int', type: 'Enum_Frame', flags: 'FrameFlags', stream: 'StreamID', frame: 'FrameType | bytes') -> 'None': ... + def __init__(self, length: 'int', type: 'Enum_Frame', flags: 'FrameFlags', stream: 'StreamID', frame: 'FrameType | bytes', payload: 'bytes | ProtocolBase | Schema') -> 'None': ... class FrameType(EnumSchema[Enum_Frame]): @@ -161,9 +174,8 @@ def post_process(self, packet: 'dict[str, Any]') -> 'Schema': # defined`` (3.10 and earlier hand back a pseudo-member instead), so # the five frame schemas that inherit it unchanged -- ``UnassignedFrame``, # ``PriorityFrame``, ``RSTStreamFrame``, ``GoawayFrame`` and - # ``WindowUpdateFrame`` -- keep the plain ``int``. All five pass - # ``flags=None`` into their data objects and never surface - # ``__flags__``, so nothing observes the difference for them. + # ``WindowUpdateFrame`` -- keep the plain ``int``, which is what their + # data objects' bare ``Flags`` carries as ``__value__``. flags = self.Flags(0) if self.Flags.__members__ else 0 for key, val in filter(lambda kv: kv[0].startswith('BIT_'), self.Flags.__members__.items()): @@ -171,7 +183,11 @@ def post_process(self, packet: 'dict[str, Any]') -> 'Schema': if packet['flags'][name]: flags |= val - self.__flags__ = flags + # Bits the frame type leaves undefined are kept as well: a receiver + # ignores them (:rfc:`9113#section-4.1`), but a rebuild must not erase + # them. ``Flags`` keeps such bits as pseudo-members. + raw = sum(packet['flags'][f'bit_{bit}'] << bit for bit in range(8)) + self.__flags__ = flags | (raw & ~int(flags)) return self @@ -354,6 +370,7 @@ class Flags(FrameType.Flags): ) #: Promised stream ID. stream: 'StreamID' = BitField(length=4, namespace={ + 'reserved': (0, 1), 'sid': (1, 31), }) #: Header block fragment. @@ -397,6 +414,7 @@ class GoawayFrame(FrameType, code=Enum_Frame.GOAWAY): #: Last stream ID. stream: 'StreamID' = BitField(length=4, namespace={ + 'reserved': (0, 1), 'sid': (1, 31), }) #: Error code. @@ -414,6 +432,7 @@ class WindowUpdateFrame(FrameType, code=Enum_Frame.WINDOW_UPDATE): #: Window size increment. size: 'WindowSize' = BitField(length=4, namespace={ + 'reserved': (0, 1), 'incr': (1, 31), }) diff --git a/tests/protocols/application/test_http2_roundtrip_unit.py b/tests/protocols/application/test_http2_roundtrip_unit.py new file mode 100644 index 0000000000..ca3b61f117 --- /dev/null +++ b/tests/protocols/application/test_http2_roundtrip_unit.py @@ -0,0 +1,179 @@ +# -*- coding: utf-8 -*- +"""HTTP/2 frames rebuild byte for byte through ``from_data``. + +The application round-trip audit (#1202) found seven ways a parsed HTTP/2 +frame lost octets on rebuild, and one way ``make`` refused its own input: + +* #1242 -- the 24-octet connection preface was skipped and never rebuilt. +* #1243 -- a frame's payload ran to the end of the buffer instead of to its + declared Length, so a following frame was swallowed into it. +* #1244 -- ``PADDED`` with a Pad Length of 0 lost both the flag and the octet. +* #1245 -- flag bits the frame type leaves undefined were cleared. +* #1246 -- the reserved R bit of every 31-bit stream field was cleared. +* #1247 -- a repeated ``SETTINGS`` identifier collapsed into one entry. +* #1248 -- ``make(frame=)`` raised :exc:`AttributeError`. +* #1251 (HTTP/2 half) -- ``PING`` opaque data of the wrong width was padded or + cut rather than refused. + +Every case builds its own octets in memory and reads no capture. Classes are +imported inside each test, after :func:`~tests._support.reimport_once_per_class`. + +""" + +import unittest + +from tests._support import reimport_once_per_class + +PREFACE = b'PRI * HTTP/2.0\r\n\r\nSM\r\n\r\n' +SETTINGS_ACK = bytes.fromhex('000000040100000000') + +#: Wire octets that must rebuild unchanged, by issue. +FRAMES = { + '1243 DATA then SETTINGS ACK': bytes.fromhex('000003000000000001616263') + SETTINGS_ACK, + '1243 HEADERS then SETTINGS ACK': bytes.fromhex('00000101040000000182') + SETTINGS_ACK, + '1243 GOAWAY then SETTINGS ACK': bytes.fromhex('000008070000000000000000010000000a') + SETTINGS_ACK, + '1244 DATA PADDED, Pad Length 0': bytes.fromhex('00000400080000000100616263'), + '1244 HEADERS PADDED, Pad Length 0': bytes.fromhex('0000020108000000010082'), + '1244 PUSH_PROMISE PADDED, Pad Length 0': bytes.fromhex('000006050800000001000000000282'), + '1245 DATA flag 0x02': bytes.fromhex('000003000200000001616263'), + '1245 PRIORITY flag 0x01': bytes.fromhex('0000050201000000010000000310'), + '1245 RST_STREAM flags 0xff': bytes.fromhex('00000403ff0000000100000008'), + '1245 unassigned type, flag 0x01': bytes.fromhex('000000fa0100000001'), + '1246 header R bit': bytes.fromhex('000003000080000001616263'), + '1246 PUSH_PROMISE R bit': bytes.fromhex('0000050504000000018000000282'), + '1246 GOAWAY R bit': bytes.fromhex('0000080700000000008000000100000002'), + '1246 WINDOW_UPDATE R bit': bytes.fromhex('00000408000000000080000001'), + '1247 SETTINGS repeated id': bytes.fromhex('00000c040000000000' '000100001000' '000100002000'), + '1247 SETTINGS A, B, A': bytes.fromhex('000012040000000000' '000100000001' '000200000000' '000100000003'), +} + + +class TestHTTP2RoundTrip(unittest.TestCase): + """Pin the rebuild of parsed HTTP/2 frames, and ``make``'s input checks.""" + + def setUp(self) -> None: + reimport_once_per_class(self) + + def test_from_data_rebuilds_byte_for_byte(self) -> None: + from pcapkit.protocols.application.httpv2 import HTTP + + for name, wire in FRAMES.items(): + with self.subTest(frame=name): + parsed = HTTP(wire, len(wire)) + self.assertEqual(bytes(HTTP.from_data(parsed.info)).hex(), wire.hex()) + + def test_preface_is_parsed_and_rebuilt(self) -> None: + from pcapkit.protocols.application.http import HTTP + + for name, wire in { + 'preface + SETTINGS': PREFACE + bytes.fromhex('000000040000000000'), + 'preface + SETTINGS + SETTINGS ACK': PREFACE + bytes.fromhex('000000040000000000') + SETTINGS_ACK, + }.items(): + with self.subTest(frame=name): + parsed = HTTP(wire, len(wire)) + self.assertEqual(parsed.info.preface, PREFACE) + self.assertEqual(bytes(HTTP.from_data(parsed.info)).hex(), wire.hex()) + + def test_payload_is_bounded_by_the_declared_length(self) -> None: + from pcapkit.protocols.application.httpv2 import HTTP + + wire = FRAMES['1243 DATA then SETTINGS ACK'] + parsed = HTTP(wire, len(wire)) + self.assertEqual(parsed.info.length, 3) + self.assertEqual(parsed.info.data, b'abc') + self.assertEqual(bytes(parsed.payload), SETTINGS_ACK) + + def test_reserved_and_undefined_bits_are_parsed(self) -> None: + from pcapkit.protocols.application.httpv2 import HTTP + + def parse(name: str) -> 'object': + wire = FRAMES[name] + return HTTP(wire, len(wire)).info + + self.assertEqual(parse('1246 header R bit').reserved, 1) + self.assertEqual(parse('1246 PUSH_PROMISE R bit').promised_reserved, 1) + self.assertEqual(parse('1246 GOAWAY R bit').last_reserved, 1) + self.assertEqual(parse('1246 WINDOW_UPDATE R bit').increment_reserved, 1) + self.assertEqual(int(parse('1245 DATA flag 0x02').flags.__value__), 0x02) + self.assertEqual(int(parse('1245 RST_STREAM flags 0xff').flags.__value__), 0xff) + + def test_repeated_settings_keep_their_order(self) -> None: + from pcapkit.const.http.setting import Setting + from pcapkit.protocols.application.httpv2 import HTTP + + wire = FRAMES['1247 SETTINGS A, B, A'] + settings = HTTP(wire, len(wire)).info.settings + self.assertEqual(list(settings.items(multi=True)), [ + (Setting.HEADER_TABLE_SIZE, 1), + (Setting.ENABLE_PUSH, 0), + (Setting.HEADER_TABLE_SIZE, 3), + ]) + + def test_repeated_settings_index_to_the_last_value(self) -> None: + """The last value of a repeated identifier wins (RFC 9113 section 6.5).""" + from pcapkit.const.http.setting import Setting + from pcapkit.protocols.application.httpv2 import HTTP + + wire = FRAMES['1247 SETTINGS A, B, A'] + info = HTTP(wire, len(wire)).info + settings = info.settings + self.assertEqual(settings[Setting.HEADER_TABLE_SIZE], 3) + self.assertEqual(settings.get(Setting.HEADER_TABLE_SIZE), 3) + self.assertEqual(dict(settings), {Setting.HEADER_TABLE_SIZE: 3, Setting.ENABLE_PUSH: 0}) + self.assertEqual(settings.to_dict(), {Setting.HEADER_TABLE_SIZE: 3, Setting.ENABLE_PUSH: 0}) + self.assertEqual(settings.getlist(Setting.HEADER_TABLE_SIZE), [1, 3]) + self.assertEqual(bytes(HTTP.from_data(info)).hex(), wire.hex()) + + def test_make_padded_with_zero_pad_length(self) -> None: + from pcapkit.const.http.frame import Frame + from pcapkit.protocols.application.httpv2 import HTTP + + made = HTTP(type=Frame.DATA, sid=1, frame={'padded': True, 'data': b'abc'}) + self.assertEqual(bytes(made), FRAMES['1244 DATA PADDED, Pad Length 0']) + + def test_make_accepts_frame_bytes(self) -> None: + from pcapkit.const.http.frame import Frame + from pcapkit.protocols.application.httpv2 import HTTP + + for frame_type, sid, payload in ( + (Frame.DATA, 1, b'abc'), + (Frame.HEADERS, 1, b'\x82'), + (Frame.PRIORITY, 1, bytes.fromhex('0000000310')), + (Frame.RST_STREAM, 1, bytes.fromhex('00000008')), + (Frame.SETTINGS, 0, bytes.fromhex('000100001000')), + (Frame.PUSH_PROMISE, 1, bytes.fromhex('0000000282')), + (Frame.PING, 0, b'01234567'), + (Frame.GOAWAY, 0, bytes.fromhex('0000000100000000')), + (Frame.WINDOW_UPDATE, 1, bytes.fromhex('00000001')), + (Frame.CONTINUATION, 1, b'\x82'), + (Frame(0xfa), 1, b'abc'), + ): + with self.subTest(type=frame_type): + made = HTTP(type=frame_type, sid=sid, frame=payload) + wire = len(payload).to_bytes(3, 'big') + bytes([int(frame_type), 0]) + sid.to_bytes(4, 'big') + payload + self.assertEqual(bytes(made), wire) + self.assertEqual(made.info.type, frame_type) + + def test_make_frame_bytes_wrong_for_the_type_raise_protocol_error(self) -> None: + from pcapkit.const.http.frame import Frame + from pcapkit.protocols.application.httpv2 import HTTP + from pcapkit.utilities.exceptions import ProtocolError + + with self.assertRaises(ProtocolError): + HTTP(type=Frame.PRIORITY, sid=1, frame=b'abc') + + def test_make_ping_rejects_wrong_width_opaque_data(self) -> None: + from pcapkit.const.http.frame import Frame + from pcapkit.protocols.application.httpv2 import HTTP + from pcapkit.utilities.exceptions import ProtocolError + + for data in (b'abc', b'0123456789'): + with self.subTest(data=data): + with self.assertRaises(ProtocolError): + HTTP(type=Frame.PING, sid=0, frame={'opaque_data': data}) + self.assertEqual(bytes(HTTP(type=Frame.PING, sid=0, frame={'opaque_data': b'01234567'}))[9:], + b'01234567') + + +if __name__ == '__main__': + unittest.main() diff --git a/tests/protocols/application/test_http_unit.py b/tests/protocols/application/test_http_unit.py index a1a6f45d0e..f548eea737 100644 --- a/tests/protocols/application/test_http_unit.py +++ b/tests/protocols/application/test_http_unit.py @@ -21,7 +21,8 @@ def http2_header(length: int, type_: object, flags: int = 0, sid: int = 1) -> Si length=length, type=type_, flags={f'bit_{bit}': (flags & (1 << bit)) >> bit for bit in range(8)}, - stream={'sid': sid}, + stream={'reserved': 0, 'sid': sid}, + get_payload=lambda: b'', ) @@ -532,14 +533,14 @@ def test_guess_version_reaches_http2_on_the_connection_preface(self) -> None: self.assertEqual(clean.version, '2') self.assertEqual([w for w in caught if issubclass(w.category, ProtocolWarning)], []) - # The preface prepended to the frame, read as one buffer, is still - # refused on the *explicit* path: ``read(version=2)`` hands the whole - # buffer to ``httpv2.HTTP``, which has no notion of the preface and must - # not read its ASCII as framing. The guess path identifies it instead -- - # see ``test_guess_version_identifies_the_http2_connection_preface``. + # The preface prepended to the frame, read as one buffer, is skipped on + # the *explicit* path too rather than read as framing, since + # ``from_data`` rebuilds a prefaced frame through ``version=2`` (#1242). + # See ``test_guess_version_identifies_the_http2_connection_preface``. raw = preface + settings - with self.assertRaises(ProtocolError): - HTTP(io.BytesIO(raw), len(raw), version=2) + prefaced = HTTP(io.BytesIO(raw), len(raw), version=2) + self.assertEqual(prefaced.info.preface, preface) + self.assertEqual(prefaced.length, len(raw)) def test_guess_version_identifies_the_http2_connection_preface(self) -> None: """The preface is identified by prefix compare, not by a parse (#800). @@ -589,7 +590,11 @@ def test_guess_version_identifies_the_http2_connection_preface(self) -> None: # reports the tail of the preface as this packet's payload, which is what # ``info`` equality below would otherwise catch. self.assertEqual(guessed.length, len(preface) + frame_only.length) - self.assertEqual(guessed.info, frame_only.info) + # Apart from the preface it records (#1242), the frame parses as it + # does alone. + self.assertEqual(guessed.info.preface, preface) + self.assertEqual({k: v for k, v in guessed.info.items() if k != 'preface'}, + dict(frame_only.info.items())) self.assertEqual(guessed.info.packet, b'') self.assertEqual(guessed.info.length, 0) self.assertNotEqual(guessed.info.length, 5265993) @@ -601,8 +606,9 @@ def test_guess_version_identifies_the_http2_connection_preface(self) -> None: settings_2 = http2_frame_bytes(0x04, 0x00, 0, b'\x00\x03\x00\x00\x00d') guessed_2 = HTTP(io.BytesIO(preface + settings_2), len(preface) + len(settings_2)) self.assertEqual(guessed_2.version, '2') - self.assertEqual(guessed_2.info, HTTP(io.BytesIO(settings_2), len(settings_2), - version=2).info) + alone_2 = HTTP(io.BytesIO(settings_2), len(settings_2), version=2).info + self.assertEqual({k: v for k, v in guessed_2.info.items() if k != 'preface'}, + dict(alone_2.items())) def test_guess_version_reports_a_preface_with_no_frame_as_such(self) -> None: """A preface with nothing after it is HTTP/2, but carries no frame. @@ -1901,17 +1907,18 @@ def test_httpv2_frame_readers_cover_successful_frames(self) -> None: with mock.patch('pcapkit.protocols.application.httpv2.warn') as warn: none = proto._read_http_none( - http2_schema(data=b'unknown'), + http2_schema(data=b'unknown', __flags__=0x01), header=http2_header(7, 250, flags=0x01, sid=3), ) self.assertEqual(none.to_dict()['data'], b'unknown') - self.assertIsNone(none.flags) + # The raw flags octet is kept for the rebuild (#1245). + self.assertEqual(none.flags.__value__, 0x01) warn.assert_called_once() with mock.patch('pcapkit.protocols.application.httpv2.warn') as warn: none_clean = proto._read_http_none( http2_schema(data=b'clean'), - header=SimpleNamespace(length=5, type=251, flags={}, stream={'sid': 4}), + header=SimpleNamespace(length=5, type=251, flags={}, stream={'reserved': 0, 'sid': 4}), ) self.assertEqual(none_clean.data, b'clean') warn.assert_not_called() @@ -1969,7 +1976,7 @@ def test_httpv2_frame_readers_cover_successful_frames(self) -> None: self.assertEqual(settings.settings[Setting.ENABLE_PUSH], 0) push = proto._read_http_push_promise( - http2_schema(pad_len=1, stream={'sid': 11}, fragment=b'push', __flags__=0x0C), + http2_schema(pad_len=1, stream={'reserved': 0, 'sid': 11}, fragment=b'push', __flags__=0x0C), header=http2_header(10, Frame.PUSH_PROMISE, flags=0x0C, sid=1), ) self.assertTrue(push.flags.to_dict()['END_HEADERS']) @@ -1985,14 +1992,14 @@ def test_httpv2_frame_readers_cover_successful_frames(self) -> None: self.assertEqual(ping.to_dict()['data'], b'12345678') goaway = proto._read_http_goaway( - http2_schema(stream={'sid': 13}, error=ErrorCode.NO_ERROR, debug=b'bye'), + http2_schema(stream={'reserved': 0, 'sid': 13}, error=ErrorCode.NO_ERROR, debug=b'bye'), header=http2_header(11, Frame.GOAWAY, sid=0), ) self.assertEqual(goaway.to_dict()['last_sid'], 13) self.assertEqual(goaway.to_dict()['debug_data'], b'bye') window = proto._read_http_window_update( - http2_schema(size={'incr': 65535}), + http2_schema(size={'reserved': 0, 'incr': 65535}), header=http2_header(4, Frame.WINDOW_UPDATE, sid=1), ) self.assertEqual(window.to_dict()['increment'], 65535) @@ -2334,7 +2341,7 @@ def test_httpv2_frame_constructors_cover_all_frame_types_and_branches(self) -> N self.assertEqual(headers_schema.weight, 4) self.assertEqual(headers_flags, 0x2D) headers_frame_schema, headers_frame_flags = proto._make_http_headers(SimpleNamespace( - flags=SimpleNamespace(PRIORITY=True, END_HEADERS=True, END_STREAM=False), + flags=SimpleNamespace(PRIORITY=True, END_HEADERS=True, END_STREAM=False, PADDED=False), pad_len=0, excl_dependency=False, stream_dependency=9, @@ -2391,8 +2398,9 @@ def test_httpv2_frame_constructors_cover_all_frame_types_and_branches(self) -> N self.assertEqual(push_schema.stream['sid'], 11) self.assertEqual(push_flags, 0x0C) push_frame_schema, push_frame_flags = proto._make_http_push_promise(SimpleNamespace( - flags=SimpleNamespace(END_HEADERS=False), + flags=SimpleNamespace(END_HEADERS=False, PADDED=False), pad_len=2, + promised_reserved=0, promised_sid=12, fragment=b'from-frame', )) @@ -2423,6 +2431,7 @@ def test_httpv2_frame_constructors_cover_all_frame_types_and_branches(self) -> N self.assertEqual(goaway_schema.stream['sid'], 13) self.assertEqual(goaway_schema.debug, b'bye') goaway_frame_schema, _ = proto._make_http_goaway(SimpleNamespace( + last_reserved=0, last_sid=14, error=ErrorCode.PROTOCOL_ERROR, debug_data=b'frame', @@ -2431,7 +2440,7 @@ def test_httpv2_frame_constructors_cover_all_frame_types_and_branches(self) -> N window_schema, _ = proto._make_http_window_update(incr=65535) self.assertEqual(window_schema.size['incr'], 65535) - window_frame_schema, _ = proto._make_http_window_update(SimpleNamespace(increment=7)) + window_frame_schema, _ = proto._make_http_window_update(SimpleNamespace(increment_reserved=0, increment=7)) self.assertEqual(window_frame_schema.size['incr'], 7) continuation_schema, continuation_flags = proto._make_http_continuation( @@ -2464,6 +2473,7 @@ def test_httpv2_frame_constructors_cover_all_frame_types_and_branches(self) -> N length=8, type=Frame.PING, flags=PingFrameFlags(ACK=True), + reserved=0, sid=0, data=b'12345678', ) @@ -2551,8 +2561,9 @@ def test_httpv2_callable_frame_registry_paths(self) -> None: original = frame_registry.get(custom) def read_frame(frame, *, header): - return SimpleNamespace(type=header.type, length=header.length, - sid=header.stream['sid'], data=frame.data) + from pcapkit.protocols.data.application.httpv2 import UnassignedFrame + return UnassignedFrame(type=header.type, length=header.length, flags=None, + reserved=0, sid=header.stream['sid'], data=frame.data) def make_frame(frame=None, *, data=b''): if frame is not None: @@ -2587,6 +2598,7 @@ def test_httpv2_schema_selector_and_frame_post_process_flags(self) -> None: field = schema_httpv2.http_frame_selector({ 'type': Frame.DATA, + 'length': 4, '__length__': 4, }) self.assertIs(field.schema, DataFrame)