fix(corekit)!: to_dict() returns an InfoDict, so a repeated IPv6 extension header survives the dict rebuild (#1453)

docs/source/pcapkit/corekit/multidict.rst

@@ -35,6 +35,15 @@ values per key, based on the `Werkzeug`_ project.
    :no-special-members: __init__
    :show-inheritance:
 
+.. autoclass:: pcapkit.corekit.multidict.InfoDict
+   :no-members:
+   :no-special-members: __init__
+   :show-inheritance:
+
+   .. automethod:: keys
+   .. automethod:: values
+   .. automethod:: items
+
 Type Variables
 --------------
 

pcapkit/corekit/infoclass.py

@@ -20,12 +20,13 @@
 from typing import TYPE_CHECKING, Generic, TypeVar
 
 from pcapkit.corekit.enum import EnumLookup
+from pcapkit.corekit.multidict import InfoDict, MultiDict
 from pcapkit.utilities.compat import Mapping, final
 from pcapkit.utilities.exceptions import InfoError, UnsupportedCall, stacklevel
 from pcapkit.utilities.warnings import InfoWarning, warn
 
 if TYPE_CHECKING:
-    from typing import Any, Iterable, Iterator, NoReturn, Optional, Type
+    from typing import Any, ItemsView, Iterable, Iterator, NoReturn, Optional, Type
 
     from typing_extensions import Self
 
@@ -137,7 +138,7 @@ def info_final(cls: 'ST', *, _finalised: 'bool' = True) -> 'ST':
     if cls is Info and cls.__dict__.get('__base_ready__'):
         return cls
 
-    temp = ['__map__', '__map_reverse__', '__builtin__', '__finalised__']
+    temp = ['__map__', '__map_reverse__', '__multi__', '__builtin__', '__finalised__']
     temp.extend(cls.__additional__)
     for obj in cls.mro():
         temp.extend(dir(obj))
@@ -306,6 +307,10 @@ class Info(Mapping[str, VT], Generic[VT], metaclass=InfoMeta):
         #: Mapping of name conflicts with builtin methods (transformed names to
         #: original names).
         __map_reverse__: 'dict[str, str]'
+        #: Values of keys held more than once, i.e. every value but the first,
+        #: keyed by the (transformed) key that was last in :attr:`__dict__` when
+        #: each was added, so :meth:`to_dict` writes it right after that key.
+        __multi__: 'dict[str, list[tuple[str, Any]]]'
         #: List of builtin methods.
         __builtin__: 'set[str]'
 
@@ -439,9 +444,20 @@ def __new__(cls, *args: 'VT', **kwargs: 'VT') -> 'Self':  # pylint: disable=unus
         # and thus being shared by all instances.
         super().__setattr__(self, '__map__', {})
         super().__setattr__(self, '__map_reverse__', {})
+        super().__setattr__(self, '__multi__', {})
 
         return self
 
+    def __copy__(self) -> 'Self':
+        # NOTE: The bookkeeping mappings are per instance, so a copy gets its
+        # own: a later :meth:`__update__` on it must not reach the original.
+        new = type(self).__new__(type(self))
+        new.__dict__.update(self.__dict__)
+        new.__dict__['__map__'] = dict(self.__map__)
+        new.__dict__['__map_reverse__'] = dict(self.__map_reverse__)
+        new.__dict__['__multi__'] = {key: list(values) for (key, values) in self.__multi__.items()}
+        return new
+
     def __post_init__(self) -> 'None':
         """Customisation method to be called after initialisation."""
 
@@ -452,16 +468,39 @@ def __update__(self, dict_: 'Optional[Mapping[str, VT] | Iterable[tuple[str, VT]
         # implicitly and internally. Such mapping information will be stored
         # within the :attr:`__map__` attribute.
 
+        # NOTE: A :class:`MultiDict` *adds* its values to the ones already held,
+        # as :meth:`MultiDict.update` does: a key already present keeps its
+        # value and records the new one in :attr:`__multi__`. Anything else
+        # replaces the values held.
+
         __name__ = type(self).__name__  # pylint: disable=redefined-builtin
 
+        multi_iter = ()  # type: Iterable[tuple[str, Any]]
         if dict_ is None:
             data_iter = kwargs.items()  # type: Iterable[tuple[str, Any]]
+        elif isinstance(dict_, MultiDict):
+            multi_iter = dict_.items(multi=True)
+            data_iter = kwargs.items()
         elif isinstance(dict_, (dict, collections.abc.Mapping)) or hasattr(dict_, 'items'):
             data_iter = itertools.chain(dict_.items(), kwargs.items())
         else:
             data_iter = itertools.chain(dict_, kwargs.items())
 
+        for (key, value) in multi_iter:
+            new_key = f'_{__name__}{key}' if key in self.__builtin__ else key
+            if new_key in self.__dict__ and new_key not in self.__excluded__:
+                anchor = next(reversed(self.__dict__))
+                self.__multi__.setdefault(anchor, []).append((new_key, value))
+            else:
+                self.__update__({key: value})
+
         for (key, value) in data_iter:
+            if self.__multi__:
+                # NOTE: A key replaced drops the values it held besides the first.
+                new_key = f'_{__name__}{key}' if key in self.__builtin__ else key
+                for values in self.__multi__.values():
+                    values[:] = [item for item in values if item[0] != new_key]
+
             if key in self.__builtin__:
                 new_key = f'_{__name__}{key}'
 
@@ -524,6 +563,27 @@ def __getitem__(self, name: 'str') -> 'VT':
         key = self.__map__.get(name, name)
         return self.__dict__[key]
 
+    def items(self, multi: 'bool' = False) -> 'ItemsView[str, VT] | Iterator[tuple[str, VT]]':  # type: ignore[override]
+        """Return the ``(key, value)`` pairs.
+
+        Args:
+            multi: If set to :obj:`True`, an iterator with a pair for each value
+                of each key, as :meth:`to_dict` writes them. Otherwise a view of
+                the pairs of the first value of each key, as
+                :meth:`Mapping.items <collections.abc.Mapping.items>` returns.
+
+        """
+        if not multi:
+            return super().items()
+        return ((self.__map_reverse__.get(key, key), value) for (key, value) in self.__items_multi())
+
+    def __items_multi(self) -> 'Iterator[tuple[str, VT]]':
+        """Every ``(key, value)`` pair, with the transformed keys, in order."""
+        for (key, value) in self.__dict__.items():
+            if key not in self.__excluded__:
+                yield key, value
+            yield from self.__multi__.get(key, ())
+
     def __setattr__(self, name: 'str', value: 'VT') -> 'NoReturn':
         raise UnsupportedCall("can't set attribute")
 
@@ -541,6 +601,10 @@ def from_dict(cls, dict_: 'Optional[Mapping[str, VT] | Iterable[tuple[str, VT]]]
           ``for k, v in dict_: self[k] = v``.
         * If ``dict_`` is not present, then does:
           ``for k, v in kwargs.items(): self[k] = v``.
+        * If ``dict_`` is a :class:`~pcapkit.corekit.multidict.MultiDict`, a
+          key it holds more than once keeps every value, as :meth:`to_dict`
+          returns them: the first is the value of the key, and
+          ``items(multi=True)`` yields all of them.
 
         A mapping given for a key whose type annotation resolves at runtime to
         an :class:`Info` subclass (or to :data:`~typing.Optional` of one) is
@@ -568,11 +632,21 @@ def from_dict(cls, dict_: 'Optional[Mapping[str, VT] | Iterable[tuple[str, VT]]]
             value = self.__dict__.get(key)
             if isinstance(value, Mapping) and not isinstance(value, Info):
                 self.__dict__[key] = info_cls.from_dict(value)
+            for values in self.__multi__.values():
+                for (index, (rkey, rvalue)) in enumerate(values):
+                    if rkey == key and isinstance(rvalue, Mapping) and not isinstance(rvalue, Info):
+                        values[index] = (rkey, info_cls.from_dict(rvalue))
         return self
 
-    def to_dict(self) -> 'dict[str, VT]':
+    def to_dict(self) -> 'InfoDict[str, VT]':
         """Convert :class:`Info` into :obj:`dict`.
 
+        The :obj:`dict` is an :class:`~pcapkit.corekit.multidict.InfoDict`. A key
+        held more than once -- as given by a
+        :class:`~pcapkit.corekit.multidict.MultiDict` to :meth:`from_dict`, e.g.
+        a repeated IPv6 extension header -- keeps every value for
+        ``items(multi=True)``, while the :obj:`dict` interface sees the first.
+
         Important:
             We only convert nested :class:`Info` objects into :obj:`dict` if
             they are the direct value of the :class:`Info` object's attribute.
@@ -585,18 +659,15 @@ def to_dict(self) -> 'dict[str, VT]':
             annotation resolves at runtime to an :class:`Info` subclass. For
             any other key -- every key of a bare :class:`Info`, and annotations
             naming a type imported only under :data:`~typing.TYPE_CHECKING` --
-            the conversion is one-way, and the nested value comes back as a
-            plain :obj:`dict`.
+            the conversion is one-way, and the nested value comes back as the
+            :class:`~pcapkit.corekit.multidict.InfoDict` it was written as.
 
         """
-        dict_ = {}  # type: dict[str, Any]
-        for (key, value) in self.__dict__.items():
-            if key in self.__excluded__:
-                continue
-
+        dict_ = InfoDict()  # type: InfoDict[str, Any]
+        for (key, value) in self.__items_multi():
             out_key = self.__map_reverse__.get(key, key)
             if isinstance(value, Info):
-                dict_[out_key] = value.to_dict()
+                dict_.add(out_key, value.to_dict())
 
             #elif isinstance(value, (tuple, list, set, frozenset)):
             #    temp = []  # type: list[Any]
@@ -608,7 +679,7 @@ def to_dict(self) -> 'dict[str, VT]':
             #    dict_[out_key] = value.__class__(temp)
 
             else:
-                dict_[out_key] = value
+                dict_.add(out_key, value)
         return dict_
 
 

pcapkit/corekit/multidict.py

@@ -12,17 +12,19 @@
 
 """
 
+import collections.abc
 import copy
 from typing import TYPE_CHECKING, Generic, TypeVar, cast, overload
 
 from pcapkit.utilities.exceptions import MissingKeyError, UnsupportedCall
 
 if TYPE_CHECKING:
-    from typing import Any, Iterable, Iterator, Mapping, NoReturn, Optional, SupportsIndex
+    from typing import (Any, ItemsView, Iterable, Iterator, KeysView, Mapping, NoReturn, Optional,
+                        SupportsIndex, ValuesView)
 
     from typing_extensions import Literal
 
-__all__ = ['MultiDict', 'OrderedMultiDict']
+__all__ = ['MultiDict', 'OrderedMultiDict', 'InfoDict']
 
 ###############################################################################
 # Type variables
@@ -636,3 +638,73 @@ def popitemlist(self) -> 'tuple[_KT, list[_VT]]':
             bucket.unlink(self)
 
         return key, [x.value for x in buckets]
+
+
+class InfoDict(OrderedMultiDict[_KT, _VT]):
+    """The :class:`OrderedMultiDict` that :meth:`Info.to_dict
+    <pcapkit.corekit.infoclass.Info.to_dict>` returns.
+
+    It is the fields of an :class:`~pcapkit.corekit.infoclass.Info` as a mapping:
+    through the :obj:`dict` interface it holds one value per key, the first, so
+    :mod:`json`, :mod:`plistlib` and the dumpers see a plain mapping. A field the
+    :class:`~pcapkit.corekit.infoclass.Info` holds more than once, such as a
+    repeated IPv6 extension header, keeps every value, in order, for
+    ``items(multi=True)``.
+
+    That sets it apart from an :class:`OrderedMultiDict` held as a field *value*,
+    e.g. a list of options, which the dumpers write as one entry per value.
+
+    Note:
+        Unlike :class:`OrderedMultiDict`, :meth:`keys`, :meth:`values` and
+        :meth:`items` return views, and it compares like a :obj:`dict`: equal to
+        a mapping with the same keys -- in any order -- when each key holds the
+        one value the mapping has, or the same values as another
+        :class:`MultiDict`.
+
+    """
+
+    def __eq__(self, other: 'Any') -> 'bool':
+        if isinstance(other, MultiDict):
+            if len(self) != len(other):
+                return False
+            return all(other.getlist(key) == values for key, values in self.lists())
+        if isinstance(other, collections.abc.Mapping):
+            if len(self) != len(other):
+                return False
+            for key, values in self.lists():
+                if key not in other or values != [other[key]]:
+                    return False
+            return True
+        return NotImplemented
+
+    def __ne__(self, other: 'Any') -> 'bool':
+        # NOTE: :obj:`dict` defines its own ``__ne__``, which compares the
+        # internal buckets rather than negating :meth:`__eq__`.
+        result = self.__eq__(other)
+        if result is NotImplemented:
+            return result
+        return not result
+
+    def __iter__(self) -> 'Iterator[_KT]':
+        return (key for key, _ in OrderedMultiDict.items(self))
+
+    def keys(self) -> 'KeysView[_KT]':  # type: ignore[override]
+        """Return a view of the keys, each once, as :meth:`dict.keys` does."""
+        return collections.abc.KeysView(self)
+
+    def values(self) -> 'ValuesView[_VT]':  # type: ignore[override]
+        """Return a view of the first value of each key, as :meth:`dict.values` does."""
+        return collections.abc.ValuesView(self)
+
+    def items(self, multi: 'bool' = False) -> 'ItemsView[_KT, _VT] | Iterator[tuple[_KT, _VT]]':  # type: ignore[override] # pylint: disable=line-too-long
+        """Return the ``(key, value)`` pairs.
+
+        Args:
+            multi: If set to :obj:`True`, an iterator with a pair for each value
+                of each key, in insertion order. Otherwise a view of the pairs
+                of the first value of each key, as :meth:`dict.items` returns.
+
+        """
+        if multi:
+            return OrderedMultiDict.items(self, multi=True)
+        return collections.abc.ItemsView(self)

pcapkit/dumpkit/common.py

@@ -30,7 +30,7 @@
 import dictdumper.tree
 
 from pcapkit.corekit.infoclass import Info
-from pcapkit.corekit.multidict import MultiDict, OrderedMultiDict
+from pcapkit.corekit.multidict import InfoDict, MultiDict, OrderedMultiDict
 from pcapkit.corekit.sentinels import NoValueType, NullType
 from pcapkit.protocols.schema.schema import Schema
 from pcapkit.utilities.exceptions import UnsupportedCall
@@ -427,7 +427,10 @@ def object_hook(self, o: 'Any') -> 'Any':
                 :func:`escape_key` instead. An
                 :class:`~pcapkit.corekit.multidict.OrderedMultiDict` becomes a
                 :class:`list` of single-key :class:`dict` objects, each of
-                which comes back through the :class:`dict` branch.
+                which comes back through the :class:`dict` branch. An
+                :class:`~pcapkit.corekit.multidict.InfoDict`, the fields of an
+                :class:`~pcapkit.corekit.infoclass.Info`, takes that branch
+                directly, so it is written with one value per key.
 
             """
             if isinstance(o, decimal.Decimal):
@@ -445,7 +448,7 @@ def object_hook(self, o: 'Any') -> 'Any':
                 result = '<NULL>'
             elif isinstance(o, NoValueType):
                 result = '<NO_VALUE>'
-            elif isinstance(o, OrderedMultiDict):
+            elif isinstance(o, OrderedMultiDict) and not isinstance(o, InfoDict):
                 # NOTE: one single-key mapping per entry, in insertion order --
                 # i.e. wire order for the option lists -- since a mapping keyed
                 # by name could only gather the repeats of a key under its first
@@ -456,7 +459,7 @@ def object_hook(self, o: 'Any') -> 'Any':
                     {render_enum(key) if isinstance(key, (enum.Enum, aenum.Enum)) else key: val}
                     for key, val in o.items(multi=True)
                 ]
-            elif isinstance(o, MultiDict):
+            elif isinstance(o, MultiDict) and not isinstance(o, InfoDict):
                 # NOTE: a :class:`MultiDict` keeps no order across keys, only
                 # among the values of each, so grouping by key loses nothing.
                 temp = collections.defaultdict(list)  # type: DefaultDict[str, list[Any]]