fix(corekit): decode as UTF-8 when a StringField's charset rejects 'replace' (#1706)

pcapkit/corekit/fields/strings.py

@@ -110,8 +110,9 @@ class StringField(_TextField[str]):
             The default is ``'strict'``. Should decoding fail under it, the field
             retries with ``'replace'`` rather than raising, so an octet the charset
             cannot decode reads as U+FFFD -- the octets themselves are not lost,
-            see below. A codec that supports no error handler at all still raises:
-            ``idna`` reports ``Unsupported error handling: replace``, measured.
+            see below. A codec that raises under ``'replace'`` as well, such as
+            ``idna``, which supports ``'strict'`` alone, decodes as UTF-8 with
+            ``'replace'`` instead, and its octets are not lost either.
             Other possible values are ``'ignore'`` and ``'replace'`` as well as any
             other name registered with :func:`codecs.register_error` that can handle
             :exc:`UnicodeDecodeError`.
@@ -206,7 +207,15 @@ def post_process(self, value: 'bytes', packet: 'dict[str, Any]') -> 'str':  # py
             try:
                 ret = value.decode(charset, self._errors)
             except UnicodeError:
-                ret = value.decode(charset, 'replace')
+                # NOTE: A codec may raise under ``'replace'`` too -- ``idna``
+                # supports ``'strict'`` alone, and ``punycode`` before Python 3.13
+                # decodes some non-ASCII octets strictly whatever the handler -- so
+                # the last resort is the fallback encoding, UTF-8, which cannot.
+                # The check below keeps the octets, as for any lossy decoding.
+                try:
+                    ret = value.decode(charset, 'replace')
+                except UnicodeError:
+                    ret = value.decode('utf-8', 'replace')
 
         # NOTE: Encoding may itself fail -- ``errors='strict'`` over text that
         # decoded under a lone-surrogate scheme, for one -- which is as much a

tests/corekit/test_stringfield_lossless_unit.py

@@ -35,6 +35,18 @@
 #: fallback through one would raise where ``'replace'`` does not.
 NO_SURROGATE_CHARSETS = ('utf-16', 'utf-16-le', 'utf-16-be', 'utf-32', 'utf-7')
 
+#: ``(octets, field keyword arguments, decoded text, whether it is a DecodedString)``
+#: for ``idna``, which raises under ``'replace'`` as well, on every Python (#1706):
+#: it supports ``'strict'`` alone, so another scheme, or octets it cannot decode,
+#: reach the UTF-8 fallback. The last row is valid ``idna`` under ``'strict'``.
+REPLACE_REJECTED = (
+    (b'a', {'errors': 'ignore'}, 'a', True),
+    (b'a', {'errors': 'replace'}, 'a', True),
+    (b'xn--caf-dma', {'errors': 'ignore'}, 'xn--caf-dma', True),
+    (b'caf\xe9', {}, 'caf�', True),
+    (b'xn--caf-dma', {}, 'café', False),
+)
+
 
 def pcapng(comment: bytes) -> bytes:
     """A minimal pcapng: SHB, IDB and one EPB, the first two carrying ``comment``.
@@ -111,6 +123,23 @@ def test_charsets_without_surrogateescape_do_not_raise(self) -> None:
                 _, packed = self.roundtrip(b'\xff\xfe\x41', encoding=charset)
                 self.assertEqual(packed, b'\xff\xfe\x41')
 
+    def test_a_codec_that_rejects_replace_does_not_raise(self) -> None:
+        from pcapkit.corekit.fields.strings import DecodedString
+
+        for octets, kwargs, text, decoded in REPLACE_REJECTED:
+            with self.subTest(octets=octets, kwargs=kwargs):
+                value, packed = self.roundtrip(octets, encoding='idna', **kwargs)  # type: ignore[arg-type]
+                self.assertEqual(packed, octets)
+                self.assertEqual(str(value), text)
+                self.assertIs(isinstance(value, DecodedString), decoded)
+
+        # ``punycode`` raises under ``'replace'`` on non-ASCII octets only before
+        # Python 3.13, so its text differs by version; the octets do not.
+        for errors in ('strict', 'ignore', 'replace'):
+            with self.subTest(charset='punycode', errors=errors):
+                _, packed = self.roundtrip(b'caf\xe9', encoding='punycode', errors=errors)
+                self.assertEqual(packed, b'caf\xe9')
+
     def test_valid_text_decodes_to_a_plain_str(self) -> None:
         value, packed = self.roundtrip('héllo'.encode())
         self.assertIs(type(value), str)