From d86d637638fcaee3f2b348cbb80904894759c9c2 Mon Sep 17 00:00:00 2001 From: "Gregory P. Smith" Date: Thu, 22 Oct 2026 00:33:53 +0000 Subject: [PATCH] gh-160185: Fix CJK StreamWriter.reset() not returning to ASCII reset() returned early when nothing was pending, so the shift-back sequence of hz and iso2022_* was never written (regression from gh-67436). Also fix a reference leak of the stream.write() result. --- Lib/codecs.py | 10 +++--- Lib/test/test_multibytecodec.py | 36 +++++++++++++++++++ ...-10-22-00-29-55.gh-issue-160185.BPz40E.rst | 5 +++ Modules/cjkcodecs/multibytecodec.c | 13 ++++--- 4 files changed, 56 insertions(+), 8 deletions(-) create mode 100644 Misc/NEWS.d/next/Library/2026-10-22-00-29-55.gh-issue-160185.BPz40E.rst diff --git a/Lib/codecs.py b/Lib/codecs.py index af6ab031157e79..b789009ace5bbd 100644 --- a/Lib/codecs.py +++ b/Lib/codecs.py @@ -402,9 +402,10 @@ def reset(self): pass def seek(self, offset, whence=0): - self.stream.seek(offset, whence) + # reset() may write to the stream: call it at the old position. if whence == 0 and offset == 0: self.reset() + self.stream.seek(offset, whence) def __getattr__(self, name, getattr=getattr): @@ -741,10 +742,11 @@ def reset(self): self.writer.reset() def seek(self, offset, whence=0): - self.stream.seek(offset, whence) - self.reader.reset() + # writer.reset() may write to the stream: call it at the old position. if whence == 0 and offset == 0: self.writer.reset() + self.stream.seek(offset, whence) + self.reader.reset() def __getattr__(self, name, getattr=getattr): @@ -864,8 +866,8 @@ def reset(self): def seek(self, offset, whence=0): # Seeks must be propagated to both the readers and writers # as they might need to reset their internal buffers. - self.reader.seek(offset, whence) self.writer.seek(offset, whence) + self.reader.seek(offset, whence) def __getattr__(self, name, getattr=getattr): diff --git a/Lib/test/test_multibytecodec.py b/Lib/test/test_multibytecodec.py index 0b34c6e69fab6b..8823d3d9f9a6dc 100644 --- a/Lib/test/test_multibytecodec.py +++ b/Lib/test/test_multibytecodec.py @@ -438,6 +438,42 @@ def test_incrementalencoder_final(self): self.assertEqual(output, self.expected_reset) self.assertEqual(encoder.encode('', final=True), b'') + def test_streamwriter_reset(self): + stream = io.BytesIO() + writer = codecs.getwriter(self.encoding)(stream) + writer.reset() + self.assertEqual(stream.getvalue(), b'') + writer.write(self.text) + self.assertEqual(stream.getvalue(), self.expected) + writer.reset() + self.assertEqual(stream.getvalue(), self.expected_reset) + writer.reset() + self.assertEqual(stream.getvalue(), self.expected_reset) + writer.write(self.text) + writer.reset() + self.assertEqual(stream.getvalue(), self.expected_reset * 2) + + def test_streamwriter_seek0(self): + # The reset sequence is written before the stream is rewound. + info = codecs.lookup(self.encoding) + for wrap in ( + info.streamwriter, + lambda stream: codecs.StreamReaderWriter( + stream, info.streamreader, info.streamwriter), + lambda stream: codecs.EncodedFile(stream, 'utf-8', self.encoding), + ): + stream = io.BytesIO() + writer = wrap(stream) + text = self.text + if isinstance(writer, codecs.StreamRecoder): + text = text.encode() + writer.write(text) + writer.seek(0) + self.assertEqual(stream.tell(), 0) + self.assertEqual(stream.getvalue(), self.expected_reset) + writer.write(text) + self.assertEqual(stream.getvalue(), self.expected_reset) + class TestHZStateful(TestStateful): text = '\u804a\u804a' encoding = 'hz' diff --git a/Misc/NEWS.d/next/Library/2026-10-22-00-29-55.gh-issue-160185.BPz40E.rst b/Misc/NEWS.d/next/Library/2026-10-22-00-29-55.gh-issue-160185.BPz40E.rst new file mode 100644 index 00000000000000..a9d7aef03214a3 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2026-10-22-00-29-55.gh-issue-160185.BPz40E.rst @@ -0,0 +1,5 @@ +Fix :meth:`codecs.StreamWriter.reset` of the stateful CJK codecs (``hz`` and +the ``iso2022_*`` family) not writing the sequence that returns the output to +ASCII. ``seek(0)`` of :class:`codecs.StreamWriter`, +:class:`codecs.StreamReaderWriter` and :class:`codecs.StreamRecoder` now +resets the writer before moving the stream position. diff --git a/Modules/cjkcodecs/multibytecodec.c b/Modules/cjkcodecs/multibytecodec.c index 5fb3ec45a38875..573b1797c13c83 100644 --- a/Modules/cjkcodecs/multibytecodec.c +++ b/Modules/cjkcodecs/multibytecodec.c @@ -1857,13 +1857,17 @@ _multibytecodec_MultibyteStreamWriter_reset_impl(MultibyteStreamWriterObject *se PyTypeObject *cls) /*[clinic end generated code: output=32ef224c2a38aa3d input=28af6a9cd38d1979]*/ { - PyObject *pwrt; + PyObject *pwrt, *pending; - if (!self->pending) - Py_RETURN_NONE; + /* even without pending characters, a stateful codec may have to + * write a sequence that returns the output to its initial state */ + pending = self->pending; + if (pending == NULL) { + pending = Py_GetConstantBorrowed(Py_CONSTANT_EMPTY_STR); + } pwrt = multibytecodec_encode(self->codec, &self->state, - self->pending, NULL, self->errors, + pending, NULL, self->errors, MBENC_FLUSH | MBENC_RESET); /* some pending buffer can be truncated when UnicodeEncodeError is * raised on 'strict' mode. but, 'reset' method is designed to @@ -1886,6 +1890,7 @@ _multibytecodec_MultibyteStreamWriter_reset_impl(MultibyteStreamWriterObject *se Py_DECREF(pwrt); return NULL; } + Py_DECREF(wr); } Py_DECREF(pwrt);