Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
10 changes: 6 additions & 4 deletions Lib/codecs.py
Original file line number Diff line number Diff line change
Expand Up @@ -402,9 +402,10 @@ def reset(self):
pass

def seek(self, offset, whence=0):
self.stream.seek(offset, whence)
# reset() may write to the stream: call it at the old position.
if whence == 0 and offset == 0:
self.reset()
self.stream.seek(offset, whence)

def __getattr__(self, name,
getattr=getattr):
Expand Down Expand Up @@ -741,10 +742,11 @@ def reset(self):
self.writer.reset()

def seek(self, offset, whence=0):
self.stream.seek(offset, whence)
self.reader.reset()
# writer.reset() may write to the stream: call it at the old position.
if whence == 0 and offset == 0:
self.writer.reset()
self.stream.seek(offset, whence)
self.reader.reset()

def __getattr__(self, name,
getattr=getattr):
Expand Down Expand Up @@ -864,8 +866,8 @@ def reset(self):
def seek(self, offset, whence=0):
# Seeks must be propagated to both the readers and writers
# as they might need to reset their internal buffers.
self.reader.seek(offset, whence)
self.writer.seek(offset, whence)
self.reader.seek(offset, whence)

def __getattr__(self, name,
getattr=getattr):
Expand Down
36 changes: 36 additions & 0 deletions Lib/test/test_multibytecodec.py
Original file line number Diff line number Diff line change
Expand Up @@ -438,6 +438,42 @@ def test_incrementalencoder_final(self):
self.assertEqual(output, self.expected_reset)
self.assertEqual(encoder.encode('', final=True), b'')

def test_streamwriter_reset(self):
stream = io.BytesIO()
writer = codecs.getwriter(self.encoding)(stream)
writer.reset()
self.assertEqual(stream.getvalue(), b'')
writer.write(self.text)
self.assertEqual(stream.getvalue(), self.expected)
writer.reset()
self.assertEqual(stream.getvalue(), self.expected_reset)
writer.reset()
self.assertEqual(stream.getvalue(), self.expected_reset)
writer.write(self.text)
writer.reset()
self.assertEqual(stream.getvalue(), self.expected_reset * 2)

def test_streamwriter_seek0(self):
# The reset sequence is written before the stream is rewound.
info = codecs.lookup(self.encoding)
for wrap in (
info.streamwriter,
lambda stream: codecs.StreamReaderWriter(
stream, info.streamreader, info.streamwriter),
lambda stream: codecs.EncodedFile(stream, 'utf-8', self.encoding),
):
stream = io.BytesIO()
writer = wrap(stream)
text = self.text
if isinstance(writer, codecs.StreamRecoder):
text = text.encode()
writer.write(text)
writer.seek(0)
self.assertEqual(stream.tell(), 0)
self.assertEqual(stream.getvalue(), self.expected_reset)
writer.write(text)
self.assertEqual(stream.getvalue(), self.expected_reset)

class TestHZStateful(TestStateful):
text = '\u804a\u804a'
encoding = 'hz'
Expand Down
Original file line number Diff line number Diff line change
@@ -0,0 +1,5 @@
Fix :meth:`codecs.StreamWriter.reset` of the stateful CJK codecs (``hz`` and
the ``iso2022_*`` family) not writing the sequence that returns the output to
ASCII. ``seek(0)`` of :class:`codecs.StreamWriter`,
:class:`codecs.StreamReaderWriter` and :class:`codecs.StreamRecoder` now
resets the writer before moving the stream position.
13 changes: 9 additions & 4 deletions Modules/cjkcodecs/multibytecodec.c
Original file line number Diff line number Diff line change
Expand Up @@ -1857,13 +1857,17 @@ _multibytecodec_MultibyteStreamWriter_reset_impl(MultibyteStreamWriterObject *se
PyTypeObject *cls)
/*[clinic end generated code: output=32ef224c2a38aa3d input=28af6a9cd38d1979]*/
{
PyObject *pwrt;
PyObject *pwrt, *pending;

if (!self->pending)
Py_RETURN_NONE;
/* even without pending characters, a stateful codec may have to
* write a sequence that returns the output to its initial state */
pending = self->pending;
if (pending == NULL) {
pending = Py_GetConstantBorrowed(Py_CONSTANT_EMPTY_STR);
}

pwrt = multibytecodec_encode(self->codec, &self->state,
self->pending, NULL, self->errors,
pending, NULL, self->errors,
MBENC_FLUSH | MBENC_RESET);
/* some pending buffer can be truncated when UnicodeEncodeError is
* raised on 'strict' mode. but, 'reset' method is designed to
Expand All @@ -1886,6 +1890,7 @@ _multibytecodec_MultibyteStreamWriter_reset_impl(MultibyteStreamWriterObject *se
Py_DECREF(pwrt);
return NULL;
}
Py_DECREF(wr);
}
Py_DECREF(pwrt);

Expand Down
Loading