From e7e54fbc6b1180677fcec9cfe915282490b70e14 Mon Sep 17 00:00:00 2001 From: Ahmed Semih Ozmekik Date: Thu, 5 Nov 2026 08:46:05 +0300 Subject: [PATCH 1/2] gh-160302: Don't encode ESC, SO and SI in the ISO-2022 codecs The ISO-2022 encoders wrote U+001B as a bare ESC byte, and iso2022_kr wrote U+000E and U+000F as bare SO and SI. Decoders take these bytes as the start of an escape sequence or as shifts, so the output was misread. Treat these characters as unencodable instead. --- Doc/whatsnew/3.16.rst | 7 ++++++ Lib/test/test_codecencodings_iso2022.py | 24 +++++++++++++++---- ...-11-05-12-00-00.gh-issue-160302.z743pv.rst | 5 ++++ Modules/cjkcodecs/_codecs_iso2022.c | 7 ++++++ 4 files changed, 39 insertions(+), 4 deletions(-) create mode 100644 Misc/NEWS.d/next/Library/2026-11-05-12-00-00.gh-issue-160302.z743pv.rst diff --git a/Doc/whatsnew/3.16.rst b/Doc/whatsnew/3.16.rst index 514ab2605e37166..896760f3258bd76 100644 --- a/Doc/whatsnew/3.16.rst +++ b/Doc/whatsnew/3.16.rst @@ -1326,6 +1326,13 @@ that may require changes to your code. raises :exc:`io.UnsupportedOperation` unless buffering is disabled. (Contributed by An Long in :gh:`86768`.) +* The ``iso2022_*`` codecs no longer encode U+001B ESCAPE, and + ``iso2022_kr`` no longer encodes U+000E SHIFT OUT and U+000F SHIFT IN. + Written as bare bytes, they start an escape or shift sequence, so any + ISO-2022 decoder misread the result. Encoding them now raises + :exc:`UnicodeEncodeError`, or calls the error handler. + (Contributed by Ahmed Semih Özmekik in :gh:`160302`.) + Build changes ============= diff --git a/Lib/test/test_codecencodings_iso2022.py b/Lib/test/test_codecencodings_iso2022.py index 117dc6b66d46b69..49659afefd833a1 100644 --- a/Lib/test/test_codecencodings_iso2022.py +++ b/Lib/test/test_codecencodings_iso2022.py @@ -8,12 +8,23 @@ (b'ab\xFFcd', 'replace', 'ab\uFFFDcd'), (b'ab\x1Bdef', 'replace', 'ab\x1Bdef'), (b'ab\x1B$def', 'replace', 'ab\uFFFD'), + # a bare ESC would start an escape sequence + ('ab\x1Bdef', 'strict', None), + ('x\x1B$B$"', 'strict', None), + ('ab\x1Bdef', 'replace', b'ab?def'), + ('ab\x1Bdef', 'ignore', b'abdef'), + ('ab\x1Bdef', 'backslashreplace', b'ab\\x1bdef'), + ) + +# SO and SI are not shifts in these encodings +NO_SHIFT_CODEC_TESTS = ( + ('a\x0Eb\x0Fc', 'strict', b'a\x0Eb\x0Fc'), ) class Test_ISO2022_JP(multibytecodec_support.TestBase, unittest.TestCase): encoding = 'iso2022_jp' tstring = multibytecodec_support.load_teststring('iso2022_jp') - codectests = COMMON_CODEC_TESTS + ( + codectests = COMMON_CODEC_TESTS + NO_SHIFT_CODEC_TESTS + ( (b'ab\x1BNdef', 'replace', 'ab\x1BNdef'), # there is no G1 in this encoding (b'ab\x1B$)Bdef', 'strict', None), @@ -24,7 +35,7 @@ class Test_ISO2022_JP(multibytecodec_support.TestBase, unittest.TestCase): class Test_ISO2022_JP2(multibytecodec_support.TestBase, unittest.TestCase): encoding = 'iso2022_jp_2' tstring = multibytecodec_support.load_teststring('iso2022_jp') - codectests = COMMON_CODEC_TESTS + ( + codectests = COMMON_CODEC_TESTS + NO_SHIFT_CODEC_TESTS + ( (b'ab\x1BNdef', 'replace', 'abdef'), # a charset can only be designated to the G set (G0, G1 or G2) # that RFC 1554 assigns it @@ -40,7 +51,7 @@ class Test_ISO2022_JP2(multibytecodec_support.TestBase, unittest.TestCase): class Test_ISO2022_JP3(multibytecodec_support.TestBase, unittest.TestCase): encoding = 'iso2022_jp_3' tstring = multibytecodec_support.load_teststring('iso2022_jp') - codectests = COMMON_CODEC_TESTS + ( + codectests = COMMON_CODEC_TESTS + NO_SHIFT_CODEC_TESTS + ( (b'ab\x1BNdef', 'replace', 'ab\x1BNdef'), (b'\x1B$(O\x2E\x23\x1B(B', 'strict', '\u3402' ), (b'\x1B$(O\x2E\x22\x1B(B', 'strict', '\U0002000B' ), @@ -63,7 +74,7 @@ class Test_ISO2022_JP3(multibytecodec_support.TestBase, unittest.TestCase): class Test_ISO2022_JP2004(multibytecodec_support.TestBase, unittest.TestCase): encoding = 'iso2022_jp_2004' tstring = multibytecodec_support.load_teststring('iso2022_jp') - codectests = COMMON_CODEC_TESTS + ( + codectests = COMMON_CODEC_TESTS + NO_SHIFT_CODEC_TESTS + ( (b'ab\x1BNdef', 'replace', 'ab\x1BNdef'), (b'\x1B$(Q\x2E\x23\x1B(B', 'strict', '\u3402' ), (b'\x1B$(Q\x2E\x22\x1B(B', 'strict', '\U0002000B' ), @@ -94,6 +105,11 @@ class Test_ISO2022_KR(multibytecodec_support.TestBase, unittest.TestCase): (b'ab\x1B$Cdefg', 'strict', None), (b'ab\x1B)Bdefg', 'strict', None), (b'ab\x1B$(Cdefg', 'replace', 'ab\ufffddefg'), + # SO and SI shift between ASCII and KS X 1001 + ('a\x0Eb', 'strict', None), + ('a\x0Fb', 'strict', None), + ('a\x0Eb\x0Fc', 'replace', b'a?b?c'), + ('\uac00\x0E', 'replace', b'\x1B$)C\x0E\x30\x21\x0F?'), ) # iso2022_kr.txt cannot be used to test "chunk coding": the escape diff --git a/Misc/NEWS.d/next/Library/2026-11-05-12-00-00.gh-issue-160302.z743pv.rst b/Misc/NEWS.d/next/Library/2026-11-05-12-00-00.gh-issue-160302.z743pv.rst new file mode 100644 index 000000000000000..1098a19eba51e60 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2026-11-05-12-00-00.gh-issue-160302.z743pv.rst @@ -0,0 +1,5 @@ +The ``iso2022_*`` codecs no longer encode U+001B ESCAPE, and ``iso2022_kr`` +no longer encodes U+000E SHIFT OUT and U+000F SHIFT IN. Written as bare +bytes, they start an escape or shift sequence, so decoders misread the +result. Encoding them now raises :exc:`UnicodeEncodeError` or calls the +error handler. diff --git a/Modules/cjkcodecs/_codecs_iso2022.c b/Modules/cjkcodecs/_codecs_iso2022.c index f0cba62e8773877..5569bade3513a55 100644 --- a/Modules/cjkcodecs/_codecs_iso2022.c +++ b/Modules/cjkcodecs/_codecs_iso2022.c @@ -187,6 +187,13 @@ ENCODER(iso2022) Py_ssize_t insize; if (c < 0x80) { + /* A bare ESC would be read as the start of an escape sequence, + and SO and SI as shifts where they are used, so these + characters cannot be encoded. */ + if (c == ESC || + ((c == SO || c == SI) && !CONFIG_ISSET(NO_SHIFT))) { + return 1; + } if (STATE_G0 != CHARSET_ASCII) { WRITEBYTE3(ESC, '(', 'B'); STATE_SETG0(CHARSET_ASCII); From 2b6f03598b8b5d27f2c494f509a0fefebbb7e78d Mon Sep 17 00:00:00 2001 From: Ahmed Semih Ozmekik Date: Thu, 5 Nov 2026 08:57:22 +0300 Subject: [PATCH 2/2] gh-160302: Don't encode the payload twice in Message.set_payload() When the input and output charsets differ, as for euc-jp and shift_jis, which are sent as iso-2022-jp, set_payload() encoded the text and set_charset() then encoded the result again. That only worked because the iso2022_jp encoder let ESC through. Leave the conversion to set_charset(). --- Lib/email/message.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/Lib/email/message.py b/Lib/email/message.py index 093b9012f31666e..850fd5523fb66f4 100644 --- a/Lib/email/message.py +++ b/Lib/email/message.py @@ -352,7 +352,11 @@ def set_payload(self, payload, charset=None): return if not isinstance(charset, Charset): charset = Charset(charset) - payload = payload.encode(charset.output_charset, 'surrogateescape') + # If the input and output charsets differ, set_charset() + # converts the text, so don't encode it twice. + if charset == charset.get_output_charset(): + payload = payload.encode(charset.output_charset, + 'surrogateescape') if hasattr(payload, 'decode'): self._payload = payload.decode('ascii', 'surrogateescape') else: