diff --git a/Doc/whatsnew/3.16.rst b/Doc/whatsnew/3.16.rst index 514ab2605e37166..896760f3258bd76 100644 --- a/Doc/whatsnew/3.16.rst +++ b/Doc/whatsnew/3.16.rst @@ -1326,6 +1326,13 @@ that may require changes to your code. raises :exc:`io.UnsupportedOperation` unless buffering is disabled. (Contributed by An Long in :gh:`86768`.) +* The ``iso2022_*`` codecs no longer encode U+001B ESCAPE, and + ``iso2022_kr`` no longer encodes U+000E SHIFT OUT and U+000F SHIFT IN. + Written as bare bytes, they start an escape or shift sequence, so any + ISO-2022 decoder misread the result. Encoding them now raises + :exc:`UnicodeEncodeError`, or calls the error handler. + (Contributed by Ahmed Semih Özmekik in :gh:`160302`.) + Build changes ============= diff --git a/Lib/email/message.py b/Lib/email/message.py index 093b9012f31666e..850fd5523fb66f4 100644 --- a/Lib/email/message.py +++ b/Lib/email/message.py @@ -352,7 +352,11 @@ def set_payload(self, payload, charset=None): return if not isinstance(charset, Charset): charset = Charset(charset) - payload = payload.encode(charset.output_charset, 'surrogateescape') + # If the input and output charsets differ, set_charset() + # converts the text, so don't encode it twice. + if charset == charset.get_output_charset(): + payload = payload.encode(charset.output_charset, + 'surrogateescape') if hasattr(payload, 'decode'): self._payload = payload.decode('ascii', 'surrogateescape') else: diff --git a/Lib/test/test_codecencodings_iso2022.py b/Lib/test/test_codecencodings_iso2022.py index 117dc6b66d46b69..49659afefd833a1 100644 --- a/Lib/test/test_codecencodings_iso2022.py +++ b/Lib/test/test_codecencodings_iso2022.py @@ -8,12 +8,23 @@ (b'ab\xFFcd', 'replace', 'ab\uFFFDcd'), (b'ab\x1Bdef', 'replace', 'ab\x1Bdef'), (b'ab\x1B$def', 'replace', 'ab\uFFFD'), + # a bare ESC would start an escape sequence + ('ab\x1Bdef', 'strict', None), + ('x\x1B$B$"', 'strict', None), + ('ab\x1Bdef', 'replace', b'ab?def'), + ('ab\x1Bdef', 'ignore', b'abdef'), + ('ab\x1Bdef', 'backslashreplace', b'ab\\x1bdef'), + ) + +# SO and SI are not shifts in these encodings +NO_SHIFT_CODEC_TESTS = ( + ('a\x0Eb\x0Fc', 'strict', b'a\x0Eb\x0Fc'), ) class Test_ISO2022_JP(multibytecodec_support.TestBase, unittest.TestCase): encoding = 'iso2022_jp' tstring = multibytecodec_support.load_teststring('iso2022_jp') - codectests = COMMON_CODEC_TESTS + ( + codectests = COMMON_CODEC_TESTS + NO_SHIFT_CODEC_TESTS + ( (b'ab\x1BNdef', 'replace', 'ab\x1BNdef'), # there is no G1 in this encoding (b'ab\x1B$)Bdef', 'strict', None), @@ -24,7 +35,7 @@ class Test_ISO2022_JP(multibytecodec_support.TestBase, unittest.TestCase): class Test_ISO2022_JP2(multibytecodec_support.TestBase, unittest.TestCase): encoding = 'iso2022_jp_2' tstring = multibytecodec_support.load_teststring('iso2022_jp') - codectests = COMMON_CODEC_TESTS + ( + codectests = COMMON_CODEC_TESTS + NO_SHIFT_CODEC_TESTS + ( (b'ab\x1BNdef', 'replace', 'abdef'), # a charset can only be designated to the G set (G0, G1 or G2) # that RFC 1554 assigns it @@ -40,7 +51,7 @@ class Test_ISO2022_JP2(multibytecodec_support.TestBase, unittest.TestCase): class Test_ISO2022_JP3(multibytecodec_support.TestBase, unittest.TestCase): encoding = 'iso2022_jp_3' tstring = multibytecodec_support.load_teststring('iso2022_jp') - codectests = COMMON_CODEC_TESTS + ( + codectests = COMMON_CODEC_TESTS + NO_SHIFT_CODEC_TESTS + ( (b'ab\x1BNdef', 'replace', 'ab\x1BNdef'), (b'\x1B$(O\x2E\x23\x1B(B', 'strict', '\u3402' ), (b'\x1B$(O\x2E\x22\x1B(B', 'strict', '\U0002000B' ), @@ -63,7 +74,7 @@ class Test_ISO2022_JP3(multibytecodec_support.TestBase, unittest.TestCase): class Test_ISO2022_JP2004(multibytecodec_support.TestBase, unittest.TestCase): encoding = 'iso2022_jp_2004' tstring = multibytecodec_support.load_teststring('iso2022_jp') - codectests = COMMON_CODEC_TESTS + ( + codectests = COMMON_CODEC_TESTS + NO_SHIFT_CODEC_TESTS + ( (b'ab\x1BNdef', 'replace', 'ab\x1BNdef'), (b'\x1B$(Q\x2E\x23\x1B(B', 'strict', '\u3402' ), (b'\x1B$(Q\x2E\x22\x1B(B', 'strict', '\U0002000B' ), @@ -94,6 +105,11 @@ class Test_ISO2022_KR(multibytecodec_support.TestBase, unittest.TestCase): (b'ab\x1B$Cdefg', 'strict', None), (b'ab\x1B)Bdefg', 'strict', None), (b'ab\x1B$(Cdefg', 'replace', 'ab\ufffddefg'), + # SO and SI shift between ASCII and KS X 1001 + ('a\x0Eb', 'strict', None), + ('a\x0Fb', 'strict', None), + ('a\x0Eb\x0Fc', 'replace', b'a?b?c'), + ('\uac00\x0E', 'replace', b'\x1B$)C\x0E\x30\x21\x0F?'), ) # iso2022_kr.txt cannot be used to test "chunk coding": the escape diff --git a/Misc/NEWS.d/next/Library/2026-11-05-12-00-00.gh-issue-160302.z743pv.rst b/Misc/NEWS.d/next/Library/2026-11-05-12-00-00.gh-issue-160302.z743pv.rst new file mode 100644 index 000000000000000..1098a19eba51e60 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2026-11-05-12-00-00.gh-issue-160302.z743pv.rst @@ -0,0 +1,5 @@ +The ``iso2022_*`` codecs no longer encode U+001B ESCAPE, and ``iso2022_kr`` +no longer encodes U+000E SHIFT OUT and U+000F SHIFT IN. Written as bare +bytes, they start an escape or shift sequence, so decoders misread the +result. Encoding them now raises :exc:`UnicodeEncodeError` or calls the +error handler. diff --git a/Modules/cjkcodecs/_codecs_iso2022.c b/Modules/cjkcodecs/_codecs_iso2022.c index f0cba62e8773877..5569bade3513a55 100644 --- a/Modules/cjkcodecs/_codecs_iso2022.c +++ b/Modules/cjkcodecs/_codecs_iso2022.c @@ -187,6 +187,13 @@ ENCODER(iso2022) Py_ssize_t insize; if (c < 0x80) { + /* A bare ESC would be read as the start of an escape sequence, + and SO and SI as shifts where they are used, so these + characters cannot be encoded. */ + if (c == ESC || + ((c == SO || c == SI) && !CONFIG_ISSET(NO_SHIFT))) { + return 1; + } if (STATE_G0 != CHARSET_ASCII) { WRITEBYTE3(ESC, '(', 'B'); STATE_SETG0(CHARSET_ASCII);