diff --git a/log.c b/log.c index bf30df98d..16649b58d 100644 --- a/log.c +++ b/log.c @@ -246,6 +246,41 @@ void logfile_reopen(void) } } +/* Decode one UTF-8 character when running in the 7-bit C/POSIX locale + * (MB_CUR_MAX == 1) where mbrtowc() rejects all bytes >= 0x80. */ +static size_t utf8_char_len(const char *buf, size_t len, int *is_control) +{ + const uchar *s = (const uchar *)buf; + uint32 cp; + size_t need; + + if (len < 2 || s[0] < 0xc2 || s[0] > 0xf4) + return 0; + if (s[0] < 0xe0) { + need = 2; + cp = s[0] & 0x1f; + } else if (s[0] < 0xf0) { + need = 3; + cp = s[0] & 0x0f; + } else { + need = 4; + cp = s[0] & 0x07; + } + if (len < need) + return 0; + for (size_t i = 1; i < need; i++) { + if ((s[i] & 0xc0) != 0x80) + return 0; + cp = (cp << 6) | (s[i] & 0x3f); + } + if ((need == 3 && cp < 0x800) || (need == 3 && cp >= 0xd800 && cp <= 0xdfff) + || (need == 4 && (cp < 0x10000 || cp > 0x10ffff))) + return 0; + + *is_control = (cp >= 0x7f && cp <= 0x9f); + return need; +} + /* Decode one character using the active locale. Invalid or incomplete input * returns 0 so its bytes are filtered individually. */ static size_t locale_char_len(const char *buf, size_t len, filter_mbstate *state, @@ -253,10 +288,27 @@ static size_t locale_char_len(const char *buf, size_t len, filter_mbstate *state { #if defined HAVE_WCHAR_H && defined HAVE_MBRTOWC wchar_t wc; - size_t char_len = mbrtowc(&wc, buf, len, state); + size_t char_len; + + if (MB_CUR_MAX == 1) { +#if defined HAVE_SETLOCALE && defined LC_CTYPE + const char *ctype = setlocale(LC_CTYPE, NULL); + if (!ctype || strcmp(ctype, "C") == 0 || strcmp(ctype, "POSIX") == 0) + return utf8_char_len(buf, len, is_control); +#else + return utf8_char_len(buf, len, is_control); +#endif + } - if (char_len == (size_t)-1 || char_len == (size_t)-2) { + char_len = mbrtowc(&wc, buf, len, state); + if (char_len == (size_t)-1 || char_len == (size_t)-2 +#ifdef __STDC_ISO_10646__ + || (wc >= 0xd800 && wc <= 0xdfff) || wc > 0x10ffff +#endif + ) { memset(state, 0, sizeof *state); + if (MB_CUR_MAX == 1) + return utf8_char_len(buf, len, is_control); return 0; } if (char_len == 0) @@ -265,11 +317,8 @@ static size_t locale_char_len(const char *buf, size_t len, filter_mbstate *state && (wc < L' ' || (wc >= 0x7f && wc <= 0x9f)); return char_len; #else - (void)buf; - (void)len; (void)state; - (void)is_control; - return 0; + return utf8_char_len(buf, len, is_control); #endif } diff --git a/testsuite/output-control-chars_test.py b/testsuite/output-control-chars_test.py index 6b5595ecd..a3d69f43a 100644 --- a/testsuite/output-control-chars_test.py +++ b/testsuite/output-control-chars_test.py @@ -69,6 +69,13 @@ def check_locale_case(label, locale_names, name, expected, rejected): 'raw_csi': b'raw_\x9b_name', 'utf8_csi': b'utf8_\xc2\x9b_name', 'valid_utf8': b'valid_\xd8\x9b_name', + 'sharp_s': b'sharp_\xc3\x9f_name', + 'valid_4byte': b'four_\xf0\x9f\x92\xa9_name', + 'inc_prefix': b'inc_\xe2\x82', + 'bad_cont': b'badcont_\xe2\x28\xa1_name', + 'overlong': b'overlong_\xc0\xaf_name', + 'surrogate': b'surrogate_\xed\xa0\x80_name', + 'above_max': b'abovemax_\xf4\x90\x80\x80_name', 'leading_cr': b'\rleading_cr_name', 'delete': b'delete_\x7f_name', } @@ -104,6 +111,22 @@ def check_locale_case(label, locale_names, name, expected, rejected): if (utf8_locale is not None and 'valid_utf8' in created and names['valid_utf8'] not in output): test_fail("valid UTF-8 containing a C1-range continuation byte was changed") +if (utf8_locale is not None and 'sharp_s' in created + and names['sharp_s'] not in output): + test_fail("valid UTF-8 sharp s (0xc3 0x9f) was changed under UTF-8 locale") +if (utf8_locale is not None and 'valid_4byte' in created + and names['valid_4byte'] not in output): + test_fail("valid 4-byte UTF-8 (f0 9f 92 a9) was changed under UTF-8 locale") +if 'inc_prefix' in created and b'inc_\xe2\\#202\n' not in output: + test_fail(f"incomplete 3-byte prefix E2 82 did not escape 0x82: {output!r}") +if 'bad_cont' in created and names['bad_cont'] not in output: + test_fail(f"invalid continuation E2 28 A1 was not handled properly: {output!r}") +if 'overlong' in created and names['overlong'] not in output: + test_fail(f"overlong C0 AF was not handled properly: {output!r}") +if 'surrogate' in created and b'surrogate_\xed\xa0\\#200_name' not in output: + test_fail(f"surrogate ED A0 80 did not escape trailing 0x80: {output!r}") +if 'above_max' in created and b'abovemax_\xf4\\#220\\#200\\#200_name' not in output: + test_fail(f"above-U+10FFFF F4 90 80 80 did not escape C1 bytes: {output!r}") if 'leading_cr' in created and names['leading_cr'] in output: test_fail("leading carriage return reached terminal output") if 'leading_cr' in created and b'\\#015leading_cr_name' not in output: @@ -113,6 +136,42 @@ def check_locale_case(label, locale_names, name, expected, rejected): if 'delete' in created and b'delete_\\#177_name' not in output: test_fail("DEL was not escaped") +# Verify C/POSIX locale (or minimal container without locale database): +dst_c = base / 'dst-c' +dst_c.mkdir() +logfile_c = base / 'rsync-c.log' +env_c = os.environ.copy() +env_c['LC_ALL'] = 'C' +proc_c = subprocess.run( + rsync_argv('-av', '--8-bit-output', f'--log-file={logfile_c}', + '--log-file-format=%f', str(src) + '/', str(dst_c) + '/'), + stdout=subprocess.PIPE, stderr=subprocess.PIPE, env=env_c) +if proc_c.returncode != 0: + test_fail(f"rsync failed under C locale: {proc_c.stderr!r}") + +c_output = proc_c.stdout + proc_c.stderr + logfile_c.read_bytes() +if 'sharp_s' in created and names['sharp_s'] not in c_output: + test_fail(f"C locale mangled valid UTF-8 sharp s: {c_output!r}") +if 'valid_utf8' in created and names['valid_utf8'] not in c_output: + test_fail(f"C locale mangled valid_utf8: {c_output!r}") +if 'valid_4byte' in created and names['valid_4byte'] not in c_output: + test_fail(f"C locale mangled valid 4-byte UTF-8: {c_output!r}") +if 'utf8_csi' in created: + if b'\xc2\x9b' in c_output: + test_fail("C locale let UTF-8-encoded CSI reach output") + if b'\\#302\\#233' not in c_output: + test_fail("C locale did not escape UTF-8-encoded CSI") +if 'inc_prefix' in created and b'inc_\xe2\\#202\n' not in c_output: + test_fail(f"C locale did not escape 0x82 in incomplete prefix E2 82: {c_output!r}") +if 'bad_cont' in created and names['bad_cont'] not in c_output: + test_fail(f"C locale mishandled E2 28 A1: {c_output!r}") +if 'overlong' in created and names['overlong'] not in c_output: + test_fail(f"C locale mishandled overlong C0 AF: {c_output!r}") +if 'surrogate' in created and b'surrogate_\xed\xa0\\#200_name' not in c_output: + test_fail(f"C locale did not escape trailing 0x80 in surrogate ED A0 80: {c_output!r}") +if 'above_max' in created and b'abovemax_\xf4\\#220\\#200\\#200_name' not in c_output: + test_fail(f"C locale did not escape C1 bytes in F4 90 80 80: {c_output!r}") + locale_cases = 0 locale_cases += check_locale_case( 'iso-8859-1', ('en_US.ISO-8859-1', 'no_NO.ISO-8859-1'),