Skip to content

Commit b490a36

Browse files
committed
Write redirected and piped output as UTF-8
Command output is rendered by Rich and routinely contains non-ASCII, but redirection targets and pipes were opened with the locale's encoding. On any system whose default is not UTF-8 -- a Windows console using a legacy code page, for instance -- redirecting output raised UnicodeEncodeError, and the user was left with an empty file and advice to set PYTHONIOENCODING. Open both with UTF-8 explicitly. Two tests that read redirected output back were relying on the locale encoding for decoding as well, so they now name it too.
1 parent 2f786cb commit b490a36

5 files changed

Lines changed: 54 additions & 10 deletions

File tree

‎CHANGELOG.md‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,10 @@
11
## 4.3.0 (TBD)
22

33
- Bug Fixes
4+
- Fixed output redirection and piping failing on systems whose default encoding is not UTF-8,
5+
such as a Windows console using a legacy code page. Command output is rendered by Rich and
6+
routinely contains non-ASCII, so redirecting it raised `UnicodeEncodeError` and left an empty
7+
file behind. Redirection targets and pipes now use UTF-8 explicitly
48
- Fixed the right prompt being redrawn beside every accepted command line in the scrollback.
59
prompt-toolkit includes it in the final frame of each prompt, which is the frame left on the
610
terminal; it is now hidden there, as the bottom toolbar already was, and stays on the live

‎cmd2/cmd2.py‎

Lines changed: 14 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -3403,9 +3403,11 @@ def _redirect_output(self, statement: Statement) -> utils.RedirectionSavedState:
34033403
# Create a pipe with read and write sides
34043404
read_fd, write_fd = os.pipe()
34053405

3406-
# Open each side of the pipe
3407-
subproc_stdin = open(read_fd) # noqa: SIM115
3408-
new_stdout: TextIO = cast(TextIO, open(write_fd, "w")) # noqa: SIM115
3406+
# Open each side of the pipe. Both ends are given an explicit encoding:
3407+
# command output is rendered by Rich and routinely contains non-ASCII, which
3408+
# the locale encoding cannot always represent.
3409+
subproc_stdin = open(read_fd, encoding="utf-8") # noqa: SIM115
3410+
new_stdout: TextIO = cast(TextIO, open(write_fd, "w", encoding="utf-8")) # noqa: SIM115
34093411

34103412
# Create pipe process in a separate group to isolate our signals from it. If a Ctrl-C event occurs,
34113413
# our sigint handler will forward it only to the most recent pipe process. This makes sure pipe
@@ -3472,8 +3474,15 @@ def _redirect_output(self, statement: Statement) -> utils.RedirectionSavedState:
34723474
# statement.output can only contain REDIRECTION_APPEND or REDIRECTION_OUTPUT
34733475
mode = "a" if statement.redirector == constants.REDIRECTION_APPEND else "w"
34743476
try:
3475-
# Use line buffering
3476-
new_stdout = cast(TextIO, open(su.strip_quotes(statement.redirect_to), mode=mode, buffering=1)) # noqa: SIM115
3477+
# Use line buffering. The encoding is explicit rather than the
3478+
# locale's: command output is rendered by Rich and routinely contains
3479+
# non-ASCII, so on a non-UTF-8 system -- a default Windows console,
3480+
# for instance -- redirection would otherwise fail and leave an empty
3481+
# file behind.
3482+
new_stdout = cast(
3483+
TextIO,
3484+
open(su.strip_quotes(statement.redirect_to), mode=mode, buffering=1, encoding="utf-8"), # noqa: SIM115
3485+
)
34773486
except OSError as ex:
34783487
raise RedirectionError("Failed to redirect output") from ex
34793488

‎tests/test_command_toolbar.py‎

Lines changed: 4 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -58,7 +58,7 @@ def test_command_toolbar_redirected_output(toolbar_app, tmp_path) -> None:
5858
destination = tmp_path / "help.txt"
5959
with app._command_toolbar_context():
6060
app.onecmd_plus_hooks(f'help > "{destination}"')
61-
text = destination.read_text()
61+
text = destination.read_text(encoding="utf-8")
6262
assert "Cmd2 Commands" in text
6363
assert "STATUS" not in text
6464
assert "Cmd2 Commands" not in output.getvalue()
@@ -79,7 +79,7 @@ def command(statement, **kwargs):
7979
app.onecmd_plus_hooks(f'custom > "{destination}"')
8080
app.poutput("terminal output")
8181

82-
assert destination.read_text() == "before\nduring\nafter\n"
82+
assert destination.read_text(encoding="utf-8") == "before\nduring\nafter\n"
8383
assert "before" not in output.getvalue()
8484
assert "during" not in output.getvalue()
8585
assert "after" not in output.getvalue()
@@ -134,7 +134,7 @@ def command(statement, **kwargs):
134134
assert running == [False]
135135
# A process given the terminal writes to it directly instead of through a captured pipe.
136136
assert readers[0]._proc.stdout is None
137-
assert "PIPED" in destination.read_text()
137+
assert "PIPED" in destination.read_text(encoding="utf-8")
138138

139139

140140
def test_command_toolbar_binary_output(toolbar_app) -> None:
@@ -773,5 +773,5 @@ def test_builtin_pager_does_not_capture_redirected_output(toolbar_app, monkeypat
773773
with mock.patch("cmd2.command_toolbar.Pager") as pager, app._command_toolbar_context():
774774
app.onecmd_plus_hooks(f'help > "{target}"')
775775
pager.assert_not_called()
776-
assert "Cmd2 Commands" in target.read_text()
776+
assert "Cmd2 Commands" in target.read_text(encoding="utf-8")
777777
assert "Cmd2 Commands" not in output.getvalue()

‎tests/test_run_pyscript.py‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -246,7 +246,7 @@ def test_run_pyscript_print_redirection(base_app, request, tmp_path, capsys) ->
246246
out, err = capsys.readouterr()
247247

248248
# Verify the output file contains what we expect from print()
249-
content = pathlib.Path(out_file).read_text()
249+
content = pathlib.Path(out_file).read_text(encoding="utf-8")
250250

251251
# Look for everything written to self.stdout
252252
assert len(content.splitlines()) == 4

‎tests/test_suite_environment.py‎

Lines changed: 31 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -9,6 +9,8 @@
99

1010
import pytest
1111

12+
import cmd2
13+
1214
#: Variables that change how output is rendered. Rich reads all of these; cmd2 reads
1315
#: NO_COLOR directly. Tests that exercise them set them explicitly instead.
1416
COLOR_ENVIRONMENT = ("NO_COLOR", "FORCE_COLOR", "TTY_COMPATIBLE", "TTY_INTERACTIVE")
@@ -20,3 +22,32 @@ def test_color_environment_does_not_leak_into_tests(name: str) -> None:
2022
assert name not in os.environ, (
2123
f"{name} leaked into the test environment; output-rendering assertions would depend on who is running the suite"
2224
)
25+
26+
27+
class EncodingProbe(cmd2.Cmd):
28+
"""Reports the encoding of whatever stream output is currently going to."""
29+
30+
def do_show_encoding(self, _: str) -> None:
31+
"""Print the current output stream's encoding."""
32+
self.poutput(f"ENCODING={getattr(self.stdout, 'encoding', None)}")
33+
34+
35+
def test_redirection_to_a_file_uses_utf8(tmp_path) -> None:
36+
"""cmd2 renders non-ASCII, so a redirect target must not use the locale encoding.
37+
38+
Opened with the locale encoding, redirecting styled output raises UnicodeEncodeError
39+
on any non-UTF-8 system -- which includes a default Windows console -- leaving the
40+
user an empty file and an error.
41+
"""
42+
app = EncodingProbe(allow_cli_args=False)
43+
target = tmp_path / "out.txt"
44+
app.onecmd_plus_hooks(f'show_encoding > "{target}"')
45+
assert "ENCODING=utf-8" in target.read_text(encoding="utf-8")
46+
47+
48+
def test_piping_uses_utf8(tmp_path) -> None:
49+
"""The same applies to the pipe the subprocess reads from."""
50+
app = EncodingProbe(allow_cli_args=False)
51+
target = tmp_path / "piped.txt"
52+
app.onecmd_plus_hooks(f'show_encoding | cat > "{target}"')
53+
assert "ENCODING=utf-8" in target.read_text(encoding="utf-8")

0 commit comments

Comments
 (0)