diff --git a/CHANGELOG.md b/CHANGELOG.md index 84cf989e2..926036b48 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,8 +1,20 @@ -## Unreleased +## 4.4.3 (TBD) - Bug Fixes - Fixed file path completion for names containing square brackets, including bracketed working directories and home directories. + - A command which prints through a `cmd2` console of its own, such as a `Cmd2GeneralConsole` + showing a Rich progress bar, no longer ends with a `BrokenPipeError` message when its output + is piped to something that exits early, as `head` does. `cmd2` now ends the command quietly + instead, printing `broken_pipe_warning` if it is set. +- Enhancements + - Added the + [rich_progress.py](https://github.com/python-cmd2/cmd2/blob/main/examples/rich_progress.py) + example, which shows how to display the progress of long-running commands with Rich status + spinners and progress bars. See the new + [Updating Output in Place](docs/features/generating_output.md#updating-output-in-place) + documentation for details, including why a carriage return embedded in a string passed to + `poutput()` does not move the cursor. ## 4.4.2 (November 11, 2026) diff --git a/README.md b/README.md index ce9464121..6fc7e4b29 100644 --- a/README.md +++ b/README.md @@ -98,7 +98,9 @@ to build a more complete application. arguments and subcommands, or plain type hints via [`@with_annotated`](https://cmd2.readthedocs.io/en/latest/features/annotated/). A single parser definition drives validation, help, and completion. -- Render Rich tables and other styled objects, with consistent helpers for +- Render Rich tables, + [progress bars](https://cmd2.readthedocs.io/en/latest/features/generating_output/#progress-bars), + and other styled objects, with consistent helpers for [normal output, errors, warnings, and paging](https://cmd2.readthedocs.io/en/latest/features/generating_output/). - Organize large applications into independently testable and dynamically loadable [CommandSets](https://cmd2.readthedocs.io/en/latest/features/modular_commands/), and customize the diff --git a/cmd2/cmd2.py b/cmd2/cmd2.py index 4107fbfc6..e945f5252 100644 --- a/cmd2/cmd2.py +++ b/cmd2/cmd2.py @@ -145,6 +145,7 @@ Cmd2ShlexError, CommandSetRegistrationError, CompletionError, + ConsoleBrokenPipeError, EmbeddedConsoleExit, EmptyStatement, IncompleteStatement, @@ -1706,8 +1707,42 @@ def print_to( # writing. If you would like your application to print a # warning message, then set the broken_pipe_warning attribute # to the message you want printed. - if self.broken_pipe_warning and file != sys.stderr: - Cmd2GeneralConsole(file=sys.stderr).print(self.broken_pipe_warning) + if file != sys.stderr: + self._print_broken_pipe_warning() + + def _print_broken_pipe_warning(self) -> None: + """Print broken_pipe_warning, if there is one, to sys.stderr.""" + if self.broken_pipe_warning: + # Should sys.stderr be a broken pipe too, print_to() says nothing about it + self.print_to(sys.stderr, self.broken_pipe_warning) + + def _handle_console_broken_pipe(self, ex: ConsoleBrokenPipeError) -> None: + """Handle a broken pipe which a cmd2 console raised while a command or one of its hooks was running. + + The print methods handle a broken pipe in print_to(). A command which prints through a cmd2 console + of its own, as one which shows a Rich progress bar does, lets it escape. Call this while self.stdout + is still what the command was writing to. + + :param ex: the exception the console raised + :raises KeyboardInterrupt: if the broken pipe replaced one + :raises SystemExit: if the broken pipe replaced one + :raises ConsoleBrokenPipeError: if it was not the command's output which broke + """ + # A console can hit the broken pipe while another exception is ending the command, as the with block + # of a progress display exits for instance. Ctrl-C and exiting still mean what they did. + context = ex.__context__ + while context is not None: + if isinstance(context, (KeyboardInterrupt, SystemExit)): + raise context + context = context.__context__ + + # Only the output of the command going away is expected. A console which was writing somewhere else, + # to a socket for instance, has a failure to report. + if not any(ex.file is stream for stream in (self.stdout, sys.stdout, sys.stderr)): + raise ex + + if ex.file is not sys.stderr: + self._print_broken_pipe_warning() def poutput( self, @@ -3089,6 +3124,8 @@ def onecmd_plus_hooks( if self.timing: self.perror(f"Elapsed: {dt.datetime.now(tz=dt.timezone.utc) - timestart}", style=None) + except ConsoleBrokenPipeError as ex: + self._handle_console_broken_pipe(ex) finally: # Get sigint protection while we restore stuff with self.sigint_protection: @@ -3118,7 +3155,10 @@ def onecmd_plus_hooks( self.pexcept(ex) finally: try: - stop = self._run_cmdfinalization_hooks(stop, statement) + try: + stop = self._run_cmdfinalization_hooks(stop, statement) + except ConsoleBrokenPipeError as ex: + self._handle_console_broken_pipe(ex) except KeyboardInterrupt: if raise_keyboard_interrupt and not stop: raise diff --git a/cmd2/exceptions.py b/cmd2/exceptions.py index a113a02df..ba95bc4b2 100644 --- a/cmd2/exceptions.py +++ b/cmd2/exceptions.py @@ -1,6 +1,9 @@ """Custom exceptions for cmd2.""" -from typing import Any +from typing import ( + IO, + Any, +) ############################################################################################################ # The following exceptions are part of the public API @@ -69,6 +72,22 @@ class Cmd2ShlexError(Exception): """Raised when shlex fails to parse a command line string in StatementParser.""" +class ConsoleBrokenPipeError(BrokenPipeError): + """Raised by a cmd2 console when the stream it writes to is a pipe whose reader has gone away. + + It names the stream, which tells cmd2 whether it was a command's output which broke, rather + than something else the command was writing to, such as a socket. + """ + + def __init__(self, file: IO[str]) -> None: + """Initialize ConsoleBrokenPipeError instance. + + :param file: the stream the console was writing to + """ + super().__init__() + self.file = file + + class EmbeddedConsoleExit(SystemExit): """Custom exception class for use with the py command.""" diff --git a/cmd2/rich_utils.py b/cmd2/rich_utils.py index 6e4192654..e2ea30563 100644 --- a/cmd2/rich_utils.py +++ b/cmd2/rich_utils.py @@ -46,6 +46,7 @@ ) from . import constants +from .exceptions import ConsoleBrokenPipeError from .styles import ( DEFAULT_ARGPARSE_STYLES, Cmd2Style, @@ -413,9 +414,12 @@ def matches_config( return self._config_key == self._build_config_key(file=file, **kwargs) def on_broken_pipe(self) -> None: - """Override which raises BrokenPipeError instead of SystemExit.""" + """Override which raises a ConsoleBrokenPipeError instead of SystemExit. + + cmd2 handles the error if it escapes the command which was printing. + """ self.quiet = True - raise BrokenPipeError + raise ConsoleBrokenPipeError(self.file) def print( self, diff --git a/docs/features/generating_output.md b/docs/features/generating_output.md index 0cb791dc3..3ac3e890e 100644 --- a/docs/features/generating_output.md +++ b/docs/features/generating_output.md @@ -56,7 +56,14 @@ conveniences: 1. Since users can pipe output to a shell command, it catches `BrokenPipeError` and outputs the contents of `self.broken_pipe_warning` to `stderr`. `self.broken_pipe_warning` defaults to an empty string so this method will just swallow the exception. If you want to show an error - message, put it in `self.broken_pipe_warning` when you initialize `cmd2.Cmd`. + message, put it in `self.broken_pipe_warning` when you initialize `cmd2.Cmd`. A command which + prints through a `cmd2` console of its own, such as the + [Cmd2GeneralConsole][cmd2.rich_utils.Cmd2GeneralConsole] used for the progress displays described + below, is covered too, with one difference. A command which uses `poutput` carries on after the + pipe breaks, but one which prints through its own console ends at the call which found the pipe + broken, and its postcommand hooks do not run. Put any cleanup such a command has to do in a + `finally` block or a context manager. Use a `cmd2` console rather than a plain + `rich.console.Console`, which exits the application when it writes to a broken pipe. 2. It examines and honors the [allow_style](./settings.md#allow_style) setting. See [Colored Output](#colored-output) below for more details. 3. It allows printing arbitrary `rich` renderable objects which can get visually quite complex. @@ -69,6 +76,134 @@ def do_echo(self, args): self.poutput(args) ``` +## Updating Output in Place + +Countdowns, progress indicators, and status lines repeatedly overwrite the same line of the +terminal. The traditional way to do that with `print()` is to embed a carriage return (`\r`) in the +string, but that does not work with `poutput` or any of the other `cmd2` print methods. `rich` +removes the carriage return, backspace, bell, vertical tab, and form feed characters from every +string it renders because they would break its width calculations, so the cursor never returns to +the start of the line. + +`rich` can manage the cursor for you instead. Its status spinners and progress bars work inside a +command when you give them a console which writes to `self.stdout`. Create one with +[Cmd2GeneralConsole][cmd2.rich_utils.Cmd2GeneralConsole] so the display uses the `cmd2` theme and +follows the [allow_style](./settings.md#allow_style) setting. The +[rich_progress.py](https://github.com/python-cmd2/cmd2/blob/main/examples/rich_progress.py) example +demonstrates everything described below. + +### Status Messages + +For work of an unknown length, show a spinner and a message with a +[status](https://rich.readthedocs.io/en/stable/console.html#status): + +```py +import time + +from cmd2.rich_utils import Cmd2GeneralConsole + + +def do_countdown(self, _): + """Count down from 5 on a single line""" + console = Cmd2GeneralConsole(file=self.stdout) + with console.status("Starting...") as status: + for remaining in range(5, 0, -1): + status.update(f"Time remaining: {remaining}s") + time.sleep(1) + self.poutput("Done!") +``` + +The status line disappears when the `with` block exits. + +### Progress Bars + +When you know how much work there is, show a +[progress bar](https://rich.readthedocs.io/en/stable/progress.html): + +```py +import time + +from rich.progress import Progress + +from cmd2.rich_utils import Cmd2GeneralConsole + + +def do_process(self, _): + """Process some files while showing a progress bar""" + files = [f"file_{i}.dat" for i in range(1, 11)] + console = Cmd2GeneralConsole(file=self.stdout) + with Progress(console=console) as progress: + task_id = progress.add_task("Processing files", total=len(files)) + for file in files: + time.sleep(0.4) + progress.console.print(f"Processed {file}") + progress.advance(task_id) + self.poutput("All files processed.") +``` + +A few things to know when using a progress bar in a command: + +- While the bar is active, print with `progress.console.print()` as shown above. The message then + appears above the bar. `poutput` and the other print methods do not know about the bar: their + output lands on the end of its line and a stale copy of the bar is left behind. Once the `with` + block exits, they are safe to use again. +- The finished bar is left on the screen. Pass `transient=True` to `Progress` to erase it instead. +- Call `add_task` more than once to show several bars at the same time, and pass + [columns](https://rich.readthedocs.io/en/stable/progress.html#columns) to `Progress` to change + what is shown for each one. +- To add a bar to a simple loop, wrap the sequence being iterated over with + `rich.progress.track(sequence, console=console)`. + +### Styles, Redirection, and Markup + +Status messages and progress bars differ from the print methods in two ways: + +- **They are only animated where `allow_style` allows it.** With the default setting of `Terminal`, + they are animated on a terminal. When the output of the command is redirected to a file or piped + to a shell command, a status writes nothing and a progress bar writes only its final state, or + just a blank line if it is transient. With `Never`, that is what happens on a terminal too. With + `Always`, redirected output also contains the escape sequences which hide and show the cursor and + style the bar. +- **Their text is parsed as [console markup](https://rich.readthedocs.io/en/stable/markup.html).** A + status message or the description of a progress bar task is searched for markup tags such as + `[bold]` and emoji codes such as `:warning:`, even though both are off by default in the print + methods. So `report[draft].txt` is displayed as `report.txt`, `user:id:42` as `user🆔42`, and an + unmatched closing tag such as `[/b]` raises a `MarkupError`. To display text you do not control, + such as a file name, exactly as it is, give a status a `rich.text.Text` object instead of a + string, and show task descriptions with a `TextColumn("{task.description}", markup=False)` + [column](https://rich.readthedocs.io/en/stable/progress.html#columns). + +### Carriage Returns + +If you would rather write the escape sequences yourself, put the carriage return in the `end` +parameter, which `rich` does not strip: + +```py +import time + +CLEAR_LINE = "\x1b[2K" + + +def do_countdown(self, _): + """Count down from 5 on a single line""" + for remaining in range(5, 0, -1): + self.poutput(f"{CLEAR_LINE}Time remaining: {remaining}s", end="\r") + time.sleep(1) + self.poutput(f"{CLEAR_LINE}Done!") +``` + +Use the `style` parameter if you want to color text printed this way. A string which already +contains ANSI style sequences is parsed as ANSI text, which discards every other kind of escape +sequence in it, including the one that clears the line. + +Unlike the `rich` displays, this always writes the escape sequences and carriage returns, including +when the output of the command is redirected to a file or pipe. + +The print methods flush the stream after every call, so there is no `flush` parameter to pass. +Strings written directly to `self.stdout` are not processed by `rich`, so control characters in them +are left alone. They are not flushed either, so call `self.stdout.flush()` after writing a line +which does not end with a newline. + ## Error Messages When an error occurs in your program, you can display it on `sys.stderr` by calling the diff --git a/examples/README.md b/examples/README.md index 96a6291dc..128080a32 100644 --- a/examples/README.md +++ b/examples/README.md @@ -88,6 +88,9 @@ each: - [remove_settable.py](https://github.com/python-cmd2/cmd2/blob/main/examples/remove_settable.py) - Shows how to remove any of the built-in cmd2 `Settables` you do not want in your cmd2 application +- [rich_progress.py](https://github.com/python-cmd2/cmd2/blob/main/examples/rich_progress.py) + - Shows how to use Rich progress bars and status spinners to display the progress of + long-running commands - [rich_tables.py](https://github.com/python-cmd2/cmd2/blob/main/examples/rich_tables.py) - Example of using Rich Tables within a cmd2 application for displaying tabular data - [rich_theme.py](https://github.com/python-cmd2/cmd2/blob/main/examples/rich_theme.py) diff --git a/examples/rich_progress.py b/examples/rich_progress.py new file mode 100755 index 000000000..9c6ca0639 --- /dev/null +++ b/examples/rich_progress.py @@ -0,0 +1,117 @@ +#!/usr/bin/env python3 +"""An example of using Rich to show the progress and status of long-running commands in a cmd2 application. + +Rich's spinners and progress bars repeatedly redraw the same lines of the terminal. To use them in a +cmd2 command, give them a console which writes to ``self.stdout``. They then honor output redirection, +the ``allow_style`` setting, and the cmd2 theme like the rest of your command's output. + +The commands in this example are: + +- ``status`` - a spinner with a status message, for work of an unknown length +- ``progress`` - a progress bar with messages printed above it while it is running +- ``downloads`` - several progress bars with custom columns updating at the same time +- ``track`` - the shortest way to add a progress bar to a loop + +Pressing Ctrl-C while any of them is running stops the display and restores the cursor. Redirect +their output to a file (e.g. ``progress > out.txt``) to see that the animation is left out of it. +""" + +import time + +from rich.progress import ( + BarColumn, + DownloadColumn, + Progress, + TextColumn, + TimeRemainingColumn, + TransferSpeedColumn, + track, +) + +import cmd2 +from cmd2.rich_utils import Cmd2GeneralConsole + +# Name, size in bytes, and bytes received per update for each simulated download +DOWNLOADS = [ + ("cmd2.tar.gz", 2_000_000, 50_000), + ("rich.tar.gz", 6_000_000, 100_000), + ("prompt_toolkit.tar.gz", 4_000_000, 40_000), +] + + +class ProgressApp(cmd2.Cmd): + """Example cmd2 application which shows the progress and status of long-running commands.""" + + def __init__(self) -> None: + super().__init__() + self.intro = "Run the status, progress, downloads, and track commands to see Rich progress displays." + + def _create_console(self) -> Cmd2GeneralConsole: + """Create a Rich console which writes to the same place as poutput().""" + return Cmd2GeneralConsole(file=self.stdout) + + def do_status(self, _: cmd2.Statement) -> None: + """Show a spinner and a status message while doing work of an unknown length.""" + console = self._create_console() + with console.status("Starting...") as status: + for step in ("Connecting to the server", "Authenticating", "Fetching data"): + status.update(f"{step}...") + time.sleep(1.5) + + # The status line is erased when the with block exits + self.poutput("Data fetched.") + + def do_progress(self, _: cmd2.Statement) -> None: + """Show a progress bar and print messages above it while it is running.""" + files = [f"file_{i}.dat" for i in range(1, 11)] + + with Progress(console=self._create_console()) as progress: + task_id = progress.add_task("Processing files", total=len(files)) + for file in files: + time.sleep(0.4) + + # While the progress bar is active, print through its console so the message + # appears above the bar. poutput() and the other print methods do not know about + # the bar: their output lands on the end of its line and a stale copy of the bar + # is left behind. + progress.console.print(f"Processed {file}") + progress.advance(task_id) + + # The finished bar is left on the screen. Once the with block exits, poutput() is safe again. + self.poutput("All files processed.") + + def do_downloads(self, _: cmd2.Statement) -> None: + """Show several progress bars with custom columns updating at the same time.""" + with Progress( + # Task descriptions are parsed as Rich markup unless the column which shows them says otherwise. + # File names can contain square brackets, so show them exactly as they are. + TextColumn("{task.description}", style=cmd2.Color.CYAN, markup=False), + BarColumn(), + DownloadColumn(), + TransferSpeedColumn(), + TimeRemainingColumn(), + console=self._create_console(), + # Erase the progress bars when the with block exits + transient=True, + ) as progress: + for name, size, chunk_size in DOWNLOADS: + # Extra keyword arguments are stored in the task's fields + progress.add_task(name, total=size, chunk_size=chunk_size) + + while not progress.finished: + time.sleep(0.05) + for task in progress.tasks: + if not task.finished: + progress.advance(task.id, task.fields["chunk_size"]) + + self.poutput(f"Downloaded {len(DOWNLOADS)} files.") + + def do_track(self, _: cmd2.Statement) -> None: + """Add a progress bar to a loop with a single function call.""" + for _number in track(range(40), description="Crunching numbers", console=self._create_console()): + time.sleep(0.05) + + +if __name__ == "__main__": + app = ProgressApp() + app.cmdloop() diff --git a/tests/test_cmd2.py b/tests/test_cmd2.py index 01efe8de7..4840b6ecd 100644 --- a/tests/test_cmd2.py +++ b/tests/test_cmd2.py @@ -9,9 +9,11 @@ import tempfile import threading from code import InteractiveConsole +from collections.abc import Iterator from concurrent.futures import ThreadPoolExecutor from typing import ( NoReturn, + TextIO, cast, ) from unittest import mock @@ -967,6 +969,138 @@ def test_pipe_to_shell_and_redirect(redirection_app, running_pipe_process) -> No os.remove(filename) +class BrokenPipeApp(cmd2.Cmd): + """Commands which hit a broken pipe without going through one of cmd2's print methods.""" + + def do_own_console(self, _: str) -> None: + """Print through a console of its own, for longer than the reader of its output stays around.""" + console = ru.Cmd2GeneralConsole(file=self.stdout) + + # More than fits in a pipe, so the command is still writing when the reader goes away + for _line in range(4096): + console.print("x" * 1024) + self.perror("the pipe never broke") + + def do_raise_broken_pipe(self, _: str) -> None: + """Raise the error a write to a closed pipe or socket raises.""" + raise BrokenPipeError + + def do_other_console(self, _: str) -> None: + """Print through a console which writes somewhere other than the command's output.""" + ru.Cmd2GeneralConsole(file=self.other_stream).print("x") + + def do_unwind(self, how: str) -> None: + """Have a console write as an exception ends the command, as the with block of a progress display does.""" + try: + raise KeyboardInterrupt if how == "interrupt" else SystemExit(5) + finally: + ru.Cmd2GeneralConsole(file=self.stdout).print("x") + + def do_quiet(self, _: str) -> None: + """Write nothing.""" + + def console_hook(self, data: cmd2.plugin.CommandFinalizationData) -> cmd2.plugin.CommandFinalizationData: + """Print through a console of its own once a command has finished.""" + ru.Cmd2GeneralConsole(file=self.stdout).print("x") + return data + + +@pytest.fixture +def broken_pipe() -> Iterator[TextIO]: + """A stream which writes to a pipe nothing reads.""" + read_fd, write_fd = os.pipe() + os.close(read_fd) + stream = open(write_fd, "w") # noqa: SIM115 + yield stream + + # Closing flushes what a failed write left in the buffer, which fails again + with contextlib.suppress(BrokenPipeError): + stream.close() + + +#: A closed pipe raises BrokenPipeError on POSIX. On Windows it can raise another OSError, which no part of cmd2 handles. +posix_broken_pipe = pytest.mark.skipif(sys.platform == "win32", reason="a write to a closed pipe raises another OSError") + + +@posix_broken_pipe +def test_broken_pipe_from_a_commands_own_console_is_ignored(broken_pipe, capsys) -> None: + """Whatever stdout is a pipe to, a command ends quietly when its reader goes away, as one which uses poutput() does.""" + app = BrokenPipeApp(stdout=broken_pipe) + app.onecmd_plus_hooks("own_console") + assert not capsys.readouterr().err + + +@posix_broken_pipe +def test_broken_pipe_from_a_commands_own_console_prints_the_warning(broken_pipe, capsys) -> None: + app = BrokenPipeApp(stdout=broken_pipe) + app.broken_pipe_warning = "The pipe broke" + app.onecmd_plus_hooks("own_console") + assert capsys.readouterr().err == "The pipe broke\n" + + +@posix_broken_pipe +def test_broken_pipe_from_a_piped_commands_own_console_is_ignored(running_pipe_process) -> None: + """The shell command a command's output is piped to exits before reading all of it. + + head cannot exit before the command has written a line, so cmd2 never finds that the pipe process ended before the + command could run, however slowly the two start. + """ + out, err = run_cmd(BrokenPipeApp(), "own_console | head -n 1") + assert out == ["x" * 1024] + assert not err + + +@posix_broken_pipe +def test_broken_pipe_from_the_console_of_a_command_in_a_piped_script_is_ignored(tmp_path, running_pipe_process) -> None: + script = tmp_path / "script.txt" + script.write_text("own_console\n") + out, err = run_cmd(BrokenPipeApp(), f"run_script {script} | head -n 1") + assert out == ["x" * 1024] + assert not err + + +@posix_broken_pipe +def test_broken_pipe_from_a_console_on_another_stream_is_reported(broken_pipe) -> None: + """A console knows which stream broke. Only the command's output going away is expected.""" + app = BrokenPipeApp() + app.other_stream = broken_pipe + _out, err = run_cmd(app, "other_console") + assert any("BrokenPipeError" in line for line in err) + + +@posix_broken_pipe +def test_broken_pipe_does_not_hide_the_interrupt_it_replaced(broken_pipe) -> None: + """Ctrl-C still stops a script when a console hits a broken pipe as the interrupt ends the command.""" + app = BrokenPipeApp(stdout=broken_pipe) + with pytest.raises(KeyboardInterrupt): + app.onecmd_plus_hooks("unwind interrupt", raise_keyboard_interrupt=True) + + +@posix_broken_pipe +def test_broken_pipe_does_not_hide_the_exit_it_replaced(broken_pipe) -> None: + app = BrokenPipeApp(stdout=broken_pipe) + assert app.onecmd_plus_hooks("unwind exit") is True + assert app.exit_code == 5 + + +@posix_broken_pipe +def test_broken_pipe_from_the_console_of_a_finalization_hook_is_ignored(broken_pipe, capsys) -> None: + app = BrokenPipeApp(stdout=broken_pipe) + app.register_cmdfinalization_hook(app.console_hook) + app.onecmd_plus_hooks("quiet") + assert not capsys.readouterr().err + + +@pytest.mark.parametrize("redirection", ["", " | sort", " > {file}"]) +def test_broken_pipe_raised_by_a_command_is_reported(tmp_path, running_pipe_process, redirection) -> None: + """Only a console knows that it was the command's output which broke. Any other BrokenPipeError is not hidden.""" + app = BrokenPipeApp() + app.broken_pipe_warning = "The pipe broke" + _out, err = run_cmd(app, "raise_broken_pipe" + redirection.format(file=tmp_path / "out.txt")) + assert "BrokenPipeError" in err[0] + assert "The pipe broke" not in err + + @pytest.mark.parametrize( "terminal", [False, pytest.param(True, marks=pytest.mark.skipif(sys.platform == "win32", reason="POSIX terminal job control"))],