diff --git a/CHANGES b/CHANGES index 6bd229c005..977b3921ba 100644 --- a/CHANGES +++ b/CHANGES @@ -47,6 +47,32 @@ _Notes on the upcoming release will go here._ ### Breaking changes +#### Plain semicolon arguments are data (#739) + +Tmux interprets an unescaped trailing `;` as command structure, including when +it is the final character of a larger value. Direct subprocess engines now +escape ordinary unescaped suffix semicolons so they round-trip literally and +cannot start a second command. An existing `\;` suffix remains tmux escape +syntax and is not escaped again. Code that intentionally grouped commands with +a plain `";"` argument must pass +{class}`~libtmux.engines.base.CommandSeparator` instead: + +```python +# Before +server.cmd("display-message", "first", ";", "display-message", "second") + +# After +from libtmux.engines import CommandSeparator + +server.cmd( + "display-message", + "first", + CommandSeparator(";"), + "display-message", + "second", +) +``` + #### `raise_if_dead()` no longer echoes tmux's error (#739) {meth}`Server.raise_if_dead() ` previously let @@ -75,7 +101,8 @@ loading. Every tmux command libtmux runs now goes through an *engine* — an object that takes a rendered argv and returns a structured result. The default, {class}`~libtmux.engines.subprocess.SubprocessEngine`, forks the tmux binary -exactly as before, so existing code is unaffected. +once per request as before. Ordinary unescaped semicolon suffixes remain data; +explicit multi-command groups now use a typed separator. Pass `engine=` to {class}`~libtmux.Server` and every command on that server runs through your object instead. {class}`~libtmux.engines.base.TmuxEngine` is a @@ -85,15 +112,14 @@ against a recorded or in-memory tmux with no server running, and it is the seam the control-mode, asyncio, and native-protocol engines plug into. This ships the seam only. {meth}`Server.cmd() ` still -returns a {class}`~libtmux.common.tmux_cmd`, arguments still reach tmux -unchanged, and nothing about the default path is new — an engine is the one -thing you can now replace. +returns a {class}`~libtmux.common.tmux_cmd`, and an engine is the one thing you +can now replace. -An engine that names no tmux server of its own adopts the server's connection, -so injecting one into a socket-scoped {class}`~libtmux.Server` cannot silently -dispatch to the ambient tmux server. Engines that name a server keep it. A -custom `tmux_bin` selects a program rather than a server, so an engine carrying -only one adopts the server's flags and keeps its own binary. +An inspectable engine must satisfy the explicit connection values on +{class}`~libtmux.Server`. Stateless engines safely adopt missing values; a +conflict or a pinned persistent connection fails before dispatch, so a Server +cannot silently operate on a different socket. Connectionless recording and +in-memory engines remain usable without inventing tmux-specific behavior. #### Observing what an engine runs (#739) @@ -116,7 +142,7 @@ own: a command group is one dispatch carrying several tmux commands, and a literal `";"` a caller meant as data is not a boundary. {class}`~libtmux.engines.connection.ServerConnection` is now the single place -the tmux binary and the `-L`/`-S`/`-f`/`-2`/`-8` flags are computed; three +the tmux binary and the `-L`/`-S`/`-f`/`-2` flags are computed; three separate copies previously disagreed about which flags to emit. It is derived from the server's public attributes on each use, so reassigning `socket_name` takes effect on the next command, and it memoizes its {func}`shutil.which` @@ -127,7 +153,7 @@ tmux version it targets via {class}`~libtmux.engines.base.SupportsTmuxVersion` capability, which callers rendering version-gated argv read to decide whether a flag is safe to send. -An engine that folds several commands into one dispatch needs to know which `;` +An engine that groups several commands into one request needs to know which `;` in an argv is a boundary and which is data. {class}`~libtmux.engines.base.CommandSeparator` marks the boundary and {func}`~libtmux.engines.base.is_command_separator` finds it, so a `;` a caller @@ -158,27 +184,28 @@ A {class}`~libtmux.experimental.ops.plan.LazyPlan` records operations and yields forward references so a later operation can target an object that does not exist yet, resolved against captured ids at execution time. How a plan becomes tmux dispatches is a pluggable {class}`~libtmux.experimental.ops.planner.Planner` -(sequential, ``;``-folding, or ``{marked}``-folding), so dispatch strategies can -be A/B tested against the same plan with identical results. +(sequential or ordered request batching), so dispatch strategies can be tested +against the same plan with identical per-operation results. A batching planner +never joins commands with `;` or borrows tmux's server-global marked pane. -#### Declarative workspace builds fold to a few tmux calls (#690) +#### Declarative workspace request batching (#690) A {class}`~libtmux.experimental.workspace.ir.Workspace` declares a session as a tree of windows and panes and lowers to a Core {class}`~libtmux.experimental.ops.plan.LazyPlan`, so a tmuxp-style spec can be analyzed, inspected, and built over any engine. {meth}`~libtmux.experimental.workspace.ir.Workspace.build` and its async twin -{meth}`~libtmux.experimental.workspace.ir.Workspace.abuild` fold the build's -dispatches by default: a multi-pane window collapses from one tmux call per -operation into a handful of ``;``-chained and ``{marked}`` dispatches, so a -session renders in a few round-trips instead of dozens. +{meth}`~libtmux.experimental.workspace.ir.Workspace.abuild` batch ready requests +by default. Every operation retains its own request, output, failure, and typed +result. Persistent control transports can pipeline a batch; subprocess engines +still start one process per request. The resulting {class}`~libtmux.experimental.ops.plan.PlanResult` is identical to -an unfolded build -- only the dispatch count changes -- because host-side steps -(per-command sleeps, the ``wait_pane`` anti-race, ``before_script``) stay hard -fold boundaries that a fold never crosses. Pass a +sequential execution because captured identifiers and host-side steps +(per-command sleeps, the ``wait_pane`` anti-race, ``before_script``) remain hard +batch boundaries. Pass a {class}`~libtmux.experimental.ops.planner.SequentialPlanner` to ``build`` for one -legible tmux call per operation when debugging. +legible engine call per operation when debugging. #### Floating panes on tmux 3.7 (#690) @@ -194,7 +221,7 @@ on a window, including a pane that overlays a different window. panes a running server has: ``filter``, ``order_by``, ``limit``, and ``map`` compose and read nothing until a terminal call. The same query commands what it selects -- ``commands()`` attaches per-pane actions (send keys, resize, select, -respawn, clear history, kill) that run as one folded tmux dispatch. A query +respawn, clear history, kill) that run as an ordered request batch. A query resolves against a live engine or a plain list of pane snapshots, so the same code runs offline in tests. @@ -226,8 +253,9 @@ command -- no sentinel string -- and whether its process exited. queries behind {attr}`~libtmux.Server.sessions` built their own connection flags and emitted only `-L`/`-S`, so a server constructed with `config_file=` or `colors=` passed those flags on some commands and not others. All paths now -share one connection. A `colors=` value other than `256` or `88` raises -{exc}`~libtmux.exc.UnknownColorOption` on those paths as well. +share one connection. Only `colors=256` is supported; every other truthy value, +including the legacy `88` mode removed before the minimum supported tmux, +raises {exc}`~libtmux.exc.UnknownColorOption` on those paths as well. ### Documentation diff --git a/docs/_ext/tmuxop/render.py b/docs/_ext/tmuxop/render.py index b6888dd732..9060f85d27 100644 --- a/docs/_ext/tmuxop/render.py +++ b/docs/_ext/tmuxop/render.py @@ -323,7 +323,7 @@ def build_operation_description( "Minimum tmux", _literal_fact(entry.min_version or "any supported version"), ), - ApiFactRow("Chainable", _boolean_fact(entry.chainable)), + ApiFactRow("Batchable", _boolean_fact(entry.batchable)), ApiFactRow( "Version-gated flags", build_chip_paragraph( diff --git a/docs/api/libtmux.engines.md b/docs/api/libtmux.engines.md index 862ca28c0f..4ff8799c25 100644 --- a/docs/api/libtmux.engines.md +++ b/docs/api/libtmux.engines.md @@ -41,7 +41,7 @@ also implement. A {class}`~libtmux.engines.connection.ServerConnection` is the pair every engine needs before it can dispatch anything: which tmux *binary* to run, and the -connection flags (`-L`/`-S`/`-f`/`-2`/`-8`) naming one tmux server. It is the +connection flags (`-L`/`-S`/`-f`/`-2`) naming one tmux server. It is the single place either is computed. ```{eval-rst} diff --git a/docs/experimental/engines.md b/docs/experimental/engines.md index 652ce63962..de2dbb0b5f 100644 --- a/docs/experimental/engines.md +++ b/docs/experimental/engines.md @@ -39,10 +39,10 @@ event loop. ## Engine boundary A synchronous engine satisfies -{class}`~libtmux.experimental.engines.base.TmuxEngine`; an async engine +{class}`~libtmux.engines.base.TmuxEngine`; an async engine satisfies {class}`~libtmux.experimental.engines.base.AsyncTmuxEngine`. Both -accept a {class}`~libtmux.experimental.engines.base.CommandRequest` and produce -a raw {class}`~libtmux.experimental.engines.base.CommandResult`. +accept a {class}`~libtmux.engines.base.CommandRequest` and produce +a raw {class}`~libtmux.engines.base.CommandResult`. {func}`~libtmux.experimental.ops.run` and {func}`~libtmux.experimental.ops.arun` own the next boundary: they render an operation and convert the raw command outcome to its declared typed result. @@ -68,16 +68,10 @@ engine-owned workflow. ## Shared API ```{eval-rst} -.. autoclass:: libtmux.experimental.engines.base.TmuxEngine - :members: - .. autoclass:: libtmux.experimental.engines.base.AsyncTmuxEngine :members: -.. autoclass:: libtmux.experimental.engines.base.CommandRequest - :members: - -.. autoclass:: libtmux.experimental.engines.base.CommandResult +.. autoclass:: libtmux.experimental.engines.base.SupportsAsyncTmuxVersion :members: .. autofunction:: libtmux.experimental.engines.registry.available_engines diff --git a/docs/experimental/engines/async-control-mode.md b/docs/experimental/engines/async-control-mode.md index b17a88d8de..0f1ef376a7 100644 --- a/docs/experimental/engines/async-control-mode.md +++ b/docs/experimental/engines/async-control-mode.md @@ -62,8 +62,20 @@ and Per-subscriber queues are bounded; {attr}`~libtmux.experimental.engines.async_control_mode.AsyncControlModeEngine.dropped_notifications` reports overflow. Connection failures and timeouts raise at the engine -boundary, while tmux command errors remain result data. Sequence anomalies are -logged. +boundary, while tmux command errors remain result data. The engine drains the +control process's stderr concurrently and includes its bounded recent tail in +connection and protocol failures. Cleanup joins that reader and reaps the +process before returning, including when the caller waiting on cleanup is +cancelled. A backwards command number or a solicited block with no pending +request raises a protocol error and restarts the connection rather than risking +result misattribution. + +Tmux does not escape command-output lines inside these blocks. Output that +resembles a nonmatching guard remains data, but output byte-for-byte identical +to its own closing guard is indistinguishable from protocol framing. This is a +tmux control-protocol limitation; use +{class}`~libtmux.experimental.engines.asyncio.AsyncSubprocessEngine` when +arbitrary output must round-trip without that ambiguity. The persistent process has normal tmux client semantics: `list-clients` shows it, `session_attached` includes it, and client attach and detach hooks can @@ -83,15 +95,40 @@ after a safe session exists; direct startup raises available. The API has no subscriber-readiness signal, so code must not assume a notification emitted before the first iteration will be delivered. +## Pane-output bytes + +{class}`~libtmux.experimental.engines.async_control_mode.ControlNotification` +keeps the encoded, human-readable control line in `raw` and its exact bytes in +`raw_bytes` for wire diagnostics. For `%output` and `%extended-output`, it also +exposes the pane ID and decoded bytes in `pane_id` and `payload`. Decode text +only at the application boundary: tmux passes pane bytes through without +validating UTF-8. + +```python +>>> from libtmux.experimental.engines import ControlNotification +>>> event = ControlNotification.parse(b"%output %7 hello\\012world\\134") +>>> event.pane_id, event.payload +('%7', b'hello\nworld\\') +>>> event.raw +'%output %7 hello\\012world\\134' +``` + +Tmux uses a backslash followed by exactly three octal digits for an escaped +byte. Shorter octal-looking text remains literal. Consumers normally read +`payload`; `raw` remains useful when diagnosing the protocol stream. + ## API ```{eval-rst} .. autoclass:: libtmux.experimental.engines.async_control_mode.AsyncControlModeEngine :members: :special-members: __aenter__, __aexit__ + +.. autoclass:: libtmux.experimental.engines.async_control_mode.ControlNotification + :members: ``` ## Related tutorial See {doc}`../tutorials/async-control-plans` to compose forward-referenced -operations and fold them into control-mode dispatches. +operations and pipeline their ordered requests over control mode. diff --git a/docs/experimental/engines/control-mode.md b/docs/experimental/engines/control-mode.md index 2d421bd576..16834a5e7e 100644 --- a/docs/experimental/engines/control-mode.md +++ b/docs/experimental/engines/control-mode.md @@ -53,7 +53,16 @@ writes the batch before collecting correlated control-mode result blocks. The engine drains unsolicited notifications so they are not mistaken for command replies. Timeouts, connection death, and write failures raise {exc}`~libtmux.experimental.engines.control_mode.ControlModeError`; tmux -`%error` blocks remain command-result data. Sequence anomalies are logged. +`%error` blocks remain command-result data. A backwards command number or a +solicited block with no pending request raises a protocol error and closes the +connection rather than risking result misattribution. + +Tmux does not escape command-output lines inside these blocks. Output that +resembles a nonmatching guard remains data, but output byte-for-byte identical +to its own closing guard is indistinguishable from protocol framing. This is a +tmux control-protocol limitation; use +{class}`~libtmux.experimental.engines.subprocess.SubprocessEngine` when +arbitrary output must round-trip without that ambiguity. The persistent process has normal tmux client semantics: `list-clients` shows it, `session_attached` includes it, and client attach and detach hooks can diff --git a/docs/experimental/engines/imsg.md b/docs/experimental/engines/imsg.md index f8d3619248..76760a2d7f 100644 --- a/docs/experimental/engines/imsg.md +++ b/docs/experimental/engines/imsg.md @@ -22,7 +22,7 @@ method. Local queries and commands that must start a missing server use the tmux binary instead. Unlike the subprocess and control-mode engines, it has no `for_server()` helper. Put a private server's raw `-L` or `-S` global argument in every -{class}`~libtmux.experimental.engines.base.CommandRequest`. +{class}`~libtmux.engines.base.CommandRequest`. ```python >>> from libtmux.experimental.engines import CommandRequest, ImsgEngine diff --git a/docs/experimental/operations/index.md b/docs/experimental/operations/index.md index 241c2aca8d..db81931f14 100644 --- a/docs/experimental/operations/index.md +++ b/docs/experimental/operations/index.md @@ -7,7 +7,7 @@ effects directly from the operation registry. ## Execution contract An engine emits a raw -{class}`~libtmux.experimental.engines.base.CommandResult`. +{class}`~libtmux.engines.base.CommandResult`. {func}`~libtmux.experimental.ops.run` converts it to the operation's declared {class}`~libtmux.experimental.ops.results.Result` subtype. Results preserve failures as data; call diff --git a/docs/experimental/plans.md b/docs/experimental/plans.md index 690eeaa995..58ea9d54d3 100644 --- a/docs/experimental/plans.md +++ b/docs/experimental/plans.md @@ -44,23 +44,24 @@ True ## Choose a planner A {class}`~libtmux.experimental.ops.planner.Planner` turns a plan into -dispatches: +ordered execution steps: - {class}`~libtmux.experimental.ops.planner.SequentialPlanner` sends one - dispatch per operation. -- {class}`~libtmux.experimental.ops.planner.FoldingPlanner` combines adjacent - chainable operations. -- {class}`~libtmux.experimental.ops.planner.MarkedPlanner` folds creation and - follow-up work by using tmux's `{marked}` register. + request per step. +- {class}`~libtmux.experimental.ops.planner.BatchingPlanner` combines adjacent + ready-to-render primitive operations while preserving one request and result + per operation. -All planners preserve per-operation results. They differ only in dispatch -shape. The callback in this live example records the dispatches: two chainable -option writes fold, while the output-bearing read stays separate. +All planners preserve per-operation results. They differ only in step shape; +the engine decides whether a batch is a control-mode pipeline or a series of +subprocess calls. The callback in this live example records one step containing +two option writes and an output-bearing read. Each result retains its own +stdout. ```python >>> from libtmux.experimental.engines import SubprocessEngine >>> from libtmux.experimental.ops import ( -... FoldingPlanner, +... BatchingPlanner, ... LazyPlan, ... SessionId, ... SetOption, @@ -79,11 +80,11 @@ option writes fold, while the output-bearing read stays separate. >>> steps = [] >>> outcome = operation_plan.execute( ... SubprocessEngine.for_server(server), -... planner=FoldingPlanner(), +... planner=BatchingPlanner(), ... on_step=lambda report: steps.append(report.step.indices), ... ).raise_for_status() >>> steps -[(0, 1), (2,)] +[(0, 1, 2)] >>> type(outcome.results[2]).__name__ 'ShowOptionsResult' >>> ( @@ -120,9 +121,9 @@ True Execution records the handle's concrete pane identifier in {attr}`~libtmux.experimental.ops.plan.PlanResult.bindings`. -See {doc}`tutorials/async-control-plans` to compose chainable operations, inspect -their compiled tmux sequence, and execute the plan over one persistent async -control-mode client. +See {doc}`tutorials/async-control-plans` to compose ordered operations, inspect +their planner steps, and execute them over one persistent async control-mode +client. ## API reference @@ -157,10 +158,7 @@ control-mode client. .. autoclass:: libtmux.experimental.ops.planner.SequentialPlanner :members: -.. autoclass:: libtmux.experimental.ops.planner.FoldingPlanner - :members: - -.. autoclass:: libtmux.experimental.ops.planner.MarkedPlanner +.. autoclass:: libtmux.experimental.ops.planner.BatchingPlanner :members: .. autoclass:: libtmux.experimental.ops.planner.BoundedPlanner diff --git a/docs/experimental/results.md b/docs/experimental/results.md index da672a94ce..14286f8ddc 100644 --- a/docs/experimental/results.md +++ b/docs/experimental/results.md @@ -32,7 +32,7 @@ All results retain the operation, rendered `argv`, `status`, `returncode`, `stdout`, and `stderr`. `ok` is true only for `complete`; `failed` identifies a tmux rejection or an incomplete composed operation. {meth}`~libtmux.experimental.ops.results.Result.raise_for_status` raises for -`failed` and `unknown`, but returns both `complete` and `skipped` results. See +`failed` and `unknown`, but returns `complete` results. See {doc}`tutorials/results-and-failures` for those paths in context. ## Choose the payload diff --git a/docs/experimental/tutorials/async-control-plans.md b/docs/experimental/tutorials/async-control-plans.md index 4304da1e52..4dec7127b9 100644 --- a/docs/experimental/tutorials/async-control-plans.md +++ b/docs/experimental/tutorials/async-control-plans.md @@ -1,46 +1,41 @@ -# Master async control-mode plans +# Async control-mode plans Build a typed operation plan synchronously, then execute it asynchronously over -one persistent tmux client. The useful mental model has three distinct layers: +one persistent tmux client. Keep three layers distinct: | Layer | Owns | Does not imply | | --- | --- | --- | | Python composition | Operation order and forward references | Any tmux I/O | -| Planning | Which operations share one tmux command sequence | Process reuse | -| Control mode | One persistent `tmux -C` client and framed replies | One reply per plan | +| Planning | Which ready requests share an execution step | Merged commands or results | +| Control mode | One persistent `tmux -C` client and framed replies | One request per step | -Keeping those layers separate lets you change dispatch policy or transport -without rewriting the operations. +This separation lets the same plan run through subprocess or control mode +without changing its result semantics. ## Compose and run one live plan This plan creates a pane, assigns two pane-local user options, then reads them -back. The pane does not exist when Python records the option operations, so +back. The pane does not exist when Python records the later operations, so {meth}`~libtmux.experimental.ops.plan.LazyPlan.add` returns a -{class}`~libtmux.experimental.ops._types.SlotRef` that stands in for its future -ID. +{class}`~libtmux.experimental.ops._types.SlotRef` for its future ID. -The same deterministic story gives an agent tool three explicit phases: +The workflow has three explicit phases: -- Observe: preview the unresolved plan and explain its two dispatch steps +- Observe: preview unresolved arguments and explain the two planner steps without contacting tmux. -- Act: execute the marked fold through +- Act: execute through {class}`~libtmux.experimental.engines.async_control_mode.AsyncControlModeEngine`. -- Verify: inspect four typed statuses, the captured `worker:ready` value, and - the live pane lookup. - -The two tabs show the same work at the Python and tmux boundaries. `@WINDOW` -stands for the live window ID. `%PANE` stands for the pane ID captured by -`split-window`. +- Verify: inspect four typed results, the captured `worker:ready` value, and the + live pane lookup. `````{tab} Python plan ```python >>> import asyncio >>> from libtmux.experimental.engines import AsyncControlModeEngine >>> from libtmux.experimental.ops import ( +... BatchingPlanner, ... DisplayMessage, ... LazyPlan, -... MarkedPlanner, ... PaneId, ... SetOption, ... SplitWindow, @@ -77,15 +72,15 @@ SlotRef(slot=0, suffix='', part='self') [None, None, None] >>> [ ... (step.kinds, step.reason) -... for step in operation_plan.explain(MarkedPlanner()) +... for step in operation_plan.explain(BatchingPlanner()) ... ] -[(('split_window', 'set_option', 'set_option'), 'marked-fold'), - (('display_message',), 'capture')] +[(('split_window',), 'creator'), + (('set_option', 'set_option', 'display_message'), 'batch')] >>> async def configure_worker(): ... async with AsyncControlModeEngine.for_server(server) as engine: ... return await operation_plan.aexecute( ... engine, -... planner=MarkedPlanner(), +... planner=BatchingPlanner(), ... ) >>> outcome = asyncio.run(configure_worker()) >>> [result.status for result in outcome.results] @@ -100,24 +95,33 @@ True ``` ````` -`````{tab} Compiled tmux sequence -The pane creation and its two decorations compile into one tmux command -sequence: +`````{tab} Tmux requests +The creator is the first planner step because its captured ID must bind before +the other requests can render: ```console $ tmux split-window \ -t @WINDOW \ -v \ -P \ - -F '#{pane_id}' \ - \; select-pane -m \ - \; set-option -t '{marked}' -p -- @role worker \ - \; set-option -t '{marked}' -p -- @state ready \ - \; select-pane -M + -F '#{pane_id}' ``` -After tmux returns the new pane ID, the output-bearing query becomes a second -dispatch: +After `%PANE` binds, the second planner step contains three distinct requests: + +```console +$ tmux set-option \ + -t %PANE \ + -p \ + -- @role worker +``` + +```console +$ tmux set-option \ + -t %PANE \ + -p \ + -- @state ready +``` ```console $ tmux display-message \ @@ -127,95 +131,80 @@ $ tmux display-message \ ``` ````` -These console commands are the direct-CLI equivalents. The control-mode engine -does not start `tmux` for either line. It removes the leading executable and -writes each command sequence to the standard input of the existing `tmux -C` -client. +These are direct-CLI equivalents. The control-mode engine removes the leading +executable and writes each command as its own newline-terminated request to the +existing `tmux -C` client. It never joins them with `;`. ## Layer one: compose Python values {meth}`~libtmux.experimental.ops.operation.Operation.then` and `>>` create an ordered {class}`~libtmux.experimental.ops._chain.OpChain`. -{meth}`~libtmux.experimental.ops.plan.LazyPlan.add_chain` records that order in -the plan. Both operations are inert: they neither contact tmux nor promise that -the operations will share a dispatch. +{meth}`~libtmux.experimental.ops.plan.LazyPlan.add_chain` records that order. +Composition is inert: it neither contacts tmux nor promises one transport +request. -That distinction matters because {meth}`~libtmux.experimental.ops.plan.LazyPlan.aexecute` defaults to -{class}`~libtmux.experimental.ops.planner.SequentialPlanner`. Pass a folding -planner explicitly when dispatch shape matters. - -## Layer two: fold safe tmux sequences +{class}`~libtmux.experimental.ops.planner.SequentialPlanner`. Pass +{class}`~libtmux.experimental.ops.planner.BatchingPlanner` when ready requests +should share a planner step. -{class}`~libtmux.experimental.ops.planner.MarkedPlanner` recognizes a focused -pane creator followed immediately by chainable operations targeting that -creator's exact forward reference. It emits: +## Layer two: batch ready requests -1. `split-window -P -F '#{pane_id}'`; -2. `select-pane -m` to mark the newly focused pane; -3. each decoration retargeted to tmux's `{marked}` special target; and -4. `select-pane -M` to clear the mark. +{class}`~libtmux.experimental.ops.planner.BatchingPlanner` groups adjacent +primitive operations whose arguments are ready to render. Every operation still +becomes one {class}`~libtmux.engines.base.CommandRequest` and one +typed result. A creator remains its own step when later requests need its +captured ID. -The first planner step therefore contains three user operations but becomes one -engine request. The mark and unmark commands are implementation details and do -not add operation results. - -The final -{class}`~libtmux.experimental.ops._ops.display_message.DisplayMessage` remains -separate because it produces output. Combining its stdout with the creator's -captured pane ID would make typed result attribution ambiguous. Creators, -capturing reads, and any operation declaring `chainable = False` remain hard -boundaries unless a specialized planner has a safe attribution rule. +The plan validates custom planner output before dispatch: steps must form an +exact ordered partition, and a batch cannot contain an ensured, composite, or +non-batchable operation. The engine must return exactly one result per request. ## Resolve references at the last responsible moment -The worker target passes through three representations: +The worker target has two representations: 1. Python records `SlotRef(slot=0)` before a pane exists. -2. The marked fold addresses the new pane as `{marked}` inside one tmux - sequence. -3. The captured `%N` binds slot `0`; subsequent operations render a concrete - {class}`~libtmux.experimental.ops._types.PaneId`. +2. The creator returns `%N`; slot `0` binds, and the second step renders three + concrete {class}`~libtmux.experimental.ops._types.PaneId` targets. -The plan awaits each planner step before rendering a dependent step. That is why -the `display-message` request contains `%PANE`, while the two option writes can -run earlier against `{marked}`. +The plan awaits the creator step before rendering the batch. It does not borrow +tmux's server-global marked-pane register, so an existing user mark remains +untouched on both success and failure. ## Reuse one asynchronous transport {class}`~libtmux.experimental.engines.async_control_mode.AsyncControlModeEngine` owns one long-lived `tmux -C attach-session -E` child while its async context is -open. Each planner step becomes one newline-terminated control-mode request. -Tmux frames every subcommand reply with `%begin` and `%end` or `%error`; the -engine correlates those blocks into raw command results, and the plan maps the -raw outcomes back to typed operation results. - -This example has four operations and two planner steps, but only one persistent -control client process. Control mode avoids one process start per request; it -does not eliminate processes entirely. +open. The example has four operations, two planner steps, four tmux requests, +and one persistent client process. -Use {meth}`~libtmux.experimental.engines.async_control_mode.AsyncControlModeEngine.run_batch` -for independent, already-rendered requests. A forward-reference plan cannot -pipeline dependent steps that way because it must capture the first result -before it can render the next target. +writes the three ready requests without waiting between writes. Tmux frames each +reply with `%begin` and `%end` or `%error`; the engine correlates each frame to +its request, and the plan builds the corresponding typed result. A failed +request does not abort later requests in the batch. + +Subprocess engines use the same batch contract but still start one process per +request. Batching changes semantics only when a transport can pipeline; it does +not claim a subprocess speedup. ## Know the boundaries - Enter the async context only after a safe session exists. Without an attachable session, the engine uses async subprocess execution to bootstrap the server. -- Keep output-bearing reads and failure-sensitive validation in their own - planner steps. Tmux stops a command sequence after an error, and a folded - result cannot provide independent stdout or exact failure attribution for - every subcommand. -- A detached pane creator cannot use the marked-pane optimization because it - does not focus the new pane. -- Host-side waits, sleeps, or callbacks are dispatch boundaries. Use a bounded - planner when host work must occur between operations. +- A forward-reference dependency ends a batch because the next target cannot + render until the creator result arrives. +- Host-side waits, sleeps, or callbacks are planner-step boundaries. Use a + {class}`~libtmux.experimental.ops.planner.BoundedPlanner` when host work must + occur between operations. +- A typed {class}`~libtmux.experimental.engines.base.CommandSeparator` creates a + deliberate tmux fail-stop command group inside one raw request. It is not a + request batch and cannot provide per-command attribution through subprocess. - Scope the engine with `async with` so its reader, supervisor, and control client close before the event loop exits. See {doc}`../engines/async-control-mode` for attachment, notification, and reconnection lifecycle details. See {doc}`results-and-failures` for typed -command failures and skipped work. +command failures and continuation. diff --git a/docs/experimental/tutorials/control-mode.md b/docs/experimental/tutorials/control-mode.md index bcd5e5f5da..0d08fca839 100644 --- a/docs/experimental/tutorials/control-mode.md +++ b/docs/experimental/tutorials/control-mode.md @@ -3,7 +3,7 @@ {class}`~libtmux.experimental.engines.control_mode.ControlModeEngine` keeps one `tmux -C` client alive once connected and correlates tmux's framed replies with submitted -{class}`~libtmux.experimental.engines.base.CommandRequest` values. +{class}`~libtmux.engines.base.CommandRequest` values. {meth}`~libtmux.experimental.engines.control_mode.ControlModeEngine.run_batch` pipelines an ordered batch instead of starting one process per request. @@ -60,7 +60,7 @@ True ``` The values are live raw -{class}`~libtmux.experimental.engines.base.CommandResult` instances because +{class}`~libtmux.engines.base.CommandResult` instances because `run_batch` is the engine boundary. Use {func}`~libtmux.experimental.ops.run` or a plan when the caller needs operation-specific result subtypes. diff --git a/docs/experimental/tutorials/imsg-parity.md b/docs/experimental/tutorials/imsg-parity.md index 601b3fe627..af91a68b3c 100644 --- a/docs/experimental/tutorials/imsg-parity.md +++ b/docs/experimental/tutorials/imsg-parity.md @@ -10,7 +10,7 @@ standard output. `ImsgEngine` has no `for_server()` helper. Include the server's `-L` socket name or `-S` socket path in every -{class}`~libtmux.experimental.engines.base.CommandRequest`. The subprocess +{class}`~libtmux.engines.base.CommandRequest`. The subprocess engine accepts the same global argument in the request, which keeps the comparison exact. diff --git a/docs/experimental/tutorials/index.md b/docs/experimental/tutorials/index.md index 18d696de07..852a0bd736 100644 --- a/docs/experimental/tutorials/index.md +++ b/docs/experimental/tutorials/index.md @@ -7,12 +7,12 @@ Each concrete engine has one tested workflow: | `SubprocessEngine` | {doc}`live-operation` returns a typed result from an isolated live server. | | `AsyncSubprocessEngine` | {doc}`async-subprocess` runs two independent live reads concurrently. | | `ControlModeEngine` | {doc}`control-mode` pipelines an ordered request batch over one synchronous connection. | -| `AsyncControlModeEngine` | {doc}`async-control-plans` resolves a forward reference and verifies the live pane in two dispatches. | +| `AsyncControlModeEngine` | {doc}`async-control-plans` resolves a forward reference and verifies the live pane in two planner steps. | | `MockEngine` and `AsyncMockEngine` | {doc}`offline-testing` distinguishes canned output and fabricated IDs from live tmux state. | | `ImsgEngine` | {doc}`imsg-parity` compares one live stdout value with the subprocess transport. | {doc}`results-and-failures` is the shared guide to typed success, command -failure, expected absence, version rejection, and skipped work. +failure, expected absence, and version rejection. ```{toctree} :hidden: diff --git a/docs/experimental/tutorials/results-and-failures.md b/docs/experimental/tutorials/results-and-failures.md index a60231064b..3737f907ff 100644 --- a/docs/experimental/tutorials/results-and-failures.md +++ b/docs/experimental/tutorials/results-and-failures.md @@ -108,15 +108,14 @@ result for diagnosis instead of constructing an object with an empty target. ('new_session', ('self',), True) ``` -## Distinguish failed and skipped plan steps +## Continue after a failed batch request -A folding planner dispatches adjacent chainable operations as one tmux command -group. If the first command fails, tmux does not run the remainder; the result -attribution marks that remainder `skipped`. +A batching planner keeps every operation as a distinct request and result. A +failed request therefore does not prevent the next request from running. ```python >>> from libtmux.experimental.engines import SubprocessEngine ->>> from libtmux.experimental.ops import FoldingPlanner, LazyPlan, RenameWindow +>>> from libtmux.experimental.ops import BatchingPlanner, LazyPlan, RenameWindow >>> from libtmux.experimental.ops._types import WindowId >>> assert window.window_id is not None >>> plan = LazyPlan() @@ -124,15 +123,21 @@ attribution marks that remainder `skipped`. ... RenameWindow(target=WindowId("@999999999"), name="unreachable") ... ) >>> _ = plan.add( -... RenameWindow(target=WindowId(window.window_id), name="not-applied") +... RenameWindow(target=WindowId(window.window_id), name="applied") ... ) >>> outcome = plan.execute( ... SubprocessEngine.for_server(server), -... planner=FoldingPlanner(), +... planner=BatchingPlanner(), ... ) >>> [result.status for result in outcome.results] -['failed', 'skipped'] +['failed', 'complete'] +>>> _ = window.refresh() +>>> window.window_name +'applied' ``` -With the default sequential planner, each step is a separate dispatch, so one -failed result does not by itself imply that the next operation was skipped. +Subprocess engines execute the requests in order; persistent control engines +may pipeline them. Both transports preserve the same per-request result model. +A raw request containing a typed command separator is different: tmux treats it +as one fail-stop command group, so subprocess cannot attribute its error to an +individual member. diff --git a/docs/topics/engines.md b/docs/topics/engines.md index 3100072a25..674049f8a0 100644 --- a/docs/topics/engines.md +++ b/docs/topics/engines.md @@ -172,14 +172,28 @@ server would silently dispatch to whichever server a flagless `tmux` reaches: ('-Lengines_doc_c',) ``` -An engine that *does* name a server is left exactly as you built it: +An engine already on the requested server is left exactly as you built it: ```python >>> from libtmux.engines import SubprocessEngine >>> from libtmux.server import Server ->>> pinned = SubprocessEngine.of(server_args=("-Lengines_doc_pinned",)) +>>> pinned = SubprocessEngine.of(server_args=("-Lengines_doc_c",)) >>> Server(socket_name="engines_doc_c", engine=pinned).engine.server_args -('-Lengines_doc_pinned',) +('-Lengines_doc_c',) +``` + +Conflicting explicit scopes fail before a command can reach the wrong server: + +```python +>>> from libtmux import exc +>>> from libtmux.engines import SubprocessEngine +>>> from libtmux.server import Server +>>> pinned = SubprocessEngine.of(server_args=("-Lengines_doc_pinned",)) +>>> try: +... Server(socket_name="engines_doc_c", engine=pinned).engine +... except exc.EngineConfigurationMismatch: +... print("connection mismatch") +connection mismatch ``` An in-memory engine has no connection at all, so neither rule applies and it is @@ -192,19 +206,26 @@ An engine may implement extra protocols. Each is optional; libtmux checks with {class}`~libtmux.engines.base.SupportsCommandLine` renders the argv an engine *would* run, which is how the full command line reaches the debug log before -dispatch. {class}`~libtmux.engines.base.SupportsConnection` marks an engine that -dispatches over a named server and can be rebound — the protocol behind the -adoption rule above. +dispatch. {class}`~libtmux.engines.base.HasConnection` exposes a transport's +server scope for validation. {class}`~libtmux.engines.base.SupportsConnection` +adds safe rebinding; stateless subprocess engines implement both, while a live +control connection implements only the read-only capability and must already +match the Server. ```python >>> from libtmux.engines import ( ... SubprocessEngine, +... HasConnection, ... SupportsCommandLine, ... SupportsConnection, ... ) >>> engine = SubprocessEngine() ->>> isinstance(engine, SupportsCommandLine), isinstance(engine, SupportsConnection) -(True, True) +>>> ( +... isinstance(engine, SupportsCommandLine), +... isinstance(engine, HasConnection), +... isinstance(engine, SupportsConnection), +... ) +(True, True, True) ``` An engine that implements neither simply is not matched: @@ -227,10 +248,11 @@ in-memory fake — omits it, and the caller assumes the newest tmux. ## Explicit command separators -tmux treats a bare `;` argument as a boundary between two commands, but only -when it arrives unquoted. A `;` that is *data* — a pane title, a shell fragment -bound for `send-keys` — must not be mistaken for one. Guessing from the string -alone cannot tell them apart, so the intent rides in the type: +Tmux's direct argv parser treats an unescaped `;` at the end of any token as a +boundary between two commands. That includes both a standalone `;` and a value +such as `title;`; an interior semicolon remains data. Guessing intent from the +string alone cannot distinguish a literal suffix from command structure, so +the intent rides in the type: {class}`~libtmux.engines.base.CommandSeparator` marks a real boundary, and {func}`~libtmux.engines.base.is_command_separator` finds it. @@ -243,8 +265,7 @@ alone cannot tell them apart, so the intent rides in the type: [False, False, True, False, False] ``` -A plain `";"` is data and stays data, so nothing an existing caller passes can -become a boundary by accident: +A plain `";"` and any other ordinary trailing semicolon are encoded as data: ```python >>> from libtmux.engines import is_command_separator @@ -252,15 +273,21 @@ become a boundary by accident: False ``` -The marker survives normalization, so an engine that chains commands into one -dispatch can find the boundaries while every other engine ignores them. The -default {class}`~libtmux.engines.subprocess.SubprocessEngine` sends one command -per dispatch and has no use for them. +An existing `\;` suffix remains tmux escape syntax and is not escaped again; +tmux consumes that backslash while producing a literal semicolon. Use the +typed marker for structure and an ordinary unescaped suffix for new literal +data. + +The marker survives request normalization. Direct subprocess engines render it +as a bare structural token while escaping ordinary suffix semicolons; control +engines render ordinary values as quoted data. Callers that intentionally used +a plain `";"` to group commands must replace it with `CommandSeparator(";")`. ## What an engine does not change An engine chooses *how* a command runs, not what libtmux does with the answer. -Arguments reach tmux exactly as they always have, results read exactly as they -always have, and {meth}`Server.cmd() ` still returns a +Unescaped suffix semicolons remain argument data while each engine applies its +transport encoding, results read as before, and +{meth}`Server.cmd() ` still returns a {class}`~libtmux.common.tmux_cmd`. Under the default engine there is nothing new -to learn and nothing to migrate. +to configure. diff --git a/scripts/bench-results/RESULTS.md b/scripts/bench-results/RESULTS.md index 5f379b7154..ef6c211ab4 100644 --- a/scripts/bench-results/RESULTS.md +++ b/scripts/bench-results/RESULTS.md @@ -1,147 +1,110 @@ -# libtmux engine build-benchmark — results +# libtmux engine benchmark status -Produced by `scripts/bench/engines.py` (a hermetic PEP 723 grid runner) plus a -one-off hyperfine end-to-end run. All builds are isolated: per-run sockets under -a throwaway dir, `TMUX` unset, servers killed on exit — the default tmux server -is never contacted. Run the default grid: +> Status: archived and not current. Do not cite the timings in `grid.json` or +> `wait.json` as evidence for planner batching. -```console -$ uv run scripts/bench/engines.py run -``` +The checked-in raw results predate `BatchingPlanner` and use an earlier workload +whose creation operations could not be batched. They measured transport and tmux +server costs, not the planner optimization their former labels implied. The raw +files remain available only as historical measurements; `STATUS.json` records +that provenance explicitly. -Compare selected engines with shell-readiness waits: +The current benchmark models cardinality as sessions x windows per session x +panes per window. A two-part `WxP` value remains shorthand for one session. Each +multi-session sample builds its sessions sequentially through one engine; the +separate concurrency experiment is not mixed into topology scaling. -```console -$ uv run scripts/bench/engines.py run --engines classic,control_mode,pipelined --wait -``` +The workload adds two adjacent ready session-option operations and checks that +sequential and batching layers emit identical tmux requests with different +planner-step shapes. Its matrix reports these quantities separately: -Profile one larger control-mode build: +- planner steps; +- engine batch calls and their request counts; +- distinct tmux requests; +- elapsed build time. + +Process-start fields carry their evidence basis. Subprocess command children are +exactly one per request. Control-mode bootstrap children remain unmeasured, +version probes are cache-dependent, and the persistent control client is labeled +as an at-most-one engine model rather than an observed count. + +Run the correctness and shape contract before collecting timings: ```console -$ uv run scripts/bench/engines.py profile --engine control_mode --shape 8x4 +$ uv run scripts/bench/engines.py contract ``` -Raw data: `grid.json` (no-wait grid), `wait.json` (wait comparison). - -## Engine grid — in-process build, median ms (xN vs classic), 20 runs - -Shape = `windows x panes-per-window`. Structural builds (no shell-readiness wait). - -| engine | 1x1 | 1x4 | 3x3 | 5x4 | 8x4 | -|---|--:|--:|--:|--:|--:| -| classic (Server/Session/Window/Pane) | 22.0 | 169.4 | 452.5 | 1442.2 | 3497.2 | -| builder / subprocess | 23.0 | 42.8 | 86.9 | 246.7 | 428.8 (8x) | -| builder / imsg | 20.6 | 31.1 | 62.6 | 153.4 | 262.0 (13x) | -| builder / control_mode | 2.5 | 9.4 | 26.5 | 103.3 | 166.7 (21x) | -| **pipelined (prototype)** | **1.4** | **7.9** | **20.2** | **65.3** | **115.7 (30x)** | -| mock (offline, in-memory) | 0.1 | 0.1 | 0.3 | 1.3 | 1.5 | - -Full percentiles at 8x4 (ms): - -| engine | min | avg | median | p90 | p95 | p99 | max | -|---|--:|--:|--:|--:|--:|--:|--:| -| classic | 2192 | 3404 | 3497 | 3931 | 4077 | 4432 | 4432 | -| subprocess | 358 | 426 | 429 | 479 | 481 | 487 | 487 | -| imsg | 222 | 283 | 262 | 342 | 421 | 455 | 455 | -| control_mode | 118 | 180 | 167 | 215 | 216 | 398 | 398 | -| pipelined | 97 | 123 | 116 | 156 | 165 | 194 | 194 | -| mock | 1 | 2 | 2 | 2 | 2 | 2 | 2 | - -Reads: - -- **control_mode** (one persistent `tmux -C`, no per-op fork) is the fastest - shipped engine: 21x classic at 32 panes. -- **imsg** (AF_UNIX one-shot per call) sits between subprocess and control_mode; - its per-call handshake makes tiny builds no faster than classic. -- **pipelined** (prototype: batch independent creates into ~3 `run_batch` - round-trips instead of ~34) is fastest overall, ~1.4x over control_mode. Not - the 11x the round-trip count implies, because the build is **tmux-server-bound** - (one shell fork per pane), not round-trip-bound. `mock` (offline, 1.5 ms) - is the Python floor: the plan/compile layer is negligible; the time is tmux. - -## With vs without shell-readiness wait - -`wait.json`, 10 runs. "wait" polls each pane until its shell has drawn a prompt. - -| shape | engine | nowait median | wait median | wait penalty | speedup vs classic | -|---|---|--:|--:|--:|--:| -| 1x4 | classic | 217.6 | 1134.5 | 5.2x | 1.0x | -| 1x4 | control_mode | 9.4 | 865.2 | 92x | 23.1x -> 1.3x | -| 1x4 | pipelined | 9.0 | 1004.3 | 112x | 24.3x -> 1.1x | -| 5x4 | classic | 1365.6 | 3252.7 | 2.4x | 1.0x | -| 5x4 | control_mode | 73.1 | 2194.5 | 30x | 18.7x -> 1.5x | -| 5x4 | pipelined | 66.6 | 2123.3 | 32x | 20.5x -> 1.5x | - -**The engine win only exists when nobody waits for shells.** Shell startup -(~0.8-2.1 s) dominates a fast build (30-112x penalty) but barely moves the slow -classic path (2-5x), so the ~20x engine advantage collapses to ~1.5x once both -sides wait. Compare engines with matching readiness policies; mixing a waiting -classic path with a no-wait builder overstates the speedup. - -## Profile — where a control_mode build spends time (32 panes x5) - -~200 ms/build; **68% is `select.epoll.poll`** in `control_mode._read_blocks` — -one round-trip per created id (each new window/pane id read back before the next -op targets it). Round-trip-latency bound, not CPU/Python bound. - -## Whole-process wall time (hyperfine, SubprocessEngine, end-to-end) - -For reference, whole-script wall time (Python start + import + build + teardown) -on `SubprocessEngine`, 50 runs: classic simple 210 ms / large 6764 ms; builder -simple 453 ms / large 1692 ms. End-to-end on the *slowest* engine understates the -builder (startup dwarfs a 3 ms build) — the in-process grid above is the clean -signal. Prefer `control_mode` and in-process timing to measure build cost. - -## Build-cost factorial — `matrix` - -The `matrix` subcommand isolates which of four independent choices drives build -cost, sweeping them as a factorial: **async** {sync, async} × **transport** -{subprocess, control_mode} × **planner** {imperative, plan-seq, plan-fold} × -**workspace** {hand `LazyPlan`, declarative `Workspace`}. Because a declarative -workspace compiles to a lazy plan, the valid expression layers are five — -`imperative`, `plan-seq`, `plan-fold`, `ws-seq`, `ws-fold` — crossed with 2 -transports × 2 modes = 20 cells, against a `classic` reference. Reproduce: +Run the exhaustive planner matrix on the supported-version-safe pane-heavy +shape. One hundred samples make p90 and p95 useful; treat p99 as descriptive, +not stable evidence: ```console -$ uv run scripts/bench/engines.py matrix --shapes 1x4,3x3,5x4 +$ uv run scripts/bench/engines.py matrix \ + --shapes 1x1x4 \ + --runs 100 \ + --json-out scripts/bench-results/matrix.json ``` -`mock` is absent from the table by design: it is the offline correctness -**oracle**, not a results row. `matrix --check` (default on) and the standalone -`contract` subcommand assert every layer × mode renders identical tmux argv to -mock, so the benchmark doubles as an ops-language contract test: +Run the practical hierarchy corpus through the public default layer: ```console -$ uv run scripts/bench/engines.py contract +$ uv run scripts/bench/engines.py matrix \ + --shapes 1x1x1,1x1x4,1x4x1,4x1x1,1x2x2 \ + --layers default \ + --runs 100 \ + --json-out scripts/bench-results/hierarchy.json ``` -Illustrative 1×4 snapshot (small run; absolute ms are machine-dependent — the -ratios are the portable signal): +The script launches every daemon with an empty tmux configuration and verifies +the live session, window, pane, name, and option postconditions outside each +timed interval. A failed operation or partial topology aborts the cell instead +of becoming an artificially fast sample. + +Samples within a cell share one warmed daemon and engine. Fixed cell order can +still track machine load or thermal drift, so these results are local +steady-state descriptions rather than independent daemon replicates. Publish a +general speed claim only after collecting seeded, interleaved rounds on an idle +host and retaining environment and round identity with the raw samples. + +The `concurrency` subcommand remains exploratory. It lacks an async-sequential +control, symmetric connection-start boundaries, interleaved strategy order, and +event-loop-lag instrumentation. Do not use its ratios as async performance or +asyncio-health evidence. -| layer × transport × mode | median ms | vs classic | -|---|--:|--:| -| classic (reference) | 168.7 | 1.0x | -| ws-fold · control_mode · sync | 7.8 | 21.5x | -| plan-fold · control_mode · sync | 6.3 | 26.8x | -| imperative · control_mode · async | 7.0 | 24.0x | -| ws-fold · subprocess · sync | 30.7 | 5.5x | +The hand-written `prototype-pipelined` implementation is opt-in and excluded +from planner parity and default engine runs. It targets objects by chosen names +without the typed plan's captured-id contract, so its timing cannot substantiate +a planner claim. -Reads: transport dominates (control_mode ~4-5× subprocess); planner choice -(imperative → plan-seq → plan-fold) and the declarative-workspace compile move -build cost only marginally — the plan/compile layer is cheap, the time is tmux. -Single-build async ≈ sync (a linear build serializes either way). +No current performance conclusion is checked in. Regenerate the matrix on an +idle, controlled host before publishing transport or planner speedups. -## Concurrency — `concurrency` +## Async control-output demos -Async's real lever is not single-build latency but overlap. `concurrency` -builds K independent sessions sync-serial vs `asyncio.gather` over one -connection: +The lossless demo scrolls a stable seeded selection of libtmux Python source +through two windows with four panes each. It verifies each pane's decoded bytes, +sequence, source hash, zero-drop delivery, follow-up engine responsiveness, and +cleanup. Increase `--lines` for a longer display: ```console -$ uv run scripts/bench/engines.py concurrency --transport control_mode --k 4 +$ uv run scripts/demo_control_output.py scroll \ + --windows 2 \ + --panes 4 \ + --lines 500 \ + --delay 0.002 \ + --seed 688 ``` -Over one `control_mode` connection, async-gather runs ~2.4× the sync-serial -wall time at K=4 — the persistent connection multiplexes the K builds' -round-trips instead of paying them end-to-end. This is the win the single-build -matrix structurally cannot show. +The overload demo deliberately stalls a one-element subscriber queue. It must +observe dropped notification frames and then prove that the same async engine +still accepts a command. This demonstrates bounded dropping, not producer +backpressure or lossless delivery: + +```console +$ uv run scripts/demo_control_output.py overload \ + --windows 2 \ + --panes 4 \ + --lines 2000 \ + --seed 688 \ + --quiet +``` diff --git a/scripts/bench-results/STATUS.json b/scripts/bench-results/STATUS.json new file mode 100644 index 0000000000..f2feed3ef3 --- /dev/null +++ b/scripts/bench-results/STATUS.json @@ -0,0 +1,10 @@ +{ + "status": "stale", + "publishable": false, + "reason": "The raw files predate BatchingPlanner and used a workload with no batchable creation run.", + "artifacts": { + "grid.json": "historical transport benchmark only", + "wait.json": "historical readiness-wait benchmark only" + }, + "regenerate_with": "uv run scripts/bench/engines.py matrix --json-out scripts/bench-results/matrix.json" +} diff --git a/scripts/bench/engines.py b/scripts/bench/engines.py index a44980119f..a2ff32f87a 100755 --- a/scripts/bench/engines.py +++ b/scripts/bench/engines.py @@ -15,9 +15,11 @@ # mypy: disable-error-code="import-not-found, untyped-decorator" """Hermetic libtmux engine build-benchmark grid. -Measures how long the experimental workspace builder (and the classic API, and a -hand-rolled pipelined prototype) take to build tmux session structures, sweeping -scenarios x engines x wait-modes. Reports min/avg/median/max/p90/p95/p99. +Measures how long the experimental workspace builder and the classic API take to +build tmux session structures, sweeping scenarios x engines x wait-modes. +Reports min/avg/median/max/p90/p95/p99. A separately named hand-written +no-capture pipeline remains available as an opt-in prototype; it is not a +planner comparison and does not participate in the parity contract. Hermetic & sandboxed: every server runs on its OWN socket under a throwaway mkdtemp dir; ``TMUX`` is unset at import so the ambient session is never touched; @@ -30,51 +32,51 @@ control_mode builder on ControlModeEngine (one persistent ``tmux -C``) imsg builder on ImsgEngine (AF_UNIX imsg, socket-injected) mock builder on MockEngine (offline, in-memory: Python floor) - pipelined prototype: batch independent creates via run_batch (control_mode) + prototype-pipelined + hand-written no-capture run_batch prototype (opt-in only) -Timing (``run`` = in-process build-only, the clean signal; ``--hyperfine`` also -runs whole-process wall time via hyperfine over the ``cell`` subcommand). +Timing: ``run`` measures in-process build-only latency. The ``cell`` subcommand +provides one whole-process build for external tools such as hyperfine. The ``matrix`` subcommand isolates *why* a build costs what it does, sweeping a four-axis factorial -- async {sync, async} x transport {subprocess, -control_mode} x planner {imperative, plan-seq, plan-fold} x workspace -{hand plan, declarative} -- as five expression layers x 2 transports x 2 modes, -against a ``classic`` reference. ``mock`` is not benchmarked here: it is the -offline correctness oracle, and ``matrix --check`` / ``contract`` assert every -layer x mode renders identical tmux argv to it, so the grid doubles as an -ops-language contract test. ``concurrency`` measures async's real lever -- K -independent builds sync-serial vs ``asyncio.gather`` over one connection. +control_mode} x planner {imperative, sequential, batching} x expression +{hand plan, declarative, workspace default}. The workload includes adjacent +ready operations, so sequential and batching planners have measurably different +step shapes while retaining identical tmux requests. The matrix reports observed +planner/request grouping separately from process-start models; unknown or +cache-dependent counts remain explicitly unmeasured. ``mock`` is the offline +correctness oracle. +``concurrency`` is an exploratory sync-serial versus ``asyncio.gather`` probe. +Its startup boundaries and strategy order are not symmetric, so it is not +evidence for async speed or event-loop health. tmux also serializes its server +command queue. Run: uv run scripts/bench/engines.py run - uv run scripts/bench/engines.py run --engines control_mode,pipelined --wait + uv run scripts/bench/engines.py run --wait \ + --engines control_mode,prototype-pipelined uv run scripts/bench/engines.py profile --engine control_mode --shape 8x4 - uv run scripts/bench/engines.py cell control_mode 8x4 # one build, for hyperfine - uv run scripts/bench/engines.py matrix --shapes 1x4,3x3,5x4 + uv run scripts/bench/engines.py cell control_mode 8x4 + uv run scripts/bench/engines.py matrix --shapes 1x1x4,1x4x1,4x1x1 uv run scripts/bench/engines.py concurrency --transport control_mode --k 4 - uv run scripts/bench/engines.py contract # parity only, for CI + uv run scripts/bench/engines.py contract """ from __future__ import annotations import asyncio -import atexit import contextlib import cProfile import dataclasses import io import itertools import json -import math import os import pathlib import pstats -import shutil -import statistics -import subprocess -import tempfile +import sys import time import typing as t -import uuid # Never inherit the ambient tmux session -- do this BEFORE importing libtmux. os.environ.pop("TMUX", None) @@ -93,15 +95,27 @@ MockEngine, SubprocessEngine, ) + +sys.path.insert(0, str(pathlib.Path(__file__).resolve().parent)) + +from primitives import ( + STAT_LABELS, + new_server, + reap_stale_scratch, + summarize, + uniq, +) + from libtmux.experimental.engines.base import CommandRequest from libtmux.experimental.ops import ( - FoldingPlanner, + BatchingPlanner, LazyPlan, NewSession, NewWindow, Planner, RenameWindow, SequentialPlanner, + SetOption, SplitWindow, arun as op_arun, run as op_run, @@ -115,149 +129,115 @@ # · mode) overflow an 80-col pipe, so render them at a fixed width so each row # stays on one line under redirection. wide_console = rich.console.Console(width=132) + +_reaped = reap_stale_scratch() +if _reaped: + console.print(f"[dim]reaped {_reaped} stale bench scratch dir(s)[/dim]") R = CommandRequest.from_args -_ctr = itertools.count() -STAT_LABELS = ("n", "min", "avg", "median", "p90", "p95", "p99", "max") +BENCH_OPTIONS = ( + ("@libtmux_bench_one", "1"), + ("@libtmux_bench_two", "2"), +) + # --------------------------------------------------------------------------- # -# Hermetic isolation # +# Scenario spec + build implementations # # --------------------------------------------------------------------------- # -#: Names the process owning a scratch dir. A concurrent run is identified by its -#: pid, not by whether a tmux happens to be running in its dir: the dir exists -#: from import, its first server only from the first `new_server`. Same filename -#: `scripts/bench/primitives.py` writes, so the two reapers spare each other. -_OWNER_PID = "owner.pid" - -#: How long a scratch dir naming no owner is left alone -- the instant between -#: creating a dir and claiming it, and dirs from before the owner file existed. -_ADOPTION_GRACE_SECONDS = 300.0 - -_SOCK_DIR = pathlib.Path( - tempfile.mkdtemp(prefix="ltbench-") -) # short: /tmp/ltbench-XXXX -(_SOCK_DIR / _OWNER_PID).write_text(f"{os.getpid()}\n", encoding="utf-8") -_SERVERS: list[Server] = [] -#: A session every bench server keeps for its whole life, so killing a cell's -#: session never drops the server to zero and trips tmux's exit-empty teardown. -_KEEPALIVE = "keepalive" - - -def new_server() -> Server: - """Return a fresh isolated server on a unique socket under the scratch dir. - - The server is pinned alive by a keepalive session. Every cell kills its - session between builds, which would otherwise drop the server to zero - sessions; under tmux's ``exit-empty`` default the server then starts - exiting, and the next build's ``new-session`` can reach the still-bound - socket mid-shutdown and fail with "server exited unexpectedly". The race is - load-dependent, so it surfaced as an intermittent create failure rather than - an obvious teardown bug. Control mode never hit it -- its ``tmux -C`` - phantom session already pinned the server -- which is exactly why only the - subprocess cells were affected. +@dataclasses.dataclass(frozen=True) +class Scenario: + """One benchmark sample's session/window/pane cardinality. + + Attributes + ---------- + sessions : int + Independent sessions built sequentially per timed sample. + windows : int + Windows created in each session. + panes : int + Tiled panes created in each window. """ - srv = Server(socket_path=str(_SOCK_DIR / f"{uuid.uuid4().hex[:8]}.sock")) - _SERVERS.append(srv) - # The keepalive has to come first: `start-server` alone leaves a server with - # zero sessions, which exits immediately under the default, so there is no - # server left to set the option on. Creating a session that is never killed - # is what actually holds the floor above zero. - srv.cmd("new-session", "-d", "-s", _KEEPALIVE) - srv.cmd("set-option", "-s", "exit-empty", "off") - return srv - - -def _cleanup() -> None: - for srv in _SERVERS: - with contextlib.suppress(Exception): - srv.kill() - # Backstop: SIGKILL any tmux server still bound to a socket in our dir. - with contextlib.suppress(Exception): - out = subprocess.run( - ["pgrep", "-f", f"tmux .*-S{_SOCK_DIR}/"], - capture_output=True, - text=True, - check=False, - ).stdout.split() - for pid in out: - with contextlib.suppress(Exception): - os.kill(int(pid), 9) - with contextlib.suppress(Exception): - shutil.rmtree(_SOCK_DIR, ignore_errors=True) + sessions: int + windows: int + panes: int -def _owner_is_alive(path: pathlib.Path) -> bool | None: - """Say whether *path*'s owning process still exists, or None if unnamed.""" - try: - pid = int((path / _OWNER_PID).read_text(encoding="utf-8").strip()) - except (OSError, ValueError): - return None - try: - os.kill(pid, 0) - except PermissionError: - return True - except OSError: - return False - return True - - -def _reap_stale_scratch() -> None: - """Remove scratch dirs left behind by runs that died before their cleanup. - - :func:`_cleanup` only knows *this* process's socket dir, so a run killed - before its ``atexit`` hook leaves its dir -- and any tmux still bound to - it -- behind for good. Those survivors keep consuming CPU and file - descriptors, and machine load is precisely what makes the server-teardown - race fire, so an unreaped leak feeds the very failure it came from. - - A dir belonging to a live run is left alone. Liveness is the owning process - named in ``_OWNER_PID``, not whether a tmux is running there: a dir exists - from the moment its run starts, while its first server appears later, and a - reaper that asked only about tmux deleted concurrent runs during that - window. An unnamed owner is judged by age, then by the tmux probe, so every - unknown resolves toward keeping the dir. + @property + def label(self) -> str: + """Return the canonical ``SxWxP`` label. + + >>> Scenario(2, 3, 4).label + '2x3x4' + """ + return f"{self.sessions}x{self.windows}x{self.panes}" + + +def parse_scenario(value: str) -> Scenario: + """Parse ``WxP`` or ``SxWxP`` cardinality into a scenario. + + >>> parse_scenario("8x4") + Scenario(sessions=1, windows=8, panes=4) + >>> parse_scenario("2x3x4") + Scenario(sessions=2, windows=3, panes=4) """ - reaped = 0 - for path in pathlib.Path(tempfile.gettempdir()).glob("ltbench-*"): - if path == _SOCK_DIR or not path.is_dir(): - continue - with contextlib.suppress(Exception): - owner = _owner_is_alive(path) - if owner: - continue - if owner is None and ( - time.time() - path.stat().st_mtime < _ADOPTION_GRACE_SECONDS - ): - continue - alive = subprocess.run( - ["pgrep", "-f", f"tmux .*-S{path}/"], - capture_output=True, - text=True, - check=False, - ).stdout.split() - if alive: - continue - shutil.rmtree(path, ignore_errors=True) - reaped += 1 - if reaped: - console.print(f"[dim]reaped {reaped} stale bench scratch dir(s)[/dim]") - - -atexit.register(_cleanup) -_reap_stale_scratch() - - -def uniq() -> str: - """Return a process-unique session name (never collides across builds).""" - return f"b{next(_ctr)}" + parts = value.lower().split("x") + if len(parts) == 2: + parts.insert(0, "1") + if len(parts) != 3: + msg = f"expected WxP or SxWxP, got {value!r}" + raise ValueError(msg) + sessions, windows, panes = (int(part) for part in parts) + if min(sessions, windows, panes) < 1: + msg = f"scenario dimensions must be positive, got {value!r}" + raise ValueError(msg) + return Scenario(sessions, windows, panes) + + +def parse_shape(value: str) -> tuple[int, int]: + """Parse a one-session ``WxP`` shape for single-session commands. + + >>> parse_shape("8x4") + (8, 4) + """ + scenario = parse_scenario(value) + if scenario.sessions != 1: + msg = f"this command accepts WxP only, got {value!r}" + raise ValueError(msg) + return scenario.windows, scenario.panes -# --------------------------------------------------------------------------- # -# Scenario spec + build implementations # -# --------------------------------------------------------------------------- # -def parse_shape(s: str) -> tuple[int, int]: - """'8x4' -> (8 windows, 4 panes-per-window).""" - w, _, p = s.lower().partition("x") - return int(w), int(p) +def parse_scenarios(value: str) -> list[Scenario]: + """Parse a comma-separated scenario option or raise a CLI usage error. + + >>> parse_scenarios("1x1,2x3x4") + [Scenario(sessions=1, windows=1, panes=1), Scenario(sessions=2, windows=3, panes=4)] + """ + try: + scenarios = [parse_scenario(item) for item in value.split(",") if item] + except ValueError as error: + raise typer.BadParameter(str(error), param_hint="--shapes") from error + if not scenarios: + msg = "at least one scenario is required" + raise typer.BadParameter(msg, param_hint="--shapes") + return scenarios + + +def _select_axis( + value: str, + choices: t.Collection[str], + option: str, +) -> list[str]: + """Parse a comma-separated matrix axis without silently dropping values.""" + selected = [item.strip() for item in value.split(",") if item.strip()] + available = set(choices) + unknown = [item for item in selected if item not in available] + if unknown: + expected = ", ".join(sorted(available)) + msg = f"unknown value(s) {', '.join(unknown)}; choose from {expected}" + raise typer.BadParameter(msg, param_hint=option) + if not selected: + msg = "at least one value is required" + raise typer.BadParameter(msg, param_hint=option) + return selected def spec(name: str, wins: int, panes: int) -> Workspace: @@ -265,6 +245,7 @@ def spec(name: str, wins: int, panes: int) -> Workspace: return Workspace( name=name, on_exists="replace", + options=dict(BENCH_OPTIONS), windows=[ Window(name=f"w{w}", panes=[Pane() for _ in range(panes)]) for w in range(wins) @@ -275,6 +256,8 @@ def spec(name: str, wins: int, panes: int) -> Workspace: def build_classic(server: Server, name: str, wins: int, panes: int) -> None: """Build the structure with the classic Server/Session/Window/Pane API.""" session = server.new_session(session_name=name, window_name="w0") + for option, value in BENCH_OPTIONS: + session.set_option(option, value) for _ in range(panes - 1): session.active_window.split() for wi in range(1, wins): @@ -284,17 +267,19 @@ def build_classic(server: Server, name: str, wins: int, panes: int) -> None: def build_pipelined(engine: t.Any, name: str, wins: int, panes: int) -> None: - """Prototype: batch INDEPENDENT creates into few run_batch round-trips. + """Prototype: batch independent named targets without capturing ids. - new-session (1) + all new-windows in one run_batch (1) + all splits in one - run_batch (1) = 3 round-trips for any shape, vs ~1-per-op for the builder. - The control-mode run_batch pipelines (write all, read all reply blocks). + This is deliberately outside the planner comparison. It hand-writes tmux + argv, targets objects by chosen names, and does not provide the per-operation + captured-id contract of the typed plan/workspace paths. """ engine.run(R("new-session", "-d", "-s", name, "-n", "w0")) - if wins > 1: - engine.run_batch( - [R("new-window", "-t", name, "-n", f"w{i}") for i in range(1, wins)] - ) + ready = [ + R("set-option", "-t", name, "--", option, value) + for option, value in BENCH_OPTIONS + ] + ready.extend(R("new-window", "-t", name, "-n", f"w{i}") for i in range(1, wins)) + engine.run_batch(ready) splits = [ R("split-window", "-t", f"{name}:w{i}") for i in range(wins) @@ -338,7 +323,7 @@ class Impl: """One benchmarked implementation: how to make its engine and build.""" name: str - kind: str # classic | builder | pipelined | offline + kind: str # classic | builder | prototype-pipelined | offline make_engine: t.Callable[[Server | None], t.Any] | None = None needs_preboot: bool = False preflight: bool = True @@ -354,8 +339,10 @@ class Impl: ), "imsg": Impl("imsg", "builder", lambda s: ImsgForServer(s), needs_preboot=True), "mock": Impl("mock", "offline", lambda s: MockEngine(), preflight=False), - "pipelined": Impl( - "pipelined", "pipelined", lambda s: ControlModeEngine.for_server(s) + "prototype-pipelined": Impl( + "prototype-pipelined", + "prototype-pipelined", + lambda s: ControlModeEngine.for_server(s), ), } @@ -366,7 +353,7 @@ def do_build( """Dispatch one build of *w* x *p* to the right implementation path.""" if impl.kind == "classic": build_classic(server, name, w, p) # type: ignore[arg-type] - elif impl.kind == "pipelined": + elif impl.kind == "prototype-pipelined": build_pipelined(engine, name, w, p) else: # builder / offline spec(name, w, p).build(engine, preflight=impl.preflight) @@ -396,19 +383,102 @@ def wait_ready( time.sleep(interval) +def _checked_stdout(server: Server, *args: str) -> tuple[str, ...]: + """Run a diagnostic tmux query and reject transport or tmux failures.""" + result = server.cmd(*args) + if result.returncode != 0: + detail = " ".join(result.stderr) or f"tmux exited {result.returncode}" + msg = f"benchmark verification command failed: {' '.join(args)}: {detail}" + raise RuntimeError(msg) + return tuple(result.stdout) + + +def _kill_session(server: Server, name: str) -> None: + """Remove a benchmark session and reject cleanup failures.""" + _checked_stdout(server, "kill-session", "-t", name) + + +def verify_topology( + server: Server, + names: t.Sequence[str], + *, + windows: int, + panes: int, +) -> None: + """Reject a partial or semantically wrong live benchmark sample.""" + live_sessions = set( + _checked_stdout(server, "list-sessions", "-F", "#{session_name}") + ) + missing = [name for name in names if name not in live_sessions] + if missing: + msg = f"missing benchmark sessions: {', '.join(missing)}" + raise RuntimeError(msg) + expected_window_names = {f"w{index}" for index in range(windows)} + for name in names: + window_names = set( + _checked_stdout( + server, + "list-windows", + "-t", + name, + "-F", + "#{window_name}", + ) + ) + if window_names != expected_window_names: + msg = ( + f"{name}: expected windows {sorted(expected_window_names)!r}, " + f"got {sorted(window_names)!r}" + ) + raise RuntimeError(msg) + pane_ids = _checked_stdout( + server, + "list-panes", + "-s", + "-t", + name, + "-F", + "#{pane_id}", + ) + expected_panes = windows * panes + if len(pane_ids) != expected_panes: + msg = f"{name}: expected {expected_panes} panes, got {len(pane_ids)}" + raise RuntimeError(msg) + for option, expected in BENCH_OPTIONS: + actual = _checked_stdout( + server, + "show-options", + "-t", + name, + "-v", + option, + ) + if actual != (expected,): + msg = f"{name}: expected {option}={expected!r}, got {actual!r}" + raise RuntimeError(msg) + + def run_cell( - impl: Impl, wins: int, panes: int, wait: bool, runs: int, warmup: int + impl: Impl, + wins: int, + panes: int, + wait: bool, + runs: int, + warmup: int, + sessions: int = 1, ) -> list[float]: - """Return per-build wall times (ms), in-process, with session cleanup.""" + """Return per-sample wall times with *sessions* sequential builds each.""" if impl.kind == "offline": engine = impl.make_engine(None) # type: ignore[misc] for _ in range(warmup): - spec(uniq(), wins, panes).build(engine, preflight=False) + for _ in range(sessions): + spec(uniq(), wins, panes).build(engine, preflight=False) samples = [] for _ in range(runs): - name = uniq() + names = [uniq() for _ in range(sessions)] t0 = time.perf_counter() - spec(name, wins, panes).build(engine, preflight=False) + for name in names: + spec(name, wins, panes).build(engine, preflight=False) samples.append((time.perf_counter() - t0) * 1000) return samples @@ -418,49 +488,41 @@ def run_cell( engine = impl.make_engine(server) if impl.make_engine else None try: for _ in range(warmup): - name = uniq() - do_build(impl, server, engine, name, wins, panes) - if wait: - wait_ready(server, name) - server.cmd("kill-session", "-t", name) + names = [uniq() for _ in range(sessions)] + for name in names: + do_build(impl, server, engine, name, wins, panes) + if wait: + wait_ready(server, name) + verify_topology(server, names, windows=wins, panes=panes) + for name in names: + _kill_session(server, name) samples = [] for _ in range(runs): - name = uniq() + names = [uniq() for _ in range(sessions)] t0 = time.perf_counter() - do_build(impl, server, engine, name, wins, panes) - if wait: - wait_ready(server, name) + for name in names: + do_build(impl, server, engine, name, wins, panes) + if wait: + wait_ready(server, name) samples.append((time.perf_counter() - t0) * 1000) - server.cmd("kill-session", "-t", name) # untimed cleanup -> no accumulation + verify_topology(server, names, windows=wins, panes=panes) + for name in names: + _kill_session(server, name) return samples finally: with contextlib.suppress(Exception): server.kill() -# --------------------------------------------------------------------------- # -# Stats (nearest-rank percentiles, like agentgrep's benchmark) # -# --------------------------------------------------------------------------- # -def percentile(sorted_vals: list[float], pct: float) -> float: - """Nearest-rank percentile of a pre-sorted sequence.""" - if not sorted_vals: - return float("nan") - rank = max(1, math.ceil(pct / 100.0 * len(sorted_vals))) - return sorted_vals[min(rank, len(sorted_vals)) - 1] +def summary_json(summary: dict[str, float]) -> dict[str, float | int]: + """Return JSON field names with a unit only on duration values. - -def summarize(samples: list[float]) -> dict[str, float]: - """Return min/avg/median/p90/p95/p99/max (and n) for *samples*.""" - s = sorted(samples) + >>> summary_json({"n": 2.0, "min": 1.5}) + {'n': 2, 'min_ms': 1.5} + """ return { - "n": float(len(s)), - "min": s[0], - "avg": statistics.fmean(s), - "median": statistics.median(s), - "p90": percentile(s, 90), - "p95": percentile(s, 95), - "p99": percentile(s, 99), - "max": s[-1], + "n": int(summary["n"]), + **{f"{name}_ms": summary[name] for name in STAT_LABELS[1:] if name in summary}, } @@ -474,12 +536,12 @@ def summarize(samples: list[float]) -> dict[str, float]: # # MODES sync | async -- which run-strategy drives a build # TRANSPORTS subprocess | control_mode -- each supplies a sync + async engine -# LAYERS imperative | plan-seq | plan-fold | ws-seq | ws-fold +# LAYERS imperative | plan-seq | plan-batch | ws-seq | ws-batch | default # -# The five LAYERS are the valid *expression* layers: a declarative Workspace -# compiles to a LazyPlan, so ws-* and plan-* share one op spine, and folding is a -# planner swap. `mock` is NOT a layer -- it is the offline correctness oracle for -# the parity contract (`check_parity`), never a results row. +# A declarative Workspace compiles to a LazyPlan, so ws-* and plan-* share one op +# spine. ``default`` exercises Workspace's public default instead of spelling a +# planner explicitly. `mock` is NOT a layer -- it is the offline correctness +# oracle for the parity contract (`check_parity`), never a results row. # # Sync/async is unified without contaminating either path with the other's # machinery: the plan/ws layers differ only by method name (`execute`/`aexecute`, @@ -507,6 +569,12 @@ def _imperative_ops( result.first_window_id, result.first_pane_id, ) + for option, value in BENCH_OPTIONS: + yield SetOption( + target=SessionId(session_id), + option=option, + value=value, + ) yield RenameWindow(target=WindowId(window0_id), name="w0") prev = pane0_id for _ in range(panes - 1): @@ -527,7 +595,8 @@ def _pump_sync(gen: t.Generator[t.Any, t.Any, None], engine: t.Any) -> None: try: op = next(gen) while True: - op = gen.send(op_run(op, engine)) + result = op_run(op, engine).raise_for_status() + op = gen.send(result) except StopIteration: pass @@ -537,7 +606,8 @@ async def _pump_async(gen: t.Generator[t.Any, t.Any, None], engine: t.Any) -> No try: op = next(gen) while True: - op = gen.send(await op_arun(op, engine)) + result = (await op_arun(op, engine)).raise_for_status() + op = gen.send(result) except StopIteration: pass @@ -546,11 +616,13 @@ def _hand_plan(name: str, wins: int, panes: int) -> LazyPlan: """Hand-author the WxP build as a ``LazyPlan`` with forward SlotRef targets. Mirrors :func:`_imperative_ops` (and the workspace compiler) op-for-op, but - records refs instead of resolving ids eagerly, so a planner can fold or - sequence the dispatch. + records refs instead of resolving ids eagerly, so a planner can batch or + sequence the execution steps. """ plan = LazyPlan() session = plan.add(NewSession(session_name=name, capture_panes=True)) + for option, value in BENCH_OPTIONS: + plan.add(SetOption(target=session, option=option, value=value)) plan.add(RenameWindow(target=session.window, name="w0")) prev: SlotRef = session.pane for _ in range(panes - 1): @@ -565,12 +637,13 @@ def _hand_plan(name: str, wins: int, panes: int) -> LazyPlan: @dataclasses.dataclass(frozen=True) class Layer: - """One expression layer: how a build is *authored* (not how it is dispatched). + """One expression layer: how a build is authored and planned. ``kind`` selects the execution shape (``imperative`` drives the generator; ``plan`` executes a hand ``LazyPlan``; ``ws`` builds the declarative :class:`~libtmux.experimental.workspace.Workspace` IR). ``planner`` is the - dispatch policy for the plan/ws kinds (``None`` for imperative). + step-grouping policy for the plan/ws kinds. ``None`` means no planner for + imperative and the public default for ``default``. """ name: str @@ -581,9 +654,10 @@ class Layer: LAYERS: dict[str, Layer] = { "imperative": Layer("imperative", "imperative"), "plan-seq": Layer("plan-seq", "plan", SequentialPlanner()), - "plan-fold": Layer("plan-fold", "plan", FoldingPlanner()), + "plan-batch": Layer("plan-batch", "plan", BatchingPlanner()), "ws-seq": Layer("ws-seq", "ws", SequentialPlanner()), - "ws-fold": Layer("ws-fold", "ws", FoldingPlanner()), + "ws-batch": Layer("ws-batch", "ws", BatchingPlanner()), + "default": Layer("default", "ws"), } @@ -592,9 +666,13 @@ def build_sync(layer: Layer, engine: t.Any, name: str, wins: int, panes: int) -> if layer.kind == "imperative": _pump_sync(_imperative_ops(name, wins, panes), engine) elif layer.kind == "plan": - _hand_plan(name, wins, panes).execute(engine, planner=layer.planner) + _hand_plan(name, wins, panes).execute( + engine, planner=layer.planner + ).raise_for_status() else: # ws - spec(name, wins, panes).build(engine, preflight=False, planner=layer.planner) + spec(name, wins, panes).build( + engine, preflight=False, planner=layer.planner + ).raise_for_status() async def build_async( @@ -604,11 +682,15 @@ async def build_async( if layer.kind == "imperative": await _pump_async(_imperative_ops(name, wins, panes), engine) elif layer.kind == "plan": - await _hand_plan(name, wins, panes).aexecute(engine, planner=layer.planner) + outcome = await _hand_plan(name, wins, panes).aexecute( + engine, planner=layer.planner + ) + outcome.raise_for_status() else: # ws - await spec(name, wins, panes).abuild( + outcome = await spec(name, wins, panes).abuild( engine, preflight=False, planner=layer.planner ) + outcome.raise_for_status() @dataclasses.dataclass(frozen=True) @@ -655,22 +737,29 @@ def _sync_cell( server: Server, wins: int, panes: int, + sessions: int, runs: int, warmup: int, ) -> list[float]: """Time *runs* synchronous builds of one cell (untimed kill-session cleanup).""" engine = transport.make_sync(server) for _ in range(warmup): - name = uniq() - build_sync(layer, engine, name, wins, panes) - server.cmd("kill-session", "-t", name) + names = [uniq() for _ in range(sessions)] + for name in names: + build_sync(layer, engine, name, wins, panes) + verify_topology(server, names, windows=wins, panes=panes) + for name in names: + _kill_session(server, name) samples: list[float] = [] for _ in range(runs): - name = uniq() + names = [uniq() for _ in range(sessions)] t0 = time.perf_counter() - build_sync(layer, engine, name, wins, panes) + for name in names: + build_sync(layer, engine, name, wins, panes) samples.append((time.perf_counter() - t0) * 1000) - server.cmd("kill-session", "-t", name) + verify_topology(server, names, windows=wins, panes=panes) + for name in names: + _kill_session(server, name) return samples @@ -682,7 +771,7 @@ async def _akill_session(server: Server, name: str) -> None: running loop measurably raises the rate of failed ``new-session`` calls, so the cleanup is offloaded to a thread. """ - await asyncio.to_thread(server.cmd, "kill-session", "-t", name) + await asyncio.to_thread(_kill_session, server, name) async def _async_cell( @@ -691,22 +780,33 @@ async def _async_cell( server: Server, wins: int, panes: int, + sessions: int, runs: int, warmup: int, ) -> list[float]: """Async twin of :func:`_sync_cell`, over one persistent async connection.""" async with _open_async(transport, server) as engine: for _ in range(warmup): - name = uniq() - await build_async(layer, engine, name, wins, panes) - await _akill_session(server, name) + names = [uniq() for _ in range(sessions)] + for name in names: + await build_async(layer, engine, name, wins, panes) + await asyncio.to_thread( + verify_topology, server, names, windows=wins, panes=panes + ) + for name in names: + await _akill_session(server, name) samples: list[float] = [] for _ in range(runs): - name = uniq() + names = [uniq() for _ in range(sessions)] t0 = time.perf_counter() - await build_async(layer, engine, name, wins, panes) + for name in names: + await build_async(layer, engine, name, wins, panes) samples.append((time.perf_counter() - t0) * 1000) - await _akill_session(server, name) + await asyncio.to_thread( + verify_topology, server, names, windows=wins, panes=panes + ) + for name in names: + await _akill_session(server, name) return samples @@ -716,6 +816,7 @@ def matrix_cell( layer: Layer, wins: int, panes: int, + sessions: int, runs: int, warmup: int, ) -> list[float]: @@ -723,9 +824,11 @@ def matrix_cell( server = new_server() try: if mode == "sync": - return _sync_cell(transport, layer, server, wins, panes, runs, warmup) + return _sync_cell( + transport, layer, server, wins, panes, sessions, runs, warmup + ) return asyncio.run( - _async_cell(transport, layer, server, wins, panes, runs, warmup) + _async_cell(transport, layer, server, wins, panes, sessions, runs, warmup) ) finally: with contextlib.suppress(Exception): @@ -735,21 +838,52 @@ def matrix_cell( # --------------------------------------------------------------------------- # # Mock-parity contract: every layer x mode renders the SAME argv as mock # # --------------------------------------------------------------------------- # +@dataclasses.dataclass(frozen=True) +class RecordedBuild: + """One mock build's exact requests and engine-call grouping.""" + + argv: tuple[tuple[str, ...], ...] + batch_sizes: tuple[int, ...] + + @property + def tmux_requests(self) -> int: + """Return the number of distinct tmux requests.""" + return len(self.argv) + + @property + def engine_calls(self) -> int: + """Return the number of ``run``/``run_batch`` engine calls.""" + return len(self.batch_sizes) + + +@dataclasses.dataclass(frozen=True) +class ExecutionShape: + """Transport-independent execution shape for one expression layer.""" + + planner_steps: int | None + engine_calls: int + tmux_requests: int + batch_sizes: tuple[int, ...] + + class _Recorder: - """Wrap a sync engine, recording each dispatched argv (the parity oracle tap).""" + """Wrap a sync engine, recording requests and engine-call grouping.""" def __init__(self, inner: t.Any) -> None: self.inner = inner self.argv: list[tuple[str, ...]] = [] + self.batch_sizes: list[int] = [] def run(self, request: CommandRequest) -> t.Any: """Record and forward one request.""" self.argv.append(tuple(request.args)) + self.batch_sizes.append(1) return self.inner.run(request) def run_batch(self, requests: t.Sequence[CommandRequest]) -> t.Any: """Record and forward a batch of requests.""" self.argv.extend(tuple(r.args) for r in requests) + self.batch_sizes.append(len(requests)) return self.inner.run_batch(requests) @@ -759,34 +893,59 @@ class _AsyncRecorder: def __init__(self, inner: t.Any) -> None: self.inner = inner self.argv: list[tuple[str, ...]] = [] + self.batch_sizes: list[int] = [] async def run(self, request: CommandRequest) -> t.Any: """Record and forward one request.""" self.argv.append(tuple(request.args)) + self.batch_sizes.append(1) return await self.inner.run(request) async def run_batch(self, requests: t.Sequence[CommandRequest]) -> t.Any: """Record and forward a batch of requests.""" self.argv.extend(tuple(r.args) for r in requests) + self.batch_sizes.append(len(requests)) return await self.inner.run_batch(requests) -def _record_sync( - layer: Layer, name: str, wins: int, panes: int -) -> list[tuple[str, ...]]: - """Render *layer* through the deterministic MockEngine, capturing its argv.""" +def _record_sync(layer: Layer, name: str, wins: int, panes: int) -> RecordedBuild: + """Render *layer* through MockEngine, recording requests and grouping.""" recorder = _Recorder(MockEngine()) build_sync(layer, recorder, name, wins, panes) - return recorder.argv + return RecordedBuild(tuple(recorder.argv), tuple(recorder.batch_sizes)) async def _record_async( layer: Layer, name: str, wins: int, panes: int -) -> list[tuple[str, ...]]: +) -> RecordedBuild: """Async twin of :func:`_record_sync` (AsyncMockEngine).""" recorder = _AsyncRecorder(AsyncMockEngine()) await build_async(layer, recorder, name, wins, panes) - return recorder.argv + return RecordedBuild(tuple(recorder.argv), tuple(recorder.batch_sizes)) + + +def execution_shape(layer: Layer, wins: int, panes: int) -> ExecutionShape: + """Return exact planner-step and request counts for one mock build.""" + recorded = _record_sync(layer, "shape", wins, panes) + return ExecutionShape( + planner_steps=(recorded.engine_calls if layer.kind != "imperative" else None), + engine_calls=recorded.engine_calls, + tmux_requests=recorded.tmux_requests, + batch_sizes=recorded.batch_sizes, + ) + + +def batch_size_summary(sizes: tuple[int, ...]) -> str: + """Format adjacent equal batch sizes with run-length encoding. + + >>> batch_size_summary((1, 1, 3, 1)) + '1x2,3,1' + """ + parts: list[str] = [] + for size, run in itertools.groupby(sizes): + count = sum(1 for _ in run) + parts.append(str(size) if count == 1 else f"{size}x{count}") + return ",".join(parts) def check_parity( @@ -794,25 +953,26 @@ def check_parity( ) -> tuple[list[tuple[str, ...]], list[tuple[str, bool]]]: """Assert every layer x mode renders the mock oracle's argv for WxP. - Mock is deterministic, so all five expression layers -- driven through it via + Mock is deterministic, so every expression layer -- driven through it via both the sync and async paths -- must emit the identical op argv sequence. The oracle is the declarative ws-seq rendering (one argv per op); returns the oracle plus a ``(label, agrees)`` row per layer x mode. """ name = "parity" - oracle = _record_sync(LAYERS["ws-seq"], name, wins, panes) + oracle = _record_sync(LAYERS["ws-seq"], name, wins, panes).argv rows: list[tuple[str, bool]] = [] for layer_name, layer in LAYERS.items(): - rows.append( - (f"{layer_name}/sync", _record_sync(layer, name, wins, panes) == oracle) - ) + sync_record = _record_sync(layer, name, wins, panes) + rows.append((f"{layer_name}/sync", sync_record.argv == oracle)) + async_record = asyncio.run(_record_async(layer, name, wins, panes)) rows.append( ( f"{layer_name}/async", - asyncio.run(_record_async(layer, name, wins, panes)) == oracle, + async_record.argv == oracle + and async_record.batch_sizes == sync_record.batch_sizes, ) ) - return oracle, rows + return list(oracle), rows # --------------------------------------------------------------------------- # @@ -825,8 +985,8 @@ def check_parity( def run( shapes: str = typer.Option("1x1,1x4,3x3,5x4,8x4", help="comma WxP shapes"), engines: str = typer.Option( - "classic,subprocess,control_mode,imsg,mock,pipelined", - help="comma engine names", + "classic,subprocess,control_mode,imsg,mock", + help="comma engine names; prototype-pipelined is opt-in", ), wait: bool = typer.Option(False, help="ALSO measure with shell-readiness wait"), runs: int = typer.Option(20, help="timed builds per cell"), @@ -835,7 +995,7 @@ def run( ) -> None: """In-process build-only benchmark grid (the clean signal).""" shape_list = [parse_shape(s) for s in shapes.split(",") if s] - engine_list = [e for e in engines.split(",") if e in IMPLS] + engine_list = _select_axis(engines, IMPLS, "--engines") wait_modes = [False, True] if wait else [False] results: list[dict[str, t.Any]] = [] @@ -848,7 +1008,7 @@ def run( table.add_column("engine", style="cyan") for label in STAT_LABELS: table.add_column(label, justify="right") - table.add_column("vs classic", justify="right", style="green") + table.add_column("vs legacy workflow", justify="right", style="green") base_median = None for name in engine_list: impl = IMPLS[name] @@ -889,7 +1049,7 @@ def run( @app.command() def cell(engine: str, shape: str, wait: bool = typer.Option(False)) -> None: - """Build ONE workspace of *shape* with *engine* (isolated). For hyperfine.""" + """Build one isolated workspace for whole-process timing tools.""" impl = IMPLS[engine] wins, panes = parse_shape(shape) if impl.kind == "offline": @@ -927,14 +1087,14 @@ def profile( warm = uniq() do_build(impl, server, eng, warm, wins, panes) # warmup if server is not None: - server.cmd("kill-session", "-t", warm) + _kill_session(server, warm) pr = cProfile.Profile() pr.enable() for _ in range(builds): name = uniq() do_build(impl, server, eng, name, wins, panes) if server is not None: - server.cmd("kill-session", "-t", name) + _kill_session(server, name) pr.disable() buf = io.StringIO() pstats.Stats(pr, stream=buf).sort_stats("cumulative").print_stats(top) @@ -948,53 +1108,118 @@ def profile( @app.command() def matrix( - shapes: str = typer.Option("1x4", help="comma WxP shapes"), + shapes: str = typer.Option("1x4", help="comma WxP or SxWxP scenarios"), layers: str = typer.Option(",".join(LAYERS), help="comma expression layers"), transports: str = typer.Option(",".join(TRANSPORTS), help="comma transports"), modes: str = typer.Option("sync,async", help="comma modes: sync,async"), - runs: int = typer.Option(10, help="timed builds per cell"), + runs: int = typer.Option(100, help="timed samples per cell"), warmup: int = typer.Option(2, help="warmup builds per cell"), check: bool = typer.Option(True, help="run the mock-parity contract first"), json_out: str = typer.Option("", help="write full JSON results here"), ) -> None: - """Factorial matrix: expression-layer x transport x mode, one table per shape. + """Factorial matrix: expression-layer x transport x mode per SxWxP scenario. Rows are *generated* by ``product`` over the axis registries (never hand-enumerated); ``classic`` is the reference row and ``mock`` never appears - (it is the parity oracle, run first when ``--check``). + (it is the parity oracle, run first when ``--check``). Separate tables keep + directly observed request grouping, process-start models, and elapsed time + distinct. """ - shape_list = [parse_shape(s) for s in shapes.split(",") if s] - layer_list = [name for name in layers.split(",") if name in LAYERS] - transport_list = [name for name in transports.split(",") if name in TRANSPORTS] - mode_list = [name for name in modes.split(",") if name in MODES] + scenarios = parse_scenarios(shapes) + layer_list = _select_axis(layers, LAYERS, "--layers") + transport_list = _select_axis(transports, TRANSPORTS, "--transports") + mode_list = _select_axis(modes, MODES, "--modes") results: list[dict[str, t.Any]] = [] - for wins, panes in shape_list: + for scenario in scenarios: + sessions, wins, panes = ( + scenario.sessions, + scenario.windows, + scenario.panes, + ) if check: _oracle, parity_rows = check_parity(wins, panes) failed = [label for label, agrees in parity_rows if not agrees] if failed: console.print( - f"[bold red]mock-parity FAILED[/bold red] for {wins}x{panes}: " + f"[bold red]mock-parity FAILED[/bold red] for {scenario.label}: " f"{', '.join(failed)}" ) raise typer.Exit(1) console.print( f"[green]mock-parity OK[/green] -- {len(parity_rows)} layer x mode " - f"agree with mock argv for {wins}x{panes}" + f"agree with mock argv for {scenario.label}" + ) + + structure = rich.table.Table( + title=f"[bold]{scenario.label} execution shape (per sample)[/bold]" + ) + structure.add_column("layer", style="cyan") + structure.add_column("planner steps", justify="right") + structure.add_column("engine calls", justify="right") + structure.add_column("tmux requests", justify="right") + structure.add_column("batch sizes (RLE)", justify="right") + layer_shapes: dict[str, ExecutionShape] = {} + for layer_name in layer_list: + shape = execution_shape(LAYERS[layer_name], wins, panes) + layer_shapes[layer_name] = shape + batch_summary = batch_size_summary(shape.batch_sizes) + if sessions > 1: + batch_summary = f"({batch_summary}) x {sessions} sessions" + structure.add_row( + layer_name, + ( + str(shape.planner_steps * sessions) + if shape.planner_steps is not None + else "-" + ), + str(shape.engine_calls * sessions), + str(shape.tmux_requests * sessions), + batch_summary, + ) + wide_console.print(structure) + wide_console.print() + + process_model = rich.table.Table( + title=( + f"[bold]{scenario.label} process-start model (not instrumented)[/bold]" ) + ) + process_model.add_column("cell", style="cyan") + process_model.add_column("command subprocesses/sample") + process_model.add_column("version probes/cell") + process_model.add_column("persistent clients/cell") + for layer_name, transport_name in itertools.product(layer_list, transport_list): + shape = layer_shapes[layer_name] + if transport_name == "subprocess": + command_model = f"{shape.tmux_requests * sessions} (exact: one/request)" + persistent_model = "0 (engine model)" + else: + command_model = "? (bootstrap-dependent)" + persistent_model = "<=1 (engine model; unmeasured)" + process_model.add_row( + f"{layer_name} · {transport_name}", + command_model, + "? (cache-dependent)", + persistent_model, + ) + wide_console.print(process_model) + wide_console.print() - classic_samples = run_cell(IMPLS["classic"], wins, panes, False, runs, warmup) + classic_samples = run_cell( + IMPLS["classic"], wins, panes, False, runs, warmup, sessions + ) base_median = summarize(classic_samples)["median"] table = rich.table.Table( - title=f"[bold]{wins} win x {panes} pane ({wins * panes} panes)" + title=f"[bold]{scenario.label} ({sessions} sessions, " + f"{wins * sessions} windows, {wins * panes * sessions} panes)" f" -- in-process build ms (async x transport x layer)[/bold]" ) table.add_column("cell", style="cyan") for label in STAT_LABELS: table.add_column(label, justify="right") - table.add_column("vs classic", justify="right", style="green") + table.add_column("vs legacy workflow", justify="right", style="green") classic_stat = summarize(classic_samples) table.add_row( @@ -1010,9 +1235,29 @@ def matrix( "transport": "classic", "mode": "sync", "shape": f"{wins}x{panes}", - "panes": wins * panes, + "scenario": scenario.label, + "sessions": sessions, + "windows_per_session": wins, + "panes_per_window": panes, + "total_windows": wins * sessions, + "total_panes": wins * panes * sessions, + "panes": wins * panes * sessions, + "planner_steps_per_session": None, + "planner_steps": None, + "engine_calls_per_session": None, + "engine_calls": None, + "tmux_requests_per_session": None, + "tmux_requests": None, + "batch_sizes_per_session": None, + "batch_sizes": None, + "command_subprocess_starts_per_sample": None, + "command_subprocess_starts_basis": "unmeasured", + "version_probe_subprocesses_per_cell": None, + "version_probe_subprocesses_basis": "unmeasured", + "persistent_tmux_clients_per_cell": None, + "persistent_tmux_clients_basis": "unmeasured", "samples_ms": classic_samples, - **{f"{k}_ms": classic_stat[k] for k in STAT_LABELS}, + **summary_json(classic_stat), } ) @@ -1025,10 +1270,22 @@ def matrix( LAYERS[layer_name], wins, panes, + sessions, runs, warmup, ) st = summarize(samples) + shape = layer_shapes[layer_name] + if transport_name == "subprocess": + command_subprocess_starts: int | None = shape.tmux_requests * sessions + command_subprocess_starts_basis = "exact-one-per-request" + persistent_client_count: int | None = 0 + persistent_clients_basis = "engine-model-no-persistent-client" + else: + command_subprocess_starts = None + command_subprocess_starts_basis = "bootstrap-dependent-unmeasured" + persistent_client_count = None + persistent_clients_basis = "at-most-one-engine-model-unmeasured" speed = ( f"{base_median / st['median']:.1f}x" if base_median and st["median"] @@ -1048,9 +1305,33 @@ def matrix( "transport": transport_name, "mode": mode, "shape": f"{wins}x{panes}", - "panes": wins * panes, + "scenario": scenario.label, + "sessions": sessions, + "windows_per_session": wins, + "panes_per_window": panes, + "total_windows": wins * sessions, + "total_panes": wins * panes * sessions, + "panes": wins * panes * sessions, + "planner_steps_per_session": shape.planner_steps, + "planner_steps": ( + shape.planner_steps * sessions + if shape.planner_steps is not None + else None + ), + "engine_calls_per_session": shape.engine_calls, + "engine_calls": shape.engine_calls * sessions, + "tmux_requests_per_session": shape.tmux_requests, + "tmux_requests": shape.tmux_requests * sessions, + "batch_sizes_per_session": shape.batch_sizes, + "batch_sizes": shape.batch_sizes * sessions, + "command_subprocess_starts_per_sample": command_subprocess_starts, + "command_subprocess_starts_basis": command_subprocess_starts_basis, + "version_probe_subprocesses_per_cell": None, + "version_probe_subprocesses_basis": "cache-dependent-unmeasured", + "persistent_tmux_clients_per_cell": persistent_client_count, + "persistent_tmux_clients_basis": persistent_clients_basis, "samples_ms": samples, - **{f"{k}_ms": st[k] for k in STAT_LABELS}, + **summary_json(st), } ) @@ -1072,8 +1353,9 @@ def _serial_build( for name in names: build_sync(layer, engine, name, wins, panes) elapsed = (time.perf_counter() - t0) * 1000 + verify_topology(server, names, windows=wins, panes=panes) for name in names: - server.cmd("kill-session", "-t", name) + _kill_session(server, name) return elapsed @@ -1088,6 +1370,9 @@ async def _gather_build( *(build_async(layer, engine, name, wins, panes) for name in names) ) elapsed = (time.perf_counter() - t0) * 1000 + await asyncio.to_thread( + verify_topology, server, names, windows=wins, panes=panes + ) for name in names: await _akill_session(server, name) return elapsed @@ -1095,19 +1380,25 @@ async def _gather_build( @app.command() def contract( - shapes: str = typer.Option("1x1,1x4,2x2,3x3,5x4", help="comma WxP shapes"), + shapes: str = typer.Option( + "1x1x1,1x1x4,1x4x1,4x1x1,1x2x2", + help="comma WxP or SxWxP scenarios", + ), ) -> None: """Run the mock-parity contract standalone; exit non-zero on divergence. The benchmark's correctness oracle without the timing -- for CI, which can assert the ops language stays consistent without paying for live builds. Every expression layer, sync and async, must render the mock oracle's argv. - A negative control confirms the equality gate is not vacuous (the oracle is - non-empty and order-sensitive, so a dropped op would be caught). + It also proves that batching reduces planner steps without reducing tmux + requests, and that Workspace's public default matches ``ws-batch``. A + negative control confirms the equality gate is not vacuous. """ - shape_list = [parse_shape(s) for s in shapes.split(",") if s] + scenarios = parse_scenarios(shapes) failures: list[str] = [] - for wins, panes in shape_list: + shape_reports: list[str] = [] + for scenario in scenarios: + wins, panes = scenario.windows, scenario.panes oracle, rows = check_parity(wins, panes) # Negative control: a non-empty oracle whose prefix differs from itself # proves the `== oracle` check below can actually catch a dropped op. @@ -1118,29 +1409,65 @@ def contract( for label, agrees in rows if not agrees ) + layer_shapes = { + name: execution_shape(layer, wins, panes) for name, layer in LAYERS.items() + } + plan_seq_steps = layer_shapes["plan-seq"].planner_steps + plan_batch_steps = layer_shapes["plan-batch"].planner_steps + ws_seq_steps = layer_shapes["ws-seq"].planner_steps + ws_batch_steps = layer_shapes["ws-batch"].planner_steps + if plan_batch_steps is None or plan_seq_steps is None: + failures.append(f"{wins}x{panes}: plan-batch did not report planner steps") + elif plan_seq_steps <= plan_batch_steps: + failures.append(f"{wins}x{panes}: plan batching did not reduce steps") + if ws_batch_steps is None or ws_seq_steps is None: + failures.append(f"{wins}x{panes}: ws-batch did not report planner steps") + elif ws_seq_steps <= ws_batch_steps: + failures.append(f"{wins}x{panes}: workspace batching did not reduce steps") + if layer_shapes["default"] != layer_shapes["ws-batch"]: + failures.append(f"{wins}x{panes}: workspace default is not ws-batch") + if len({shape.tmux_requests for shape in layer_shapes.values()}) != 1: + failures.append(f"{wins}x{panes}: layers emit different request counts") + if ( + plan_seq_steps is None + or plan_batch_steps is None + or ws_seq_steps is None + or ws_batch_steps is None + ): + continue + scale = scenario.sessions + shape_reports.append( + f"{scenario.label}: {len(oracle) * scale} requests; " + f"plan {plan_seq_steps * scale}->{plan_batch_steps * scale} steps; " + f"workspace {ws_seq_steps * scale}->{ws_batch_steps * scale} steps" + ) if failures: for problem in failures: console.print(f"[red]FAIL[/red] {problem}") raise typer.Exit(1) console.print( f"[green]contract OK[/green] -- every layer x {{sync,async}} renders the " - f"mock oracle's argv across {len(shape_list)} shape(s); negative control passed" + f"mock oracle's argv across {len(scenarios)} scenario(s); batching shape and " + "negative controls passed" ) + for report in shape_reports: + console.print(f"[dim]{report}[/dim]") @app.command() def concurrency( shape: str = typer.Option("1x4", help="WxP shape of each session"), transport: str = typer.Option("control_mode", help="subprocess | control_mode"), - layer: str = typer.Option("ws-fold", help="expression layer"), + layer: str = typer.Option("default", help="expression layer"), k: int = typer.Option(4, help="independent sessions to build"), runs: int = typer.Option(5, help="timed repeats"), warmup: int = typer.Option(1, help="warmup repeats"), ) -> None: - """Build K independent sessions: sync-serial vs async-gather wall time. + """Explore K independent sessions: sync serial vs async gather wall time. - Async should actually win here: one async connection pipelines K builds' - round-trips instead of blocking on each in turn. + This probe has asymmetric engine-startup boundaries, fixed strategy order, + and no async-sequential or event-loop-lag control. Its output is diagnostic, + not evidence for async speed or health. """ wins, panes = parse_shape(shape) tp = TRANSPORTS[transport] @@ -1177,7 +1504,7 @@ def timed_gather() -> float: table.add_column("strategy", style="cyan") for label in STAT_LABELS: table.add_column(label, justify="right") - table.add_column("speedup", justify="right", style="green") + table.add_column("serial / gather", justify="right") table.add_row( "sync-serial", f"{int(serial_stat['n'])}", diff --git a/scripts/demo_control_output.py b/scripts/demo_control_output.py new file mode 100755 index 0000000000..37fb823be4 --- /dev/null +++ b/scripts/demo_control_output.py @@ -0,0 +1,790 @@ +#!/usr/bin/env -S uv run --script +# /// script +# requires-python = ">=3.10" +# dependencies = ["libtmux"] +# +# [tool.uv.sources] +# libtmux = { path = "..", editable = true } +# /// +"""Demonstrate decoded async control-mode pane output on an isolated server. + +``scroll`` renders deterministic Python source through a grid of panes and +proves every decoded byte arrived in order. ``overload`` deliberately stalls a +size-one subscriber queue, reports dropped control-notification frames, and +then proves the same engine still accepts commands. Overload is not producer +backpressure: the engine's bounded queues discard their oldest notification. +""" + +from __future__ import annotations + +import argparse +import asyncio +import contextlib +import dataclasses +import hashlib +import json +import os +import pathlib +import shlex +import shutil +import sys +import tempfile +import typing as t + +# Never inherit an ambient client while constructing the isolated Server. +os.environ.pop("TMUX", None) +os.environ.pop("TMUX_PANE", None) + +from libtmux.experimental.engines import AsyncControlModeEngine, ControlNotification +from libtmux.experimental.engines.base import CommandRequest +from libtmux.server import Server + +_SESSION_NAME = "libtmux-source-scroll" +_RESPONSIVE_TEXT = "control-output-responsive" +_PRODUCER_LINGER_SECONDS = 300 +_PRODUCER_CODE = """ +import os +import pathlib +import subprocess +import sys +import termios +import time + +tmux_bin, socket_path, gate, payload_path, done_path, delay, linger = sys.argv[1:] +subprocess.run( + [tmux_bin, "-S", socket_path, "wait-for", gate], + check=True, + stdout=subprocess.DEVNULL, + stderr=subprocess.DEVNULL, +) +attrs = termios.tcgetattr(1) +attrs[1] &= ~termios.OPOST +termios.tcsetattr(1, termios.TCSANOW, attrs) +with pathlib.Path(payload_path).open("rb") as payload: + for frame in payload: + os.write(1, frame) + if float(delay): + time.sleep(float(delay)) +pathlib.Path(done_path).touch() +time.sleep(float(linger)) +""".strip() + + +@dataclasses.dataclass(frozen=True) +class SourceFile: + """One source file selected for a producer pane. + + Attributes + ---------- + relative_path : str + POSIX path relative to the selected source root. + sha256 : str + SHA-256 digest of the complete source file. + lines : tuple[bytes, ...] + Source lines without newline terminators. + """ + + relative_path: str + sha256: str + lines: tuple[bytes, ...] + + +@dataclasses.dataclass +class PaneStream: + """Expected stream and runtime identity for one producer pane. + + Attributes + ---------- + logical_id : str + Stable topology label, independent of tmux object allocation. + window_index : int + Zero-based window position. + pane_index : int + Zero-based pane position within the window. + source : SourceFile + Source assigned by the seeded selection. + expected : bytes + Complete framed byte stream the pane must emit. + payload_path : pathlib.Path + Private scratch file read by the producer. + done_path : pathlib.Path + Private scratch marker touched after the last write. + gate : str + Unique tmux ``wait-for`` channel released after subscription. + pane_id : str + Runtime tmux pane identifier populated during topology creation. + """ + + logical_id: str + window_index: int + pane_index: int + source: SourceFile + expected: bytes + payload_path: pathlib.Path + done_path: pathlib.Path + gate: str + pane_id: str = "" + + +@dataclasses.dataclass(frozen=True) +class DemoOptions: + """Validated command-line options for one demo run. + + Attributes + ---------- + mode : str + Either ``scroll`` or ``overload``. + source_root : pathlib.Path + Root searched recursively for Python source. + seed : int + Seed mixed into the stable source ordering. + windows : int + Number of windows in the single session. + panes : int + Panes created in each window. + lines : int + Framed source lines emitted per pane. + delay : float + Seconds between producer frames. + timeout : float + Maximum seconds for setup and collection stages. + quiet : bool + Suppress the readable source stream when true. + json_out : pathlib.Path or None + Optional report destination. + """ + + mode: str + source_root: pathlib.Path + seed: int + windows: int + panes: int + lines: int + delay: float + timeout: float + quiet: bool + json_out: pathlib.Path | None + + +def _selected_sources(root: pathlib.Path, seed: int, count: int) -> list[SourceFile]: + """Return a stable seeded ordering of Python files, cycling when needed. + + Ranking each relative path by SHA-256 avoids depending on implementation + details of :mod:`random`, so the same corpus and seed retain their order + across supported Python versions. + + Parameters + ---------- + root : pathlib.Path + Source tree to scan. + seed : int + Stable ordering seed. + count : int + Number of pane assignments required. + + Returns + ------- + list[SourceFile] + Selected source metadata in pane-assignment order. + """ + paths = sorted(path for path in root.rglob("*.py") if path.is_file()) + if not paths: + msg = "source root contains no Python files" + raise ValueError(msg) + + ranked = sorted( + paths, + key=lambda path: ( + hashlib.sha256( + f"{seed}\0{path.relative_to(root).as_posix()}".encode() + ).digest(), + path.relative_to(root).as_posix(), + ), + ) + selected: list[SourceFile] = [] + for index in range(count): + path = ranked[index % len(ranked)] + content = path.read_bytes() + selected.append( + SourceFile( + relative_path=path.relative_to(root).as_posix(), + sha256=hashlib.sha256(content).hexdigest(), + lines=tuple(content.splitlines()) or (b"",), + ) + ) + return selected + + +def _framed_stream(logical_id: str, source: SourceFile, frames: int) -> bytes: + """Render readable, sequence-numbered source frames for one pane. + + Parameters + ---------- + logical_id : str + Stable pane label. + source : SourceFile + Selected source metadata and lines. + frames : int + Number of frames to render. + + Returns + ------- + bytes + Exact bytes expected from the pane after PTY output processing is off. + """ + rendered = bytearray() + for sequence in range(frames): + line = source.lines[sequence % len(source.lines)] + prefix = ( + f"[{logical_id} {sequence:06d} {source.relative_path} " + f"{source.sha256[:12]}] " + ).encode() + rendered.extend(prefix) + rendered.extend(line) + rendered.extend(b"\n") + return bytes(rendered) + + +def _pane_streams(options: DemoOptions, scratch: pathlib.Path) -> list[PaneStream]: + """Build expected streams and private producer files for the topology.""" + count = options.windows * options.panes + sources = _selected_sources(options.source_root, options.seed, count) + streams: list[PaneStream] = [] + for flat_index, source in enumerate(sources): + window_index, pane_index = divmod(flat_index, options.panes) + logical_id = f"w{window_index:02d}p{pane_index:02d}" + expected = _framed_stream(logical_id, source, options.lines) + payload_path = scratch / f"{logical_id}.payload" + payload_path.write_bytes(expected) + streams.append( + PaneStream( + logical_id=logical_id, + window_index=window_index, + pane_index=pane_index, + source=source, + expected=expected, + payload_path=payload_path, + done_path=scratch / f"{logical_id}.done", + gate=f"libtmux-source-scroll-{logical_id}", + ) + ) + return streams + + +def _producer_command( + stream: PaneStream, + *, + tmux_bin: str, + socket_path: pathlib.Path, + delay: float, +) -> str: + """Render one shell-safe direct pane command for the bounded producer.""" + return shlex.join( + ( + sys.executable, + "-c", + _PRODUCER_CODE, + tmux_bin, + str(socket_path), + stream.gate, + str(stream.payload_path), + str(stream.done_path), + str(delay), + str(_PRODUCER_LINGER_SECONDS), + ) + ) + + +def _require_success(result: t.Any, action: str) -> tuple[str, ...]: + """Return stdout from a successful classic command or raise with context.""" + if result.returncode != 0: + detail = "; ".join(result.stderr) or f"exit {result.returncode}" + msg = f"{action} failed: {detail}" + raise RuntimeError(msg) + return tuple(result.stdout) + + +def _create_topology( + server: Server, + streams: list[PaneStream], + *, + tmux_bin: str, + socket_path: pathlib.Path, + delay: float, +) -> None: + """Create one session with the requested windows and producer panes.""" + first = streams[0] + first_output = _require_success( + server.cmd( + "new-session", + "-d", + "-x", + "120", + "-y", + "40", + "-s", + _SESSION_NAME, + "-P", + "-F", + "#{session_id} #{window_id} #{pane_id}", + _producer_command( + first, + tmux_bin=tmux_bin, + socket_path=socket_path, + delay=delay, + ), + ), + "create session", + ) + session_id, current_window, first.pane_id = first_output[0].split() + _require_success( + server.cmd("set-option", "-t", session_id, "destroy-unattached", "off"), + "disable destroy-unattached", + ) + _require_success( + server.cmd("set-option", "-s", "exit-empty", "off"), + "disable exit-empty", + ) + + by_position = { + (stream.window_index, stream.pane_index): stream for stream in streams + } + for window_index in range(len({stream.window_index for stream in streams})): + if window_index: + initial = by_position[(window_index, 0)] + output = _require_success( + server.cmd( + "new-window", + "-d", + "-t", + session_id, + "-n", + f"source-{window_index}", + "-P", + "-F", + "#{window_id} #{pane_id}", + _producer_command( + initial, + tmux_bin=tmux_bin, + socket_path=socket_path, + delay=delay, + ), + ), + f"create window {window_index}", + ) + current_window, initial.pane_id = output[0].split() + + pane_count = len( + [stream for stream in streams if stream.window_index == window_index] + ) + for pane_index in range(1, pane_count): + stream = by_position[(window_index, pane_index)] + output = _require_success( + server.cmd( + "split-window", + "-d", + "-t", + current_window, + "-P", + "-F", + "#{pane_id}", + _producer_command( + stream, + tmux_bin=tmux_bin, + socket_path=socket_path, + delay=delay, + ), + ), + f"create pane {stream.logical_id}", + ) + stream.pane_id = output[0] + _require_success( + server.cmd("select-layout", "-t", current_window, "tiled"), + f"layout window {window_index}", + ) + + +async def _wait_for_one_subscriber( + engine: AsyncControlModeEngine, + *, + timeout: float, +) -> None: + """Wait until the one permitted demo subscriber is registered.""" + deadline = asyncio.get_running_loop().time() + timeout + while True: + subscribers = getattr(engine, "_subscribers", ()) + if len(subscribers) == 1: + return + if len(subscribers) > 1: + msg = "demo registered more than one control-output subscriber" + raise RuntimeError(msg) + if asyncio.get_running_loop().time() >= deadline: + msg = "control-output subscriber registration timed out" + raise TimeoutError(msg) + await asyncio.sleep(0) + + +async def _release_producers( + engine: AsyncControlModeEngine, + streams: list[PaneStream], +) -> None: + """Release every per-pane gate through the connected control engine.""" + requests = [ + CommandRequest.from_args("wait-for", "-S", stream.gate) for stream in streams + ] + results = await engine.run_batch(requests) + failed = [result for result in results if result.returncode != 0] + if failed: + msg = "failed to release one or more source producers" + raise RuntimeError(msg) + + +async def _wait_for_producers( + streams: list[PaneStream], + *, + timeout: float, +) -> None: + """Wait for every producer to record that its final write completed.""" + deadline = asyncio.get_running_loop().time() + timeout + while not all(stream.done_path.exists() for stream in streams): + if asyncio.get_running_loop().time() >= deadline: + missing = [ + stream.logical_id for stream in streams if not stream.done_path.exists() + ] + msg = f"source producers timed out: {', '.join(missing)}" + raise TimeoutError(msg) + await asyncio.sleep(0.01) + + +def _render_pane_report(stream: PaneStream, *, verified: bool) -> dict[str, t.Any]: + """Return public, path-safe verification metadata for one pane.""" + return { + "logical_id": stream.logical_id, + "window": stream.window_index, + "pane": stream.pane_index, + "source": stream.source.relative_path, + "source_sha256": stream.source.sha256, + "stream_sha256": hashlib.sha256(stream.expected).hexdigest(), + "frames": len(stream.expected.splitlines()), + "first_sequence": 0, + "last_sequence": len(stream.expected.splitlines()) - 1, + "verified": verified, + } + + +async def _next_notification( + task: asyncio.Future[ControlNotification], + *, + deadline: float, +) -> ControlNotification: + """Await a pending notification within an absolute monotonic deadline.""" + remaining = deadline - asyncio.get_running_loop().time() + if remaining <= 0: + msg = "control-output collection timed out" + raise TimeoutError(msg) + try: + return await asyncio.wait_for(task, timeout=remaining) + except asyncio.TimeoutError: + msg = "control-output collection timed out" + raise TimeoutError(msg) from None + + +async def _prove_responsive(engine: AsyncControlModeEngine) -> bool: + """Return whether a follow-up command succeeds on the same engine.""" + result = await engine.run( + CommandRequest.from_args("display-message", "-p", _RESPONSIVE_TEXT) + ) + return result.returncode == 0 and result.stdout == (_RESPONSIVE_TEXT,) + + +async def _run_scroll( + engine: AsyncControlModeEngine, + stream: t.AsyncIterator[ControlNotification], + first: asyncio.Future[ControlNotification], + panes: list[PaneStream], + *, + quiet: bool, + timeout: float, +) -> tuple[int, bool]: + """Collect and byte-verify the complete lossless pane streams.""" + expected = {pane.pane_id: pane.expected for pane in panes} + received = {pane.pane_id: bytearray() for pane in panes} + deadline = asyncio.get_running_loop().time() + timeout + pending = first + observed = 0 + while any(bytes(received[pane_id]) != value for pane_id, value in expected.items()): + notification = await _next_notification(pending, deadline=deadline) + payload = notification.payload + pane_id = notification.pane_id + if payload is not None and pane_id in received: + observed += 1 + if not quiet: + sys.stdout.buffer.write(payload) + sys.stdout.buffer.flush() + received[pane_id].extend(payload) + buffered = bytes(received[pane_id]) + if not expected[pane_id].startswith(buffered): + msg = f"decoded output diverged for {pane_id}" + raise RuntimeError(msg) + if all( + bytes(received[pane_id]) == value for pane_id, value in expected.items() + ): + break + pending = asyncio.ensure_future(anext(stream)) + + await _wait_for_producers(panes, timeout=timeout) + if engine.dropped_notifications: + msg = f"lossless scroll dropped {engine.dropped_notifications} frame(s)" + raise RuntimeError(msg) + return observed, await _prove_responsive(engine) + + +async def _run_overload( + engine: AsyncControlModeEngine, + stream: t.AsyncIterator[ControlNotification], + first: asyncio.Future[ControlNotification], + panes: list[PaneStream], + *, + timeout: float, +) -> tuple[int, bool]: + """Stall a bounded subscriber, prove drops, then probe engine liveness.""" + await _wait_for_producers(panes, timeout=timeout) + # Give the control reader time to offer the final PTY chunks while this + # consumer remains deliberately stalled at a size-one queue. + await asyncio.sleep(0.1) + responsive = await _prove_responsive(engine) + deadline = asyncio.get_running_loop().time() + timeout + observed = int( + (await _next_notification(first, deadline=deadline)).payload is not None + ) + try: + notification = await asyncio.wait_for(anext(stream), timeout=0.2) + except (StopAsyncIteration, asyncio.TimeoutError): + pass + else: + observed += int(notification.payload is not None) + if engine.dropped_notifications <= 0: + msg = "overload did not fill the bounded subscriber queue" + raise RuntimeError(msg) + return observed, responsive + + +async def _exercise_engine( + server: Server, + panes: list[PaneStream], + options: DemoOptions, +) -> dict[str, t.Any]: + """Register one subscriber, release producers, and run the selected proof.""" + queue_size = ( + 1 if options.mode == "overload" else max(4096, len(panes) * options.lines) + ) + async with AsyncControlModeEngine.for_server( + server, + event_queue_size=queue_size, + timeout=options.timeout, + ) as engine: + notifications = engine.subscribe() + first: asyncio.Future[ControlNotification] = asyncio.ensure_future( + anext(notifications) + ) + try: + await _wait_for_one_subscriber(engine, timeout=options.timeout) + await _release_producers(engine, panes) + if options.mode == "scroll": + observed, responsive = await _run_scroll( + engine, + notifications, + first, + panes, + quiet=options.quiet, + timeout=options.timeout, + ) + lossless = True + else: + observed, responsive = await _run_overload( + engine, + notifications, + first, + panes, + timeout=options.timeout, + ) + lossless = False + dropped = engine.dropped_notifications + finally: + if not first.done(): + first.cancel() + await asyncio.gather(first, return_exceptions=True) + generator = t.cast( + "t.AsyncGenerator[ControlNotification, None]", + notifications, + ) + with contextlib.suppress(RuntimeError): + await generator.aclose() + + return { + "mode": options.mode, + "topology": { + "windows": options.windows, + "panes_per_window": options.panes, + "total_panes": len(panes), + }, + "panes": [_render_pane_report(pane, verified=lossless) for pane in panes], + "observed_frames": observed, + "dropped_frames": dropped, + "lossless": lossless, + "responsive": responsive, + } + + +def run_demo(options: DemoOptions) -> dict[str, t.Any]: + """Run one hermetic demo and return its post-cleanup report.""" + scratch = pathlib.Path(tempfile.mkdtemp(prefix="ltout-")) + socket_path = scratch / "tmux.sock" + server = Server(socket_path=str(socket_path), config_file=os.devnull) + report: dict[str, t.Any] | None = None + server_stopped = False + try: + streams = _pane_streams(options, scratch) + tmux_bin = shutil.which(server.tmux_bin or "tmux") + if tmux_bin is None: + msg = "tmux executable was not found" + raise RuntimeError(msg) + producer_delay = options.delay if options.mode == "scroll" else 0.001 + _create_topology( + server, + streams, + tmux_bin=tmux_bin, + socket_path=socket_path, + delay=producer_delay, + ) + report = asyncio.run(_exercise_engine(server, streams, options)) + finally: + with contextlib.suppress(Exception): + if server.is_alive(): + server.kill() + with contextlib.suppress(Exception): + server_stopped = not server.is_alive() + shutil.rmtree(scratch, ignore_errors=True) + scratch_removed = not scratch.exists() + + if report is None: + msg = "demo stopped before producing a report" + raise RuntimeError(msg) + report["cleanup"] = { + "scratch_removed": scratch_removed, + "server_stopped": server_stopped, + } + return report + + +def _positive(parser: argparse.ArgumentParser, name: str, value: int) -> int: + """Validate a strictly positive integer option.""" + if value <= 0: + parser.error(f"{name} must be greater than zero") + return value + + +def _options(parser: argparse.ArgumentParser, args: argparse.Namespace) -> DemoOptions: + """Validate parsed arguments and build immutable demo options.""" + delay = float(args.delay) + timeout = float(args.timeout) + if delay < 0: + parser.error("--delay must be non-negative") + if timeout <= 0: + parser.error("--timeout must be greater than zero") + return DemoOptions( + mode=t.cast("str", args.mode), + source_root=pathlib.Path(args.source_root).resolve(), + seed=int(args.seed), + windows=_positive(parser, "--windows", int(args.windows)), + panes=_positive(parser, "--panes", int(args.panes)), + lines=_positive(parser, "--lines", int(args.lines)), + delay=delay, + timeout=timeout, + quiet=bool(args.quiet), + json_out=pathlib.Path(args.json_out) if args.json_out else None, + ) + + +def _add_common_arguments(parser: argparse.ArgumentParser, *, overload: bool) -> None: + """Add the shared topology, corpus, timing, and report arguments.""" + default_root = pathlib.Path(__file__).parents[1] / "src" / "libtmux" + parser.add_argument( + "--source-root", + default=default_root, + help="Python source tree used as the deterministic scrolling corpus", + ) + parser.add_argument( + "--seed", + type=int, + default=2026, + help="stable source-selection seed", + ) + parser.add_argument("--windows", type=int, default=2, help="number of windows") + parser.add_argument( + "--panes", + type=int, + default=2, + help="panes per window", + ) + parser.add_argument( + "--lines", + type=int, + default=2000 if overload else 120, + help="source frames emitted by each pane", + ) + parser.add_argument( + "--delay", + type=float, + default=0.01, + help="seconds between pane writes in scroll mode", + ) + parser.add_argument( + "--timeout", + type=float, + default=30.0, + help="maximum seconds for each setup or collection stage", + ) + parser.add_argument("--json-out", help="write the verification report to this path") + parser.add_argument( + "--quiet", + action="store_true", + help="suppress source frames and print only the report", + ) + + +def _parser() -> argparse.ArgumentParser: + """Build the two-mode command-line parser.""" + parser = argparse.ArgumentParser(description=__doc__) + modes = parser.add_subparsers(dest="mode", required=True) + scroll = modes.add_parser( + "scroll", + help="readable lossless source scrolling with byte verification", + ) + _add_common_arguments(scroll, overload=False) + overload = modes.add_parser( + "overload", + help="intentional bounded-queue drops followed by a liveness probe", + ) + _add_common_arguments(overload, overload=True) + return parser + + +def main(argv: t.Sequence[str] | None = None) -> int: + """Run the selected demo and emit its machine-readable verification report.""" + parser = _parser() + options = _options(parser, parser.parse_args(argv)) + try: + report = run_demo(options) + except Exception as error: # noqa: BLE001 - concise CLI error boundary + parser.exit(1, f"error: {error}\n") + rendered = json.dumps(report, indent=2, sort_keys=True) + if options.json_out is not None: + options.json_out.write_text(f"{rendered}\n") + else: + print(rendered) + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/src/libtmux/_internal/tmux_argv.py b/src/libtmux/_internal/tmux_argv.py new file mode 100644 index 0000000000..ab10aa6618 --- /dev/null +++ b/src/libtmux/_internal/tmux_argv.py @@ -0,0 +1,234 @@ +"""Parse and normalize arguments passed directly to the tmux executable.""" + +from __future__ import annotations + +import typing as t + +from libtmux.engines.base import CommandSeparator, is_command_separator + + +class DirectArgv(t.NamedTuple): + """Tmux arguments split at the client-option boundary. + + Attributes + ---------- + global_args : tuple[str, ...] + Client-global options and their values. + command_argv : tuple[str | CommandSeparator, ...] + Command names, flags, data, and explicit separators. + """ + + global_args: tuple[str, ...] + command_argv: tuple[str | CommandSeparator, ...] + + +class ClientOption(t.NamedTuple): + """One parsed tmux client-global option. + + Attributes + ---------- + name : str + Single-character option name without ``-``. + value : str or None + Attached or separate value for value-taking options. + """ + + name: str + value: str | None + + +_CLIENT_OPTION_VALUE_CHARACTERS = frozenset("cfLST") +_CLIENT_FLAG_CHARACTERS = frozenset("2CDdhlNquUvV") + + +def _scan_client_prefix( + raw_args: tuple[object, ...], +) -> tuple[int, tuple[ClientOption, ...]]: + """Return the client-prefix boundary and its normalized options.""" + if any("\0" in str(arg) for arg in raw_args): + msg = "tmux command arguments cannot contain NUL" + raise ValueError(msg) + options: list[ClientOption] = [] + index = 0 + while index < len(raw_args): + arg = raw_args[index] + if type(arg) is CommandSeparator: + break + token = str(arg) + if "\0" in token: + msg = "tmux command arguments cannot contain NUL" + raise ValueError(msg) + if token == "--": + index += 1 + break + if len(token) <= 1 or not token.startswith("-"): + break + + option_start = len(options) + is_client_option = True + position = 0 + cluster = token[1:] + while position < len(cluster): + option = cluster[position] + if option in _CLIENT_FLAG_CHARACTERS: + options.append(ClientOption(option, None)) + position += 1 + continue + if option in _CLIENT_OPTION_VALUE_CHARACTERS: + attached = cluster[position + 1 :] + if attached: + value: str | None = attached + elif index + 1 < len(raw_args): + index += 1 + value = str(raw_args[index]) + else: + value = None + options.append(ClientOption(option, value)) + break + is_client_option = False + del options[option_start:] + break + if not is_client_option: + break + index += 1 + return index, tuple(options) + + +def split_direct_argv(args: t.Iterable[object]) -> DirectArgv: + """Separate tmux client-global options from command arguments. + + Parameters + ---------- + args : iterable[object] + Arguments passed after the tmux executable. + + Returns + ------- + DirectArgv + Client-global prefix and command region. + + Examples + -------- + >>> parsed = split_direct_argv(("-L", "socket", "display-message", ";")) + >>> parsed.global_args + ('-L', 'socket') + >>> parsed.command_argv + ('display-message', ';') + """ + raw_args = tuple(args) + command_start, _options = _scan_client_prefix(raw_args) + + global_args = tuple(str(arg) for arg in raw_args[:command_start]) + command_argv = tuple( + arg if type(arg) is CommandSeparator else str(arg) + for arg in raw_args[command_start:] + ) + if any("\0" in str(arg) for arg in command_argv): + msg = "tmux command arguments cannot contain NUL" + raise ValueError(msg) + if any( + type(arg) is CommandSeparator and not is_command_separator(arg) + for arg in command_argv + ): + msg = "a command separator must be exactly ';'" + raise ValueError(msg) + return DirectArgv(global_args=global_args, command_argv=command_argv) + + +def parse_client_options(args: t.Iterable[object]) -> tuple[ClientOption, ...]: + """Parse normalized options from the leading tmux client argv region. + + Parameters + ---------- + args : iterable[object] + Arguments passed after the tmux executable. + + Returns + ------- + tuple[ClientOption, ...] + Options in command-line order, with clusters and values expanded. + + Examples + -------- + >>> options = parse_client_options(("-uqLwork", "list-sessions")) + >>> [(option.name, option.value) for option in options] + [('u', None), ('q', None), ('L', 'work')] + >>> parse_client_options(("-f", "/tmp/tmux.conf", "list-sessions"))[-1] + ClientOption(name='f', value='/tmp/tmux.conf') + """ + _boundary, options = _scan_client_prefix(tuple(args)) + return options + + +def encode_command_argv(args: t.Iterable[object]) -> tuple[str, ...]: + r"""Encode argv already known to be in tmux's command parser region. + + Parameters + ---------- + args : iterable[object] + Command names, flags, values, and typed separators. + + Returns + ------- + tuple[str, ...] + Command argv with literal suffix semicolons protected. + + Examples + -------- + >>> encode_command_argv(("-L", "value;", CommandSeparator(";"))) + ('-L', 'value\\;', ';') + """ + command_argv = tuple( + arg if type(arg) is CommandSeparator else str(arg) for arg in args + ) + if any("\0" in str(arg) for arg in command_argv): + msg = "tmux command arguments cannot contain NUL" + raise ValueError(msg) + if any( + type(arg) is CommandSeparator and not is_command_separator(arg) + for arg in command_argv + ): + msg = "a command separator must be exactly ';'" + raise ValueError(msg) + + encoded: list[str] = [] + for arg in command_argv: + if is_command_separator(arg): + encoded.append(str.__str__(arg)) + elif arg.endswith(";") and not arg.endswith(r"\;"): + encoded.append(f"{arg[:-1]}\\;") + else: + encoded.append(arg) + return tuple(encoded) + + +def encode_direct_argv(args: t.Iterable[object]) -> tuple[str, ...]: + r"""Render direct tmux argv with untyped trailing semicolons as data. + + An ordinary string ending in a bare semicolon receives the tmux-level + escape needed to keep it literal. Existing ``\;`` suffixes are preserved, + while a :class:`~libtmux.engines.base.CommandSeparator` renders as command + structure. Client-global option values remain unchanged because tmux + removes them before parsing command separators. + + Parameters + ---------- + args : iterable[object] + Arguments passed after the tmux executable. + + Returns + ------- + tuple[str, ...] + Normalized subprocess arguments. + + Examples + -------- + >>> encode_direct_argv(("-Lsocket;", "display-message", "value;")) + ('-Lsocket;', 'display-message', 'value\\;') + >>> encode_direct_argv(("display-message", CommandSeparator(";"))) + ('display-message', ';') + >>> encode_direct_argv(("display-message", r"already\;")) + ('display-message', 'already\\;') + """ + direct = split_direct_argv(args) + return (*direct.global_args, *encode_command_argv(direct.command_argv)) diff --git a/src/libtmux/engines/__init__.py b/src/libtmux/engines/__init__.py index bba2ae94c4..3dc952d7d4 100644 --- a/src/libtmux/engines/__init__.py +++ b/src/libtmux/engines/__init__.py @@ -30,7 +30,7 @@ >>> engine.seen [('list-sessions',)] -The connection flags (``-L``/``-S``/``-f``/``-2``/``-8``) are *not* part of a +The connection flags (``-L``/``-S``/``-f``/``-2``) are *not* part of a request: they belong to the engine's :class:`~libtmux.engines.connection.ServerConnection`, so every engine sees the same request regardless of which tmux server it targets. @@ -42,6 +42,7 @@ CommandRequest, CommandResult, CommandSeparator, + HasConnection, SupportsCommandLine, SupportsConnection, SupportsTmuxVersion, @@ -63,6 +64,7 @@ "CommandResult", "CommandSeparator", "CountingSink", + "HasConnection", "InstrumentedEngine", "ServerConnection", "Sink", diff --git a/src/libtmux/engines/base.py b/src/libtmux/engines/base.py index 224b56859b..921381a387 100644 --- a/src/libtmux/engines/base.py +++ b/src/libtmux/engines/base.py @@ -19,16 +19,17 @@ from typing_extensions import Self + from libtmux.engines.connection import ServerConnection + class CommandSeparator(str): """A caller-authored command boundary, distinct from a literal ``";"``. - tmux treats a bare ``;`` argument as a command separator only when it - arrives unquoted, so a ``";"`` that is *data* -- a pane title, a shell - fragment passed to ``send-keys`` -- must not be mistaken for one. Marking - the boundary with its own type keeps the distinction in the value rather - than in a parsing convention, so an engine that chains commands can find - the real boundaries and every other engine can ignore them. + Tmux's direct argv parser treats any unescaped trailing ``;`` as command + structure, including a suffix on a larger value. A literal suffix -- a pane + title or shell fragment passed to ``send-keys`` -- must not become one. + Marking a real boundary with its own type lets direct engines escape + ordinary values while rendering intentional command groups structurally. Examples -------- @@ -127,7 +128,7 @@ class CommandRequest: """A tmux command, ready for an engine to execute. Carries the subcommand and its arguments only. Connection flags - (``-L``/``-S``/``-f``/``-2``/``-8``) belong to the engine's + (``-L``/``-S``/``-f``/``-2``) belong to the engine's :class:`~libtmux.engines.connection.ServerConnection`, so every engine sees the same request no matter which tmux server it targets. @@ -338,13 +339,33 @@ def command_line(self, request: CommandRequest) -> tuple[str, ...]: @t.runtime_checkable -class SupportsConnection(t.Protocol): - """An engine that dispatches over a named tmux server and can be rebound. +class HasConnection(t.Protocol): + """An engine whose tmux connection can be inspected without changing it. + + Persistent engines implement this read-only capability even though their + live transport cannot be cloned safely. In-memory engines with no tmux + connection omit it. + + Examples + -------- + >>> from libtmux.engines import HasConnection, SubprocessEngine + >>> isinstance(SubprocessEngine(), HasConnection) + True + """ + + @property + def connection(self) -> ServerConnection: + """Return the tmux binary and global flags this engine uses.""" + ... + + +@t.runtime_checkable +class SupportsConnection(HasConnection, t.Protocol): + """An inspectable engine that can safely return a rebound equivalent. - Optional capability. :attr:`Server.engine ` reads it - so an injected engine that names no server of its own adopts the server's - connection instead of silently reaching the ambient tmux server. In-memory - engines have no connection and simply do not implement it. + Optional capability. Stateless subprocess engines implement it. Persistent + transports expose :class:`HasConnection` only, because cloning one could + duplicate or abandon live connection state. Examples -------- @@ -363,12 +384,7 @@ class SupportsConnection(t.Protocol): False """ - @property - def connection(self) -> t.Any: - """Return the tmux binary and flags this engine dispatches over.""" - ... - - def with_connection(self, connection: t.Any) -> TmuxEngine: + def with_connection(self, connection: ServerConnection) -> Self: """Return an equivalent engine bound to *connection*.""" ... diff --git a/src/libtmux/engines/connection.py b/src/libtmux/engines/connection.py index 318c4619e1..6405753005 100644 --- a/src/libtmux/engines/connection.py +++ b/src/libtmux/engines/connection.py @@ -1,7 +1,7 @@ """The connection an engine talks to: which tmux binary, which tmux server. Every engine needs the same two things before it can dispatch anything: a tmux -*binary* to exec, and the *connection flags* (``-L``/``-S``/``-f``/``-2``/``-8``) +*binary* to exec, and the *connection flags* (``-L``/``-S``/``-f``/``-2``) that point at one particular tmux server. :class:`ServerConnection` is that pair as one frozen value, and it is the only place in libtmux where either is computed -- :meth:`libtmux.Server.cmd`, :meth:`libtmux.Server.raise_if_dead` and @@ -26,6 +26,238 @@ from collections.abc import Sequence +_GLOBAL_OPTIONS_WITH_VALUE = frozenset({"c", "f", "L", "S", "T"}) +_GLOBAL_OPTIONS_WITHOUT_VALUE = frozenset( + {"2", "C", "D", "d", "h", "l", "N", "q", "u", "U", "v", "V"}, +) + + +@dataclass(frozen=True) +class _ConnectionSettings: + """Effective tmux globals relevant to Server connection constraints. + + Attributes + ---------- + socket_name : str or None + Final ``-L`` socket name, before ``-S`` precedence is applied. + socket_path : str or None + Final ``-S`` socket path. Any value overrides ``socket_name``. + config_files : tuple[str, ...] + Every cumulative ``-f`` value in command-line order. + colors : int or None + Requested client color mode, if declared. + """ + + socket_name: str | None = None + socket_path: str | None = None + config_files: tuple[str, ...] = () + colors: int | None = None + + @property + def socket(self) -> tuple[str, str] | None: + """Return tmux's effective socket selector. + + ``-S`` wins regardless of where ``-L`` appears, matching tmux's global + option semantics. + + Examples + -------- + >>> _ConnectionSettings(socket_name="ignored", socket_path="/tmp/s").socket + ('path', '/tmp/s') + >>> _ConnectionSettings(socket_name="work").socket + ('name', 'work') + """ + if self.socket_path is not None: + return ("path", self.socket_path) + if self.socket_name is not None: + return ("name", self.socket_name) + return None + + +@dataclass(frozen=True) +class _ConnectionResolution: + """A constraint overlay plus the fields that could not be reconciled. + + Attributes + ---------- + connection : ServerConnection + Rebound target after adding non-conflicting missing requirements. + conflicts : tuple[str, ...] + Explicit settings whose engine and Server values disagree. + missing : tuple[str, ...] + Server settings the engine must add through safe rebinding. + """ + + connection: ServerConnection + conflicts: tuple[str, ...] = () + missing: tuple[str, ...] = () + + +def _connection_settings(args: Sequence[str]) -> _ConnectionSettings: + """Parse effective connection globals from tmux's short-option argv. + + Attached and separate values normalize to the same settings, repeated + values use the last occurrence, and ``-S`` makes every ``-L`` ineffective. + + Parameters + ---------- + args : Sequence[str] + Tmux client-global arguments. + + Returns + ------- + _ConnectionSettings + Effective settings used for constraint comparison. + + Examples + -------- + >>> _connection_settings(("-2q", "-f", "/tmp/c", "-Lwork")) + _ConnectionSettings(socket_name='work', socket_path=None, + config_files=('/tmp/c',), colors=256) + >>> _connection_settings(("-S/tmp/s", "-Lignored")).socket + ('path', '/tmp/s') + """ + socket_name: str | None = None + socket_path: str | None = None + config_files: list[str] = [] + colors: int | None = None + index = 0 + + while index < len(args): + token = args[index] + if token == "--": + break + if not token.startswith("-") or token == "-": + break + + cluster = token[1:] + option_index = 0 + while option_index < len(cluster): + option = cluster[option_index] + # tmux removed ``-8`` before libtmux's minimum supported version. + # Recognize it only so an injected engine cannot hide a conflicting + # legacy color constraint; ServerConnection never emits it. + if option == "8": + colors = 88 + option_index += 1 + continue + if option in _GLOBAL_OPTIONS_WITH_VALUE: + attached = cluster[option_index + 1 :] + value: str | None + if attached: + value = attached + elif index + 1 < len(args): + index += 1 + value = args[index] + else: + value = None + + if value is not None: + if option == "f": + config_files.append(value) + elif option == "L": + socket_name = value + elif option == "S": + socket_path = value + break + + if option not in _GLOBAL_OPTIONS_WITHOUT_VALUE: + break + if option == "2": + colors = 256 + elif option == "8": + colors = 88 + option_index += 1 + index += 1 + + return _ConnectionSettings( + socket_name=socket_name, + socket_path=socket_path, + config_files=tuple(config_files), + colors=colors, + ) + + +def _merge_connection_constraints( + connection: ServerConnection, + constraints: ServerConnection, +) -> _ConnectionResolution: + """Overlay explicit Server constraints onto an engine connection. + + Existing engine settings survive when the Server is silent. Missing + settings are appended in canonical tmux order. Contradictory explicit + settings are reported instead of choosing an authority silently. + + Parameters + ---------- + connection : ServerConnection + The injected engine's current connection. + constraints : ServerConnection + Explicit values declared by the Server. + + Returns + ------- + _ConnectionResolution + Target connection, conflicting fields, and fields requiring a rebind. + + Examples + -------- + >>> current = ServerConnection.of(args=("-q", "-Lwork")) + >>> required = ServerConnection.of(args=("-2", "-f/tmp/c")) + >>> _merge_connection_constraints(current, required).connection.args + ('-q', '-Lwork', '-2', '-f/tmp/c') + >>> _merge_connection_constraints( + ... ServerConnection.of(args=("-Lone",)), + ... ServerConnection.of(args=("-Ltwo",)), + ... ).conflicts + ('socket',) + """ + current = _connection_settings(connection.args) + required = _connection_settings(constraints.args) + conflicts: list[str] = [] + missing: list[str] = [] + additions: list[str] = [] + + tmux_bin = connection.tmux_bin + if constraints.tmux_bin is not None: + if tmux_bin is None: + tmux_bin = constraints.tmux_bin + missing.append("binary") + elif tmux_bin != constraints.tmux_bin: + conflicts.append("binary") + + if required.colors is not None: + if current.colors is None: + additions.append("-2" if required.colors == 256 else "-8") + missing.append("color") + elif current.colors != required.colors: + conflicts.append("color") + + if required.config_files: + if not current.config_files: + additions.extend(f"-f{path}" for path in required.config_files) + missing.append("config") + elif current.config_files != required.config_files: + conflicts.append("config") + + if required.socket is not None: + if current.socket is None: + kind, value = required.socket + additions.append(f"-{'L' if kind == 'name' else 'S'}{value}") + missing.append("socket") + elif current.socket != required.socket: + conflicts.append("socket") + + return _ConnectionResolution( + connection=ServerConnection.of( + tmux_bin=tmux_bin, + args=(*connection.args, *additions), + ), + conflicts=tuple(conflicts), + missing=tuple(missing), + ) + + class _BinaryResolver: """Memoized tmux-binary resolution and ``tmux -V`` probe. @@ -198,7 +430,7 @@ def from_server(cls, server: t.Any) -> ServerConnection: Raises ------ :exc:`~libtmux.exc.UnknownColorOption` - ``colors`` is truthy but is neither ``256`` nor ``88``. + ``colors`` is truthy but is not ``256``. Examples -------- @@ -213,7 +445,7 @@ def from_server(cls, server: t.Any) -> ServerConnection: ... ServerConnection.from_server(types.SimpleNamespace(colors=16)) ... except exc.UnknownColorOption as e: ... print(e) - Server.colors must equal 88 or 256 + Server.colors must equal 256 """ args: list[str] = [] @@ -221,8 +453,6 @@ def from_server(cls, server: t.Any) -> ServerConnection: if colors: if colors == 256: args.append("-2") - elif colors == 88: - args.append("-8") else: raise exc.UnknownColorOption @@ -261,18 +491,11 @@ def is_unconfigured(self) -> bool: @property def names_server(self) -> bool: - """Whether this connection carries connection flags of its own. + """Whether this connection effectively selects a tmux server. - :attr:`Server.engine ` reads this on the - *engine's* side of adoption: an engine that already carries flags knows - which tmux server it talks to and is left alone, while one that carries - none is bound to the server's flags so it cannot silently dispatch to - the ambient server. - - :attr:`tmux_bin` deliberately does not count. It selects which tmux - *program* to exec, which says nothing about which server that program - connects to -- a custom binary with no ``-L``/``-S`` reaches the same - ambient server as the stock one. + Only effective ``-L`` and ``-S`` values count. ``-S`` overrides ``-L`` + exactly as tmux documents. The binary, color, configuration and quiet + globals affect execution but do not select a socket. Returns ------- @@ -289,8 +512,10 @@ def names_server(self) -> bool: >>> ServerConnection.of(tmux_bin="/usr/bin/tmux").names_server False + >>> ServerConnection.of(args=("-2", "-f/dev/null", "-q")).names_server + False """ - return bool(self.args) + return _connection_settings(self.args).socket is not None def resolve_bin(self) -> str: """Return the tmux binary path (memoized). diff --git a/src/libtmux/engines/subprocess.py b/src/libtmux/engines/subprocess.py index 886dded420..374e5f4095 100644 --- a/src/libtmux/engines/subprocess.py +++ b/src/libtmux/engines/subprocess.py @@ -14,6 +14,7 @@ import typing as t from libtmux import exc +from libtmux._internal.tmux_argv import encode_direct_argv from libtmux.engines.base import CommandResult from libtmux.engines.connection import ServerConnection @@ -211,7 +212,10 @@ def command_line(self, request: CommandRequest) -> tuple[str, ...]: ... ) ('tmux', '-Lwork', 'send-keys', 'echo hi') """ - return self._conn.argv(*request.args, tmux_bin=request.tmux_bin) + return self._conn.argv( + *encode_direct_argv(request.args), + tmux_bin=request.tmux_bin, + ) def run(self, request: CommandRequest) -> CommandResult: """Execute one tmux command via :mod:`subprocess` and return its result. diff --git a/src/libtmux/exc.py b/src/libtmux/exc.py index 0dd2326cd0..76bf46a65c 100644 --- a/src/libtmux/exc.py +++ b/src/libtmux/exc.py @@ -10,6 +10,7 @@ import typing as t if t.TYPE_CHECKING: + from libtmux.engines.connection import ServerConnection from libtmux.neo import ListExtraArgs @@ -160,6 +161,58 @@ def __init__(self, engine: object, method: str, *args: object) -> None: super().__init__(msg, *args) +class EngineConfigurationMismatch(ValueError): + """An inspectable engine contradicts explicit Server configuration. + + Raised when dispatch cannot honor both the injected engine's declared + connection and the values explicitly set on :class:`libtmux.Server`. It is + also raised when satisfying missing Server constraints would require + cloning a pinned transport, or when a claimed safe rebind reports the wrong + connection. + + Parameters + ---------- + engine : object + Injected engine whose connection cannot satisfy the Server. + reason : str + Concrete conflicting or unsafe configuration detail. + server_connection : ServerConnection, optional + Connection constraints derived from the Server at dispatch time. + engine_connection : object, optional + Connection reported by the injected engine. + *args : object + Forwarded to :class:`ValueError`. + + Examples + -------- + >>> from libtmux import exc + >>> print(exc.EngineConfigurationMismatch(object(), "socket conflict")) + object configuration does not satisfy Server: socket conflict + + It is a configuration error, not a tmux command failure: + + >>> issubclass(exc.EngineConfigurationMismatch, exc.LibTmuxException) + False + """ + + def __init__( + self, + engine: object, + reason: str, + *args: object, + server_connection: ServerConnection | None = None, + engine_connection: object | None = None, + ) -> None: + self.engine = engine + self.reason = reason + self.server_connection = server_connection + self.engine_connection = engine_connection + super().__init__( + f"{type(engine).__name__} configuration does not satisfy Server: {reason}", + *args, + ) + + class NotInsideTmux(LibTmuxException): """Raised when the process is not running inside a tmux pane. @@ -396,7 +449,7 @@ class UnknownColorOption(UnknownOption): """Unknown color option.""" def __init__(self, *args: object) -> None: - super().__init__("Server.colors must equal 88 or 256") + super().__init__("Server.colors must equal 256") class InvalidOption(OptionError): diff --git a/src/libtmux/experimental/_wait_pane.py b/src/libtmux/experimental/_wait_pane.py index 8e64650022..e73f316ff6 100644 --- a/src/libtmux/experimental/_wait_pane.py +++ b/src/libtmux/experimental/_wait_pane.py @@ -5,8 +5,8 @@ fix both builders use is the same: poll ``#{cursor_x},#{cursor_y}`` until the cursor leaves the origin, then proceed. -This is a *host* step: it must run between tmux dispatches, never inside a fold, -so both drivers replay it from their plan's ``on_step`` hook (the workspace runner +This is a *host* step: it must run between request batches, so both drivers +replay it from their plan's ``on_step`` hook (the workspace runner through a compiled :class:`~.workspace.compiler.HostStep`, the fluent builder through its recorded host action). Only the poll itself is shared here. """ @@ -21,6 +21,8 @@ from libtmux.experimental.ops.plan import _resolve if t.TYPE_CHECKING: + from collections.abc import Mapping + from libtmux.experimental.engines.base import AsyncTmuxEngine, TmuxEngine from libtmux.experimental.ops._types import Target from libtmux.experimental.ops.operation import Operation @@ -57,7 +59,7 @@ def pane_ready(cursor: str) -> bool: def _cursor_probe( pane: Target, - bindings: dict[int | tuple[int, str], str], + bindings: Mapping[int | tuple[int, str], str], ) -> Operation[t.Any]: """Build the cursor read for *pane*, resolving a forward ref against bindings.""" return _resolve(DisplayMessage(target=pane, message=CURSOR_FMT), bindings) @@ -66,7 +68,7 @@ def _cursor_probe( def wait_pane( engine: TmuxEngine, pane: Target, - bindings: dict[int | tuple[int, str], str], + bindings: Mapping[int | tuple[int, str], str], version: str | None = None, ) -> bool: """Poll *pane* until its prompt draws; return whether it did within the budget. @@ -86,7 +88,7 @@ def wait_pane( async def await_pane( engine: AsyncTmuxEngine, pane: Target, - bindings: dict[int | tuple[int, str], str], + bindings: Mapping[int | tuple[int, str], str], version: str | None = None, ) -> bool: """Async sibling of :func:`wait_pane` (same budget, same exhaustion contract).""" diff --git a/src/libtmux/experimental/engines/__init__.py b/src/libtmux/experimental/engines/__init__.py index 6b9d6b7c9e..10228af22b 100644 --- a/src/libtmux/experimental/engines/__init__.py +++ b/src/libtmux/experimental/engines/__init__.py @@ -21,6 +21,9 @@ CommandResult, EngineKind, EngineSpec, + HasConnection, + SupportsAsyncTmuxVersion, + SupportsConnection, SupportsTmuxVersion, TmuxEngine, ) @@ -52,10 +55,13 @@ "ControlNotification", "EngineKind", "EngineSpec", + "HasConnection", "ImsgEngine", "MockEngine", "ServerConnection", "SubprocessEngine", + "SupportsAsyncTmuxVersion", + "SupportsConnection", "SupportsTmuxVersion", "TmuxEngine", "available_engines", diff --git a/src/libtmux/experimental/engines/async_control_mode.py b/src/libtmux/experimental/engines/async_control_mode.py index e09cb47bfc..855b18ffda 100644 --- a/src/libtmux/experimental/engines/async_control_mode.py +++ b/src/libtmux/experimental/engines/async_control_mode.py @@ -12,10 +12,10 @@ the I/O layer differs from the sync engine (``await stdout.read`` instead of ``selectors``). - Command correlation is a FIFO of futures resolved in block-arrival order. A - block that arrives with *no* pending command is **unsolicited** (a hook- - triggered command, or the startup ACK) and is skipped, so correlation never - desyncs. The startup ACK is consumed synchronously in :meth:`_spawn` before - the reader runs, closing the startup race. + block with unsolicited flags (a hook-triggered command or startup ACK) is + skipped. A solicited block with no pending command is a protocol error rather + than a result to guess at. The startup ACK is consumed synchronously in + :meth:`_spawn` before the reader runs, closing the startup race. - A supervisor owns the process lifecycle. :meth:`start` launches it once; it attaches to an exact existing session without updating its environment, replays the desired subscriptions, and runs the reader inline (one reader at @@ -32,7 +32,7 @@ - A reader failure or EOF marks the engine *dead* and fails every pending command, rather than hanging; the supervisor then reconnects. - Notifications go to a bounded queue; on overflow the oldest is dropped and - counted (backpressure), mirroring control mode's own ``%pause`` philosophy. + counted. Producers are not slowed. """ from __future__ import annotations @@ -46,7 +46,11 @@ from libtmux import exc from libtmux.experimental.engines.asyncio import AsyncSubprocessEngine -from libtmux.experimental.engines.base import CommandRequest, render_control_line +from libtmux.experimental.engines.base import ( + CommandRequest, + render_control_line, + unescape_control_output, +) from libtmux.experimental.engines.connection import ServerConnection from libtmux.experimental.engines.control_mode import ( BlockSequenceMonitor, @@ -71,6 +75,9 @@ _DEFAULT_TIMEOUT = 30.0 _STARTUP_TIMEOUT = 5.0 _STOP_TIMEOUT = 2.0 +_STDERR_TAIL_LINES = 20 +_STDERR_TAIL_BYTES = 16 * 1024 +_STDERR_LOG_BYTES = 4 * 1024 # A connection must survive at least this long to count as healthy and reset the # reconnect backoff; a shorter-lived one is treated as a failed attempt so a # persistently flapping proc escalates instead of fork-storming. @@ -90,12 +97,20 @@ class ControlNotification: args : tuple[str, ...] Whitespace-separated notification arguments. raw : str - Decoded control-mode line before tokenization. + Control-mode line before tokenization and pane-output unescaping. + raw_bytes : bytes + Exact control-mode wire line for diagnostics that require byte fidelity. + pane_id : str or None + Pane identifier for an output notification, otherwise ``None``. + payload : bytes or None + Octal-unescaped pane bytes for ``%output`` and ``%extended-output``; + ``None`` for every other notification. Examples -------- - >>> ControlNotification.parse(b"%window-add @3") - ControlNotification(kind='window-add', args=('@3',), raw='%window-add @3') + >>> notification = ControlNotification.parse(b"%window-add @3") + >>> notification.kind, notification.args, notification.payload + ('window-add', ('@3',), None) >>> ControlNotification.parse(b"%output %1 hello world").kind 'output' """ @@ -103,15 +118,43 @@ class ControlNotification: kind: str args: tuple[str, ...] raw: str + raw_bytes: bytes = b"" + pane_id: str | None = None + payload: bytes | None = None @classmethod def parse(cls, line: bytes) -> ControlNotification: - """Parse a raw ``%``-notification line.""" + r"""Parse one notification and decode pane data at the wire boundary. + + Tmux encodes every non-printable pane byte and backslash as exactly + three octal digits. The human-readable :attr:`raw` line remains encoded + for diagnostics while :attr:`payload` carries the original pane bytes. + """ text = line.decode(errors="replace") body = text.removeprefix("%") parts = body.split(" ") kind = parts[0] if parts else "" - return cls(kind=kind, args=tuple(parts[1:]), raw=text) + pane_id: str | None = None + payload: bytes | None = None + if line.startswith(b"%output "): + output_parts = line.split(b" ", 2) + if len(output_parts) == 3: + pane_id = output_parts[1].decode(errors="replace") + payload = unescape_control_output(output_parts[2]) + elif line.startswith(b"%extended-output "): + metadata, separator, payload_wire = line.partition(b" : ") + fields = metadata.split(b" ") + if separator and len(fields) >= 3: + pane_id = fields[1].decode(errors="replace") + payload = unescape_control_output(payload_wire) + return cls( + kind=kind, + args=tuple(parts[1:]), + raw=text, + raw_bytes=line, + pane_id=pane_id, + payload=payload, + ) @dataclass(slots=True) @@ -161,7 +204,7 @@ def _force_put(queue: asyncio.Queue[t.Any], item: t.Any) -> None: Like :func:`_offer` but drop-count-free: used to land the stream-end sentinel even on a queue already at ``maxsize``, so a slow consumer that hit - backpressure still gets closed instead of hanging on ``queue.get()``. Pulled + overflow still gets closed instead of hanging on ``queue.get()``. Pulled out of the broadcast loop so the ``try``/``except`` stays out of it. """ try: @@ -198,7 +241,7 @@ class AsyncControlModeEngine: timeout : float Seconds to await a command's result before failing it. event_queue_size : int - Bounded size of the notification queue (backpressure). + Bounded notification count; overflow drops the oldest item. Notes ----- @@ -226,6 +269,9 @@ def __init__( self._subscribers: set[asyncio.Queue[t.Any]] = set() self._dropped_notifications = 0 self._proc: asyncio.subprocess.Process | None = None + self._stderr_tail = bytearray() + self._stderr_proc: asyncio.subprocess.Process | None = None + self._stderr_task: asyncio.Task[None] | None = None self._start_lock = asyncio.Lock() self._write_lock = asyncio.Lock() self._started = False @@ -265,12 +311,22 @@ def tmux_version(self) -> str | None: """Report the connected server's tmux version (``tmux -V``), memoized. Implements - :class:`~libtmux.experimental.engines.base.SupportsTmuxVersion` so + :class:`~libtmux.engines.base.SupportsTmuxVersion` so version-gated operations render correctly over control mode; in-memory engines omit it and resolution assumes latest. """ return self._conn.tmux_version() + async def atmux_version(self) -> str | None: + """Probe the connected tmux version without blocking the event loop. + + Examples + -------- + >>> asyncio.run(AsyncControlModeEngine().atmux_version()) is not None + True + """ + return await self._bootstrap.atmux_version() + def add_subscription(self, spec: str) -> None: """Record a desired ``refresh-client -B`` subscription (idempotent). @@ -380,7 +436,7 @@ async def _spawn(self) -> None: # exited, so this is a no-op there. old = self._proc if old is not None: - await self._stop_process(old) + await self._stop_connection(old) target = self._next_attach_target self._next_attach_target = None if target is None: @@ -398,13 +454,18 @@ async def _spawn(self) -> None: "-t", target, ) - try: - proc = await asyncio.create_subprocess_exec( + creation = asyncio.create_task( + asyncio.create_subprocess_exec( *cmd, stdin=asyncio.subprocess.PIPE, stdout=asyncio.subprocess.PIPE, stderr=asyncio.subprocess.PIPE, - ) + ), + name="libtmux-async-control-process-start", + ) + cancelled = await self._wait_for_owned_task(creation) + try: + proc = creation.result() except FileNotFoundError: raise exc.TmuxCommandNotFound from None self._proc = proc @@ -413,11 +474,19 @@ async def _spawn(self) -> None: # run_batch must hit the dead guard instead of writing a reply that # _consume_startup would drain and discard. try: - await self._consume_startup() - except BaseException: - await self._stop_process(proc) - self._proc = None + self._start_stderr_reader(proc) + if not cancelled: + await self._consume_startup() + except BaseException as error: + await self._stop_connection(proc) + if isinstance(error, asyncio.CancelledError): + raise + if isinstance(error, ControlModeError): + raise self._died(str(error)) from error raise + if cancelled: + await self._stop_connection(proc) + raise asyncio.CancelledError self._dead = None async def _consume_startup(self) -> None: @@ -542,7 +611,9 @@ async def run_batch( # so the reader can drain any replies, but make their futures # terminal so close/reconnect cannot publish unobserved errors. for queued in appended: - if not queued.future.done(): + if queued.future.done(): + _swallow_future(queued.future) + else: queued.future.cancel() raise except (BrokenPipeError, OSError) as error: @@ -552,7 +623,9 @@ async def run_batch( for queued in appended: with contextlib.suppress(ValueError): self._pending.remove(queued) - if not queued.future.done(): + if queued.future.done(): + _swallow_future(queued.future) + else: queued.future.cancel() raise cm_error from error @@ -632,20 +705,20 @@ async def aclose(self) -> None: self._aclose_impl(), name="libtmux-async-control-close", ) - cancelled = await self._wait_for_close_cleanup(cleanup) + cancelled = await self._wait_for_owned_task(cleanup) cleanup.result() if cancelled: raise asyncio.CancelledError @staticmethod - async def _wait_for_close_cleanup(cleanup: asyncio.Task[None]) -> bool: + async def _wait_for_owned_task(task: asyncio.Task[t.Any]) -> bool: """Wait through every caller cancellation; report whether one occurred.""" try: - await asyncio.wait((cleanup,)) + await asyncio.wait((task,)) except asyncio.CancelledError: # Recursion gives each accepted ``cancel()`` its own handler without - # propagating cancellation into the independently owned cleanup task. - await AsyncControlModeEngine._wait_for_close_cleanup(cleanup) + # propagating cancellation into the independently owned task. + await AsyncControlModeEngine._wait_for_owned_task(task) return True return False @@ -682,9 +755,7 @@ async def _close_locked(self) -> None: self._fail_pending(ControlModeError("control-mode engine closed")) proc = self._proc if proc is not None: - await self._stop_process(proc) - if self._proc is proc: - self._proc = None + await self._stop_connection(proc) async def __aenter__(self) -> Self: """Start when a safe session exists; otherwise remain lazy to bootstrap.""" @@ -785,17 +856,18 @@ async def _supervisor( proc = self._proc if proc is not None: try: - await self._stop_process(proc) + await self._stop_connection(proc) except Exception as cleanup_error: failure = ControlModeError( f"control-mode supervisor failed: {error}; " f"process cleanup failed: {cleanup_error}" ) - else: - if self._proc is proc: - self._proc = None await self._finish_failed_start(first_attempt, failure) finally: + proc = self._proc + if proc is not None: + with contextlib.suppress(Exception): + await self._stop_connection(proc) if first_attempt is not None and not first_attempt.done(): first_attempt.set_exception( ControlModeError("control-mode engine closed before connecting"), @@ -857,6 +929,107 @@ async def _stop_process(proc: asyncio.subprocess.Process) -> None: msg = "tmux control process did not exit after kill" raise ControlModeError(msg) from None + def _start_stderr_reader(self, proc: asyncio.subprocess.Process) -> None: + """Start the one stderr drain task owned by *proc*'s generation.""" + if self._stderr_task is not None: + msg = "control-mode stderr reader already exists" + raise ControlModeError(msg) + self._stderr_tail.clear() + if getattr(proc, "stderr", None) is None: + return + self._stderr_proc = proc + self._stderr_task = asyncio.create_task( + self._drain_stderr(proc), + name="libtmux-async-control-stderr", + ) + + async def _drain_stderr(self, proc: asyncio.subprocess.Process) -> None: + """Continuously drain *proc* stderr into a bounded diagnostic tail.""" + stderr = proc.stderr + if stderr is None: + return + try: + while True: + chunk = await stderr.read(_READ_CHUNK) + if not chunk: + return + self._append_stderr(chunk) + preview = chunk[-_STDERR_LOG_BYTES:].decode(errors="replace") + logger.debug( + "tmux control-mode stderr", + extra={"tmux_stderr": [preview]}, + ) + except asyncio.CancelledError: + raise + except Exception as error: + self._append_stderr(f"stderr read failed: {error}\n".encode()) + + def _append_stderr(self, chunk: bytes) -> None: + """Append stderr while retaining bounded bytes and logical lines.""" + self._stderr_tail.extend(chunk) + overflow = len(self._stderr_tail) - _STDERR_TAIL_BYTES + if overflow > 0: + del self._stderr_tail[:overflow] + lines = self._stderr_tail.splitlines(keepends=True) + if len(lines) > _STDERR_TAIL_LINES: + self._stderr_tail[:] = b"".join(lines[-_STDERR_TAIL_LINES:]) + + def _stderr_lines(self) -> tuple[str, ...]: + """Return the retained non-empty stderr lines for diagnostics.""" + return tuple( + line.decode(errors="replace") + for line in self._stderr_tail.splitlines() + if line.strip() + ) + + async def _finish_stderr_reader( + self, + proc: asyncio.subprocess.Process, + ) -> None: + """Join and forget *proc*'s stderr task without leaking on cancellation.""" + if self._stderr_proc is not proc: + return + task = self._stderr_task + try: + if task is not None: + waiter = asyncio.gather(task, return_exceptions=True) + try: + await asyncio.wait_for(waiter, timeout=_STOP_TIMEOUT) + except asyncio.TimeoutError: + task.cancel() + await asyncio.gather(task, return_exceptions=True) + except asyncio.CancelledError: + if task is not None and not task.done(): + task.cancel() + await asyncio.gather(task, return_exceptions=True) + raise + finally: + if self._stderr_proc is proc: + self._stderr_proc = None + self._stderr_task = None + + async def _stop_connection(self, proc: asyncio.subprocess.Process) -> None: + """Finish connection cleanup even when this caller is cancelled.""" + cleanup = asyncio.create_task( + self._stop_connection_impl(proc), + name="libtmux-async-control-process-close", + ) + cancelled = await self._wait_for_owned_task(cleanup) + cleanup.result() + if cancelled: + raise asyncio.CancelledError + + async def _stop_connection_impl(self, proc: asyncio.subprocess.Process) -> None: + """Terminate and reap *proc*, then join its stderr drain task.""" + try: + await self._stop_process(proc) + finally: + try: + await self._finish_stderr_reader(proc) + finally: + if self._proc is proc: + self._proc = None + async def _replay_subscriptions(self) -> None: """Re-issue every desired subscription to the freshly connected proc. @@ -961,7 +1134,8 @@ async def _reader(self) -> None: while True: chunk = await stdout.read(_READ_CHUNK) if not chunk: - self._mark_dead(ControlModeError("tmux -C closed stdout")) + await self._stop_connection(proc) + self._mark_dead(self._died("tmux -C closed stdout")) return self._parser.feed(chunk) for block in self._parser.blocks(): @@ -971,35 +1145,28 @@ async def _reader(self) -> None: except asyncio.CancelledError: raise except Exception as error: - self._mark_dead(ControlModeError(f"control-mode reader failed: {error}")) + with contextlib.suppress(Exception): + await self._stop_connection(proc) + self._mark_dead(self._died(f"control-mode reader failed: {error}")) def _dispatch_block(self, block: ControlModeBlock) -> None: """Accumulate a solicited block; resolve the command once it has them all. - A ``;``-folded command emits one block per sub-command; unsolicited blocks + A semicolon command group emits one block per member; unsolicited blocks (hook-triggered commands, the startup ACK) carry flags 0 and are skipped, so FIFO correlation never desyncs. """ if block.flags != 1: return # unsolicited (hook-triggered command or startup ACK): skip if not self._pending: - # A solicited reply with no command waiting: its command's future was - # already resolved, cancelled, or failed. FIFO is now one block off. - logger.warning( - "control-mode dropped solicited block #%s with no pending command", - block.number, - extra={ - "tmux_stdout": [ - line.decode(errors="replace") for line in block.body - ], - "tmux_stdout_len": len(block.body), - }, - ) - return + msg = "control-mode received a solicited result with no pending request" + raise ControlModeError(msg) pending = self._pending[0] - self._sequence.check(block, pending.argv) + if not self._sequence.check(block, pending.argv): + msg = "control-mode command block sequence moved backwards" + raise ControlModeError(msg) pending.blocks.append(block) - if len(pending.blocks) < pending.expected: + if not block.is_error and len(pending.blocks) < pending.expected: return self._pending.popleft() if not pending.future.done(): @@ -1019,7 +1186,7 @@ def _broadcast_stream_end(self) -> None: """Push the stream-end sentinel to every subscriber, then clear them. Uses :func:`_force_put` so the sentinel lands even on a queue already at - ``maxsize`` (a slow consumer that hit backpressure); otherwise the + ``maxsize`` (a slow consumer that hit overflow); otherwise the sentinel would be lost and the consumer would hang forever on ``queue.get()`` -- the exact bug this guards against. """ @@ -1034,6 +1201,11 @@ def _mark_dead(self, error: BaseException) -> None: self._fail_pending(error) self._broadcast_stream_end() + def _died(self, message: str) -> ControlModeError: + """Build a connection error carrying tmux's bounded stderr tail.""" + tail = "; ".join(self._stderr_lines()) + return ControlModeError(f"{message}: {tail}" if tail else message) + def _fail_pending(self, error: BaseException) -> None: """Fail every queued command future with *error*.""" while self._pending: diff --git a/src/libtmux/experimental/engines/asyncio.py b/src/libtmux/experimental/engines/asyncio.py index 00944008e5..f08a783016 100644 --- a/src/libtmux/experimental/engines/asyncio.py +++ b/src/libtmux/experimental/engines/asyncio.py @@ -12,17 +12,137 @@ import asyncio import contextlib +import re +import sys import typing as t from libtmux import exc -from libtmux.experimental.engines.base import CommandResult, encode_direct_argv +from libtmux.experimental.engines.base import ( + CommandRequest, + CommandResult, + encode_direct_argv, +) from libtmux.experimental.engines.connection import ServerConnection if t.TYPE_CHECKING: import pathlib from collections.abc import Sequence - from libtmux.experimental.engines.base import CommandRequest + +_TERMINATE_TIMEOUT = 1.0 +_KILL_TIMEOUT = 1.0 + + +def _version_from_result(result: CommandResult) -> str | None: + """Normalize native async ``tmux -V`` output like the sync probe. + + Examples + -------- + >>> _version_from_result( + ... CommandResult(cmd=("tmux", "-V"), stdout=("tmux 3.4a",)) + ... ) + '3.4' + >>> _version_from_result(CommandResult(cmd=("tmux", "-V"), stderr=("bad",))) + """ + from libtmux.common import TMUX_MAX_VERSION + + if result.stderr: + if ( + sys.platform.startswith("openbsd") + and result.stderr[0] == "tmux: unknown option -- V" + ): + return f"{TMUX_MAX_VERSION}-openbsd" + return None + if not result.stdout: + return None + _prefix, separator, version = result.stdout[0].partition("tmux ") + if not separator or not version: + return None + if version == "master": + return f"{TMUX_MAX_VERSION}-master" + return re.sub(r"[a-z-]", "", version) or None + + +class _AsyncProcessOwner: + """Own one subprocess and every task required to communicate or reap it.""" + + __slots__ = ("_argv",) + + def __init__(self, argv: Sequence[str]) -> None: + self._argv = tuple(argv) + + async def communicate(self) -> tuple[bytes, bytes, int]: + """Spawn, communicate, and own exceptional cleanup for one process. + + Examples + -------- + >>> asyncio.run(_AsyncProcessOwner(("tmux", "-V")).communicate())[2] + 0 + """ + process = await asyncio.create_subprocess_exec( + *self._argv, + stdout=asyncio.subprocess.PIPE, + stderr=asyncio.subprocess.PIPE, + ) + communication = asyncio.create_task( + process.communicate(), + name="libtmux-async-subprocess-communicate", + ) + try: + stdout, stderr = await asyncio.shield(communication) + except BaseException: + + async def finish_communication(timeout: float) -> bool: + """Await pipe draining without cancelling it on timeout.""" + if communication.done(): + await asyncio.gather(communication, return_exceptions=True) + return True + done, _pending = await asyncio.wait( + (communication,), + timeout=timeout, + ) + if not done: + return False + await asyncio.gather(communication, return_exceptions=True) + return True + + async def cleanup_process() -> None: + """Drain pipes while performing bounded TERM-to-KILL cleanup.""" + if process.returncode is None: + with contextlib.suppress(Exception): + process.terminate() + drained = await finish_communication(_TERMINATE_TIMEOUT) + if drained and process.returncode is not None: + return + if process.returncode is None: + with contextlib.suppress(Exception): + process.kill() + drained = await finish_communication(_KILL_TIMEOUT) + if drained: + return + # A killed direct child closes both pipes. This fallback bounds + # pathological inherited-pipe cases where a descendant keeps an + # fd open after the child has exited. + communication.cancel() + await asyncio.gather(communication, return_exceptions=True) + + cleanup_task = asyncio.create_task( + cleanup_process(), + name="libtmux-async-subprocess-cleanup", + ) + while not cleanup_task.done(): + try: + await asyncio.shield(cleanup_task) + except asyncio.CancelledError: # noqa: PERF203 + continue + with contextlib.suppress(asyncio.CancelledError, Exception): + cleanup_task.result() + raise + return ( + stdout, + stderr, + process.returncode if process.returncode is not None else -1, + ) class AsyncSubprocessEngine: @@ -52,6 +172,9 @@ def __init__( server_args: Sequence[str] = (), ) -> None: self._conn = ServerConnection.of(tmux_bin, server_args) + self._async_version: str | None = None + self._async_version_probed = False + self._async_version_lock = asyncio.Lock() @property def connection(self) -> ServerConnection: @@ -76,30 +199,42 @@ def tmux_version(self) -> str | None: """ return self._conn.tmux_version() + async def atmux_version(self) -> str | None: + """Probe ``tmux -V`` natively asynchronously and memoize the result. + + A cancelled probe remains retryable. + + Examples + -------- + >>> asyncio.run(AsyncSubprocessEngine().atmux_version()) is not None + True + """ + if self._async_version_probed: + return self._async_version + async with self._async_version_lock: + if not self._async_version_probed: + try: + result = await self.run(CommandRequest.from_args("-V")) + except exc.LibTmuxException: + version = None + else: + version = _version_from_result(result) + self._async_version = version + self._async_version_probed = True + return self._async_version + async def run(self, request: CommandRequest) -> CommandResult: """Execute one tmux command asynchronously and return its result.""" argv = encode_direct_argv(request.args) cmd = self._conn.argv(*argv, tmux_bin=request.tmux_bin) try: - process = await asyncio.create_subprocess_exec( - *cmd, - stdout=asyncio.subprocess.PIPE, - stderr=asyncio.subprocess.PIPE, - ) + stdout_bytes, stderr_bytes, returncode = await _AsyncProcessOwner( + cmd, + ).communicate() except FileNotFoundError: raise exc.TmuxCommandNotFound from None - try: - stdout_bytes, stderr_bytes = await process.communicate() - except asyncio.CancelledError: - # The child may have already exited (terminate races the reap); - # suppress so the cancellation propagates, not ProcessLookupError. - with contextlib.suppress(ProcessLookupError): - process.terminate() - await process.wait() - raise - stdout = stdout_bytes.decode(errors="backslashreplace") stderr = stderr_bytes.decode(errors="backslashreplace") @@ -112,7 +247,7 @@ async def run(self, request: CommandRequest) -> CommandResult: cmd=tuple(cmd), stdout=tuple(stdout_lines), stderr=tuple(stderr_lines), - returncode=process.returncode if process.returncode is not None else -1, + returncode=returncode, ) async def run_batch( @@ -122,6 +257,34 @@ async def run_batch( """Execute requests sequentially (preserving tmux command ordering).""" return [await self.run(req) for req in requests] + def with_connection(self, connection: ServerConnection) -> AsyncSubprocessEngine: + """Return an equivalent stateless async engine bound to *connection*. + + Parameters + ---------- + connection : ServerConnection + The connection the returned engine dispatches over. + + Returns + ------- + AsyncSubprocessEngine + A new engine; this engine is unchanged. + + Examples + -------- + >>> from libtmux.experimental.engines import ServerConnection + >>> engine = AsyncSubprocessEngine() + >>> rebound = engine.with_connection( + ... ServerConnection.of(args=("-Lwork",)) + ... ) + >>> rebound.server_args, engine.server_args + (('-Lwork',), ()) + """ + return type(self)( + tmux_bin=connection.tmux_bin, + server_args=connection.args, + ) + @classmethod def for_server(cls, server: t.Any) -> AsyncSubprocessEngine: """Build an async engine bound to a live :class:`libtmux.Server`'s socket.""" diff --git a/src/libtmux/experimental/engines/base.py b/src/libtmux/experimental/engines/base.py index 006d5d62f3..2d6870865a 100644 --- a/src/libtmux/experimental/engines/base.py +++ b/src/libtmux/experimental/engines/base.py @@ -22,10 +22,17 @@ import typing as t from dataclasses import dataclass +from libtmux._internal.tmux_argv import ( + DirectArgv, + encode_direct_argv, + split_direct_argv, +) from libtmux.engines.base import ( CommandRequest, CommandResult, CommandSeparator, + HasConnection, + SupportsConnection, SupportsTmuxVersion, TmuxEngine, is_command_separator, @@ -38,14 +45,6 @@ #: tmux escapes a byte in ``%output`` as a backslash plus three octal digits. _CONTROL_OCTAL = re.compile(rb"\\([0-7]{3})") -# tmux parses these options with getopt before its command argv reaches -# cmd_parse_from_arguments. Only values in the latter have structural -# trailing-semicolon semantics. -_GLOBAL_OPTIONS_WITH_VALUE = frozenset({"c", "f", "L", "S", "T"}) -_GLOBAL_OPTIONS_WITHOUT_VALUE = frozenset( - {"2", "8", "C", "D", "d", "h", "l", "N", "q", "u", "U", "v", "V"}, -) - __all__ = ( "AsyncTmuxEngine", "CommandRequest", @@ -54,6 +53,9 @@ "DirectArgv", "EngineKind", "EngineSpec", + "HasConnection", + "SupportsAsyncTmuxVersion", + "SupportsConnection", "SupportsTmuxVersion", "TmuxEngine", "encode_direct_argv", @@ -64,116 +66,6 @@ ) -class DirectArgv(t.NamedTuple): - """The client-global and command portions of direct tmux argv. - - Attributes - ---------- - global_args : tuple[str, ...] - Leading options consumed by tmux's client-level ``getopt`` parser. - command_argv : tuple[str, ...] - The subcommand and arguments passed to ``cmd_parse_from_arguments``. - """ - - global_args: tuple[str, ...] - command_argv: tuple[str, ...] - - -def _global_option_consumes_next(token: str) -> bool | None: - """Return a global option's separate-value arity, or ``None`` if unknown. - - Examples - -------- - >>> _global_option_consumes_next("-L") - True - >>> _global_option_consumes_next("-Lwork") - False - >>> _global_option_consumes_next("list-sessions") is None - True - """ - if not token.startswith("-") or token in {"-", "--"}: - return None - cluster = token[1:] - if not cluster: - return None - for index, option in enumerate(cluster): - if option in _GLOBAL_OPTIONS_WITH_VALUE: - return index == len(cluster) - 1 - if option not in _GLOBAL_OPTIONS_WITHOUT_VALUE: - return None - return False - - -def split_direct_argv(argv: Sequence[str]) -> DirectArgv: - """Split raw tmux argv at the client-global/command parser boundary. - - The split follows tmux's leading short-option ``getopt`` grammar, including - attached values and ``--``. Global values remain byte-for-byte data because - tmux removes them before parsing command separators. - - Examples - -------- - >>> split_direct_argv(("-L", "socket;", "display-message", "text;")) - DirectArgv(global_args=('-L', 'socket;'), command_argv=('display-message', 'text;')) - >>> split_direct_argv(("-Lsocket;", "--", "display-message")) - DirectArgv(global_args=('-Lsocket;', '--'), command_argv=('display-message',)) - """ - args = tuple(argv) - if any("\0" in token for token in args): - msg = "tmux command arguments cannot contain NUL" - raise ValueError(msg) - - index = 0 - while index < len(args): - token = args[index] - if token == "--": - index += 1 - break - consumes_next = _global_option_consumes_next(token) - if consumes_next is None: - break - index += 2 if consumes_next and index + 1 < len(args) else 1 - return DirectArgv(global_args=args[:index], command_argv=args[index:]) - - -def _encode_command_argv(argv: Sequence[str]) -> tuple[str, ...]: - r"""Escape literal separators in argv already known to be command-scoped. - - Examples - -------- - >>> _encode_command_argv(("display-message", "literal;")) - ('display-message', 'literal\\;') - """ - encoded: list[str] = [] - for token in argv: - if not is_command_separator(token) and token.endswith(";"): - token = f"{token[:-1]}\\;" - encoded.append(str(token)) - return tuple(encoded) - - -def encode_direct_argv(argv: Sequence[str]) -> tuple[str, ...]: - r"""Encode literal arguments for tmux's direct argv parser. - - Tmux first removes client-global options, then routes only the remaining - command argv through ``cmd_parse_from_arguments``, where a final ``;`` is - structural. Prefixing that final byte with one backslash preserves it as - data. Global option values remain unchanged, and a - :class:`CommandSeparator` remains structural. - - Examples - -------- - >>> encode_direct_argv(("send-keys", "text;")) - ('send-keys', 'text\\;') - >>> encode_direct_argv(("-L", "socket;", "send-keys", "text;")) - ('-L', 'socket;', 'send-keys', 'text\\;') - >>> encode_direct_argv(("a", CommandSeparator(";"), "b")) - ('a', ';', 'b') - """ - direct = split_direct_argv(argv) - return (*direct.global_args, *_encode_command_argv(direct.command_argv)) - - def _quote_control_token(token: str) -> str: r"""Quote one literal token for tmux's line-oriented control parser.""" if "\0" in token: @@ -208,7 +100,7 @@ def render_control_line(argv: Sequence[str]) -> str: ) -def unescape_control_output(payload: str) -> bytes: +def unescape_control_output(payload: str | bytes) -> bytes: r"""Decode a control-mode ``%output`` payload back to the bytes the pane wrote. tmux does not forward pane output verbatim: in a ``%output`` notification it @@ -217,8 +109,9 @@ def unescape_control_output(payload: str) -> bytes: this first, or it can never match: an ``ESC`` (``0x1b``) arrives on the wire as the four *characters* ``\``, ``0``, ``3``, ``3``. - Bytes tmux left alone pass through untouched, so feeding this an already-raw - payload is harmless. + Bytes outside an exact three-digit escape pass through untouched. Apply the + decoder once to wire data; it is not idempotent when decoded pane bytes + themselves contain a backslash followed by three octal digits. Examples -------- @@ -237,7 +130,11 @@ def unescape_control_output(payload: str) -> bytes: >>> unescape_control_output(r"caf\303\251").decode() 'café' """ - raw = payload.encode("utf-8", "surrogateescape") + raw = ( + payload.encode("utf-8", "surrogateescape") + if isinstance(payload, str) + else payload + ) return _CONTROL_OCTAL.sub(lambda m: bytes((int(m.group(1), 8),)), raw) @@ -305,6 +202,24 @@ def imsg(cls, *, protocol_version: int | None = None) -> EngineSpec: return cls(kind=EngineKind.IMSG, protocol_version=protocol_version) +@t.runtime_checkable +class SupportsAsyncTmuxVersion(t.Protocol): + """An async engine that can report its tmux version without blocking. + + Examples + -------- + >>> class Versioned: + ... async def atmux_version(self): + ... return "3.4" + >>> isinstance(Versioned(), SupportsAsyncTmuxVersion) + True + """ + + async def atmux_version(self) -> str | None: + """Return the engine's tmux version string, or ``None`` if unknown.""" + ... + + @t.runtime_checkable class AsyncTmuxEngine(t.Protocol): """An asynchronous executor of tmux commands.""" diff --git a/src/libtmux/experimental/engines/connection.py b/src/libtmux/experimental/engines/connection.py index 59d4244676..97e5d49807 100644 --- a/src/libtmux/experimental/engines/connection.py +++ b/src/libtmux/experimental/engines/connection.py @@ -3,7 +3,7 @@ :class:`~libtmux.engines.connection.ServerConnection` graduated to Core as part of the command execution seam, so the experimental engines dispatch over the same connection value the object API does: one tmux binary resolution, one set -of ``-L``/``-S``/``-f``/``-2``/``-8`` flags, one ``tmux -V`` probe. +of ``-L``/``-S``/``-f``/``-2`` flags, one ``tmux -V`` probe. This module is the import path the experimental engines have always used; it carries no definition of its own. diff --git a/src/libtmux/experimental/engines/control_mode.py b/src/libtmux/experimental/engines/control_mode.py index ad740ad807..d4657815bf 100644 --- a/src/libtmux/experimental/engines/control_mode.py +++ b/src/libtmux/experimental/engines/control_mode.py @@ -78,12 +78,15 @@ class ControlModeBlock: Whether tmux closed the block with ``%error``. body : tuple[bytes, ...] Raw output lines between the opening and closing guards. + timestamp : int + Timestamp echoed by both guards for exact block correlation. """ number: int flags: int is_error: bool body: tuple[bytes, ...] + timestamp: int = 0 @dataclasses.dataclass(slots=True) @@ -98,11 +101,14 @@ class _PendingBlock: Flags from the opening control-mode guard. body : list[bytes] Raw output lines collected before the closing guard. + timestamp : int + Timestamp from the opening guard. """ number: int flags: int body: list[bytes] + timestamp: int class ControlModeParser: @@ -159,30 +165,31 @@ def notifications(self) -> list[bytes]: def _handle_line(self, line: bytes) -> None: if self._pending is not None: - if _matches_pending_close(line, self._pending.number): + if _matches_pending_close(line, self._pending): self._close_block(line) return self._pending.body.append(line) return if line.startswith(_BEGIN_PREFIX): self._open_block(line) + elif line.startswith((_END_PREFIX, _ERROR_PREFIX)): + msg = "control-mode received a closing guard without %begin" + raise ControlModeError(msg) elif line.startswith(b"%"): self._notifications.append(line) def _open_block(self, line: bytes) -> None: - number, flags = _parse_guard(line, _BEGIN_PREFIX) - if number is None: - # A %begin whose guard will not parse is dropped: its body lines then - # fall through as notifications and a LATER command's block fills the - # count, silently mis-attributing output. Never observed in practice - # (tmux always writes "%begin