diff --git a/README.md b/README.md index 5e53ffb..da2cbfb 100644 --- a/README.md +++ b/README.md @@ -18,19 +18,19 @@ Download OS-specific installers/portable packages from [Releases](https://github **Located diagnostics:** static review and execution evidence list original filenames and local line locations, with snapshot hashes. Python checks include auxiliary-file syntax, possibly unused imports/locals/functions and directly unreachable statements. Every other supported execution language now has native unused-code candidates and actual language-specific coverage. UI/HTML show missed source lines, genuine branch arcs or native branch IDs/counts, and explicit partial/unknown states. SQL measures completed statement starts, Bash command-entry lines and assembly instruction-backed debug lines—not universal statement/branch coverage. Unsupported locations and wrong-answer failures without source references stay unlocated. See the [language-by-language tools and limits](docs/polyglot-locations.md), [file/line contract](docs/located-diagnostics.md) and [validation](docs/validation-polyglot.md). No tool guarantees discovery or root-cause localization of every bug. -| Feature | Implemented contract | -|---|---| -| Private accounts | Real alias/password registration, hashed sessions and one-use login recovery. Independent browser-derived vault key, authenticated AES-256-GCM locked source/history/provider archives and per-session unlock proof. No developer plaintext read API. Trusted local service and OS can access active plaintext; not browser-only E2EE. | -| Desktop application | Electron shell and bundled Python service; one-click Windows installer plus OS-specific Linux/macOS builds. No IntelliJ or separately installed Python required for the UI. Unsigned development packages. | -| Web application | Same responsive editor/API served on loopback. Narrow-screen browser support; not a native Android/iOS installer or public hosted service. | -| Polyglot execution | Fixed CLI adapters for Python, JavaScript, TypeScript, C/C++, Java, C#, Rust, PHP, Bash, SQLite SQL and GNU x86-64 assembly. Actual toolchains live inside the worker image. | -| Testing | Approved CLI oracles and reviewed pytest harnesses, including subprocess harnesses for non-Python targets. 48-category applicability catalog; human, hardware, cloud/load and specialized tool checks need independent integrations. | -| Coverage/mutation | Exact original source locations from native collectors for all twelve adapters; per-file hashes, missed/executed lines, language-specific metric labels and honest unknown/partial states. Separate from baseline timing/RSS. Python additionally supports up to eight operator mutants. Approved CLI coverage excludes generated pytest; no fabricated MC/DC, universal dead-code or universal mutation scores. | -| Fuzzing | Seeded black-box generation, mutation, dictionary, boundaries and CSV/SQL grammar; instrumented libFuzzer/AFL++ for reviewed C/C++ harnesses and Atheris for Python harnesses. Differential/metamorphic/stateful relations require custom oracles. | -| AI routing | Account-owned OpenAI-compatible, Anthropic and Ollama endpoints/keys, four task routes and destination-version-bound consent. Keys stay server-side after explicit settings submission, outside histories/exports/workers. Protocol tests use mocks, not a claim of live model quality. | -| Metrics/ranking | Repeated process wall times, peak child RSS, source bytes and transparent long-line proxy. Passing independent cases is mandatory. No syntax-based Big-O claims or automatic “best” model claims. | -| Training | Rights-checked, project-disjoint JSONL preparation, actual small causal-model creation or approved local-model fine-tuning, safe-tensor output and authenticated local inference API. Tiny synthetic smoke training is not a useful general coding model. | -| CI | SHA-pinned, read-only Actions validate controller/browser behavior, real worker scenarios, training smoke and native packages. Push analysis retains generated tests/candidate evidence when a provider is explicitly configured; never auto-commits model changes. | +| Feature | Implemented contract | +| ------------------- | ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| Private accounts | Real alias/password registration, hashed sessions and one-use login recovery. Independent browser-derived vault key, authenticated AES-256-GCM locked source/history/provider archives and per-session unlock proof. No developer plaintext read API. Trusted local service and OS can access active plaintext; not browser-only E2EE. | +| Desktop application | Electron shell and bundled Python service; one-click Windows installer plus OS-specific Linux/macOS builds. No IntelliJ or separately installed Python required for the UI. Unsigned development packages. | +| Web application | Same responsive editor/API served on loopback. Narrow-screen browser support; not a native Android/iOS installer or public hosted service. | +| Polyglot execution | Fixed CLI adapters for Python, JavaScript, TypeScript, C/C++, Java, C#, Rust, PHP, Bash, SQLite SQL and GNU x86-64 assembly. Actual toolchains live inside the worker image. | +| Testing | Approved CLI oracles and reviewed pytest harnesses, including subprocess harnesses for non-Python targets. 48-category applicability catalog; human, hardware, cloud/load and specialized tool checks need independent integrations. | +| Coverage/mutation | Exact original source locations from native collectors for all twelve adapters; per-file hashes, missed/executed lines, language-specific metric labels and honest unknown/partial states. Separate from baseline timing/RSS. Python additionally supports up to eight operator mutants. Approved CLI coverage excludes generated pytest; no fabricated MC/DC, universal dead-code or universal mutation scores. | +| Fuzzing | Seeded black-box generation, mutation, dictionary, boundaries and CSV/SQL grammar; instrumented libFuzzer/AFL++ for reviewed C/C++ harnesses and Atheris for Python harnesses. Differential/metamorphic/stateful relations require custom oracles. | +| AI routing | Account-owned OpenAI-compatible, Anthropic and Ollama endpoints/keys, four task routes and destination-version-bound consent. Keys stay server-side after explicit settings submission, outside histories/exports/workers. Protocol tests use mocks, not a claim of live model quality. | +| Metrics/ranking | Repeated process wall times, peak child RSS, source bytes and transparent long-line proxy. Passing independent cases is mandatory. No syntax-based Big-O claims or automatic “best” model claims. | +| Training | Rights-checked, project-disjoint JSONL preparation, actual small causal-model creation or approved local-model fine-tuning, safe-tensor output and authenticated local inference API. Tiny synthetic smoke training is not a useful general coding model. | +| CI | SHA-pinned, read-only Actions validate controller/browser behavior, real worker scenarios, training smoke and native packages. Push analysis retains generated tests/candidate evidence when a provider is explicitly configured; never auto-commits model changes. | Binary formats can be retained as fixtures. There is no promise to understand every document, image, encrypted archive or binary. Supply text requirements and explicit harnesses for formats without a parser; unsupported extraction must not silently succeed. @@ -49,49 +49,50 @@ Source-development instructions, provider setup, first-run diagnostics and packa ## Source map -| Location | Responsibility | -|---|---| -| `studio/accounts.py`, `studio/permissions.py` | scrypt logins, session/recovery digests, bounded attempts and private per-OS data permissions. | -| `studio/private_api.py` | Account-only HTTP/CSRF boundary, own provider settings, per-session key proof, backup/recovery/deletion and scoped controller proxy. | -| `studio/private_workspace.py`, `studio/vault.py` | Single-instance ownership, account-specific engine lifecycle, authenticated streaming encryption, SQLite online snapshots and safe restoration. | -| `studio/private_cli.py` | Token-free loopback account service, dedicated data root and exact-parent graceful shutdown monitor. | -| `ui/Account.tsx` | Sign-in/unlock/recovery, contextual help, encrypted backup, private AI configuration and creator contact. | -| `studio/models.py` | Validated flat-file, project, case, preference and AI proposal schemas; size/path limits. | -| `studio/api.py` | Authenticated loopback API, bounded jobs, history, reports, comparisons and AI tasks. | -| `studio/store.py` | SQLite transactions, append-only revisions, immutable baseline oracles and interrupted-job recovery. | -| `studio/execution.py` | Fixed container flags, readiness checks, output/deadline bounds and exact owned-container cleanup. | -| `studio/analysis.py` | Non-executing source heuristics and correctness-gated weighted comparisons. | -| `studio/source_checks.py`, `studio/locations.py`, `ui/Locations.tsx` | Python syntax/unused/unreachable candidates, bounded tool-output source references, file/category filters and clickable exact-location/coverage evidence views. | -| `ui/source-navigation.ts`, `ui/SourceInspector.tsx` | Exact UTF-8 SHA-256 verification, bounded immutable source inspection, nearby-line browsing and guarded navigation to an unchanged current draft. | -| `studio/providers.py` | Private provider configuration, task routes, consent, bounded requests and proposal schema validation. | -| `studio/catalog.py` | All 48 supplied test categories with applicability/integration requirements. | -| `studio/reports.py` | Escaped, inert HTML/JSON/ZIP evidence exports. | -| `studio/cli.py` | Shared web launcher and manifest-to-report CI command. | -| `runner/Dockerfile`, `runner/worker.py` | Trusted compiler image and isolated compile/test/coverage/mutation/fuzz/benchmark execution. | -| `runner/language_checks.py`, `runner/node_checks.cjs` | Bounded native tool adapters, source binding, fixed ESLint configuration and V8 source-map conversion. | -| `runner/sql_runtime.py`, `runner/php_coverage.php`, `runner/bash_coverage.sh`, `runner/java-unused.xml` | SQLite statement anchors, pre-compilation Xdebug hook, transient Bash trace collector and fixed PMD unused rules. | -| `runner/tools/package.json`, `runner/tools/package-lock.json` | Pinned/audited worker-only JS analyzer dependencies; not desktop runtime dependencies. | -| `ui/App.tsx` | Workspace, cases, priorities, live suggestions, proposal approval, evidence and history views. | -| `ui/main.tsx`, `ui/api.ts` | Offline editor worker registry and authenticated same-origin client/downloads. | -| `ui/demo.ts`, `ui/types.ts` | Synthetic CSV example and renderer contracts. | -| `ui/index.html`, `ui/style.css` | Local HTML entry and responsive accessible layout. | -| `desktop/main.cjs` | Sandboxed Electron renderer and private bundled-service lifecycle. | -| `training/dataset.py`, `training/train.py`, `training/serve.py` | Provenance/split validation, parameterized real training, local model API. | -| `examples/projects.py`, `examples/web-demo.project.json` | Compiler-contract fixtures and the exact synthetic CSV source/manifest used by the UI, checked in CI. | -| `examples/language_locations.py`, `tests/test_native_locations.py` | Independent polyglot unused/missed-line oracles and pure native-report parser/security regressions. | -| `examples/training.json`, `examples/training.jsonl` | Tiny rights-owned training smoke parameters and data; no quality claims. | -| `scripts/frozen_entry.py` | Frozen-service entry point for installers. | -| `scripts/verify_electron.cjs` | Fail-fast pinned Electron path/embedded-version verification; official SDK installation happens before timed native tests. | -| `scripts/worker_checks.py` | Actual isolated compiler/fuzz/security-boundary integration scenarios. | -| `runner/sanitizer_probe.py`, `scripts/worker_analyzer_checks.cjs` | Trusted same-container sanitizer-startup/fault controls and independent exact-line JS/TS checks against the separately audited worker dependency graph. | -| `scripts/private_worker_checks.py` | Real twelve-adapter execution through authenticated account API and encrypted-history reopen. | -| `scripts/license_bundle.py`, `scripts/release_archive.py` | Generated third-party notices, portable app packages and SHA-256 checksums; never package user accounts. | -| `scripts/release_gate.py`, `scripts/draft_lookup.py` | Current-main source/CodeQL prerequisites, strict authenticated numeric draft identity, embedded source/worker provenance, bounded fixed-host uploads and draft-only final verification. | -| `tests/test_release_gate.py`, `tests/test_draft_lookup.py`, `tests/node-report.test.cjs` | Independent GitHub response/policy mutation fixtures, real archive/transport negative oracles and actual descriptor-bound worker JSON read regressions. | -| `scripts/push_analysis.py` | Push generation/test/candidate workflow without automatic source promotion. | -| `tests/` | Controller/security/history/provider/metric tests, browser flows and native launch smoke. | -| `.github/workflows/` | Validation, real isolated workers, training and native OS-package workflows. | -| `pyproject.toml`, `package.json`, lockfile, TypeScript/Vite/Playwright configs | Dependency and reproducible build/test definitions. | +| Location | Responsibility | +| ------------------------------------------------------------------------------------------------------- | --------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| `studio/accounts.py`, `studio/permissions.py` | scrypt logins, session/recovery digests, bounded attempts and private per-OS data permissions. | +| `studio/private_api.py` | Account-only HTTP/CSRF boundary, own provider settings, per-session key proof, backup/recovery/deletion and scoped controller proxy. | +| `studio/private_workspace.py`, `studio/vault.py` | Single-instance ownership, account-specific engine lifecycle, authenticated streaming encryption, SQLite online snapshots and safe restoration. | +| `studio/private_cli.py` | Token-free loopback account service, dedicated data root and exact-parent graceful shutdown monitor. | +| `ui/Account.tsx` | Sign-in/unlock/recovery, contextual help, encrypted backup, private AI configuration and creator contact. | +| `studio/models.py` | Validated flat-file, project, case, preference and AI proposal schemas; size/path limits. | +| `studio/api.py` | Authenticated loopback API, bounded jobs, history, reports, comparisons and AI tasks. | +| `studio/store.py` | SQLite transactions, append-only revisions, immutable baseline oracles and interrupted-job recovery. | +| `studio/execution.py` | Fixed container flags, readiness checks, output/deadline bounds and exact owned-container cleanup. | +| `studio/analysis.py` | Non-executing source heuristics and correctness-gated weighted comparisons. | +| `studio/source_checks.py`, `studio/locations.py`, `ui/Locations.tsx` | Python syntax/unused/unreachable candidates, bounded tool-output source references, file/category filters and clickable exact-location/coverage evidence views. | +| `ui/source-navigation.ts`, `ui/SourceInspector.tsx` | Exact UTF-8 SHA-256 verification, bounded immutable source inspection, nearby-line browsing and guarded navigation to an unchanged current draft. | +| `studio/providers.py` | Private provider configuration, task routes, consent, bounded requests and proposal schema validation. | +| `studio/catalog.py` | All 48 supplied test categories with applicability/integration requirements. | +| `studio/reports.py` | Escaped, inert HTML/JSON/ZIP evidence exports. | +| `studio/cli.py` | Shared web launcher and manifest-to-report CI command. | +| `runner/Dockerfile`, `runner/worker.py` | Trusted compiler image and isolated compile/test/coverage/mutation/fuzz/benchmark execution. | +| `runner/language_checks.py`, `runner/node_checks.cjs` | Bounded native tool adapters, source binding, fixed ESLint configuration and V8 source-map conversion. | +| `runner/sql_runtime.py`, `runner/php_coverage.php`, `runner/bash_coverage.sh`, `runner/java-unused.xml` | SQLite statement anchors, pre-compilation Xdebug hook, transient Bash trace collector and fixed PMD unused rules. | +| `runner/tools/package.json`, `runner/tools/package-lock.json` | Pinned/audited worker-only JS analyzer dependencies; not desktop runtime dependencies. | +| `ui/App.tsx` | Workspace, cases, priorities, live suggestions, proposal approval, evidence and history views. | +| `ui/main.tsx`, `ui/api.ts` | Offline editor worker registry and authenticated same-origin client/downloads. | +| `ui/demo.ts`, `ui/types.ts` | Synthetic CSV example and renderer contracts. | +| `ui/index.html`, `ui/style.css` | Local HTML entry and responsive accessible layout. | +| `desktop/main.cjs` | Sandboxed Electron renderer and private bundled-service lifecycle. | +| `studio/native_server.py`, `studio/completed_offload.py` | Finite HTTP drain, completed cancellation cleanup and coherent account/vault transitions before normal sealing. | +| `training/dataset.py`, `training/train.py`, `training/serve.py` | Provenance/split validation, parameterized real training, local model API. | +| `examples/projects.py`, `examples/web-demo.project.json` | Compiler-contract fixtures and the exact synthetic CSV source/manifest used by the UI, checked in CI. | +| `examples/language_locations.py`, `tests/test_native_locations.py` | Independent polyglot unused/missed-line oracles and pure native-report parser/security regressions. | +| `examples/training.json`, `examples/training.jsonl` | Tiny rights-owned training smoke parameters and data; no quality claims. | +| `scripts/frozen_entry.py` | Frozen-service entry point for installers. | +| `scripts/verify_electron.cjs` | Fail-fast pinned Electron path/embedded-version verification; official SDK installation happens before timed native tests. | +| `scripts/worker_checks.py` | Actual isolated compiler/fuzz/security-boundary integration scenarios. | +| `runner/sanitizer_probe.py`, `scripts/worker_analyzer_checks.cjs` | Trusted same-container sanitizer-startup/fault controls and independent exact-line JS/TS checks against the separately audited worker dependency graph. | +| `scripts/private_worker_checks.py` | Real twelve-adapter execution through authenticated account API and encrypted-history reopen. | +| `scripts/license_bundle.py`, `scripts/release_archive.py` | Generated third-party notices, portable app packages and SHA-256 checksums; never package user accounts. | +| `scripts/release_gate.py`, `scripts/draft_lookup.py` | Current-main source/CodeQL prerequisites, strict authenticated numeric draft identity, embedded source/worker provenance, bounded fixed-host uploads and draft-only final verification. | +| `tests/test_release_gate.py`, `tests/test_draft_lookup.py`, `tests/node-report.test.cjs` | Independent GitHub response/policy mutation fixtures, real archive/transport negative oracles and actual descriptor-bound worker JSON read regressions. | +| `scripts/push_analysis.py` | Push generation/test/candidate workflow without automatic source promotion. | +| `tests/` | Controller/security/history/provider/metric tests, browser flows and native launch smoke. | +| `.github/workflows/` | Validation, real isolated workers, training and native OS-package workflows. | +| `pyproject.toml`, `package.json`, lockfile, TypeScript/Vite/Playwright configs | Dependency and reproducible build/test definitions. | Indexed Python/TypeScript/JavaScript headers point to the checked symbol/variable line map; the small Bash/PHP hooks describe their local variables inline. [Architecture](docs/architecture.md) and the [polyglot decision](docs/adr-polyglot-evidence.md) explain trade-offs; [testing](docs/testing.md) separates test claims from evidence. @@ -101,6 +102,11 @@ problems remain visible as failures until the actual isolated acceptance checks Historical 0.1 bearer-token CLI mode remains explicitly opt-in for existing automation; it is not account mode. No automatic import of legacy plaintext projects or global AI keys occurs. +Native shutdown uses a five-second HTTP-drain budget and completes owned disk and +vault transitions before normal sealing. Critical I/O is never abandoned to meet +a fabricated fixed exit time. The [dated native-drain report](docs/reports/native-http-drain-2026-11-10.md) +separates the actual observed defect, real cancellation/recovery controls and +local packaged evidence from later hosted/release acceptance. ## CodeQL source security review diff --git a/docs/code-index.md b/docs/code-index.md index 639166c..d34fb26 100644 --- a/docs/code-index.md +++ b/docs/code-index.md @@ -1389,6 +1389,23 @@ Generated declaration locations. Python uses AST; JS/TS uses declaration-line ma | variable | `failed` | 65 | | variable | `failed` | 70 | +## studio/completed_offload.py + +| Kind | Declaration | Line | +|---|---|---| +| function | `completed_offload(function)` | 24 | +| parameter | `args` | 24 | +| parameter | `function` | 24 | +| parameter | `kwargs` | 24 | +| function | `completed_operation(operation)` | 38 | +| parameter | `operation` | 38 | +| function | `capture()` | 45 | +| variable | `work` | 52 | +| variable | `cancelled` | 53 | +| variable | `result` | 56 | +| variable | `succeeded` | 56 | +| variable | `cancelled` | 63 | + ## studio/execution.py | Kind | Declaration | Line | @@ -1570,8 +1587,34 @@ Generated declaration locations. Python uses AST; JS/TS uses declaration-line ma | Kind | Declaration | Line | |---|---|---| -| function | `native_stage(stage)` | 17 | -| parameter | `stage` | 17 | +| function | `native_stage(stage, connections, tasks)` | 18 | +| parameter | `connections` | 18 | +| parameter | `stage` | 18 | +| parameter | `tasks` | 18 | +| variable | `counts` | 21 | +| variable | `count` | 26 | +| variable | `name` | 26 | + +## studio/native_server.py + +| Kind | Declaration | Line | +|---|---|---| +| variable | `HTTP_DRAIN_SECONDS` | 22 | +| function | `native_config(app, port)` | 25 | +| parameter | `app` | 25 | +| parameter | `port` | 25 | +| class | `NativeServer` | 34 | +| function | `shutdown(self, sockets)` | 37 | +| parameter | `self` | 37 | +| parameter | `sockets` | 37 | +| function | `stage(self, name)` | 43 | +| parameter | `name` | 43 | +| parameter | `self` | 43 | +| function | `_wait_tasks_to_complete(self)` | 51 | +| parameter | `self` | 51 | +| variable | `pending` | 63 | +| variable | `task` | 63 | +| variable | `task` | 64 | ## studio/permissions.py @@ -1727,61 +1770,65 @@ Generated declaration locations. Python uses AST; JS/TS uses declaration-line ma | parameter | `data` | 421 | | variable | `db` | 425 | | variable | `row` | 426 | -| variable | `fresh` | 432 | -| function | `account(request)` | 438 | -| parameter | `request` | 438 | -| variable | `owner` | 440 | -| variable | `session` | 440 | -| function | `unlock(request, data)` | 448 | -| parameter | `data` | 448 | -| parameter | `request` | 448 | -| variable | `owner` | 450 | -| variable | `session` | 450 | -| function | `activity(request, data)` | 459 | -| parameter | `data` | 459 | -| parameter | `request` | 459 | -| function | `save(request)` | 467 | -| parameter | `request` | 467 | -| function | `lock(request)` | 474 | -| parameter | `request` | 474 | -| variable | `owner` | 477 | -| variable | `session` | 477 | -| function | `logout(request)` | 482 | -| parameter | `request` | 482 | -| variable | `owner` | 485 | -| variable | `session` | 485 | -| variable | `response` | 489 | -| function | `delete(request, data)` | 494 | -| parameter | `data` | 494 | -| parameter | `request` | 494 | -| variable | `_` | 497 | -| variable | `owner` | 497 | -| variable | `db` | 498 | -| variable | `row` | 499 | -| variable | `response` | 505 | -| function | `backup(request)` | 510 | -| parameter | `request` | 510 | -| variable | `_` | 512 | -| variable | `owner` | 512 | -| variable | `path` | 513 | -| function | `get_providers(request)` | 521 | -| parameter | `request` | 521 | -| function | `providers(request, data)` | 528 | -| parameter | `data` | 528 | -| parameter | `request` | 528 | -| variable | `keys` | 532 | -| variable | `old` | 532 | -| variable | `rows` | 533 | -| variable | `provider` | 534 | -| variable | `row` | 535 | -| variable | `p` | 537 | -| variable | `previous` | 537 | -| variable | `config` | 542 | -| variable | `router` | 543 | -| variable | `k` | 545 | -| variable | `p` | 545 | -| variable | `v` | 545 | -| variable | `p` | 548 | +| function | `commit_reset()` | 431 | +| variable | `fresh` | 436 | +| function | `account(request)` | 442 | +| parameter | `request` | 442 | +| variable | `owner` | 444 | +| variable | `session` | 444 | +| function | `unlock(request, data)` | 452 | +| parameter | `data` | 452 | +| parameter | `request` | 452 | +| variable | `owner` | 454 | +| variable | `session` | 454 | +| function | `activity(request, data)` | 463 | +| parameter | `data` | 463 | +| parameter | `request` | 463 | +| function | `save(request)` | 471 | +| parameter | `request` | 471 | +| function | `lock(request)` | 478 | +| parameter | `request` | 478 | +| variable | `owner` | 481 | +| variable | `session` | 481 | +| function | `logout(request)` | 486 | +| parameter | `request` | 486 | +| variable | `owner` | 489 | +| variable | `session` | 489 | +| function | `commit_logout()` | 490 | +| variable | `response` | 497 | +| function | `delete(request, data)` | 502 | +| parameter | `data` | 502 | +| parameter | `request` | 502 | +| variable | `_` | 505 | +| variable | `owner` | 505 | +| variable | `db` | 506 | +| variable | `row` | 507 | +| function | `commit_delete()` | 510 | +| variable | `response` | 517 | +| function | `backup(request)` | 522 | +| parameter | `request` | 522 | +| variable | `_` | 524 | +| variable | `owner` | 524 | +| variable | `path` | 525 | +| function | `get_providers(request)` | 533 | +| parameter | `request` | 533 | +| function | `providers(request, data)` | 540 | +| parameter | `data` | 540 | +| parameter | `request` | 540 | +| variable | `keys` | 544 | +| variable | `old` | 544 | +| variable | `rows` | 545 | +| variable | `provider` | 546 | +| variable | `row` | 547 | +| variable | `p` | 549 | +| variable | `previous` | 549 | +| variable | `config` | 554 | +| variable | `router` | 555 | +| variable | `k` | 557 | +| variable | `p` | 557 | +| variable | `v` | 557 | +| variable | `p` | 560 | +| function | `commit_providers()` | 562 | ## studio/private_cli.py @@ -1805,11 +1852,11 @@ Generated declaration locations. Python uses AST; JS/TS uses declaration-line ma | variable | `port` | 76 | | variable | `app` | 77 | | variable | `server` | 84 | -| function | `shutdown()` | 97 | -| function | `run_server()` | 102 | -| variable | `thread` | 111 | -| variable | `deadline` | 115 | -| variable | `url` | 120 | +| function | `shutdown()` | 86 | +| function | `run_server()` | 91 | +| variable | `thread` | 100 | +| variable | `deadline` | 104 | +| variable | `url` | 109 | ## studio/private_workspace.py @@ -1858,33 +1905,42 @@ Generated declaration locations. Python uses AST; JS/TS uses declaration-line ma | parameter | `salt` | 150 | | parameter | `self` | 150 | | parameter | `session` | 150 | -| variable | `key` | 154 | -| variable | `current` | 155 | -| variable | `config` | 156 | -| variable | `proof` | 159 | -| variable | `nonce` | 170 | -| variable | `config` | 171 | -| variable | `keys` | 200 | -| variable | `settings` | 200 | -| function | `require(self, owner, session)` | 214 | -| parameter | `owner` | 214 | -| parameter | `self` | 214 | -| parameter | `session` | 214 | -| function | `save(self, force)` | 220 | -| parameter | `force` | 220 | -| parameter | `self` | 220 | -| variable | `current` | 228 | -| function | `remove_active(self, owner)` | 235 | -| parameter | `owner` | 235 | -| parameter | `self` | 235 | -| function | `lock(self, owner)` | 247 | -| parameter | `owner` | 247 | -| parameter | `self` | 247 | -| function | `delete(self, owner)` | 265 | -| parameter | `owner` | 265 | -| parameter | `self` | 265 | -| function | `close(self)` | 273 | +| function | `_unlock(self, owner, session, encoded, salt)` | 154 | +| parameter | `encoded` | 154 | +| parameter | `owner` | 154 | +| parameter | `salt` | 154 | +| parameter | `self` | 154 | +| parameter | `session` | 154 | +| variable | `key` | 158 | +| variable | `current` | 159 | +| variable | `config` | 160 | +| variable | `proof` | 163 | +| variable | `nonce` | 174 | +| variable | `config` | 175 | +| variable | `keys` | 204 | +| variable | `settings` | 204 | +| function | `require(self, owner, session)` | 218 | +| parameter | `owner` | 218 | +| parameter | `self` | 218 | +| parameter | `session` | 218 | +| function | `save(self, force)` | 224 | +| parameter | `force` | 224 | +| parameter | `self` | 224 | +| variable | `current` | 232 | +| function | `remove_active(self, owner)` | 239 | +| parameter | `owner` | 239 | +| parameter | `self` | 239 | +| function | `lock(self, owner)` | 251 | +| parameter | `owner` | 251 | +| parameter | `self` | 251 | +| function | `_lock(self, owner)` | 255 | +| parameter | `owner` | 255 | +| parameter | `self` | 255 | +| function | `delete(self, owner)` | 273 | +| parameter | `owner` | 273 | | parameter | `self` | 273 | +| function | `close(self)` | 281 | +| parameter | `self` | 281 | ## studio/providers.py @@ -2250,14 +2306,15 @@ Generated declaration locations. Python uses AST; JS/TS uses declaration-line ma | variable | `statuses` | 195 | | variable | `lifecycle` | 260 | | variable | `records` | 261 | -| variable | `launcher` | 262 | -| variable | `line` | 265 | -| variable | `service` | 270 | +| variable | `launcher` | 268 | | variable | `line` | 271 | -| variable | `diagnostic` | 278 | -| variable | `records` | 282 | -| variable | `stage` | 283 | -| variable | `pid` | 294 | +| variable | `service` | 276 | +| variable | `line` | 277 | +| variable | `diagnostic` | 284 | +| variable | `records` | 288 | +| variable | `stage` | 289 | +| variable | `record` | 301 | +| variable | `pid` | 314 | ## tests/electron-readiness.test.cjs @@ -2445,6 +2502,31 @@ Generated declaration locations. Python uses AST; JS/TS uses declaration-line ma | variable | `_` | 107 | | variable | `saved` | 108 | +## tests/test_completed_offload.py + +| Kind | Declaration | Line | +|---|---|---| +| function | `test_native_cancellation_cannot_release_mutex_before_actual_worker(cancels, fail)` | 24 | +| parameter | `cancels` | 24 | +| parameter | `fail` | 24 | +| function | `offload_scenario(cancels, fail)` | 29 | +| parameter | `cancels` | 29 | +| parameter | `fail` | 29 | +| variable | `release` | 31 | +| variable | `started` | 31 | +| variable | `mutex` | 32 | +| variable | `events` | 33 | +| variable | `unhandled` | 33 | +| variable | `loop` | 34 | +| variable | `original_handler` | 35 | +| parameter | `_loop` | 36 | +| parameter | `context` | 36 | +| function | `worker()` | 38 | +| function | `caller()` | 48 | +| variable | `request` | 56 | +| variable | `_` | 59 | +| variable | `outcome` | 68 | + ## tests/test_contracts.py | Kind | Declaration | Line | @@ -2751,6 +2833,95 @@ Generated declaration locations. Python uses AST; JS/TS uses declaration-line ma | variable | `data` | 327 | | variable | `result` | 333 | +## tests/test_native_drain.py + +| Kind | Declaration | Line | +|---|---|---| +| function | `test_actual_native_drain_preserves_authenticated_workspace(tmp_path, project, kind, monkeypatch)` | 33 | +| parameter | `kind` | 33 | +| parameter | `monkeypatch` | 33 | +| parameter | `project` | 33 | +| parameter | `tmp_path` | 33 | +| function | `native_scenario(tmp_path, project, kind, monkeypatch)` | 38 | +| parameter | `kind` | 38 | +| parameter | `monkeypatch` | 38 | +| parameter | `project` | 38 | +| parameter | `tmp_path` | 38 | +| variable | `listener` | 40 | +| variable | `port` | 44 | +| variable | `base` | 45 | +| variable | `token` | 46 | +| variable | `key` | 47 | +| variable | `app` | 48 | +| variable | `manager` | 49 | +| variable | `_` | 50 | +| variable | `budget` | 50 | +| variable | `entered` | 50 | +| variable | `ready` | 50 | +| variable | `release` | 50 | +| variable | `cancellation_deferred` | 51 | +| variable | `_` | 52 | +| variable | `thread_entered` | 52 | +| variable | `thread_finished` | 52 | +| variable | `thread_release` | 52 | +| variable | `counts` | 53 | +| variable | `events` | 53 | +| variable | `original_stage` | 54 | +| function | `observe_offload(stage)` | 56 | +| parameter | `stage` | 56 | +| function | `held(request)` | 65 | +| parameter | `request` | 65 | +| variable | `original_save` | 76 | +| variable | `original_decrypt` | 77 | +| function | `held_save()` | 79 | +| parameter | `args` | 79 | +| variable | `result` | 86 | +| function | `held_decrypt()` | 92 | +| parameter | `args` | 92 | +| variable | `result` | 98 | +| function | `observed_app(scope, receive, send)` | 104 | +| parameter | `receive` | 104 | +| parameter | `scope` | 104 | +| parameter | `send` | 104 | +| variable | `partial` | 108 | +| function | `observe_receive()` | 110 | +| variable | `message` | 114 | +| variable | `partial` | 116 | +| class | `ObservedServer` | 121 | +| function | `startup(self, sockets)` | 124 | +| parameter | `self` | 124 | +| parameter | `sockets` | 124 | +| function | `stage(self, name)` | 129 | +| parameter | `name` | 129 | +| parameter | `self` | 129 | +| variable | `server` | 137 | +| variable | `running` | 139 | +| variable | `pending` | 140 | +| variable | `writer` | 141 | +| variable | `client` | 147 | +| variable | `response` | 148 | +| variable | `response` | 152 | +| variable | `response` | 156 | +| variable | `owner` | 158 | +| variable | `revision` | 158 | +| variable | `archive` | 159 | +| variable | `_` | 161 | +| variable | `writer` | 161 | +| variable | `cookie` | 162 | +| variable | `name` | 162 | +| variable | `value` | 162 | +| variable | `pending` | 173 | +| variable | `pending` | 179 | +| variable | `pending` | 182 | +| variable | `response` | 185 | +| variable | `response` | 199 | +| variable | `_` | 205 | +| variable | `name` | 205 | +| variable | `tasks` | 205 | +| variable | `restored` | 213 | +| function | `test_private_cli_selects_finite_http_drain()` | 234 | +| variable | `config` | 237 | + ## tests/test_native_lifecycle.py | Kind | Declaration | Line | @@ -2761,13 +2932,18 @@ Generated declaration locations. Python uses AST; JS/TS uses declaration-line ma | variable | `payload` | 29 | | variable | `prefix` | 29 | | variable | `value` | 31 | -| function | `test_actual_launcher_shutdown_denials_cannot_invoke_callback(tmp_path)` | 37 | -| parameter | `tmp_path` | 37 | -| variable | `app` | 39 | -| variable | `called` | 40 | -| variable | `headers` | 42 | -| variable | `client` | 43 | -| variable | `response` | 50 | +| function | `test_native_stage_counts_are_only_nonnegative_integers(monkeypatch, capsys)` | 37 | +| parameter | `capsys` | 37 | +| parameter | `monkeypatch` | 37 | +| variable | `value` | 41 | +| variable | `value` | 44 | +| function | `test_actual_launcher_shutdown_denials_cannot_invoke_callback(tmp_path)` | 48 | +| parameter | `tmp_path` | 48 | +| variable | `app` | 50 | +| variable | `called` | 51 | +| variable | `headers` | 53 | +| variable | `client` | 54 | +| variable | `response` | 61 | ## tests/test_native_locations.py @@ -2957,6 +3133,51 @@ Generated declaration locations. Python uses AST; JS/TS uses declaration-line ma | parameter | `project` | 334 | | variable | `identity` | 340 | +## tests/test_private_cancellation.py + +| Kind | Declaration | Line | +|---|---|---| +| function | `test_cancelled_authorized_transition_is_completed_under_request_mutex(tmp_path, monkeypatch, kind)` | 29 | +| parameter | `kind` | 29 | +| parameter | `monkeypatch` | 29 | +| parameter | `tmp_path` | 29 | +| function | `transition_scenario(tmp_path, monkeypatch, kind)` | 34 | +| parameter | `kind` | 34 | +| parameter | `monkeypatch` | 34 | +| parameter | `tmp_path` | 34 | +| variable | `base` | 36 | +| variable | `password` | 37 | +| variable | `key` | 38 | +| variable | `app` | 39 | +| variable | `manager` | 40 | +| variable | `entered` | 41 | +| variable | `release` | 41 | +| variable | `deferred` | 42 | +| variable | `original_stage` | 43 | +| function | `stage(name)` | 45 | +| parameter | `name` | 45 | +| variable | `request` | 51 | +| variable | `client` | 55 | +| variable | `owner` | 60 | +| variable | `archive` | 61 | +| variable | `config` | 62 | +| variable | `original` | 67 | +| function | `held_write()` | 69 | +| parameter | `args` | 69 | +| variable | `request` | 77 | +| variable | `original` | 79 | +| function | `held_delete()` | 81 | +| parameter | `args` | 81 | +| variable | `request` | 89 | +| variable | `disk` | 99 | +| variable | `router` | 100 | +| variable | `name` | 104 | +| variable | `value` | 104 | +| variable | `row` | 105 | +| variable | `public` | 108 | +| variable | `restored` | 111 | +| variable | `db` | 115 | + ## tests/test_providers.py | Kind | Declaration | Line | diff --git a/docs/private-accounts.md b/docs/private-accounts.md index 3e6b2d8..2d4683a 100644 --- a/docs/private-accounts.md +++ b/docs/private-accounts.md @@ -16,11 +16,11 @@ Import flat source/data files or start from the clearly synthetic CSV draft. Cho Default roots: -| OS | Dedicated private root | -|---|---| -| Windows | `%LOCALAPPDATA%/CodeFlowStudioAccounts` | -| macOS | `~/Library/Application Support/CodeFlowStudioAccounts` | -| Linux | `$XDG_DATA_HOME/CodeFlowStudioAccounts` or `~/.local/share/CodeFlowStudioAccounts` | +| OS | Dedicated private root | +| ------- | ---------------------------------------------------------------------------------- | +| Windows | `%LOCALAPPDATA%/CodeFlowStudioAccounts` | +| macOS | `~/Library/Application Support/CodeFlowStudioAccounts` | +| Linux | `$XDG_DATA_HOME/CodeFlowStudioAccounts` or `~/.local/share/CodeFlowStudioAccounts` | The native launcher has a separate `CodeFlowStudioLauncher` directory for nonsecret startup logs. The new account root never opens or migrates older capstone/native/developer history. A trusted operator can explicitly override `CFS_ACCOUNT_DATA` to an empty dedicated directory; this is not a browser-selectable path. A held OS advisory lock prevents two services sharing it. @@ -61,6 +61,14 @@ python -m studio.private_cli The printed URL is nonsecret. Ctrl+C requests sealing. Legacy `python -m studio.cli serve` intentionally remains a different bearer-token developer mode; do not expose either service to a network. +Normal native shutdown stops accepting requests and gives existing HTTP work five +seconds to drain. Remaining HTTP tasks are cancelled and their owned cleanup is +joined before vault sealing. Already-started local disk/account transitions finish +under the request mutex before cancellation is reported; arbitrary network calls +and request-body reads remain cancellable. Slow or failed critical I/O can still +delay completion: preserve recovery data and never force-delete an active account +directory. See the [actual cancellation and native acceptance report](reports/native-http-drain-2026-11-10.md). + Limits: 20 flat files/1 MiB project; 192 KiB browser file import; 64 cases; 4 MiB HTTP request; 500 revisions/128 MiB history; 512 MiB/2,000 archived workspace files; eight providers; one unlocked engine; bounded worker output/deadlines/resources. Unexpected/unsupported formats fail explicitly. Run hostile unknown code in a dedicated disposable VM with independent review; containers are not a certified hostile multi-tenant boundary. The 128 MiB history threshold stops new revisions and run queues. A 160 MiB recovery reserve keeps existing history readable after a final write/in-flight result crosses the soft threshold. Reads, backups, lock/reopen and existing report exports remain available; no history is automatically erased to satisfy a quota. diff --git a/docs/reports/native-http-drain-2026-11-10.md b/docs/reports/native-http-drain-2026-11-10.md new file mode 100644 index 0000000..2627be2 --- /dev/null +++ b/docs/reports/native-http-drain-2026-11-10.md @@ -0,0 +1,79 @@ +# Native HTTP drain and coherent local cancellation — 2026-11-10 + +By Jason Stys. + +## Observed defect and scope + +The [Windows native job](https://github.com/JasonStys/code-flow-studio/actions/runs/42796136482/job/129761676463) +at documentation PR head `7fc7693aba990925d35010fc0644108644e8eb3c` completed account, +editor, privacy and explicit workspace-lock checks. Its actual shutdown response +was HTTP 200 with `stopping: true`, and the frozen process logged the shutdown +callback. It did not reach the lifespan-close milestone before the unchanged +30-second launcher wait. Acknowledgement was not proof of service termination. + +The pinned Uvicorn 0.38 server was using its default unlimited graceful HTTP drain. +Real owned-loopback experiments reproduced that pre-lifespan wait with both a held +handler and an incomplete POST body. A finite drain cancels those requests and +allows normal vault sealing. These controls establish the mechanism; they do not +identify which request remained open in the historical Windows failure. + +## Correction + +The account entry point now uses a five-second HTTP-drain budget. `NativeServer` +retains Uvicorn's listener, connection and normal lifespan handling. At drain +expiry it cancels and joins its own remaining HTTP tasks before lifespan shutdown. +There is no `force_exit`, process termination, skipped sealing, deadline increase +or test retry. + +Raw `asyncio.Task.cancel()` can otherwise release an awaiting request's mutex while +Starlette/AnyIO thread work continues. Retained shielded operations now finish real +disk work before propagating cancellation, including repeated cancellation. +Unlock and lock complete their full marker/controller/key and seal/remove transitions. +Provider publication completes disk settings, actual in-memory destinations/keys +and snapshot together; authorized deletion, logout and recovery reset also finish +their local transitions under the existing request mutex. Network requests, AI +calls and request-body reads are not made uncancellable. + +The HTTP budget is not a universal process-exit deadline: critical disk work, +owned-container cleanup and authenticated sealing must finish safely. A stalled +disk or unavailable cleanup can still prevent clean termination; recovery data +must be preserved rather than forcibly deleted. + +## Verification + +The real HTTP tests create and authenticate fictional accounts, retain an actual +revision, and hold a handler, partial body, real snapshot or authenticated restore. +At genuine cancellation they verify the request mutex remains held until offloaded +work completes. Normal lifespan then seals the workspace, removes plaintext and +the owner marker, clears keys, and restores the exact revision from its real +AES-GCM archive. Separate repeated-cancel/success/error controls check actual +thread completion. Real provider/deletion tests check consent/destination and +data/account-row coherence after cancellation. + +Local Windows validation used Python 3.14.6, pinned Uvicorn 0.38, Electron 44.10.2 +and Playwright 1.64.1. The full controller/training gate, original coverage floor, +strict TypeScript build, three-engine controller/account journeys and native +acceptance remain required. Results and counts are recorded in the integration +handoff rather than inferred from source inspection. + +A freshly frozen service passed the actual Electron SDK journey. The freshly +compiled unpacked Windows application passed the unchanged account/editor/privacy/ +lock journey in 6.2 seconds. It reported six connections and zero tasks before +normal drain, then zero of each afterward, completed workspace/lifespan/server/ +main milestones, and both owned Python-service and bootloader PIDs had exited. +This is local dirty-build evidence, not a published release or clean-machine +installation claim. Exact-source hosted checks on all three operating systems, +full-main CodeQL and later fresh draft/download acceptance remain separate gates. + +## References and retained boundaries + +- [Pinned Uvicorn shutdown implementation](https://github.com/encode/uvicorn/blob/0.38.0/uvicorn/server.py) + drains connections/tasks before lifespan shutdown. +- [Python cancellation and shielding](https://docs.python.org/3.14/library/asyncio-task.html#shielding-from-cancellation) + distinguishes a protected child from its still-cancelled caller and requires + retaining a strong task reference. +- [Private account operation](../private-accounts.md) describes local trust, + recovery and the required container-only compiler boundary. + +No user account, installed application, existing public release, Docker engine, +source-consent rule or worker-isolation permission was changed by these tests. diff --git a/studio/completed_offload.py b/studio/completed_offload.py new file mode 100644 index 0000000..98b960b --- /dev/null +++ b/studio/completed_offload.py @@ -0,0 +1,74 @@ +# +# Generated definition map; @L is the exact declaration line. +# Function/class docs explain behavior; variables/parameters are scoped inputs, outputs or state. +# functions: completed_offload(function)@L24; completed_operation(operation)@L38; capture()@L45 +# variables: work@L52; cancelled@L53; result@L56; succeeded@L56; cancelled@L63 +# parameters: args@L24; function@L24; kwargs@L24; operation@L38 +# + +"""Jason Stys: complete owned blocking work before propagating HTTP-task cancellation. + +Symbol locations: docs/code-index.md. Never release a surrounding vault/request +mutex while its worker can still read or replace the protected filesystem state. +""" + +import asyncio +from collections.abc import Awaitable, Callable +from typing import cast + +from starlette.concurrency import run_in_threadpool + +from studio.native_lifecycle import native_stage + + +async def completed_offload[**P, T](function: Callable[P, T], *args: P.args, **kwargs: P.kwargs) -> T: + """Defer even repeated outer Task.cancel until the real owned thread operation finishes. + + Starlette/AnyIO's default cancel scope does not protect against raw asyncio + Task.cancel. A separately retained shielded task keeps the caller inside its + existing mutex. Worker errors are transported as data to avoid Python 3.14 + shield logging a child failure after outer cancellation; the sole caller + then receives the actual error or the original cancellation with its cause. + This helper never cancels its child or uses a timeout to abandon disk work. + """ + + return await completed_operation(run_in_threadpool(function, *args, **kwargs)) + + +async def completed_operation[T](operation: Awaitable[T]) -> T: + """Finish one bounded owned state transition before propagating outer cancellation. + + Use only for disk work or coherent workspace transitions, never an unbounded + network request/body. The surrounding caller retains its existing mutex. + """ + + async def capture() -> tuple[bool, T | Exception]: + """Return ordinary errors as data for the sole awaiting task, without shield child-error logging.""" + try: + return True, await operation + except Exception as error: + return False, error + + work = asyncio.create_task(capture()) + cancelled = None + while True: + try: + succeeded, result = await asyncio.shield(work) + break + except asyncio.CancelledError as error: + if work.cancelled(): + raise RuntimeError("Owned offload was independently cancelled") from error + if cancelled is None: + native_stage("offload-cancellation-deferred") + cancelled = cancelled or error + except BaseException as error: + if cancelled is not None: + raise cancelled from error + raise + if cancelled is not None: + if not succeeded: + raise cancelled from cast(Exception, result) + raise cancelled + if not succeeded: + raise cast(Exception, result) + return cast(T, result) diff --git a/studio/native_lifecycle.py b/studio/native_lifecycle.py index 0f8c31a..1e501fc 100644 --- a/studio/native_lifecycle.py +++ b/studio/native_lifecycle.py @@ -1,8 +1,9 @@ # # Generated definition map; @L is the exact declaration line. # Function/class docs explain behavior; variables/parameters are scoped inputs, outputs or state. -# functions: native_stage(stage)@L17 -# parameters: stage@L17 +# functions: native_stage(stage, connections, tasks)@L18 +# variables: counts@L21; count@L26; name@L26 +# parameters: connections@L18; stage@L18; tasks@L18 # """Jason Stys: opt-in native-test lifecycle milestones containing no account or source values. @@ -14,11 +15,15 @@ import os -def native_stage(stage: str) -> None: +def native_stage(stage: str, *, connections: int | None = None, tasks: int | None = None) -> None: """Expose a fixed caller-supplied milestone and owned process identities only in native test mode.""" if os.environ.get("STUDIO_DESKTOP_TEST") == "1": + counts = {"connections": connections, "tasks": tasks} print( "NATIVE_SERVICE_STAGE", - json.dumps({"stage": stage, "pid": os.getpid(), "parent_pid": os.getppid()}), + json.dumps({ + "stage": stage, "pid": os.getpid(), "parent_pid": os.getppid(), + **{name: count for name, count in counts.items() if type(count) is int and count >= 0}, + }), flush=True, ) diff --git a/studio/native_server.py b/studio/native_server.py new file mode 100644 index 0000000..7fa2a92 --- /dev/null +++ b/studio/native_server.py @@ -0,0 +1,69 @@ +# +# Generated definition map; @L is the exact declaration line. +# Function/class docs explain behavior; variables/parameters are scoped inputs, outputs or state. +# classes: NativeServer@L34 +# functions: native_config(app, port)@L25; shutdown(self, sockets)@L37; stage(self, name)@L43; _wait_tasks_to_complete(self)@L51 +# variables: HTTP_DRAIN_SECONDS@L22; pending@L63; task@L63; task@L64 +# parameters: app@L25; port@L25; self@L37; sockets@L37; name@L43; self@L43; self@L51 +# + +"""Jason Stys: finite HTTP drain followed by completed request cleanup and normal vault sealing. + +Symbol locations: docs/code-index.md. Pinned Uvicorn owns sockets and lifespan; +this subclass only joins its own canceled HTTP tasks before the lifespan can close. +""" + +import asyncio + +import uvicorn + +from studio.native_lifecycle import native_stage + +HTTP_DRAIN_SECONDS = 5.0 + + +def native_config(app, port: int) -> uvicorn.Config: + """Use identical literal-loopback and bounded-concurrency configuration in the shipping app and real tests.""" + return uvicorn.Config( + app, host="127.0.0.1", port=port, access_log=False, log_level="warning", + proxy_headers=False, limit_concurrency=50, timeout_keep_alive=5, + timeout_graceful_shutdown=HTTP_DRAIN_SECONDS, + ) + + +class NativeServer(uvicorn.Server): + """Bound idle/held HTTP work without abandoning active account filesystem operations.""" + + async def shutdown(self, sockets=None): + """Keep Uvicorn's normal listener/connection/lifespan cleanup, with value-free stage counts.""" + self.stage("http-drain-started") + await super().shutdown(sockets=sockets) + self.stage("http-drain-complete") + + def stage(self, name): + """Observe counts only; no request routes, account data, source or authentication values.""" + native_stage( + name, + connections=len(self.server_state.connections), + tasks=len(self.server_state.tasks), + ) + + async def _wait_tasks_to_complete(self): + """On the configured drain timeout, finish cancellation cleanup before normal lifespan sealing. + + Uvicorn 0.38 otherwise cancels HTTP tasks and immediately starts lifespan + shutdown. Joining those exact tasks lets protected offloads finish while + their request locks are still held. Disk operations retain data safety over + a fabricated fixed process-exit guarantee; no task is killed or detached. + """ + try: + await super()._wait_tasks_to_complete() + except asyncio.CancelledError: + self.stage("http-drain-budget-exhausted") + pending = tuple(task for task in self.server_state.tasks if not task.done()) + for task in pending: + task.cancel("Native HTTP drain deadline reached; complete owned cleanup") + if pending: + await asyncio.gather(*pending, return_exceptions=True) + self.stage("http-request-cleanup-complete") + raise diff --git a/studio/private_api.py b/studio/private_api.py index e8a3e43..98c1495 100644 --- a/studio/private_api.py +++ b/studio/private_api.py @@ -2,9 +2,9 @@ # Generated definition map; @L is the exact declaration line. # Function/class docs explain behavior; variables/parameters are scoped inputs, outputs or state. # classes: Boundary@L42; Closed@L142; Credentials@L148; Recovery@L155; Unlock@L161; Activity@L168; Delete@L174; Provider@L181; Settings@L194; Proxy@L228 -# functions: __init__(self, app, manager, port)@L45; __call__(self, scope, receive, send)@L51; guarded(message)@L58; deny(status, detail)@L79; replay()@L131; coherent(self)@L203; public_settings(manager)@L215; __init__(self, manager)@L231; __call__(self, scope, receive, send)@L235; create_private_app(root, port, ui, launcher_token, nonce)@L286; lifespan(app)@L294; idle()@L297; invalid(request, error)@L337; bad_value(request, error)@L342; bad_disk(request, error)@L347; identify(request)@L353; signed(owner, data)@L357; auth_limit(alias)@L370; health()@L378; shutdown(request)@L389; register(data)@L402; login(data)@L412; reset(data)@L421; account(request)@L438; unlock(request, data)@L448; activity(request, data)@L459; save(request)@L467; lock(request)@L474; logout(request)@L482; delete(request, data)@L494; backup(request)@L510; get_providers(request)@L521; providers(request, data)@L528 -# variables: CSP@L38; HISTORY_STOP_BYTES@L39; raw@L55; headers@L56; k@L56; v@L56; k@L63; v@L63; _@L85; k@L85; cookies@L103; token@L108; account@L109; chunks@L117; length@L117; message@L119; data@L122; length@L123; delivered@L129; delivered@L135; model_config@L145; alias@L151; password@L152; recovery@L158; key@L164; salt@L165; idle_seconds@L171; password@L177; confirm@L178; id@L184; kind@L185; url@L186; model@L187; key@L188; retain_key@L189; token_parameter@L190; temperature@L191; providers@L197; routes@L198; ids@L205; p@L205; value@L206; p@L209; keys@L217; settings@L217; data@L218; p@L220; manager@L239; app@L242; path@L247; db@L253; count@L254; headers@L266; forwarded@L275; k@L276; v@L276; manager@L290; task@L311; app@L325; response@L359; callback@L395; owner@L406; recovery@L406; owner@L415; db@L425; row@L426; fresh@L432; owner@L440; session@L440; owner@L450; session@L450; owner@L477; session@L477; owner@L485; session@L485; response@L489; _@L497; owner@L497; db@L498; row@L499; response@L505; _@L512; owner@L512; path@L513; keys@L532; old@L532; rows@L533; provider@L534; row@L535; p@L537; previous@L537; config@L542; router@L543; k@L545; p@L545; v@L545; p@L548 -# parameters: app@L45; manager@L45; port@L45; self@L45; receive@L51; scope@L51; self@L51; send@L51; message@L58; detail@L79; status@L79; self@L203; manager@L215; manager@L231; self@L231; receive@L235; scope@L235; self@L235; send@L235; launcher_token@L287; nonce@L287; port@L287; root@L287; ui@L287; app@L294; error@L337; request@L337; error@L342; request@L342; error@L347; request@L347; request@L353; data@L357; owner@L357; alias@L370; request@L389; data@L402; data@L412; data@L421; request@L438; data@L448; request@L448; data@L459; request@L459; request@L467; request@L474; request@L482; data@L494; request@L494; request@L510; request@L521; data@L528; request@L528 +# functions: __init__(self, app, manager, port)@L45; __call__(self, scope, receive, send)@L51; guarded(message)@L58; deny(status, detail)@L79; replay()@L131; coherent(self)@L203; public_settings(manager)@L215; __init__(self, manager)@L231; __call__(self, scope, receive, send)@L235; create_private_app(root, port, ui, launcher_token, nonce)@L286; lifespan(app)@L294; idle()@L297; invalid(request, error)@L337; bad_value(request, error)@L342; bad_disk(request, error)@L347; identify(request)@L353; signed(owner, data)@L357; auth_limit(alias)@L370; health()@L378; shutdown(request)@L389; register(data)@L402; login(data)@L412; reset(data)@L421; commit_reset()@L431; account(request)@L442; unlock(request, data)@L452; activity(request, data)@L463; save(request)@L471; lock(request)@L478; logout(request)@L486; commit_logout()@L490; delete(request, data)@L502; commit_delete()@L510; backup(request)@L522; get_providers(request)@L533; providers(request, data)@L540; commit_providers()@L562 +# variables: CSP@L38; HISTORY_STOP_BYTES@L39; raw@L55; headers@L56; k@L56; v@L56; k@L63; v@L63; _@L85; k@L85; cookies@L103; token@L108; account@L109; chunks@L117; length@L117; message@L119; data@L122; length@L123; delivered@L129; delivered@L135; model_config@L145; alias@L151; password@L152; recovery@L158; key@L164; salt@L165; idle_seconds@L171; password@L177; confirm@L178; id@L184; kind@L185; url@L186; model@L187; key@L188; retain_key@L189; token_parameter@L190; temperature@L191; providers@L197; routes@L198; ids@L205; p@L205; value@L206; p@L209; keys@L217; settings@L217; data@L218; p@L220; manager@L239; app@L242; path@L247; db@L253; count@L254; headers@L266; forwarded@L275; k@L276; v@L276; manager@L290; task@L311; app@L325; response@L359; callback@L395; owner@L406; recovery@L406; owner@L415; db@L425; row@L426; fresh@L436; owner@L444; session@L444; owner@L454; session@L454; owner@L481; session@L481; owner@L489; session@L489; response@L497; _@L505; owner@L505; db@L506; row@L507; response@L517; _@L524; owner@L524; path@L525; keys@L544; old@L544; rows@L545; provider@L546; row@L547; p@L549; previous@L549; config@L554; router@L555; k@L557; p@L557; v@L557; p@L560 +# parameters: app@L45; manager@L45; port@L45; self@L45; receive@L51; scope@L51; self@L51; send@L51; message@L58; detail@L79; status@L79; self@L203; manager@L215; manager@L231; self@L231; receive@L235; scope@L235; self@L235; send@L235; launcher_token@L287; nonce@L287; port@L287; root@L287; ui@L287; app@L294; error@L337; request@L337; error@L342; request@L342; error@L347; request@L347; request@L353; data@L357; owner@L357; alias@L370; request@L389; data@L402; data@L412; data@L421; request@L442; data@L452; request@L452; data@L463; request@L463; request@L471; request@L478; request@L486; data@L502; request@L502; request@L522; request@L533; data@L540; request@L540 # """Jason Stys: loopback-only real accounts, session-proven vaults, private AI settings and scoped controller. @@ -27,10 +27,10 @@ from fastapi.responses import FileResponse, JSONResponse from fastapi.staticfiles import StaticFiles from pydantic import BaseModel, ConfigDict, Field, model_validator -from starlette.concurrency import run_in_threadpool from studio import __version__ from studio.accounts import digest, password_matches +from studio.completed_offload import completed_offload, completed_operation from studio.native_lifecycle import native_stage from studio.private_workspace import Workspaces, atomic_json from studio.providers import Providers @@ -91,7 +91,7 @@ async def deny(status, detail): or headers.get("sec-fetch-site") == "cross-site" ): return await deny(403, "Untrusted origin") - if not await run_in_threadpool( + if not await completed_offload( self.manager.accounts.throttle, "request:" + str(scope.get("client", ("local",))[0]), 1000 ): return await deny(429, "Too many requests; wait before retrying") @@ -106,7 +106,7 @@ async def deny(status, detail): except CookieError: return await deny(401, "Invalid session") token = cookies[self.cookie].value if self.cookie in cookies else "" - account = await run_in_threadpool(self.manager.accounts.owner, token) if token else None + account = await completed_offload(self.manager.accounts.owner, token) if token else None scope.setdefault("state", {}).update(account=account, token=token, session=digest(token)) if ( scope["path"].startswith("/private/") @@ -278,7 +278,7 @@ async def __call__(self, scope, receive, send): ] await app(forwarded, receive, send) try: - await run_in_threadpool(manager.save) + await completed_offload(manager.save) except (OSError, ValueError): manager.failed = True @@ -304,7 +304,7 @@ async def idle(): if time.monotonic() - manager.last_human > manager.idle_seconds: await manager.lock(manager.owner) else: - await run_in_threadpool(manager.save) + await completed_offload(manager.save) except (OSError, ValueError, sqlite3.Error): manager.failed = True @@ -369,7 +369,7 @@ def signed(owner, data): async def auth_limit(alias): """Persist global/alias auth throttles without logging submitted secrets.""" - if not await run_in_threadpool(manager.accounts.throttle, "auth", 40) or not await run_in_threadpool( + if not await completed_offload(manager.accounts.throttle, "auth", 40) or not await completed_offload( manager.accounts.throttle, "alias:" + alias, 10 ): raise HTTPException(429, "Too many attempts; wait five minutes") @@ -403,7 +403,7 @@ async def register(data: Credentials): """Create empty real accounts and show a random recovery code once, with no seeded/demo passwords.""" await auth_limit(data.alias) try: - owner, recovery = await run_in_threadpool(manager.accounts.register, data.alias, data.password) + owner, recovery = await completed_offload(manager.accounts.register, data.alias, data.password) except sqlite3.IntegrityError: raise HTTPException(409, "Alias unavailable") from None return signed(owner, {"recovery": recovery}) @@ -412,7 +412,7 @@ async def register(data: Credentials): async def login(data: Credentials): """Authenticate a password without unlocking its independent source vault.""" await auth_limit(data.alias) - owner = await run_in_threadpool(manager.accounts.login, data.alias, data.password) + owner = await completed_offload(manager.accounts.login, data.alias, data.password) if not owner: raise HTTPException(401, "Alias or password is incorrect") return signed(owner, {"signed_in": True}) @@ -428,8 +428,12 @@ async def reset(data: Recovery): ).fetchone() if not row or not secrets.compare_digest(row["recovery"], digest(data.recovery)): raise HTTPException(401, "Recovery credentials invalid") - await manager.lock(row["id"]) - fresh = await run_in_threadpool(manager.accounts.reset, data.alias, data.recovery, data.password) + async def commit_reset(): + """Complete the authorized seal and recovery-code replacement as one local transition.""" + await manager.lock(row["id"]) + return await completed_offload(manager.accounts.reset, data.alias, data.recovery, data.password) + + fresh = await completed_operation(commit_reset()) if not fresh: raise HTTPException(409, "Recovery credentials changed") return {"recovery": fresh} @@ -448,7 +452,7 @@ def account(request: Request): async def unlock(request: Request, data: Unlock): """Prove this session's key and intentionally reset the human-idle clock.""" owner, session = identify(request) - if not await run_in_threadpool(manager.accounts.throttle, "unlock:" + owner, 10): + if not await completed_offload(manager.accounts.throttle, "unlock:" + owner, 10): raise HTTPException(429, "Too many unlock attempts; wait five minutes") async with manager.requests: await manager.unlock(owner, session, data.key, data.salt) @@ -468,7 +472,7 @@ async def save(request: Request): """Explicit complete encrypted backup requires no active compiler jobs.""" async with manager.requests: manager.require(*identify(request)) - return await run_in_threadpool(manager.save, True) + return await completed_offload(manager.save, True) @app.post("/private/lock") async def lock(request: Request): @@ -483,9 +487,13 @@ async def logout(request: Request): """Seal a proven account or revoke an unproven login; never cancel another account's jobs.""" async with manager.requests: owner, session = identify(request) - if manager.owner == owner and session in manager.sessions: - await manager.lock(owner) - await run_in_threadpool(manager.accounts.revoke, request.state.token) + async def commit_logout(): + """Complete authorized local sealing and token revocation before acknowledging cancellation.""" + if manager.owner == owner and session in manager.sessions: + await manager.lock(owner) + await completed_offload(manager.accounts.revoke, request.state.token) + + await completed_operation(commit_logout()) response = JSONResponse({"signed_out": True}) response.delete_cookie(manager.cookie, path="/private") return response @@ -497,10 +505,14 @@ async def delete(request: Request, data: Delete): owner, _ = identify(request) with manager.accounts.connection() as db: row = db.execute("SELECT password FROM accounts WHERE id=?", (owner,)).fetchone() - if not row or not await run_in_threadpool(password_matches, data.password, row[0]): + if not row or not await completed_offload(password_matches, data.password, row[0]): raise HTTPException(401, "Current password incorrect") - await manager.delete(owner) - if not await run_in_threadpool(manager.accounts.delete, owner, data.password): + async def commit_delete(): + """Keep authorized data removal and account-row removal coherent under the same request mutex.""" + await manager.delete(owner) + return await completed_offload(manager.accounts.delete, owner, data.password) + + if not await completed_operation(commit_delete()): raise HTTPException(409, "Account changed; sign in again") response = JSONResponse({"deleted": True}) response.delete_cookie(manager.cookie, path="/private") @@ -547,10 +559,14 @@ async def providers(request: Request, data: Settings): }, credentials={p["id"]: p["key"] for p in rows}, ) - await run_in_threadpool(atomic_json, manager.active / "providers.private.json", config) - manager.app.state.providers.config = router.config - manager.app.state.providers.credentials = router.credentials - await run_in_threadpool(manager.save) + async def commit_providers(): + """Publish matching disk/in-memory destination/consent state and snapshot before cancellation.""" + await completed_offload(atomic_json, manager.active / "providers.private.json", config) + manager.app.state.providers.config = router.config + manager.app.state.providers.credentials = router.credentials + await completed_offload(manager.save) + + await completed_operation(commit_providers()) return public_settings(manager) app.mount("/private/api", Proxy(manager)) diff --git a/studio/private_cli.py b/studio/private_cli.py index 8ffba9c..0cfbe72 100644 --- a/studio/private_cli.py +++ b/studio/private_cli.py @@ -1,8 +1,8 @@ # # Generated definition map; @L is the exact declaration line. # Function/class docs explain behavior; variables/parameters are scoped inputs, outputs or state. -# functions: data_directory()@L27; parent_guard(server)@L40; main()@L61; shutdown()@L97; run_server()@L102 -# variables: base@L32; base@L34; base@L36; pid@L42; started@L43; parent@L47; parser@L63; args@L68; bundle@L71; ui@L72; listener@L73; port@L76; app@L77; server@L84; thread@L111; deadline@L115; url@L120 +# functions: data_directory()@L27; parent_guard(server)@L40; main()@L61; shutdown()@L86; run_server()@L91 +# variables: base@L32; base@L34; base@L36; pid@L42; started@L43; parent@L47; parser@L63; args@L68; bundle@L71; ui@L72; listener@L73; port@L76; app@L77; server@L84; thread@L100; deadline@L104; url@L109 # parameters: server@L40 # @@ -18,9 +18,9 @@ from pathlib import Path import psutil -import uvicorn from studio.native_lifecycle import native_stage +from studio.native_server import NativeServer, native_config from studio.private_api import create_private_app @@ -81,18 +81,7 @@ def main(): os.environ.get("CFS_LAUNCH_TOKEN", ""), os.environ.get("CFS_LAUNCH_NONCE", ""), ) - server = uvicorn.Server( - uvicorn.Config( - app, - host="127.0.0.1", - port=port, - access_log=False, - log_level="warning", - proxy_headers=False, - limit_concurrency=50, - timeout_keep_alive=5, - ) - ) + server = NativeServer(native_config(app, port)) def shutdown(): """Record actual accepted launcher shutdown before requesting the existing graceful server stop.""" diff --git a/studio/private_workspace.py b/studio/private_workspace.py index e80ec2b..4d7da81 100644 --- a/studio/private_workspace.py +++ b/studio/private_workspace.py @@ -2,9 +2,9 @@ # Generated definition map; @L is the exact declaration line. # Function/class docs explain behavior; variables/parameters are scoped inputs, outputs or state. # classes: InstanceLock@L50; Workspaces@L78 -# functions: atomic_json(path, value)@L37; __init__(self, path)@L53; close(self)@L73; __init__(self, root)@L81; archive(self, owner)@L116; status(self, owner, session)@L124; providers(self)@L136; unlock(self, owner, session, encoded, salt)@L150; require(self, owner, session)@L214; save(self, force)@L220; remove_active(self, owner)@L235; lock(self, owner)@L247; delete(self, owner)@L265; close(self)@L273 -# variables: HISTORY_RECOVERY_BYTES@L34; descriptor@L39; name@L39; stream@L41; allowed@L87; p@L96; c@L119; config@L126; path@L138; data@L139; data@L143; k@L145; p@L145; v@L145; p@L148; key@L154; current@L155; config@L156; proof@L159; nonce@L170; config@L171; keys@L200; settings@L200; current@L228 -# parameters: path@L37; value@L37; path@L53; self@L53; self@L73; root@L81; self@L81; owner@L116; self@L116; owner@L124; self@L124; session@L124; self@L136; encoded@L150; owner@L150; salt@L150; self@L150; session@L150; owner@L214; self@L214; session@L214; force@L220; self@L220; owner@L235; self@L235; owner@L247; self@L247; owner@L265; self@L265; self@L273 +# functions: atomic_json(path, value)@L37; __init__(self, path)@L53; close(self)@L73; __init__(self, root)@L81; archive(self, owner)@L116; status(self, owner, session)@L124; providers(self)@L136; unlock(self, owner, session, encoded, salt)@L150; _unlock(self, owner, session, encoded, salt)@L154; require(self, owner, session)@L218; save(self, force)@L224; remove_active(self, owner)@L239; lock(self, owner)@L251; _lock(self, owner)@L255; delete(self, owner)@L273; close(self)@L281 +# variables: HISTORY_RECOVERY_BYTES@L34; descriptor@L39; name@L39; stream@L41; allowed@L87; p@L96; c@L119; config@L126; path@L138; data@L139; data@L143; k@L145; p@L145; v@L145; p@L148; key@L158; current@L159; config@L160; proof@L163; nonce@L174; config@L175; keys@L204; settings@L204; current@L232 +# parameters: path@L37; value@L37; path@L53; self@L53; self@L73; root@L81; self@L81; owner@L116; self@L116; owner@L124; self@L124; session@L124; self@L136; encoded@L150; owner@L150; salt@L150; self@L150; session@L150; encoded@L154; owner@L154; salt@L154; self@L154; session@L154; owner@L218; self@L218; session@L218; force@L224; self@L224; owner@L239; self@L239; owner@L251; self@L251; owner@L255; self@L255; owner@L273; self@L273; self@L281 # """Jason Stys: exclusive account-scoped controller, encrypted history/providers, and recoverable shutdown. @@ -22,10 +22,10 @@ from pathlib import Path from cryptography.hazmat.primitives.ciphers.aead import AESGCM -from starlette.concurrency import run_in_threadpool from studio.accounts import Accounts, decode_field from studio.api import create_app +from studio.completed_offload import completed_offload, completed_operation from studio.permissions import protect_directory from studio.providers import Providers from studio.vault import AAD, decrypt_workspace, encrypt_workspace, members, signature @@ -148,6 +148,10 @@ def providers(self) -> tuple[dict, dict]: }, {p["id"]: p.get("key", "") for p in data["providers"]} async def unlock(self, owner: str, session: str, encoded: str, salt: str | None): + """Complete decrypt/marker/controller/key setup while the caller retains the request mutex.""" + return await completed_operation(self._unlock(owner, session, encoded, salt)) + + async def _unlock(self, owner: str, session: str, encoded: str, salt: str | None): """Authenticate the independent key, recover only its owner, then enter its genuine controller lifecycle.""" if self.owner is not None and self.owner != owner: raise ValueError("Another account has this local engine unlocked; it must lock first") @@ -186,7 +190,7 @@ async def unlock(self, owner: str, session: str, encoded: str, salt: str | None) else: if self.archive(owner).exists(): try: - await run_in_threadpool(decrypt_workspace, self.archive(owner), self.active, key) + await completed_offload(decrypt_workspace, self.archive(owner), self.active, key) except Exception: raise ValueError("Cannot unlock damaged archive; existing files were preserved") from None else: @@ -209,7 +213,7 @@ async def unlock(self, owner: str, session: str, encoded: str, salt: str | None) self.owner, self.key = owner, key self.sessions = {session} self.saved_signature = None - await run_in_threadpool(self.save, True) + await completed_offload(self.save, True) def require(self, owner: str, session: str): """A login cookie alone cannot join an already-unlocked account without key proof.""" @@ -245,6 +249,10 @@ def remove_active(self, owner: str): self.marker.unlink() async def lock(self, owner: str): + """Complete container cleanup, snapshot and plaintext removal before reporting caller cancellation.""" + return await completed_operation(self._lock(owner)) + + async def _lock(self, owner: str): """Cancel/clean only this account's containers before sealing; preserve all recovery data on failure.""" if self.owner != owner: return {"locked": True} @@ -253,8 +261,8 @@ async def lock(self, owner: str): await self.context.__aexit__(None, None, None) self.context = None self.app.state.active.clear() - await run_in_threadpool(self.save, True) - await run_in_threadpool(self.remove_active, owner) + await completed_offload(self.save, True) + await completed_offload(self.remove_active, owner) except Exception: self.failed = True raise @@ -267,7 +275,7 @@ async def delete(self, owner: str): if self.owner == owner: await self.lock(owner) elif self.marker.exists() and json.loads(self.marker.read_text())["owner"] == owner: - await run_in_threadpool(self.remove_active, owner) + await completed_offload(self.remove_active, owner) self.archive(owner).unlink(missing_ok=True) async def close(self): diff --git a/tests/desktop.spec.ts b/tests/desktop.spec.ts index db17076..bf882a9 100644 --- a/tests/desktop.spec.ts +++ b/tests/desktop.spec.ts @@ -1,7 +1,7 @@ /* * Generated definition map; @L is the exact declaration line. * Function/class docs explain behavior; variables/parameters are scoped inputs, outputs or state. - * variables: { verifyElectron }@L24; { readDiagnosticLog }@L27; executable@L33; app@L36; launcherDirectory@L54; acceptedPackage@L57; resources@L64; file@L65; metadata@L98; page@L118; accountStartedAt@L121; accountRoutes@L122; route@L124; route@L136; browserErrors@L148; health@L152; state@L160; directory@L191; serviceLog@L194; statuses@L195; lifecycle@L260; records@L261; launcher@L262; line@L265; service@L270; line@L271; diagnostic@L278; records@L282; stage@L283; pid@L294 + * variables: { verifyElectron }@L24; { readDiagnosticLog }@L27; executable@L33; app@L36; launcherDirectory@L54; acceptedPackage@L57; resources@L64; file@L65; metadata@L98; page@L118; accountStartedAt@L121; accountRoutes@L122; route@L124; route@L136; browserErrors@L148; health@L152; state@L160; directory@L191; serviceLog@L194; statuses@L195; lifecycle@L260; records@L261; launcher@L268; line@L271; service@L276; line@L277; diagnostic@L284; records@L288; stage@L289; record@L301; pid@L314 * */ /** Verify the actual Electron shell and bundled Python service, without faking readiness. @@ -258,7 +258,13 @@ test("bundled desktop authenticates, seals history and provides a sandboxed edit } finally { console.info("NATIVE_JOURNEY_STAGE", "owned-close:start"); const lifecycle = () => { - const records: { stage: string; pid: number; parent_pid: number }[] = []; + const records: { + stage: string; + pid: number; + parent_pid: number; + connections?: number; + tasks?: number; + }[] = []; const launcher = readDiagnosticLog( join(launcherDirectory, "native-lifecycle.log"), ); @@ -281,6 +287,8 @@ test("bundled desktop authenticates, seals history and provides a sandboxed edit console.info("NATIVE_JOURNEY_STAGE", "owned-close:complete"); const records = lifecycle(); for (const stage of [ + "http-drain-started", + "http-drain-complete", "workspace-close-complete", "lifespan-close-complete", "server-thread-stopped", @@ -290,6 +298,18 @@ test("bundled desktop authenticates, seals history and provides a sandboxed edit records.some((record) => record.stage === stage), `Actual frozen service milestone: ${stage}`, ).toBe(true); + for (const record of records.filter((record) => + ["http-drain-started", "http-drain-complete"].includes(record.stage), + )) { + expect( + Number.isSafeInteger(record.connections) && record.connections! >= 0, + ).toBe(true); + expect(Number.isSafeInteger(record.tasks) && record.tasks! >= 0).toBe( + true, + ); + if (record.stage === "http-drain-complete") + expect(record.tasks).toBe(0); + } // Signal zero observes identity existence without sending a termination signal or touching unrelated processes. for (const pid of new Set( records.flatMap((record) => [record.pid, record.parent_pid]), diff --git a/tests/test_completed_offload.py b/tests/test_completed_offload.py new file mode 100644 index 0000000..822d650 --- /dev/null +++ b/tests/test_completed_offload.py @@ -0,0 +1,84 @@ +# +# Generated definition map; @L is the exact declaration line. +# Function/class docs explain behavior; variables/parameters are scoped inputs, outputs or state. +# functions: test_native_cancellation_cannot_release_mutex_before_actual_worker(cancels, fail)@L24; offload_scenario(cancels, fail)@L29; worker()@L38; caller()@L48 +# variables: release@L31; started@L31; mutex@L32; events@L33; unhandled@L33; loop@L34; original_handler@L35; request@L56; _@L59; outcome@L68 +# parameters: cancels@L24; fail@L24; cancels@L29; fail@L29; _loop@L36; context@L36 +# + +"""Jason Stys: actual thread completion under raw repeated asyncio cancellation. + +Symbol locations: docs/code-index.md. All operations are disposable synchronized +fixtures, not account files, mocked threadpool completion or arbitrary-code execution. +""" + +import asyncio +import threading + +import pytest + +from studio.completed_offload import completed_offload + + +@pytest.mark.parametrize("cancels,fail", [(0, False), (1, False), (2, False), (2, True), (0, True)]) +def test_native_cancellation_cannot_release_mutex_before_actual_worker(cancels, fail): + """Preserve successful results/errors and finish real work before cancellation and lock release.""" + asyncio.run(offload_scenario(cancels, fail)) + + +async def offload_scenario(cancels, fail): + """Order real worker completion before caller finally, even after repeated raw cancellation.""" + started, release = threading.Event(), threading.Event() + mutex = asyncio.Lock() + events, unhandled = [], [] + loop = asyncio.get_running_loop() + original_handler = loop.get_exception_handler() + loop.set_exception_handler(lambda _loop, context: unhandled.append(context)) + + def worker(): + """Wait for a controlled release, then produce an actual result or ordinary worker error.""" + started.set() + if not release.wait(5): + raise AssertionError("Test did not release its worker") + events.append("worker-finished") + if fail: + raise ValueError("fictional-worker-error") + return 42 + + async def caller(): + """Represent a vault operation whose surrounding lock must remain held until real I/O finishes.""" + try: + async with mutex: + return await completed_offload(worker) + finally: + events.append("caller-finally") + + request = asyncio.create_task(caller()) + try: + assert await asyncio.to_thread(started.wait, 3) + for _ in range(cancels): + assert request.cancel() + # Give actual CancelledError/shield callbacks their event-loop turns; + # this is neither a duration-based wait nor worker cancellation. + await asyncio.sleep(0) + await asyncio.sleep(0) + assert mutex.locked() and not request.done() and events == [] + release.set() + if cancels: + with pytest.raises(asyncio.CancelledError) as outcome: + await request + assert not fail or isinstance(outcome.value.__cause__, ValueError) + elif fail: + with pytest.raises(ValueError, match="fictional-worker-error"): + await request + else: + assert await request == 42 + assert events == ["worker-finished", "caller-finally"] + assert not mutex.locked() and not unhandled + finally: + release.set() + if not request.done(): + request.cancel() + with pytest.raises(asyncio.CancelledError): + await request + loop.set_exception_handler(original_handler) diff --git a/tests/test_native_drain.py b/tests/test_native_drain.py new file mode 100644 index 0000000..2330ad2 --- /dev/null +++ b/tests/test_native_drain.py @@ -0,0 +1,240 @@ +# +# Generated definition map; @L is the exact declaration line. +# Function/class docs explain behavior; variables/parameters are scoped inputs, outputs or state. +# classes: ObservedServer@L121 +# functions: test_actual_native_drain_preserves_authenticated_workspace(tmp_path, project, kind, monkeypatch)@L33; native_scenario(tmp_path, project, kind, monkeypatch)@L38; observe_offload(stage)@L56; held(request)@L65; held_save()@L79; held_decrypt()@L92; observed_app(scope, receive, send)@L104; observe_receive()@L110; startup(self, sockets)@L124; stage(self, name)@L129; test_private_cli_selects_finite_http_drain()@L234 +# variables: listener@L40; port@L44; base@L45; token@L46; key@L47; app@L48; manager@L49; _@L50; budget@L50; entered@L50; ready@L50; release@L50; cancellation_deferred@L51; _@L52; thread_entered@L52; thread_finished@L52; thread_release@L52; counts@L53; events@L53; original_stage@L54; original_save@L76; original_decrypt@L77; result@L86; result@L98; partial@L108; message@L114; partial@L116; server@L137; running@L139; pending@L140; writer@L141; client@L147; response@L148; response@L152; response@L156; owner@L158; revision@L158; archive@L159; _@L161; writer@L161; cookie@L162; name@L162; value@L162; pending@L173; pending@L179; pending@L182; response@L185; response@L199; _@L205; name@L205; tasks@L205; restored@L213; config@L237 +# parameters: kind@L33; monkeypatch@L33; project@L33; tmp_path@L33; kind@L38; monkeypatch@L38; project@L38; tmp_path@L38; stage@L56; request@L65; args@L79; args@L92; receive@L104; scope@L104; send@L104; self@L124; sockets@L124; name@L129; self@L129 +# + +"""Jason Stys: real owned-loopback HTTP cancellation, authenticated accounts and vault recovery. + +Symbol locations: docs/code-index.md. These disposable tests never execute code, +use Docker, connect user accounts or replace installed applications. +""" + +import asyncio +import base64 +import contextlib +import socket +import threading + +import httpx +import pytest +from fastapi import Request + +from studio import completed_offload, private_workspace +from studio.native_server import HTTP_DRAIN_SECONDS, NativeServer, native_config +from studio.private_api import create_private_app +from studio.vault import decrypt_workspace + + +@pytest.mark.parametrize("kind", ["locked-handler", "partial-upload", "offloaded-save", "offloaded-unlock"]) +def test_actual_native_drain_preserves_authenticated_workspace(tmp_path, project, kind, monkeypatch): + """Finite real HTTP drain cancels held work; cancellation cleanup finishes before authenticated sealing.""" + asyncio.run(native_scenario(tmp_path, project, kind, monkeypatch)) + + +async def native_scenario(tmp_path, project, kind, monkeypatch): + """Own one socket/server, create a real synthetic account and recover its exact stored revision.""" + listener = socket.socket(socket.AF_INET, socket.SOCK_STREAM) + listener.bind(("127.0.0.1", 0)) + listener.listen(128) + listener.setblocking(False) + port = listener.getsockname()[1] + base = f"http://127.0.0.1:{port}" + token = "fictional-native-drain-launch-secret" + key = b"k" * 32 + app = create_private_app(tmp_path / "accounts", port, launcher_token=token) + manager = app.state.manager + entered, release, ready, budget = (asyncio.Event() for _ in range(4)) + cancellation_deferred = asyncio.Event() + thread_entered, thread_release, thread_finished = (threading.Event() for _ in range(3)) + events, counts = [], [] + original_stage = completed_offload.native_stage + + def observe_offload(stage): + """Signal only after genuine outer-task cancellation reached the protected offload.""" + original_stage(stage) + if stage == "offload-cancellation-deferred": + cancellation_deferred.set() + + monkeypatch.setattr(completed_offload, "native_stage", observe_offload) + + @app.get("/private/synthetic-held") + async def held(request: Request): + """Use the actual authenticated boundary and request mutex before a test-controlled suspension.""" + async with manager.requests: + manager.require(request.state.account["id"], request.state.session) + entered.set() + try: + await release.wait() + finally: + events.append("request-lock-released") + return {"released": True} + + original_save = manager.save + original_decrypt = private_workspace.decrypt_workspace + + def held_save(*args): + """Hold a genuine snapshot offload until the actual cancellation stage has been observed.""" + thread_entered.set() + try: + if not thread_release.wait(15): + raise AssertionError("Test did not release its own snapshot worker") + assert manager.active.is_dir() and manager.owner is not None + result = original_save(*args) + events.append("snapshot-worker-finished") + return result + finally: + thread_finished.set() + + def held_decrypt(*args): + """Hold actual authenticated restore, then allow its marker/controller/key transition to finish.""" + thread_entered.set() + try: + if not thread_release.wait(15): + raise AssertionError("Test did not release its own restore worker") + result = original_decrypt(*args) + events.append("restore-worker-finished") + return result + finally: + thread_finished.set() + + async def observed_app(scope, receive, send): + """Observe a real partial HTTP body without replacing the actual request/security handler.""" + if scope["type"] != "http" or scope["path"] != "/private/activity": + return await app(scope, receive, send) + partial = False + + async def observe_receive(): + nonlocal partial + if partial: + entered.set() + message = await receive() + if message["type"] == "http.request" and message.get("more_body"): + partial = True + return message + + return await app(scope, observe_receive, send) + + class ObservedServer(NativeServer): + """Observe actual Uvicorn startup/drain rather than substituting fake service readiness.""" + + async def startup(self, sockets=None): + """A ready event follows the real listener/lifespan startup.""" + await super().startup(sockets) + ready.set() + + def stage(self, name): + """Retain actual numeric work counts and signal the real configured drain deadline.""" + super().stage(name) + events.append(name) + counts.append((name, len(self.server_state.connections), len(self.server_state.tasks))) + if name == "http-drain-budget-exhausted": + budget.set() + + server = ObservedServer(native_config(observed_app, port)) + app.state.shutdown = lambda: setattr(server, "should_exit", True) + running = asyncio.create_task(server.serve(sockets=[listener])) + pending = None + writer = None + try: + await asyncio.wait_for(ready.wait(), 10) + async with httpx.AsyncClient( + base_url=base, headers={"Origin": base, "X-CFS-Request": "v1"}, + timeout=15, trust_env=False, + ) as client: + response = await client.post("/private/register", json={ + "alias": "fictional_drain", "password": "synthetic-drain-password-only-2026", + }) + assert response.status_code == 200 + response = await client.post("/private/unlock", json={ + "key": base64.b64encode(key).decode(), "salt": base64.b64encode(b"s" * 16).decode(), + }) + assert response.status_code == 200 + response = await client.post("/private/api/revisions", json=project.model_dump()) + assert response.status_code == 200 + revision, owner = response.json()["id"], manager.owner + archive = manager.archive(owner) + if kind == "partial-upload": + _, writer = await asyncio.open_connection("127.0.0.1", port) + cookie = "; ".join(f"{name}={value}" for name, value in client.cookies.items()) + writer.write(( + f"POST /private/activity HTTP/1.1\r\nHost: 127.0.0.1:{port}\r\n" + f"Origin: {base}\r\nCookie: {cookie}\r\nX-CFS-Request: v1\r\n" + "Content-Type: application/json\r\nContent-Length: 30\r\n\r\n{" + ).encode()) + await writer.drain() + elif kind == "offloaded-unlock": + assert (await client.post("/private/lock", json={})).status_code == 200 + assert not manager.active.exists() and manager.owner is None + monkeypatch.setattr(private_workspace, "decrypt_workspace", held_decrypt) + pending = asyncio.create_task(client.post("/private/unlock", json={ + "key": base64.b64encode(key).decode(), "salt": base64.b64encode(b"s" * 16).decode(), + })) + assert await asyncio.to_thread(thread_entered.wait, 10) + elif kind == "offloaded-save": + manager.save = held_save + pending = asyncio.create_task(client.post("/private/save", json={})) + assert await asyncio.to_thread(thread_entered.wait, 10) + else: + pending = asyncio.create_task(client.get("/private/synthetic-held")) + if kind not in {"offloaded-save", "offloaded-unlock"}: + await asyncio.wait_for(entered.wait(), 10) + response = await client.post("/launcher/shutdown", json={}, headers={"Authorization": "Bearer " + token}) + assert response.status_code == 200 and response.json() == {"stopping": True} + await asyncio.wait_for(budget.wait(), 10) + if kind in {"offloaded-save", "offloaded-unlock"}: + # The original raw asyncio cancellation released this mutex while + # the real snapshot worker was still using active plaintext files. + await asyncio.wait_for(cancellation_deferred.wait(), 5) + assert manager.requests.locked() + assert kind != "offloaded-save" or manager.active.is_dir() + assert not thread_finished.is_set() and not running.done() + thread_release.set() + await asyncio.wait_for(asyncio.shield(running), 10) + if pending: + try: + response = await asyncio.wait_for(pending, 5) + assert response.status_code == 500 + except httpx.RemoteProtocolError: + # Cancellation may close the HTTP connection without a response; + # a successful response is never accepted for the canceled action. + pass + assert any(name == "http-drain-budget-exhausted" and tasks >= 1 for name, _, tasks in counts) + assert events.index("http-request-cleanup-complete") < events.index("http-drain-complete") + if kind == "offloaded-save": + assert events.index("snapshot-worker-finished") < events.index("http-request-cleanup-complete") + if kind == "offloaded-unlock": + assert events.index("restore-worker-finished") < events.index("http-request-cleanup-complete") + assert not manager.active.exists() and not manager.marker.exists() + assert manager.owner is None and manager.key is None + restored = tmp_path / "restored" + decrypt_workspace(archive, restored, key) + from studio.store import Store + assert Store(restored / "history.sqlite").revision(revision)["project"] == project.model_dump() + finally: + release.set() + thread_release.set() + manager.save = original_save + if writer: + writer.close() + await writer.wait_closed() + server.should_exit = True + if not running.done(): + await asyncio.wait_for(asyncio.shield(running), 20) + if pending and not pending.done(): + pending.cancel() + with contextlib.suppress(asyncio.CancelledError): + await pending + listener.close() + + +def test_private_cli_selects_finite_http_drain(): + """The shipping configuration retains network/concurrency limits and finite HTTP drain.""" + assert HTTP_DRAIN_SECONDS == 5.0 + config = native_config(None, 8090) + assert config.timeout_graceful_shutdown == HTTP_DRAIN_SECONDS + assert config.host == "127.0.0.1" and config.port == 8090 + assert config.proxy_headers is False and config.limit_concurrency == 50 diff --git a/tests/test_native_lifecycle.py b/tests/test_native_lifecycle.py index 5480323..b748038 100644 --- a/tests/test_native_lifecycle.py +++ b/tests/test_native_lifecycle.py @@ -1,9 +1,9 @@ # # Generated definition map; @L is the exact declaration line. # Function/class docs explain behavior; variables/parameters are scoped inputs, outputs or state. -# functions: test_native_stage_is_opt_in_and_has_only_fixed_identity_fields(monkeypatch, capsys)@L22; test_actual_launcher_shutdown_denials_cannot_invoke_callback(tmp_path)@L37 -# variables: payload@L29; prefix@L29; value@L31; app@L39; called@L40; headers@L42; client@L43; response@L50 -# parameters: capsys@L22; monkeypatch@L22; tmp_path@L37 +# functions: test_native_stage_is_opt_in_and_has_only_fixed_identity_fields(monkeypatch, capsys)@L22; test_native_stage_counts_are_only_nonnegative_integers(monkeypatch, capsys)@L37; test_actual_launcher_shutdown_denials_cannot_invoke_callback(tmp_path)@L48 +# variables: payload@L29; prefix@L29; value@L31; value@L41; value@L44; app@L50; called@L51; headers@L53; client@L54; response@L61 +# parameters: capsys@L22; monkeypatch@L22; capsys@L37; monkeypatch@L37; tmp_path@L48 # """Jason Stys: opt-in native diagnostic privacy and genuine launcher acknowledgement. @@ -34,6 +34,17 @@ def test_native_stage_is_opt_in_and_has_only_fixed_identity_fields(monkeypatch, assert value["pid"] > 0 and value["parent_pid"] > 0 +def test_native_stage_counts_are_only_nonnegative_integers(monkeypatch, capsys): + """Fixed numeric drain observations never accept booleans, negative values or arbitrary strings.""" + monkeypatch.setenv("STUDIO_DESKTOP_TEST", "1") + native_stage("http-drain-started", connections=2, tasks=0) + value = json.loads(capsys.readouterr().out.strip().split(" ", 1)[1]) + assert value["connections"] == 2 and value["tasks"] == 0 + native_stage("http-drain-started", connections=True, tasks=-1) + value = json.loads(capsys.readouterr().out.strip().split(" ", 1)[1]) + assert set(value) == {"stage", "pid", "parent_pid"} + + def test_actual_launcher_shutdown_denials_cannot_invoke_callback(tmp_path): """Accepted shutdown means the real authenticated route actually invoked its graceful-stop callback.""" app = create_private_app(tmp_path / "accounts", 8090, launcher_token="fictional-native-token") diff --git a/tests/test_private_cancellation.py b/tests/test_private_cancellation.py new file mode 100644 index 0000000..abcba12 --- /dev/null +++ b/tests/test_private_cancellation.py @@ -0,0 +1,122 @@ +# +# Generated definition map; @L is the exact declaration line. +# Function/class docs explain behavior; variables/parameters are scoped inputs, outputs or state. +# functions: test_cancelled_authorized_transition_is_completed_under_request_mutex(tmp_path, monkeypatch, kind)@L29; transition_scenario(tmp_path, monkeypatch, kind)@L34; stage(name)@L45; held_write()@L69; held_delete()@L81 +# variables: base@L36; password@L37; key@L38; app@L39; manager@L40; entered@L41; release@L41; deferred@L42; original_stage@L43; request@L51; client@L55; owner@L60; archive@L61; config@L62; original@L67; request@L77; original@L79; request@L89; disk@L99; router@L100; name@L104; value@L104; row@L105; public@L108; restored@L111; db@L115 +# parameters: kind@L29; monkeypatch@L29; tmp_path@L29; kind@L34; monkeypatch@L34; tmp_path@L34; name@L45; args@L69; args@L81 +# + +"""Jason Stys: real authorized local mutations remain coherent after native request cancellation. + +Symbol locations: docs/code-index.md. Only fictional accounts and inert provider +destinations are used; no provider API, worker, private account or external network. +""" + +import asyncio +import base64 +import json +import threading + +import httpx +import pytest + +from studio import completed_offload, private_api +from studio.private_api import create_private_app +from studio.vault import decrypt_workspace + + +@pytest.mark.parametrize("kind", ["providers", "delete"]) +def test_cancelled_authorized_transition_is_completed_under_request_mutex(tmp_path, monkeypatch, kind): + """Raw cancellation cannot split consent/destination publication or data/account deletion.""" + asyncio.run(transition_scenario(tmp_path, monkeypatch, kind)) + + +async def transition_scenario(tmp_path, monkeypatch, kind): + """Use actual route authorization and real disk/SQLite work with event-controlled cancellation.""" + base = "http://127.0.0.1:8090" + password = "synthetic-transition-password-2026" + key = b"k" * 32 + app = create_private_app(tmp_path / "accounts", 8090) + manager = app.state.manager + entered, release = threading.Event(), threading.Event() + deferred = asyncio.Event() + original_stage = completed_offload.native_stage + + def stage(name): + """Wait for actual cancellation handling, rather than inferring it from elapsed time.""" + original_stage(name) + deferred.set() + + monkeypatch.setattr(completed_offload, "native_stage", stage) + request = None + async with app.router.lifespan_context(app), httpx.AsyncClient( + transport=httpx.ASGITransport(app=app), base_url=base, + headers={"Origin": base, "X-CFS-Request": "v1"}, trust_env=False, + ) as client: + assert (await client.post("/private/register", json={"alias": "fictional_transition", "password": password})).status_code == 200 + assert (await client.post("/private/unlock", json={ + "key": base64.b64encode(key).decode(), "salt": base64.b64encode(b"s" * 16).decode(), + })).status_code == 200 + owner = manager.owner + archive = manager.archive(owner) + config = { + "providers": [{"id": "owned", "kind": "openai", "url": "https://model.example/api", "model": "fixture", "key": "synthetic-provider-key-only"}], + "routes": {"generate": "owned"}, + } + if kind == "providers": + original = private_api.atomic_json + + def held_write(*args): + """Stop after the actual settings bytes changed but before the in-memory router can change.""" + original(*args) + entered.set() + if not release.wait(5): + raise AssertionError("Test did not release provider publication") + + monkeypatch.setattr(private_api, "atomic_json", held_write) + request = asyncio.create_task(client.put("/private/providers", json=config)) + else: + original = manager.accounts.delete + + def held_delete(*args): + """Stop the actual final account-row removal after authorized workspace removal.""" + entered.set() + if not release.wait(5): + raise AssertionError("Test did not release account deletion") + return original(*args) + + monkeypatch.setattr(manager.accounts, "delete", held_delete) + request = asyncio.create_task(client.post("/private/delete", json={"password": password, "confirm": True})) + try: + assert await asyncio.to_thread(entered.wait, 3) + assert request.cancel() + await asyncio.wait_for(deferred.wait(), 3) + assert manager.requests.locked() and not request.done() + release.set() + with pytest.raises(asyncio.CancelledError): + await request + if kind == "providers": + disk = json.loads((manager.active / "providers.private.json").read_text(encoding="utf-8")) + router = manager.app.state.providers + assert router.config["providers"][0]["url"] == disk["providers"][0]["url"] == config["providers"][0]["url"] + assert router.config["routes"] == disk["routes"] + assert router.config["providers"] == [ + {name: value for name, value in row.items() if name != "key"} + for row in disk["providers"] + ] + assert router.credentials["owned"] == disk["providers"][0]["key"] + public = await client.get("/private/providers") + assert public.status_code == 200 and public.json()["revision"] == disk["revision"] + assert config["providers"][0]["key"] not in public.text + restored = tmp_path / "restored" + decrypt_workspace(archive, restored, key) + assert json.loads((restored / "providers.private.json").read_text()) == disk + else: + with manager.accounts.connection() as db: + assert db.execute("SELECT id FROM accounts WHERE id=?", (owner,)).fetchone() is None + assert not archive.exists() and not manager.active.exists() and not manager.marker.exists() + finally: + release.set() + if request and not request.done(): + with pytest.raises(asyncio.CancelledError): + await request