From e860b7024399487bb128c8382582114d5547581c Mon Sep 17 00:00:00 2001 From: JeffreyChen Date: Fri, 2 Oct 2026 18:40:47 +0800 Subject: [PATCH 01/18] docs: design shared test records and LoadDensity upgrades --- .../specs/2026-10-02-shared-record-design.md | 173 ++++++++++++++++++ .../2026-10-02-testing-platform-roadmap.md | 163 +++++++++++++++++ docs/updates/2026-10.md | 8 + docs/updates/README.md | 1 + progress.md | 11 +- 5 files changed, 354 insertions(+), 2 deletions(-) create mode 100644 docs/superpowers/specs/2026-10-02-shared-record-design.md create mode 100644 docs/superpowers/specs/2026-10-02-testing-platform-roadmap.md diff --git a/docs/superpowers/specs/2026-10-02-shared-record-design.md b/docs/superpowers/specs/2026-10-02-shared-record-design.md new file mode 100644 index 0000000..1e91e15 --- /dev/null +++ b/docs/superpowers/specs/2026-10-02-shared-record-design.md @@ -0,0 +1,173 @@ +# 共用 request record v1 設計草案 + +狀態：待審閱。此文件是第一個子專案的規格；整批範圍見 +[測試平台設計](2026-10-02-testing-platform-roadmap.md)。 + +## 目的與邊界 + +讓 APITestka 的功能結果、LoadDensity 的 Locust／asyncio 結果與 WebRunner synthetic check +可用同一套 JSON 格式保存、比較與追蹤。第一階段只定義並接上 request record； +單一腳本三模式、分散式恢復、UI 與 extras CI 依 roadmap 分別交付。 +WebRunner 的 browser action／screenshot／trace 舊紀錄不轉成 HTTP response； +monitor 保留自身結果，新增關聯 run_id 與 request 結果來源。 + +## 所有權與架構 + +ActionCore 新增純標準函式庫的 request record 型別、驗證與序列化工具，保留零外部相依。 +它不 import Locust、requests、httpx、Qt 或雲端 SDK，也不管理 framework 的 singleton。 +APITestka 與 LoadDensity 的 adapter 負責將各自 response／exception 轉成 canonical record。 +schema_version=`1` 是結果契約版本，與各套件發版號獨立。 + +ActionCore 先發佈具有該介面的版本，再提高使用端相依下限；測試期間可指定隔離 checkout。 +不能直接改共用 executor 的 action execution record，亦不能改它原有的 repr、key 或 error 行為。 + +## Canonical 欄位 + +每一筆成功或失敗使用相同物件結構，以下欄位必須出現；未知的量值用 null，不捏造零值。 + +| 欄位 | 型別／規則 | +| --- | --- | +| schema_version | integer，固定 1 | +| record_id | string，run 內唯一；同一結果重送保留同一 ID | +| run_id | 非空 string，每次執行重新建立 | +| source | `apitestka`／`loaddensity`／`webrunner` | +| phase | `functional`／`load`／`synthetic` | +| engine | 非空 string，例如 `requests`／`httpx`／`locust`／`asyncio` | +| worker_id | string 或 null，本機可為 null | +| scenario_id | string 或 null | +| step_id | string 或 null，對應腳本的穩定步驟識別 | +| protocol | 非空 string，例如 `http`；其他協定以 extension 承載特定資訊 | +| request_method | string，HTTP method 大寫 | +| request_url | string，原 request endpoint；不得以 request name 取代 | +| name | string，request label，缺少時使用 method 與 endpoint | +| status_code | integer 或 null；transport failure 為 null | +| start_time | 有限 float 或 null，UTC epoch seconds | +| end_time | 有限 float 或 null，UTC epoch seconds；存在時不得小於 start_time | +| response_time_ms | 有限、非負 float 或 null，monotonic 計算的耗時 | +| response_length | 非負 integer 或 null，實際 response bytes 長度 | +| outcome | `passed`／`failed` | +| error | null 或 `{kind: string, message: string}`，failed 時必須有 error | +| assertions | JSON array，每筆含 type、passed；失敗附 message | +| extensions | JSON object，保留協定或來源特有資訊，不覆寫 canonical 欄位 | + +HTTP response 若實際回傳 status，即使 assertion 不通過仍保存原 status 與 latency。 +record 工具只保存 runner 的判定，不自行把 status 改判為成功或失敗。 +第一階段保留各 runner 目前的判定；後續引擎與腳本對等階段才統一 +「HTTP 4xx／5xx 預設 failed，明確 status expectation 命中時可 passed，其他 assertions 仍需通過」。 +Transport failure 保存實際已耗時間，error.kind 區分 timeout／connection／tls／其他 transport。 +設定錯誤在 preflight 丟例外，不製造零毫秒「request 已執行」結果。 +assertion failure、HTTP status failure 與 runner internal error 使用不同 kind。 +新紀錄的 end_time 使用 start_time 加上 monotonic 耗時，避免 wall clock 校正產生負持續時間。 + +選填 payload 欄位為 `text`、`headers`、`content_base64`、`request_body`。 +bytes 使用 base64；headers 轉為 JSON object；datetime 統一轉 epoch；timedelta 轉為毫秒。 +token、Authorization、Cookie 等敏感資料沿用各 runner 的遮罩策略；壓力模式預設不存完整 body。 +擴充欄位不要求保存任意 Python object，不能以 `str(response)` 當成正常序列化。 + +## 範例 + +```json +{ + "schema_version": 1, + "record_id": "run-001-worker-0-1", + "run_id": "run-001", + "source": "loaddensity", + "phase": "load", + "engine": "asyncio", + "worker_id": "worker-0", + "scenario_id": "checkout", + "step_id": "get-health", + "protocol": "http", + "request_method": "GET", + "request_url": "http://127.0.0.1:8080/health", + "name": "health", + "status_code": 200, + "start_time": 1790899200.0, + "end_time": 1790899200.025, + "response_time_ms": 25.0, + "response_length": 12, + "outcome": "passed", + "error": null, + "assertions": [{"type": "status_code", "passed": true}], + "extensions": {} +} +``` + +## 相容性與資料流 + +1. 各 runner 建立 RequestRecord，集中交給該 run 的 RecordSink。 +2. sink 將 canonical record 保存至 JSON／SQLite，並更新 bounded live statistics。 +3. 舊的 `test_record_list`／`error_record_list` 介面由 legacy adapter 供既有報表與 consumer 使用。 + LoadDensity 保留 `Method`、`test_url`、字串 status 與 error 等既有型別； + APITestka 保留 response-data return shape 與舊失敗清單，不在第一階段強制下游改格式。 +4. 新的 canonical export 明確版本化；既有 summary 的 `totals` 與 + `latency_overall.p95_ms` 以及 `.json` 不改名、不改意義。 +5. legacy import 使用來源專用 adapter：LoadDensity 的 status `"0"` 作 transport sentinel， + APITestka 失敗清單能取出的 method／URL 仍保存，沒有時間資訊便填 null。 + malformed record 回報帶位置的 validation error，不靜默丟掉。 + +明確指定 RunContext 隔離每次執行；全域 legacy singleton 只服務既有呼叫模式。 +canonical sink 不能每次輸出都掃描全域成功／失敗清單，也不能因上一輪資料殘留改變本輪 summary。 +跨 worker 不平均百分位數；本階段完整 raw record 可供離線統計，後續 distributed live +需要另外選擇可合併 histogram，該項未完成前不標示為精確的全域 live percentile。 + +## 預定檔案與介面責任 + +| 專案／路徑 | 責任 | +| --- | --- | +| ActionCore：`je_action_core/request_record.py` | RequestRecord／RecordError 型別、validate、JSON-safe serialization | +| LoadDensity：`utils/test_record/contract.py` | Locust／async record adapters、legacy import／export | +| LoadDensity：`utils/test_record/run_context.py` | RunContext、run-scoped RecordSink、record IDs 與統計隔離 | +| LoadDensity：`wrapper/event/request_hook.py` | 將 request event 交給 sink，保留 legacy hook 相容性 | +| LoadDensity：`engine/asyncio_engine.py` | 以同一 sink 記錄成功／失敗，補上真實 failure latency | +| LoadDensity：`utils/test_record/sqlite_persistence.py` | 版本化 canonical export 與既有資料庫相容讀取 | +| APITestka：`utils/test_record/contract.py` | requests／httpx／exception adapters，不改 transport return 值 | +| APITestka：`requests_wrapper/`、`httpx_wrapper/` | 接上 request sink，保存 assertion 與 HTTP failure | +| 各方：`architecture.md` §6、README、Sphinx、contract tests | 發版相依、schema 與 consumer 相容性 | + +以上是路徑與責任規劃，實作計畫需在各 repo 規則與工作樹狀態檢查後固定確切 signature。 + +## 驗收 + +- 三條路徑對同一個本機 endpoint 輸出的 canonical keys／types 相同，engine 與來源可區分。 +- 200／201、404／500、內容 assertion failure、timeout、connection refused 均保存 runner 的原 outcome。 + 預期 404 的共同判定另由 roadmap 第 3／5 子專案驗收，不以 record adapter 偷改既有語意。 +- response bytes、非 ASCII headers／錯誤訊息、datetime、timedelta 能 JSON round-trip。 +- 兩個 run、並行使用者與 Lambda warm invocation 的 record 不交叉污染；重送 ID 可去重。 +- legacy fixtures 的 success、failure list、status sentinel 與缺時間欄位皆可匯入。 +- 舊 JSON／JUnit／summary／SQLite consumer、CLI 旗標與 APITestka load bridge 回歸測試通過。 +- 新格式未知 schema version、負 latency、NaN、缺必要欄位與 malformed error 明確拒絕。 +- ActionCore 的套件相依不增加，原 action executor 與其 consumer 測試全部通過。 + +## 後續單一腳本契約 + +以下是第 5 子專案的預定文件形狀，用以檢查 record 的 phase／step_id 是否足夠； +不屬於本階段已可使用的 CLI 功能： + +```json +{ + "api_testka": [ + ["AT_test_api_method", { + "http_method": "get", + "test_url": "http://127.0.0.1:8080/health", + "result_check_dict": {"status_code": 200} + }] + ], + "execution_profiles": { + "functional": {}, + "load": { + "engine": "asyncio", "user_count": 20, "spawn_rate": 5, "test_time": 30, + "sla": {"min_requests": 1, "max_failure_rate": 0, "max_p95_ms": 200} + }, + "synthetic": { + "interval_seconds": 60, "failure_threshold": 2, "recovery_threshold": 1 + } + } +} +``` + +request 只寫一次；每種模式產生新 run_id，step_id 由來源 action index 或明確 name 穩定建立。 +以獨立子行程啟動 LoadDensity，避免 Locust patch 污染 APITestka 或 monitor。 +monitor 執行功能檢查，不在每個 interval 重跑容量測試。 +依既有 APITestka bridge 擴展 request、assertion、extractor 轉換；無法轉換必須在啟動前報錯。 +功能 runner 不能只看 exit code，需讀取結構化 outcome；synthetic 的 check 遇到 failed 要丟例外。 diff --git a/docs/superpowers/specs/2026-10-02-testing-platform-roadmap.md b/docs/superpowers/specs/2026-10-02-testing-platform-roadmap.md new file mode 100644 index 0000000..fdcf2f2 --- /dev/null +++ b/docs/superpowers/specs/2026-10-02-testing-platform-roadmap.md @@ -0,0 +1,163 @@ +# LoadDensity 與跨專案測試流程設計草案 + +狀態：待審閱；以下描述預定行為，不代表功能已實作。 + +## 目標與範圍 + +一份版本控制中的測試腳本分別用於功能測試、壓力測試與上線後合成監控： +APITestka 判定功能正確性，LoadDensity 測量容量與 SLA，WebRunner 的 +`synthetic_monitoring` 負責週期執行、失敗／恢復門檻與狀態變化通知。 +這批工作也涵蓋 UI 重新設計、煙霧測試、Docker extras 安裝矩陣、百分位數帶狀圖、 +async 引擎對等、雲端後端測試，以及分散式健康檢查與失聯重新分配。 + +假設：「百位數」指「百分位數」；UI 範圍包含 PySide6 GUI 與現有 web dashboard。 +功能測試與監控共用同一套正確性檢查；壓力測試另外套用負載與 SLA 設定。 + +## 現況與相依關係 + +- `engine/asyncio_engine.py` 只支援 HTTP 請求的部分欄位，未接入 `start_test`。 + 它共用程序全域 record，使用一個 client；scenario、extractor 與個別使用者狀態尚未對等。 +- `__init__.py` 直接載入 Locust。async 子模組也會經過套件初始化，必須處理 gevent + monkey patch 對 asyncio、執行緒與 TLS 的影響，不能只增加一個引擎參數。 +- Locust 的成功／失敗 record 是字典；APITestka 成功 record 使用 + `request_method`、`request_url`、`request_time_sec` 等欄位，失敗 record 常為二元素清單。 +- `APITestka/je_api_testka/integrations/load_density*.py` 已有腳本轉換與子行程執行， + 目前轉換部分 request 指令與 status assertion，其他內容會列在 `skipped`。 +- `WebRunner/.../synthetic_monitoring/monitor.py` 已提供 `from_action_files(files, runner)`、 + `tick_once()`、`run_for()`。check 沒有丟例外便視為通過，因此整合必須將測試失敗轉為例外。 +- `_wait_for_workers()` 最多等 60 秒，逾時記警告後仍開跑；只計算 clients 的數量。 +- GUI 將 RPS 與 p95 畫在同一條數值軸；離線 chart report 使用 `ts` 或 request index， + 與 request hook 提供的 `start_time` 不一致。 +- `test/test_cloud_launchers.py` 已有成功路徑與 SDK 缺少測試；四個後端仍需補錯誤與邊界情境。 +- CI 有 Windows／Ubuntu、Python 3.10–3.14 的單元測試，沒有獨立 extras 容器安裝矩陣。 + +## 可選方向 + +| 方向 | 優點 | 代價 | +| --- | --- | --- | +| 共用版本化契約，各專案保留 runner（建議） | 延伸既有 APITestka 橋接；結果能比較；引擎彼此隔離 | 需要協調 schema 與相容性測試 | +| 只在 LoadDensity 加轉換層 | 跨 repo 改動較少 | APITestka 與監控仍有多套 record 真實來源 | +| 由 TestPioneer 統一編排所有模式 | 重用既有多 runner 編排 | 擴大本批範圍；不能單靠編排解決 script 與 record 語意差異 | + +建議採第一種。TestPioneer 之後可以呼叫這些 runner，本批不新增另一套編排平台。 +record 契約與正規化的共用基礎放在既有零相依的 ActionCore；HTTP transport adapter、 +報表與監控策略留在各專案。ActionCore 的「action 執行紀錄」與「request 結果」保持不同型別。 + +## 子專案與交付順序 + +每一列都有獨立的設計、實作計畫與驗收；不把整批視為一個不可拆分的改動。 + +| 順序 | 子專案 | 完成條件 | +| --- | --- | --- | +| 1 | 共用 record 契約 | APITestka、Locust 與 asyncio 的成功、assertion failure、transport failure 都能輸出同一 schema，舊介面仍可讀 | +| 2 | 煙霧測試與 Docker extras CI | checkout 建置的 wheel 在乾淨容器安裝、`pip check`、實際能力檢查；本機 HTTP 的 CLI／報表／SLA 閉環能通過及正確失敗 | +| 3 | Async 引擎對等 | 同一腳本從 Python、action、CLI、GUI 進入兩引擎；scenario、狀態隔離、負載控制、取消、輸出、監控與分散式語意均有對等測試 | +| 4 | 分散式恢復與雲端後端 | ready gate、心跳、失聯重新分配、重連與結果去重；四個雲端 adapter 的失敗／部分失敗測試完整 | +| 5 | 共用腳本三種模式 | 一份來源實際執行功能、容量與兩輪以上監控；正確性失敗在每種模式都可觀察，沒有默默丟棄 assertion | +| 6 | UI 與百分位數圖 | GUI／dashboard 顯示一致的 run 狀態、容量與延遲；時間窗百分位數在即時圖與輸出報表一致 | + +第 2 項可在第 1 項前先建立現況回歸基準。第 4 項的雲端契約測試可獨立交付， +分散式恢復需使用第 1 項的 run／worker／record 識別。第 6 項依賴已穩定的 run lifecycle 與統計資料。 + +## 煙霧測試與 extras 安裝矩陣 + +新增 `test/smoke/`：本機測試服務在獨立子行程啟動，避免 gevent／asyncio 互相影響。 +涵蓋 wheel 安裝後的 CLI、`LD_start_test`、summary／JSON／JUnit、SQLite、SLA gate、 +dashboard JSON／SSE、async 路徑與 offscreen GUI 啟動、執行、取消。測試不能只驗證 import 成功。 + +Docker job 從 `pyproject.toml` 產生 `base`、每個 extra、`all` 的矩陣，不維護另一份手寫 extras 清單。 +每格使用新環境，只安裝該 wheel 與指定 extra；安裝、`pip check`、CLI 與相應 capability probe 任何一步失敗都讓 job 失敗。 +probe 不可用 `importorskip` 掩蓋宣告應有的功能；沒有相依的 `mcp` extra 仍需協定握手。 + +PR 執行代表性 Python 3.12 的完整 extras 集合與 base 的 3.10／3.14；排程補齊 3.10–3.14。 +若某個 extra 在宣告支援的版本裝不起來，應修正依賴／支援聲明，不能未經記錄排除。 +GUI 容器安裝需要的系統函式庫並使用 offscreen；協定整合測試以 Compose 的健康檢查等候服務就緒。 +`all` 另外驗證套件共存。extras 的真實網路服務測試與安裝 probe 分開報告，不能互相替代。 + +CI Stable／Dev 共用同一個可重用工作流程，保留 SHA 鎖定 actions、最小權限、job timeout， +並讓既有發版 job 等候必要的煙霧／extras 檢查。建置及發版工具仍沿用 hash lock。 + +## Async 引擎對等 + +`start_test` 保留預設 Locust 與既有參數，新增明確的 engine 選擇；`run_async_load` 保留可 await 的介面。 +使用 Engine／RunHandle 邊界承接 start、stop、wait、snapshot，runner 的 I/O 排程各自實作。 +套件 facade 延後載入 Locust／其 request hook，async 入口不可觸發 gevent patch。 + +對等清單至少涵蓋 request kwargs、labels、預期 status、全部現有 assertions／extractors、 +sequence／weighted／conditional、retry、think time、throttle、CSV／DB／變數解析、 +per-user cookie／session、spawn rate、stages／spike／soak、SLA、報表、SQLite、metrics exporters、 +取消與例外清理，以及 local／master／worker 的生命週期。 +目前全域 resolver 的使用者隔離問題（`progress.md` #15）是前置工作。 + +先交付 HTTP／HTTP2 的對等測試，再逐協定擴展 capability table。未支援的 protocol／設定在啟動前報錯； +不以同步 blocking call 包裝成 async 支援。只完成 HTTP 子階段時，不宣告整個 async 對等項目完成。 +測試除結果數字外，也驗證並行上限、ramp、停止時不殘留 client／task／socket，以及每次 run 統計互不污染。 + +## 分散式健康與失聯重新分配 + +先核對目前安裝的 Locust 對 heartbeat、missing client 與 rebalance 的支援，重用它的公開機制， +LoadDensity 補上明確策略、可觀察狀態與測試，避免再造相互競爭的心跳／使用者分配器。 + +分開 target health 與 worker health。設定採 `worker_startup_timeout=60`、 +`worker_heartbeat_interval=5`、`worker_lost_timeout=15`；皆需驗證正數，lost timeout 大於 interval。 +`expected_workers` 是啟動 gate，只有 ready 且未失聯的 worker 算入。 +預設逾時失敗並清理 runner；明確選擇 `worker_startup_policy="degraded"` 才降級開跑。 + +worker 狀態為 connecting／ready／running／lost／stopped。失聯時將「虛擬使用者容量」 +重新分配到健康 worker，維持目標總使用者數與既有 ramp 約束；不重播可能已對目標產生副作用的單筆 HTTP request。 +剩餘 worker 無法承擔時將 run 標成 degraded 或 failed，而不是宣稱維持目標容量。 + +有限工作分片使用 run_id、assignment_id、generation 與 lease；回收時 generation 遞增， +拒絕舊 lease 的遲到結果。record_id／worker_id 用於結果去重；重連不能造成 double assignment。 +容量調整不保證跨機器搬移已登入 session，也不宣稱 exactly-once request；報告必須揭露受影響的 journey。 +測試涵蓋啟動逾時、worker 被終止、網路中斷、遲到心跳、重連、全員失聯、負載形狀變化與 master 清理。 + +## 雲端後端測試 + +延伸現有 fake SDK 測試，覆蓋 Fargate／Lambda／Cloud Run／ACI 的非法數量／資源參數、 +credentials failure、permission denied、timeout、限流、服務錯誤、部分成功與 malformed response。 +特別驗證 Fargate 回傳 `failures`、Lambda `FunctionError`／非 JSON／Event 回應／warm invocation 隔離、 +Cloud Run token refresh 與錯誤 HTTP 回應、ACI poller failure 與不同 worker 的唯一名稱。 +測試 SDK 呼叫參數與結果判定，不能只驗證 fake 回傳值原封不動傳出。 + +合約測試使用可控 SDK stub／本機 HTTP，不申請真實雲端資源。實作時依官方文件核對 API payload； +若新增 retry，對可能已被服務接受的啟動請求必須使用服務支援的冪等識別，不能盲目重送。 + +## 單一腳本 + +保留 APITestka 的 action script 作為 request 與 correctness 的來源；可選的執行 profile +補充 load 與 synthetic 設定。延伸既有 APITestka→LoadDensity bridge，不複製 request 清單。 +完整契約與 record 的第一階段設計見 [共用 record 設計](2026-10-02-shared-record-design.md)。 + +功能模式執行一次；load 模式將共用步驟套入負載引擎；synthetic 模式以 WebRunner 的 monitor +週期呼叫功能 runner。功能／assertion failure 必須轉成失敗結果並讓 monitor 的 check 丟例外， +不能把 executor 回傳或程序 exit code 0 視為充分的成功證據。 +未知指令或無法轉換的檢查在 preflight 阻擋；需要 mode-specific 的動作明確標註，不靜默跳過。 + +## UI 與百分位數帶狀圖 + +GUI 主畫面：左側腳本／target／engine／負載設定；右側依序為 run 狀態與 Start／Stop、 +使用者／RPS／failure rate／p50／p95／p99 指標、圖表、request 結果／logs／run history。 +載入 action file、停止與多語系延伸既有元件；背景 runner 的更新只透過訊號進入 Qt UI。 +running 時避免重複 Start，停止可見 stopping，失敗／完成保留結果供檢視。 +web dashboard 以相同資訊順序呈現，窄螢幕堆疊顯示；動態 request 名稱透過 text node 安全呈現。 + +延遲圖採每秒時間窗的 p50 線、p50–p95 帶與 p95–p99 帶；RPS 使用獨立圖／軸。 +GUI 用 QtCharts 的 area series；離線報表用 matplotlib `fill_between`；dashboard 用原生 SVG／canvas。 +共用統計 helper 將成功與失敗合併並以 `start_time` epoch 排序；耗時使用 monotonic clock。 +百分位數沿用既有排序索引規則：`index = int(round((p / 100) * (n - 1)))`， +其中 round 使用 Python 的 ties-to-even；取排序後該索引的值，不由各 UI 另算。 +空窗顯示資料缺口而非零延遲，單樣本所有百分位數相同，RPS 為 count／實際 bucket 秒數。 +不把平均數／已算好的 worker p95 再取平均當成全域百分位數。 +即時資料採有界時間窗，歷史結果由持久化讀取，避免長時間監控無限累積記憶體。 + +## 驗證與文件 + +每個子專案補實際行為測試；既有 legacy CLI、APITestka bridge summary keys、record lists 保持相容。 +跨 repo 的公開契約與 consumer 測試同步更新；涉及 WebRunner 工作樹前遵守 workspace 的即時 consumer 規則。 +使用者行為變動在同一階段更新三份 README、Sphinx 與各 repo 的 `architecture.md` §6。 +只有完成的項目才從 `progress.md` 移除並寫入 `docs/updates/`。 + +2026-10-02 基準：`.venv/Scripts/python.exe -m pytest` 執行 asyncio、cloud、chart report、 +dashboard、record、GUI 的既有測試，27 passed；另有既有 gevent TLS monkey patch 與 pytest collection warning。 +本機 Docker CLI 可呼叫，但 Docker Desktop Linux daemon 未啟動；容器驗證尚未執行。 diff --git a/docs/updates/2026-10.md b/docs/updates/2026-10.md index 06d0506..0b068bd 100644 --- a/docs/updates/2026-10.md +++ b/docs/updates/2026-10.md @@ -178,3 +178,11 @@ Index and query commands: [README.md](README.md). New entries go at the end. - **Docs**: `architecture.md` §3 "Release channels" names the backend among the locked tools, the `--no-isolation` build and the guards. `CLAUDE.md` keeps `python -m build` under "Development Commands": that is the local build, not the publish jobs. No README or `docs/source/` page describes the release jobs, so none changed. - **Files**: `.github/requirements/publish.in`, `.github/requirements/publish.txt` (generated), `.github/workflows/ci-dev.yml`, `.github/workflows/publish-pypi.yml`, `test/test_workflow_actions.py`, `test/test_dev_release.py`, `architecture.md`. - **Open items**: none. `publish-pypi.yml` runs only after `CI Stable` on `main`, so its build step is first exercised when `dev` is merged into `main`. + +## U-20261002-01 · 2026-10-02 · Testing platform design draft and outstanding work · #docs #design + +- **What**: prepared a proposed six-subproject roadmap and a request-record v1 design for the functional/load/synthetic workflow. Expanded outstanding async and UI work, and added smoke tests, Docker extras CI, percentile bands, cloud failures, shared records, distributed recovery and the single-script integration to `progress.md`. The designs are awaiting review; no runtime behavior changed. +- **Baseline**: the existing asyncio, cloud launcher, chart report, dashboard, record and GUI tests passed: 27 passed, 2 warnings. Docker Desktop's Linux daemon was unavailable, so no container verification was run. +- **Files**: `docs/superpowers/specs/2026-10-02-testing-platform-roadmap.md`, `docs/superpowers/specs/2026-10-02-shared-record-design.md`, `progress.md`. +- **Evidence**: `.venv/Scripts/python.exe -m pytest test/test_asyncio_engine.py test/test_cloud_launchers.py test/test_chart_report.py test/test_live_dashboard.py test/test_test_record.py test/test_gui_tabs.py -q`; `docker version --format '{{.Server.Version}}'`. +- **Open items**: `progress.md` #16, #18 and #24–#30. diff --git a/docs/updates/README.md b/docs/updates/README.md index 00772c6..b8a24c1 100644 --- a/docs/updates/README.md +++ b/docs/updates/README.md @@ -58,6 +58,7 @@ In the same commit: delete the item from `progress.md`, add a `#done` entry here | ID | Date | Title | Tags | Batch | |---|---|---|---|---| +| U-20261002-01 | 2026-10-02 | Testing platform design draft and outstanding work | #docs #design | [2026-10](2026-10.md) | | U-20261001-12 | 2026-10-01 | Publish jobs build with the locked setuptools | #ci #security #X-13 | [2026-10](2026-10.md) | | U-20261001-11 | 2026-10-01 | Publish jobs install hash-locked build tooling | #done #ci #security #X-13 | [2026-10](2026-10.md) | | U-20261001-10 | 2026-10-01 | The sdist carries no tests | #done #packaging #X-13 | [2026-10](2026-10.md) | diff --git a/progress.md b/progress.md index ef26c43..b19148a 100644 --- a/progress.md +++ b/progress.md @@ -8,9 +8,16 @@ Cross-repo and workspace items live in `D:\Codes\progress.md` (relevant here: X- - **#14** FSM scenario runner integration: `FsmRunner` (`je_load_density/utils/scenario/fsm.py`) is exported by `__init__.py` but `run_scenario` (`wrapper/user_template/scenario_runner.py`) does not handle `mode="fsm"`. Furthermore, `fsm.py:56` terminates on self-transitions (`target == current_state`). Fix self-transitions with a loop/retry limit, wire `FsmRunner` into `run_scenario` and user templates, and add scenario tests. - **#15** Virtual user session context & parameter isolation: `parameter_resolver` (`je_load_density/utils/parameterization/parameter_resolver.py`) stores variables in a single global dictionary. When multiple concurrent users extract values (e.g. auth tokens, session IDs) via `task["extract"]`, they overwrite each other. Introduce session-scoped / user-local context support (`${session.VAR}` or per-user local resolution) so concurrent user journeys maintain isolated state. -- **#16** Asyncio engine (`bench`) feature parity: `je_load_density/engine/asyncio_engine.py` only passes `headers` and `json` body in `_send_one`. Add query parameters (`params`), form data (`data`), request labels (`name`), custom `timeout`, `expect_status` (instead of unconditionally failing status >= 400), and basic assertions (`status_code`, `contains`, `json_path`) so the lightweight engine can run realistic API tests. +- **#16** Asyncio engine as a first-class runner: `je_load_density/engine/asyncio_engine.py` is not dispatched by `wrapper/start_wrapper/start_test.py`, and package initialisation imports Locust even for async callers. Add engine selection across Python/action/CLI/GUI, lazy Locust loading, request kwargs and all assertions/extractors, scenario/retry/pacing/load shapes, per-user sessions (#15), run isolation, cancellation, reports/persistence/exporters and local/master/worker parity. Track protocol capabilities explicitly; unsupported options must fail preflight. Deliver HTTP parity first without closing full parity prematurely. Design: `docs/superpowers/specs/2026-10-02-testing-platform-roadmap.md`. - **#17** Mixed workload multi-user support: `start_test` and `prepare_env` (`wrapper/start_wrapper/start_test.py`, `wrapper/create_locust_env/create_locust_env.py`) only instantiate a single user class from `_USER_REGISTRY`. Extend `user_detail_dict` to accept a list of user types with weights (`users: [{"user": "http_user", "weight": 8}, {"user": "websocket_user", "weight": 2}]`) and pass them to Locust's `Environment(user_classes=[...])` for realistic multi-protocol simulation. -- **#18** GUI test abort control and action file execution: `je_load_density/gui/main_widget.py` has no "Stop Test" button (tests cannot be interrupted before duration completes), and only supports single-URL inputs. Add a stop button to signal the running thread/runner, add an action file picker to run `.json` test scripts directly in the GUI, and update translations in `gui/language_wrapper/`. +- **#18** Redesign GUI and web dashboard: `je_load_density/gui/main_widget.py` lacks cancellation and action-file execution; `utils/dashboard/live_dashboard.py` lacks the shared run-state/chart view. Add script/target/engine/load controls, Start/Stop lifecycle, metric cards and result/log/history views, responsive dashboard layout and translations. Use the run lifecycle from #16 and chart data from #26. Design: `docs/superpowers/specs/2026-10-02-testing-platform-roadmap.md`. - **#19** Markdown summary report generator (`LD_generate_markdown_report`): Add `generate_markdown_report.py` under `utils/generate_report/` to render GFM summary tables (RPS, latencies, SLA breaches, error clusters) suitable for GitHub Actions `$GITHUB_STEP_SUMMARY` and GitLab MR comments. Register the command in `action_executor.py`, export in `__init__.py`, and add tests. - **#20** Pre-flight health check gate: Add a target health probe before ramping load in `start_test` / `bench`. If the target endpoint is unreachable, DNS fails, or TLS handshake fails, abort early with a descriptive error instead of spawning hundreds of workers that flood logs with connection refused errors. - **#23** [BLOCKED: two releases must ship the warning first] Flip the package gate's default to refuse packages outside the allowlist: set `self.allow_arbitrary_packages = False` in `PackageManager.__init__` (`je_load_density/utils/package_manager/package_manager_class.py`); the warning branch is je_action_core's `_check_allowed` and stays for the other projects. Update the "Package gate" paragraph in the three READMEs and the "Package Gate" section of `docs/source/{En,Zh}/doc/package_manager/package_manager_doc.rst`. The warning is first released in the version after 0.0.77 (`origin/main`); flip once two releases after that one have shipped it. Decide first how a user who only runs action files (`loaddensity`, `-e`, the socket server) allows a package without a Python host to call `executor.allow_packages(...)`. +- **#24** End-to-end smoke tests: add `test/smoke/` for installed-wheel CLI/action execution against a subprocess-local HTTP service, success and failure reports/SLA/SQLite, async, dashboard JSON/SSE and offscreen GUI execution/cancellation. Existing `test_expansion*_smoke.py` does not cover that complete lifecycle. +- **#25** Docker extras installation matrix: `.github/workflows/ci-dev.yml` and `ci-stable.yml` lack isolated installation/capability checks for base, every declared extra and all. Generate the matrix from `pyproject.toml`, test a checkout-built wheel with `pip check` and non-skipping probes, include GUI system libraries and Compose health checks, and gate publishing on required checks. Local execution needs the Docker Desktop Linux daemon running. +- **#26** Percentile bands: `gui/chart_panel.py` mixes RPS and latency on one axis; `utils/generate_report/generate_chart_report.py` uses `ts`/index instead of request `start_time`. Share per-window percentile aggregation across GUI/dashboard/report, show p50 with p50–p95 and p95–p99 bands, separate RPS units and handle gaps, failures and actual bucket duration. +- **#27** Cloud backend failure coverage: extend `test/test_cloud_launchers.py` for invalid inputs, auth/permissions, timeout/rate limits, partial failures, Fargate `failures`, Lambda `FunctionError`/Event/warm reuse, Cloud Run refresh/error responses and ACI poller failure. Use SDK stubs/local HTTP and assert request/response semantics; no live provisioning is needed. +- **#28** Shared APITestka record format: `utils/test_record/`, `wrapper/event/request_hook.py` and `engine/asyncio_engine.py` need a versioned canonical request record and run-scoped sink, with legacy list/report compatibility and adapters for APITestka's response dicts and failure pairs. Coordinate the proposed stdlib contract in ActionCore and consumer dependency releases. Design: `docs/superpowers/specs/2026-10-02-shared-record-design.md`. +- **#29** Distributed worker health and recovery: `wrapper/create_locust_env/create_locust_env.py::_wait_for_workers` counts clients and starts after timeout. Add configurable ready/heartbeat/lost policy, fail or explicitly degrade on startup timeout, reassign virtual-user capacity after loss, handle reconnect/stale assignments and deduplicate results; reuse Locust heartbeat/rebalance support where possible. Target health is separately tracked by #20. +- **#30** One script for functional/load/synthetic modes: extend APITestka's existing `integrations/load_density*.py` bridge and WebRunner's `utils/synthetic_monitoring/monitor.py` so common request/assertion steps are authored once and execution profiles choose functional correctness, capacity/SLA or periodic monitoring. Unsupported conversions must fail preflight and failed checks must reach monitor transitions. Depends on #28 and async parity #16; preserve existing CLI/action/summary contracts. From 8dac6cc9a11656694efde03b4d37d0b80f6a50b9 Mon Sep 17 00:00:00 2001 From: JeffreyChen Date: Fri, 2 Oct 2026 18:56:09 +0800 Subject: [PATCH 02/18] feat: capture canonical request records in load engines --- README.md | 11 ++ README/README_zh-CN.md | 9 ++ README/README_zh-TW.md | 9 ++ architecture.md | 7 ++ .../En/doc/test_record/test_record_doc.rst | 12 ++ .../Zh/doc/test_record/test_record_doc.rst | 11 ++ .../plans/2026-10-02-shared-record.md | 103 +++++++++++++++ .../specs/2026-10-02-shared-record-design.md | 2 +- .../2026-10-02-testing-platform-roadmap.md | 2 +- docs/updates/2026-10.md | 7 ++ docs/updates/README.md | 1 + je_load_density/engine/asyncio_engine.py | 68 ++++++---- je_load_density/utils/test_record/contract.py | 88 +++++++++++++ .../utils/test_record/run_context.py | 13 ++ .../create_locust_env/create_locust_env.py | 16 ++- je_load_density/wrapper/event/request_hook.py | 27 ++-- test/test_record_contract.py | 66 ++++++++++ test/test_record_engine_integration.py | 118 ++++++++++++++++++ 18 files changed, 537 insertions(+), 33 deletions(-) create mode 100644 docs/superpowers/plans/2026-10-02-shared-record.md create mode 100644 je_load_density/utils/test_record/contract.py create mode 100644 je_load_density/utils/test_record/run_context.py create mode 100644 test/test_record_contract.py create mode 100644 test/test_record_engine_integration.py diff --git a/README.md b/README.md index a706d1d..f022f12 100644 --- a/README.md +++ b/README.md @@ -781,6 +781,17 @@ Legacy single-flag form (`-e/-d/-c/--execute_str`) is still accepted for backwar ## Test Record +Canonical request records are available as an opt-in API with an ActionCore release that provides +`je_action_core.request_context` (or the coordinated development checkout). Legacy record lists and reports +continue to work with the existing dependency floor. Create +`RunContext(source="loaddensity", phase="load", engine="asyncio")` from +`je_load_density.utils.test_record.run_context`, pass it as `run_context` to `run_async_load`, and call +`context.to_json()` for versioned results. For Locust, pass `run_context` to `start_test`/`prepare_env`/`create_env`; +the context is bound to isolated environment events, including requests fired by greenlets. +`use_run_context(context)` also captures direct request events in the current scope. +Canonical records use numeric/null status, measured milliseconds, structured errors and run identities; +full response payloads are disabled by default. Async return summaries count only their own invocation. + `test_record_instance.test_record_list` and `error_record_list` collect every request with `Method`, `test_url`, `name`, `status_code`, `response_time_ms`, `response_length`, `start_time` (epoch seconds, so reports can restore request order across the two lists), and (for failures) `error`. Reports and the SQLite sink read directly from these lists. ## Exception Handling diff --git a/README/README_zh-CN.md b/README/README_zh-CN.md index 4f7c733..0e88d2c 100644 --- a/README/README_zh-CN.md +++ b/README/README_zh-CN.md @@ -781,6 +781,15 @@ python -m je_load_density serve [--host ...] # start the control socket ## 测试记录 +Canonical request 记录是可选 API，需要提供 `je_action_core.request_context` 的 ActionCore 版本 +或协调的开发工作树。原记录列表与报告仍支持现有依赖下限。从 +`je_load_density.utils.test_record.run_context` 创建 +`RunContext(source="loaddensity", phase="load", engine="asyncio")`，以 `run_context` 传给 +`run_async_load`，再用 `context.to_json()` 输出版本化结果。Locust 可将 `run_context` 传给 +`start_test`／`prepare_env`／`create_env`；context 绑定隔离的 environment 事件，覆盖 greenlet 的请求。 +`use_run_context(context)` 也可捕获当前范围的直接 request 事件。新格式使用数字／null 状态码、 +实测毫秒、结构化错误与 run 标识；默认不保存完整响应内容。Async 返回摘要只计算本次调用。 + `test_record_instance.test_record_list` 与 `error_record_list` 收集每次请求,内含 `Method`、`test_url`、`name`、`status_code`、`response_time_ms`、`response_length`、`start_time`(epoch 秒,因此报告可跨两份 list 还原请求顺序),失败时还带 `error`。报告与 SQLite sink 直接从这些 list 读取。 ## 异常处理 diff --git a/README/README_zh-TW.md b/README/README_zh-TW.md index 4d85967..44617c3 100644 --- a/README/README_zh-TW.md +++ b/README/README_zh-TW.md @@ -781,6 +781,15 @@ python -m je_load_density serve [--host ...] # start the control socket ## 測試紀錄 +Canonical request 紀錄為選用 API，需要提供 `je_action_core.request_context` 的 ActionCore 版本 +或協調的開發工作樹。原紀錄清單與報表仍支援既有相依下限。從 +`je_load_density.utils.test_record.run_context` 建立 +`RunContext(source="loaddensity", phase="load", engine="asyncio")`，以 `run_context` 傳給 +`run_async_load`，再用 `context.to_json()` 輸出版本化結果。Locust 可將 `run_context` 傳給 +`start_test`／`prepare_env`／`create_env`；context 綁定隔離的 environment 事件，涵蓋 greenlet 的請求。 +`use_run_context(context)` 也可擷取目前範圍的直接 request 事件。新格式使用數字／null 狀態碼、 +實測毫秒、結構化錯誤與 run 識別；預設不保存完整回應內容。Async 回傳摘要只計算這次呼叫。 + `test_record_instance.test_record_list` 與 `error_record_list` 蒐集每次請求,內含 `Method`、`test_url`、`name`、`status_code`、`response_time_ms`、`response_length`、`start_time`(epoch 秒,因此報告可跨兩份 list 還原請求順序),失敗時還帶 `error`。報告與 SQLite sink 直接從這些 list 讀取。 ## 例外處理 diff --git a/architecture.md b/architecture.md index 0fea70a..ca76107 100644 --- a/architecture.md +++ b/architecture.md @@ -135,6 +135,13 @@ MCP `load_density.list_executor_commands` tool all read the `LD_*` names from `e ## 6. Cross-project boundaries +Canonical request records are an opt-in contract supplied by ActionCore's ``request_record``/ +``request_context`` APIs. ``utils/test_record/contract.py`` adapts legacy results and +``run_context.py`` exposes explicit run scopes. Locust environments bind the selected context +to isolated events; asyncio propagates it to its tasks. Canonical APIs require the coordinated +core release/checkouts, while existing imports and legacy report shapes remain compatible with +the published dependency floor. The old action-executor record contract is separate and unchanged. + - **PyBreeze (subprocess)** runs `python -m je_load_density --execute_str ` or `--execute_file ` (`PyBreeze/pybreeze/extend/process_executor/python_task_process_manager.py`; the package name is in `.../process_executor/load_density/load_density_process.py`). The hidden legacy flags and the diff --git a/docs/source/En/doc/test_record/test_record_doc.rst b/docs/source/En/doc/test_record/test_record_doc.rst index f9b11a5..7dc615e 100644 --- a/docs/source/En/doc/test_record/test_record_doc.rst +++ b/docs/source/En/doc/test_record/test_record_doc.rst @@ -45,3 +45,15 @@ SQLite persistence ------------------ See :doc:`../../../En/doc/sqlite_persistence/sqlite_persistence_doc`. + +Canonical run-scoped results +--------------------------- + +The opt-in ``utils.test_record.run_context`` module re-exports ActionCore's +``RunContext`` and ``use_run_context``. It requires the coordinated core request-record API; +legacy package imports and lists remain compatible with earlier core releases. +Create a context with ``source="loaddensity"``, ``phase="load"`` and the chosen engine. +Pass ``run_context=context`` to ``run_async_load`` or ``start_test``. Locust binds the context +to isolated environment events so greenlets do not depend on ContextVar inheritance. +``context.to_json()`` exports v1 records with identities, numeric/null status and structured errors. +Payload capture is disabled by default. Async summaries exclude previous invocations' records. diff --git a/docs/source/Zh/doc/test_record/test_record_doc.rst b/docs/source/Zh/doc/test_record/test_record_doc.rst index 4318561..0df72d0 100644 --- a/docs/source/Zh/doc/test_record/test_record_doc.rst +++ b/docs/source/Zh/doc/test_record/test_record_doc.rst @@ -37,3 +37,14 @@ SQLite 持久化 ------------- 見 :doc:`../sqlite_persistence/sqlite_persistence_doc`。 + +隔離的 canonical 執行紀錄 +------------------------- + +選用模組 ``utils.test_record.run_context`` 提供 ActionCore 的 ``RunContext`` 與 +``use_run_context``，需要協調的 core request-record API；原套件 import 與清單維持舊版本相容。 +建立 context 時指定 ``source="loaddensity"``、``phase="load"`` 與引擎。 +以 ``run_context=context`` 傳給 ``run_async_load`` 或 ``start_test``；Locust 將 context +綁定隔離的 environment 事件，greenlet 不依賴 ContextVar 繼承。 +``context.to_json()`` 輸出 v1 紀錄，包含識別、數字／null 狀態碼與結構化錯誤。 +預設不擷取完整內容；async 回傳摘要不納入前次呼叫的紀錄。 diff --git a/docs/superpowers/plans/2026-10-02-shared-record.md b/docs/superpowers/plans/2026-10-02-shared-record.md new file mode 100644 index 0000000..e41f8cc --- /dev/null +++ b/docs/superpowers/plans/2026-10-02-shared-record.md @@ -0,0 +1,103 @@ +# Shared request record Implementation Plan + +> **For agentic workers:** REQUIRED SUB-SKILL: Use superpowers:executing-plans to implement this plan task-by-task. Steps use checkbox (`- [ ]`) syntax for tracking. + +**Goal:** Publish a stdlib request-record contract and connect APITestka and LoadDensity without breaking legacy consumers. + +**Architecture:** ActionCore owns the JSON-safe, versioned contract. Framework adapters feed an explicitly scoped RunContext; existing response returns, record lists and summary fields remain compatible. New consumers are developed against isolated checkouts; release-dependent imports are lazy until the ActionCore release is available. + +**Tech Stack:** Python 3.10+, TypedDict, ContextVar, threading locks, SQLite, pytest, Locust, requests and httpx. + +**Spec:** `docs/superpowers/specs/2026-10-02-shared-record-design.md`; overall roadmap: `2026-10-02-testing-platform-roadmap.md` in the same directory. + +## Global Constraints + +- ActionCore dependencies remain empty; it must not import any framework or HTTP client. +- schema_version is integer 1; unknown values, NaN and negative measured values are rejected. +- Existing CLI flags, response return shapes, record lists and summary keys remain supported. +- Adapter outcome preserves each runner's existing judgement; no new status policy in this stage. +- No manual package-version bump. The core release precedes consumer dependency-floor changes. +- Framework source changes, tests, three README languages and architecture contracts ship together. +- Execute inline in isolated feature worktrees; the user approved the spec and explicitly requested implementation. + +## Review Focus + +- Error response objects can be falsey: retain their status/body rather than treating them as absent. +- Failed requests before a response exists need measured elapsed time and nullable status, not fake zeros. +- Context isolation must survive asyncio task inheritance and synchronous runner boundaries. +- Repeated record IDs with conflicting content must fail; identical retries must not double count. +- SDK release order must not make existing imports fail while only the older ActionCore is installed. + +--- + +### Task 1: ActionCore record contract + +**Files:** Create `ActionCore/je_action_core/request_record.py`, `ActionCore/test/test_request_record.py`; modify public exports, three READMEs and `architecture.md`. + +**Interfaces:** Produces `RequestRecord`, `RecordError`, `RequestRecordError`, `validate_request_record(record: Mapping[str, object]) -> RequestRecord`, `serialize_request_record(record: Mapping[str, object]) -> str` and `request_record_schema() -> dict[str, object]`. + +- [ ] Write tests proving missing/version/type/outcome/latency/time/assertion errors are rejected; complete success/failure records round-trip; nested bytes/datetime/timedelta serialize safely without arbitrary-object stringification. Hand-derived fixtures assert status 500 remains 500 and the contract has no client imports. +- [ ] Run `D:/Codes/ActionCore/.venv/Scripts/python.exe -m pytest test/test_request_record.py -q`; expect missing-module failure before implementation. +- [ ] Implement the TypedDict and validators, JSON conversion and schema in the named module. Keep individual validators focused and the public normalization surface independent of transport. +- [ ] Run the new tests and `python -m pytest -q` with the ActionCore interpreter; expect all tests pass. Run static checks for the new module. +- [ ] Document the new API and commit the ActionCore stage. Do not publish or change a version by hand. + +### Task 2: Run context and LoadDensity legacy adapters + +**Files:** Create `LoadDensity/je_load_density/utils/test_record/{run_context,contract}.py`, `LoadDensity/test/test_record_contract.py`. + +**Interfaces:** Consumes Task 1. Produces `RunContext(source='loaddensity', phase='load', engine='locust', worker_id=None)`, `RunContext.append(record: Mapping[str, object]) -> bool`, `RunContext.snapshot() -> list[RequestRecord]`, `RunContext.to_json() -> str`, `use_run_context(context)` and `get_run_context()`. Adapter: `from_legacy_record(record, context, outcome) -> RequestRecord`; legacy conversion never creates fake timestamps. + +- [ ] Test two contexts remain isolated, task-local nesting restores prior contexts, identical IDs deduplicate, conflicting IDs fail, input mutations cannot alter snapshots, and LoadDensity/APITestka legacy fixtures normalize status/time/bytes correctly. +- [ ] Run `pytest test/test_record_contract.py -q` with the isolated ActionCore on PYTHONPATH; expect missing-module failure. +- [ ] Implement the context/sink and source-specific adapters. Import the new core only when a canonical operation is called; missing API raises a clear upgrade error and does not break legacy imports. +- [ ] Run the targeted suite; expect pass including malformed import positions, nullable unknown measurements and credential redaction. +- [ ] Keep this task's commit together with Task 3 because its public adapter needs a real runner consumer and user documentation. + +### Task 3: Locust and async recording + +**Files:** Modify `LoadDensity/je_load_density/wrapper/event/request_hook.py`, `engine/asyncio_engine.py`, `utils/test_record/test_record_class.py`; add `test/test_record_engine_integration.py`; update three READMEs, Sphinx record docs and architecture. + +**Interfaces:** Consumes Task 2. Locust reads the active context; `run_async_load(..., run_context=None)` explicitly propagates the selected run to asyncio workers. Canonical records are separate from existing legacy lists. + +- [ ] Test a Locust response with status 500 and falsey truthiness retains status; async success/status failure/transport failure capture distinct records with real elapsed time; consecutive runs' return summaries exclude prior records; scoped recording leaves legacy report keys unchanged. +- [ ] Run integration tests; expect failure on the currently missing context output and contaminated run summaries. +- [ ] Route measured request outcomes to the context alongside the existing legacy append. Give run_async_load per-invocation counters and propagate cancellation. Keep existing return request/failure semantics. +- [ ] Run new tests, existing asyncio/report/SLA/SQLite tests, then the full LoadDensity suite; expect pass. +- [ ] Document opt-in canonical collection and commit the LoadDensity recording stage. Record remaining schema-release coordination in progress.md rather than inventing a released version. + +### Task 4: APITestka adapters and recording + +**Files:** Create `APITestka/je_api_testka/utils/test_record/contract.py`, `run_context.py`, `test/test_utils/test_record_contract.py`; modify requests/httpx sync/async wrappers and record class; update READMEs, Sphinx and architecture. + +**Interfaces:** Same canonical context API as Task 2, with source='apitestka' and phase='functional'; context implementation is shared in ActionCore if it proves reusable rather than copied. `record_response(response_data, request, error, engine, elapsed_ms)` appends only when a context is active. + +- [ ] Add tests using real local requests/httpx calls and failures; assert native return values and failure pairs remain identical while canonical records retain response information and status. Test recording disabled and context cleanup. +- [ ] Run those tests with the isolated core; expect missing-module/API failure. +- [ ] Wire a scope-aware capture helper into all three wrappers, including before-response errors and clean_record behavior. Preserve caught-error behavior and record_request_info semantics. +- [ ] Run the targeted tests, load bridge tests and full APITestka suite; expect pass. Separately run with released ActionCore to prove existing imports still work. +- [ ] Document API and commit this repository independently. + +### Task 5: Versioned JSON and SQLite export + +**Files:** Modify `LoadDensity/je_load_density/utils/test_record/sqlite_persistence.py`; create `test/test_canonical_persistence.py`; update record/export documentation. + +**Interfaces:** Consumes contexts. Produces `persist_canonical_records(database_path: str, context: RunContext) -> str`, `fetch_canonical_records(database_path: str, run_id: str) -> list[RequestRecord]`; existing tables and functions remain unchanged. + +- [ ] Test empty database and existing legacy database, JSON round-trip, multiple runs, duplicate/conflicting records and rollback on invalid records. +- [ ] Run the new test; expect missing-function failure. +- [ ] Add separate versioned canonical tables and parameterized transactional writes; never migrate or overwrite legacy data implicitly. +- [ ] Run persistence and legacy regression tests; expect pass. +- [ ] Document the explicit export APIs and commit the persistence stage. + +### Task 6: Cross-project verification and release handoff + +**Files:** Contract documentation and test fixtures in the three repositories; LoadDensity progress and update log. + +**Interfaces:** All Task 1–5 interfaces and the existing APITestka load subprocess bridge. + +- [ ] Exercise both projects against a local service, confirm matching keys/types for success and failures and distinct run IDs; run all changed-repository suites and relevant ActionCore consumers. +- [ ] Check three-language docs, schemas/examples, public exports, static analysis and `git diff --check`. +- [ ] Review the whole branch using the requesting-code-review workflow; repair any substantive issues with regression tests. +- [ ] Prepare concrete feature branches/PRs for core and consumers. Consumer dependency-floor activation waits for the real core release; do not mark that rollout finished beforehand. +- [ ] Finish the shared-record stage, retain remaining roadmap items, and continue to smoke/extras CI rather than ending the overall task. diff --git a/docs/superpowers/specs/2026-10-02-shared-record-design.md b/docs/superpowers/specs/2026-10-02-shared-record-design.md index 1e91e15..0720faa 100644 --- a/docs/superpowers/specs/2026-10-02-shared-record-design.md +++ b/docs/superpowers/specs/2026-10-02-shared-record-design.md @@ -1,6 +1,6 @@ # 共用 request record v1 設計草案 -狀態：待審閱。此文件是第一個子專案的規格；整批範圍見 +狀態：2026-10-02 已確認，依使用者指示開始實作。此文件是第一個子專案的規格；整批範圍見 [測試平台設計](2026-10-02-testing-platform-roadmap.md)。 ## 目的與邊界 diff --git a/docs/superpowers/specs/2026-10-02-testing-platform-roadmap.md b/docs/superpowers/specs/2026-10-02-testing-platform-roadmap.md index fdcf2f2..7f49ce6 100644 --- a/docs/superpowers/specs/2026-10-02-testing-platform-roadmap.md +++ b/docs/superpowers/specs/2026-10-02-testing-platform-roadmap.md @@ -1,6 +1,6 @@ # LoadDensity 與跨專案測試流程設計草案 -狀態：待審閱；以下描述預定行為，不代表功能已實作。 +狀態：2026-10-02 已確認；以下描述目標行為，各項實作狀態依 progress.md 與 updates 記錄。 ## 目標與範圍 diff --git a/docs/updates/2026-10.md b/docs/updates/2026-10.md index 0b068bd..e482141 100644 --- a/docs/updates/2026-10.md +++ b/docs/updates/2026-10.md @@ -186,3 +186,10 @@ Index and query commands: [README.md](README.md). New entries go at the end. - **Files**: `docs/superpowers/specs/2026-10-02-testing-platform-roadmap.md`, `docs/superpowers/specs/2026-10-02-shared-record-design.md`, `progress.md`. - **Evidence**: `.venv/Scripts/python.exe -m pytest test/test_asyncio_engine.py test/test_cloud_launchers.py test/test_chart_report.py test/test_live_dashboard.py test/test_test_record.py test/test_gui_tabs.py -q`; `docker version --format '{{.Server.Version}}'`. - **Open items**: `progress.md` #16, #18 and #24–#30. + +## U-20261002-02 · 2026-10-02 · Capture canonical records in Locust and asyncio runs · #migration #records + +- **What**: connected opt-in canonical RunContext recording to Locust request events and asyncio requests, with legacy adapters and numeric/null status conversion. Locust binds the context to per-environment events across greenlets; existing report lists remain compatible. Falsey HTTP error responses retain their status, protocol-only events no longer require an HTTP response object, timeout durations are measured, and async return counts exclude earlier runs. +- **Tests**: 1237 passed, 2 skipped in the complete suite against the coordinated core checkout (14 new adapter/integration tests). Legacy installed-core regression: 8 passed, 2 canonical modules skipped. New source modules passed ruff E/F/W/C90/I; git diff --check passed. +- **Files**: utils/test_record/{contract,run_context}.py, engine/asyncio_engine.py, wrapper/event/request_hook.py, wrapper/create_locust_env/create_locust_env.py, tests, three READMEs, Sphinx record docs and architecture.md. +- **Open items**: progress.md #28 still requires APITestka integration, canonical SQLite exports and coordinated core rollout; #16 remains the full async parity work. diff --git a/docs/updates/README.md b/docs/updates/README.md index b8a24c1..fb3734a 100644 --- a/docs/updates/README.md +++ b/docs/updates/README.md @@ -58,6 +58,7 @@ In the same commit: delete the item from `progress.md`, add a `#done` entry here | ID | Date | Title | Tags | Batch | |---|---|---|---|---| +| U-20261002-02 | 2026-10-02 | Capture canonical records in Locust and asyncio runs | #migration #records | [2026-10](2026-10.md) | | U-20261002-01 | 2026-10-02 | Testing platform design draft and outstanding work | #docs #design | [2026-10](2026-10.md) | | U-20261001-12 | 2026-10-01 | Publish jobs build with the locked setuptools | #ci #security #X-13 | [2026-10](2026-10.md) | | U-20261001-11 | 2026-10-01 | Publish jobs install hash-locked build tooling | #done #ci #security #X-13 | [2026-10](2026-10.md) | diff --git a/je_load_density/engine/asyncio_engine.py b/je_load_density/engine/asyncio_engine.py index 2a9203c..6cc429e 100644 --- a/je_load_density/engine/asyncio_engine.py +++ b/je_load_density/engine/asyncio_engine.py @@ -19,10 +19,12 @@ import asyncio import time +from contextlib import nullcontext from typing import Any, Dict, List, Optional from je_load_density.utils.logging.loggin_instance import load_density_logger from je_load_density.utils.parameterization import parameter_resolver +from je_load_density.utils.test_record.contract import record_request from je_load_density.utils.test_record.test_record_class import test_record_instance @@ -37,7 +39,7 @@ def _import_httpx(): return httpx -def _record(method: str, url: str, status: int, elapsed_ms: float, length: int, started: float) -> None: +def _record(method: str, url: str, status: int, elapsed_ms: float, length: int, started: float) -> bool: """Record one response. A 4xx/5xx is a failure, as Locust's HTTP users count it, so SLA gates and failure-rate reports mean the same thing whichever engine produced the records.""" entry = { @@ -52,25 +54,32 @@ def _record(method: str, url: str, status: int, elapsed_ms: float, length: int, if status >= 400: entry["error"] = f"HTTP {status}" test_record_instance.error_record_list.append(entry) - return + record_request({**entry, "error_kind": "http_status"}, "failed") + return False entry["error"] = None test_record_instance.test_record_list.append(entry) + record_request(entry, "passed") + return True -def _record_error(method: str, url: str, error: str, started: float) -> None: - test_record_instance.error_record_list.append({ +def _record_error(method: str, url: str, error: Exception, started: float, elapsed_ms: float) -> bool: + entry = { "Method": method.upper(), "test_url": url, "name": url, "status_code": "0", - "response_time_ms": 0.0, + "response_time_ms": elapsed_ms, "response_length": 0, - "error": error, + "error": repr(error), "start_time": started, - }) + } + test_record_instance.error_record_list.append(entry) + kind = "timeout" if "timeout" in type(error).__name__.lower() else "connection" + record_request({**entry, "error_kind": kind}, "failed") + return False -async def _send_one(client, raw_task: Dict[str, Any]) -> None: +async def _send_one(client, raw_task: Dict[str, Any]) -> bool: task = parameter_resolver.resolve(raw_task) method = str(task.get("method", "get")).lower() url = task["request_url"] @@ -84,9 +93,9 @@ async def _send_one(client, raw_task: Dict[str, Any]) -> None: timeout=float(task.get("timeout", 10.0)), ) elapsed_ms = (time.monotonic() - start) * 1000 - _record(method, url, response.status_code, elapsed_ms, len(response.content), started) + return _record(method, url, response.status_code, elapsed_ms, len(response.content), started) except Exception as error: # noqa: BLE001 - _record_error(method, url, repr(error), started) + return _record_error(method, url, error, started, (time.monotonic() - start) * 1000) async def _worker( @@ -94,13 +103,17 @@ async def _worker( tasks: List[Dict[str, Any]], deadline: float, semaphore: asyncio.Semaphore, -) -> None: +) -> tuple[int, int]: + successes = failures = 0 while time.monotonic() < deadline: for task in tasks: if time.monotonic() >= deadline: - return + return successes, failures async with semaphore: - await _send_one(client, task) + passed = await _send_one(client, task) + successes += int(passed) + failures += int(not passed) + return successes, failures async def run_async_load( @@ -109,10 +122,23 @@ async def run_async_load( duration_seconds: float = 10.0, http2: bool = False, max_in_flight: Optional[int] = None, + run_context=None, ) -> Dict[str, Any]: """Run a pure-asyncio HTTP load test. Returns a summary dict.""" httpx = _import_httpx() semaphore = asyncio.Semaphore(max_in_flight or max(users, 1)) + scope = nullcontext() + if run_context is not None: + from je_load_density.utils.test_record.run_context import use_run_context + scope = use_run_context(run_context) + with scope: + results = await _run_workers(httpx, tasks, users, duration_seconds, http2, semaphore) + summary = {"requests": sum(result[0] for result in results), "failures": sum(result[1] for result in results)} + load_density_logger.info(f"asyncio engine done — {summary['requests']} ok, {summary['failures']} err") + return summary + + +async def _run_workers(httpx, tasks, users, duration_seconds, http2, semaphore): async with httpx.AsyncClient(http2=http2) as client: # Start the clock once the client exists, so duration_seconds is time under load and a # slow client setup cannot eat a short run's whole budget. @@ -121,12 +147,10 @@ async def run_async_load( asyncio.create_task(_worker(client, tasks, deadline, semaphore)) for _ in range(users) ] - await asyncio.gather(*workers, return_exceptions=True) - load_density_logger.info( - f"asyncio engine done — {len(test_record_instance.test_record_list)} ok, " - f"{len(test_record_instance.error_record_list)} err" - ) - return { - "requests": len(test_record_instance.test_record_list), - "failures": len(test_record_instance.error_record_list), - } + try: + return await asyncio.gather(*workers) + finally: + for worker in workers: + if not worker.done(): + worker.cancel() + await asyncio.gather(*workers, return_exceptions=True) diff --git a/je_load_density/utils/test_record/contract.py b/je_load_density/utils/test_record/contract.py new file mode 100644 index 0000000..e8ec515 --- /dev/null +++ b/je_load_density/utils/test_record/contract.py @@ -0,0 +1,88 @@ +"""Adapt existing LoadDensity records without changing legacy report consumers.""" + +from __future__ import annotations + +import base64 +from collections.abc import Mapping +from typing import TYPE_CHECKING + +if TYPE_CHECKING: + from je_action_core.request_context import RunContext + from je_action_core.request_record import RequestRecord + +_SENSITIVE_HEADERS = frozenset(("authorization", "proxy-authorization", "cookie", "set-cookie", "x-api-key")) +_HTTP_METHODS = frozenset(("GET", "POST", "PUT", "PATCH", "DELETE", "HEAD", "OPTIONS", "CONNECT", "TRACE")) + + +def _number(record: Mapping[str, object], field: str, converter): + value = record.get(field) + if value is None: + return None + try: + return converter(value) + except (TypeError, ValueError, OverflowError) as error: + raise ValueError(f"{field}: invalid measurement") from error + + +def _fields(record: Mapping[str, object], outcome: str) -> dict[str, object]: + method = str(record.get("Method") or "GET").upper() + url = str(record.get("test_url") or "") + start = _number(record, "start_time", float) + elapsed = _number(record, "response_time_ms", float) + status = _number(record, "status_code", int) + failure = None if outcome == "passed" else { + "kind": str(record.get("error_kind") or "request"), + "message": str(record.get("error") or "request failed"), + } + protocol = record.get("protocol") or ("http" if method in _HTTP_METHODS else method.lower()) + return {"protocol": str(protocol), "request_method": method, + "request_url": url, "name": str(record.get("name") or f"{method} {url}"), + "status_code": status if status != 0 else None, "start_time": start, + "end_time": start + elapsed / 1000.0 if start is not None and elapsed is not None else None, + "response_time_ms": elapsed, "response_length": _number(record, "response_length", int), + "outcome": outcome, "error": failure, "assertions": record.get("assertions") or [], + "step_id": record.get("step_id"), "scenario_id": record.get("scenario_id")} + + +def _payload(record: Mapping[str, object]) -> dict[str, object]: + fields: dict[str, object] = {} + if "text" in record: + fields["text"] = record["text"] + headers = record.get("headers") + if isinstance(headers, Mapping): + fields["headers"] = { + str(key): "[REDACTED]" if str(key).lower() in _SENSITIVE_HEADERS else str(value) + for key, value in headers.items() + } + content = record.get("content") + if isinstance(content, bytes): + fields["content_base64"] = base64.b64encode(content).decode("ascii") + return fields + + +def from_legacy_record(record: Mapping[str, object], context: RunContext, outcome: str, + capture_payload: bool = False) -> RequestRecord: + """Import one legacy result into the selected run, preserving unknown measurements as null.""" + fields = _fields(record, outcome) + if capture_payload: + fields.update(_payload(record)) + return context.capture(fields) + + +def get_optional_run_context() -> RunContext | None: + """Return a selected context, keeping legacy-only imports usable with earlier core releases.""" + try: + from je_action_core.request_context import get_run_context + except ModuleNotFoundError as error: + if error.name != "je_action_core.request_context": + raise + return None + return get_run_context() + + +def record_request(record: Mapping[str, object], outcome: str, capture_payload: bool = False) -> RequestRecord | None: + """Capture only when a run scope is active; older core versions retain legacy-only behavior.""" + context = get_optional_run_context() + if context is None: + return None + return from_legacy_record(record, context, outcome, capture_payload) diff --git a/je_load_density/utils/test_record/run_context.py b/je_load_density/utils/test_record/run_context.py new file mode 100644 index 0000000..9740879 --- /dev/null +++ b/je_load_density/utils/test_record/run_context.py @@ -0,0 +1,13 @@ +"""Opt-in canonical recording; legacy imports do not require the new core API.""" + +try: + from je_action_core.request_context import RunContext, get_run_context, use_run_context +except ModuleNotFoundError as error: + if error.name != "je_action_core.request_context": + raise + raise RuntimeError( + "Canonical records require the ActionCore request-record API; upgrade je_action_core " + "or use the coordinated development checkout." + ) from error + +__all__ = ["RunContext", "get_run_context", "use_run_context"] diff --git a/je_load_density/wrapper/create_locust_env/create_locust_env.py b/je_load_density/wrapper/create_locust_env/create_locust_env.py index 875f34c..5494243 100644 --- a/je_load_density/wrapper/create_locust_env/create_locust_env.py +++ b/je_load_density/wrapper/create_locust_env/create_locust_env.py @@ -1,8 +1,10 @@ +from functools import partial from typing import Any, Dict, List, Optional import gevent from locust import User, events from locust.env import Environment +from locust.event import Events from locust.log import setup_logging from locust.stats import stats_history, stats_printer @@ -36,6 +38,10 @@ def prepare_env( master_host = kwargs.pop("master_host", "127.0.0.1") master_port = kwargs.pop("master_port", 5557) expected_workers = kwargs.pop("expected_workers", 0) + run_context = kwargs.pop("run_context", None) + if run_context is None: + from je_load_density.utils.test_record.contract import get_optional_run_context + run_context = get_optional_run_context() load_density_logger.info( f"prepare_env mode={runner_mode}, user_class={user_class}, user_count={user_count}, " @@ -46,7 +52,7 @@ def prepare_env( env = create_env(user_class, runner_mode=runner_mode, master_bind_host=master_bind_host, master_bind_port=master_bind_port, master_host=master_host, master_port=master_port, - load_shape=load_shape, shape_config=shape_config) + load_shape=load_shape, shape_config=shape_config, run_context=run_context) if runner_mode == "worker": env.runner.greenlet.join() @@ -81,6 +87,7 @@ def create_env( master_port: int = 5557, load_shape: Optional[str] = None, shape_config: Optional[Dict[str, Any]] = None, + **kwargs, ): """ 建立 Locust Environment 並依模式建立 runner。 @@ -90,6 +97,13 @@ def create_env( f"create_env mode={runner_mode}, user_class={user_class}, another_event={another_event}" ) shape_class = _resolve_shape(load_shape, shape_config) + run_context = kwargs.pop("run_context", None) + if run_context is not None: + from je_load_density.wrapper.event.request_hook import request_hook + if another_event is not events: + raise ValueError("a canonical run context requires an isolated default event environment") + another_event = Events() + another_event.request.add_listener(partial(request_hook, record_run=run_context)) env = Environment(user_classes=[user_class], events=another_event, shape_class=shape_class) diff --git a/je_load_density/wrapper/event/request_hook.py b/je_load_density/wrapper/event/request_hook.py index 536b4ba..9333966 100644 --- a/je_load_density/wrapper/event/request_hook.py +++ b/je_load_density/wrapper/event/request_hook.py @@ -1,6 +1,8 @@ -from je_load_density.utils.test_record.test_record_class import test_record_instance from locust import events +from je_load_density.utils.test_record.contract import from_legacy_record, record_request +from je_load_density.utils.test_record.test_record_class import test_record_instance + def _epoch_seconds(start_time): """ @@ -38,10 +40,10 @@ def request_hook( "Method": str(request_type), "test_url": str(url), "name": str(name), - "status_code": str(response.status_code), - "text": str(response.text), - "content": str(response.content), - "headers": str(response.headers), + "status_code": str(response.status_code) if response is not None else None, + "text": str(response.text) if response is not None else None, + "content": str(response.content) if response is not None else None, + "headers": str(response.headers) if response is not None else None, "response_time_ms": float(response_time or 0), "response_length": int(response_length or 0), "error": None, @@ -55,11 +57,20 @@ def request_hook( "Method": str(request_type), "test_url": str(url), "name": str(name), - "status_code": str(response.status_code) if response else None, - "text": str(response.text) if response else None, + "status_code": str(response.status_code) if response is not None else None, + "text": str(response.text) if response is not None else None, "response_time_ms": float(response_time or 0), "response_length": int(response_length or 0), "error": str(exception), "start_time": _epoch_seconds(start_time), } - ) \ No newline at end of file + ) + + outcome = "passed" if exception is None else "failed" + records = test_record_instance.test_record_list if exception is None else test_record_instance.error_record_list + entry = records[-1] + selected_run = kwargs.get("record_run") + if selected_run is not None: + from_legacy_record(entry, selected_run, outcome) + else: + record_request(entry, outcome) diff --git a/test/test_record_contract.py b/test/test_record_contract.py new file mode 100644 index 0000000..41502c8 --- /dev/null +++ b/test/test_record_contract.py @@ -0,0 +1,66 @@ +import json + +import pytest + +pytest.importorskip("je_action_core.request_context", reason="coordinated ActionCore request-record release") + +from je_load_density.utils.test_record.contract import from_legacy_record, record_request +from je_load_density.utils.test_record.run_context import RunContext, get_run_context, use_run_context + + +def context(): + return RunContext(source="loaddensity", phase="load", engine="locust") + + +def test_legacy_success_maps_types_without_inventing_timestamps(): + result = from_legacy_record({"Method": "get", "test_url": "http://localhost/", "name": "home", + "status_code": "200", "response_time_ms": 25, "response_length": 2}, + context(), "passed") + assert result["request_method"] == "GET" + assert result["status_code"] == 200 + assert result["start_time"] is None + assert result["end_time"] is None + assert result["response_time_ms"] == 25.0 + + +def test_transport_sentinel_and_missing_latency_remain_unknown(): + result = from_legacy_record({"Method": "GET", "test_url": "http://localhost/", "status_code": "0", + "error": "connection refused"}, context(), "failed") + assert result["status_code"] is None + assert result["response_time_ms"] is None + assert result["error"]["message"] == "connection refused" + + +def test_request_capture_redacts_sensitive_headers_and_retains_binary_body(): + run = context() + with use_run_context(run): + record_request({"Method": "GET", "test_url": "http://localhost/", "status_code": "200", + "response_time_ms": 25, "start_time": 100.0, "content": b"\x00\xff", + "headers": {"Authorization": "secret", "Content-Type": "application/json", + "Set-Cookie": "session=secret"}}, "passed", capture_payload=True) + record = json.loads(run.to_json())[0] + assert record["end_time"] == 100.025 + assert record["content_base64"] == "AP8=" + assert record["headers"] == {"Authorization": "[REDACTED]", "Content-Type": "application/json", + "Set-Cookie": "[REDACTED]"} + + +def test_recording_without_explicit_context_is_a_no_op(): + assert get_run_context() is None + assert record_request({"Method": "GET"}, "passed") is None + + +def test_malformed_legacy_record_reports_field_location(): + with pytest.raises(ValueError, match="status_code"): + from_legacy_record({"Method": "GET", "test_url": "http://localhost/", "status_code": "bad"}, + context(), "passed") + + +def test_live_payload_capture_is_disabled_by_default(): + run = context() + with use_run_context(run): + record_request({"Method": "GET", "test_url": "http://localhost/", "status_code": "200", + "content": b"secret", "text": "secret"}, "passed") + result = run.snapshot()[0] + assert "content_base64" not in result + assert "text" not in result diff --git a/test/test_record_engine_integration.py b/test/test_record_engine_integration.py new file mode 100644 index 0000000..4bd7fdd --- /dev/null +++ b/test/test_record_engine_integration.py @@ -0,0 +1,118 @@ +import asyncio +from types import SimpleNamespace + +import pytest + +pytest.importorskip("je_action_core.request_context", reason="coordinated ActionCore request-record release") + +from je_load_density.engine.asyncio_engine import _send_one, run_async_load +from je_load_density.utils.test_record.run_context import RunContext, use_run_context +from je_load_density.utils.test_record.test_record_class import test_record_instance +from je_load_density.wrapper.event.request_hook import request_hook + +import httpx +import requests + + +@pytest.fixture(autouse=True) +def clean_records(): + test_record_instance.clear_records() + yield + test_record_instance.clear_records() + + +def test_falsey_locust_error_response_retains_status_in_both_formats(): + response = requests.Response() + response.status_code = 500 + response._content = b"down" + run = RunContext(source="loaddensity", phase="load", engine="locust") + with use_run_context(run): + request_hook(100.0, "http://localhost/", "GET", "home", {}, response, ValueError("HTTP 500"), 4, 25) + assert test_record_instance.error_record_list[0]["status_code"] == "500" + assert run.snapshot()[0]["status_code"] == 500 + assert run.snapshot()[0]["response_time_ms"] == 25.0 + + +@pytest.mark.parametrize("status,outcome", [(200, "passed"), (500, "failed")]) +def test_async_response_captures_canonical_and_legacy_result(status, outcome): + run = RunContext(source="loaddensity", phase="load", engine="asyncio") + + async def execute(): + transport = httpx.MockTransport(lambda request: httpx.Response(status, content=b"ok")) + with use_run_context(run): + async with httpx.AsyncClient(transport=transport) as client: + await _send_one(client, {"method": "get", "request_url": "http://localhost/"}) + + asyncio.run(execute()) + assert run.snapshot()[0]["outcome"] == outcome + assert run.snapshot()[0]["status_code"] == status + assert run.snapshot()[0]["response_length"] == 2 + + +def test_async_timeout_records_measured_elapsed_and_nullable_canonical_status(monkeypatch): + from je_load_density.engine import asyncio_engine + + clock = iter((10.0, 10.025)) + monkeypatch.setattr(asyncio_engine, "time", SimpleNamespace(time=lambda: 100.0, monotonic=lambda: next(clock))) + run = RunContext(source="loaddensity", phase="load", engine="asyncio") + + async def handle(request): + await asyncio.sleep(0) + raise httpx.ReadTimeout("deadline", request=request) + + async def execute(): + with use_run_context(run): + async with httpx.AsyncClient(transport=httpx.MockTransport(handle)) as client: + await _send_one(client, {"request_url": "http://localhost/"}) + + asyncio.run(execute()) + assert test_record_instance.error_record_list[0]["response_time_ms"] == pytest.approx(25.0) + record = run.snapshot()[0] + assert record["response_time_ms"] == pytest.approx(25.0) + assert record["status_code"] is None + assert record["error"]["kind"] == "timeout" + + +def test_new_async_run_summary_excludes_old_global_records(): + test_record_instance.test_record_list.append({"name": "old"}) + result = asyncio.run(run_async_load(tasks=[{"request_url": "http://localhost/"}], users=0)) + assert result == {"requests": 0, "failures": 0} + + +def test_async_run_propagates_explicit_context_to_workers(): + run = RunContext(source="loaddensity", phase="load", engine="asyncio") + result = asyncio.run(run_async_load(tasks=[{"request_url": "http://127.0.0.1:9/", "timeout": 0.1}], + users=1, duration_seconds=0.2, run_context=run)) + assert result["failures"] >= 1 + assert len(run.snapshot()) == result["failures"] + + +def test_locust_environment_captures_context_across_greenlet_boundary(): + import gevent + from locust import HttpUser + from je_load_density.wrapper.create_locust_env.create_locust_env import create_env + + run = RunContext(source="loaddensity", phase="load", engine="locust") + env = create_env(HttpUser, run_context=run) + response = requests.Response() + response.status_code = 200 + response._content = b"ok" + try: + greenlet = gevent.spawn(env.events.request.fire, start_time=100.0, url="http://localhost/", + request_type="GET", name="home", context={}, response=response, + exception=None, response_length=2, response_time=25.0) + greenlet.get(timeout=2) + assert len(run.snapshot()) == 1 + assert run.snapshot()[0]["status_code"] == 200 + finally: + env.runner.quit() + + +def test_non_http_locust_event_keeps_protocol_and_unknown_status(): + run = RunContext(source="loaddensity", phase="load", engine="locust") + with use_run_context(run): + request_hook(None, "localhost:9000", "TCP", "echo", {}, None, None, 2, 25) + result = run.snapshot()[0] + assert result["protocol"] == "tcp" + assert result["status_code"] is None + assert result["start_time"] is None From a067b1e83bae0b894d6bc35a2ae514f44a7ad602 Mon Sep 17 00:00:00 2001 From: JeffreyChen Date: Fri, 2 Oct 2026 19:13:29 +0800 Subject: [PATCH 03/18] feat: persist canonical request records transactionally --- README.md | 2 + README/README_zh-CN.md | 2 + README/README_zh-TW.md | 2 + architecture.md | 2 + .../En/doc/test_record/test_record_doc.rst | 2 + .../Zh/doc/test_record/test_record_doc.rst | 2 + .../plans/2026-10-02-shared-record.md | 52 +++++----- docs/updates/2026-10.md | 4 + docs/updates/README.md | 1 + .../utils/test_record/sqlite_persistence.py | 87 ++++++++++++++++- test/test_canonical_persistence.py | 97 +++++++++++++++++++ 11 files changed, 226 insertions(+), 27 deletions(-) create mode 100644 test/test_canonical_persistence.py diff --git a/README.md b/README.md index f022f12..de0465b 100644 --- a/README.md +++ b/README.md @@ -781,6 +781,8 @@ Legacy single-flag form (`-e/-d/-c/--execute_str`) is still accepted for backwar ## Test Record +Canonical SQLite export uses `persist_canonical_records(database_path, context)` and `fetch_canonical_records(database_path, run_id)` from `utils.test_record.sqlite_persistence`. Separate `request_runs_v1` / `request_records_v1` tables preserve legacy runs. Writes validate snapshots, deduplicate identical IDs, reject conflicting retries and roll back the entire batch on failure. Reads revalidate stored records. JSON export is `context.to_json()`. + Canonical request records are available as an opt-in API with an ActionCore release that provides `je_action_core.request_context` (or the coordinated development checkout). Legacy record lists and reports continue to work with the existing dependency floor. Create diff --git a/README/README_zh-CN.md b/README/README_zh-CN.md index 0e88d2c..564835c 100644 --- a/README/README_zh-CN.md +++ b/README/README_zh-CN.md @@ -781,6 +781,8 @@ python -m je_load_density serve [--host ...] # start the control socket ## 测试记录 +Canonical SQLite 导出使用 `utils.test_record.sqlite_persistence` 的 `persist_canonical_records(database_path, context)` 与 `fetch_canonical_records(database_path, run_id)`。独立的 `request_runs_v1`／`request_records_v1` 表保留旧记录。写入验证 snapshot、去除相同 ID 的重复项、拒绝冲突重送，失败时回滚整批；读取重新验证保存的记录。JSON 导出为 `context.to_json()`。 + Canonical request 记录是可选 API，需要提供 `je_action_core.request_context` 的 ActionCore 版本 或协调的开发工作树。原记录列表与报告仍支持现有依赖下限。从 `je_load_density.utils.test_record.run_context` 创建 diff --git a/README/README_zh-TW.md b/README/README_zh-TW.md index 44617c3..b33213f 100644 --- a/README/README_zh-TW.md +++ b/README/README_zh-TW.md @@ -781,6 +781,8 @@ python -m je_load_density serve [--host ...] # start the control socket ## 測試紀錄 +Canonical SQLite 匯出使用 `utils.test_record.sqlite_persistence` 的 `persist_canonical_records(database_path, context)` 與 `fetch_canonical_records(database_path, run_id)`。獨立的 `request_runs_v1`／`request_records_v1` 表保留舊紀錄。寫入驗證 snapshot、去除相同 ID 的重複項、拒絕衝突重送，失敗時回復整批；讀取重新驗證保存的紀錄。JSON 匯出為 `context.to_json()`。 + Canonical request 紀錄為選用 API，需要提供 `je_action_core.request_context` 的 ActionCore 版本 或協調的開發工作樹。原紀錄清單與報表仍支援既有相依下限。從 `je_load_density.utils.test_record.run_context` 建立 diff --git a/architecture.md b/architecture.md index ca76107..816b817 100644 --- a/architecture.md +++ b/architecture.md @@ -135,6 +135,8 @@ MCP `load_density.list_executor_commands` tool all read the `LD_*` names from `e ## 6. Cross-project boundaries +Canonical persistence exports `persist_canonical_records(database_path, context)` and `fetch_canonical_records(database_path, run_id)` from `utils.test_record.sqlite_persistence`. Versioned tables are separate from legacy runs; transaction rollback, JSON-aware retry conflicts and read-time schema/identity validation preserve the shared contract. + Canonical request records are an opt-in contract supplied by ActionCore's ``request_record``/ ``request_context`` APIs. ``utils/test_record/contract.py`` adapts legacy results and ``run_context.py`` exposes explicit run scopes. Locust environments bind the selected context diff --git a/docs/source/En/doc/test_record/test_record_doc.rst b/docs/source/En/doc/test_record/test_record_doc.rst index 7dc615e..829d665 100644 --- a/docs/source/En/doc/test_record/test_record_doc.rst +++ b/docs/source/En/doc/test_record/test_record_doc.rst @@ -57,3 +57,5 @@ Pass ``run_context=context`` to ``run_async_load`` or ``start_test``. Locust bin to isolated environment events so greenlets do not depend on ContextVar inheritance. ``context.to_json()`` exports v1 records with identities, numeric/null status and structured errors. Payload capture is disabled by default. Async summaries exclude previous invocations' records. + +Canonical SQLite export uses `persist_canonical_records(database_path, context)` and `fetch_canonical_records(database_path, run_id)` from `utils.test_record.sqlite_persistence`. Separate `request_runs_v1` / `request_records_v1` tables preserve legacy runs. Writes validate snapshots, deduplicate identical IDs, reject conflicting retries and roll back the entire batch on failure. Reads revalidate stored records. JSON export is `context.to_json()`. diff --git a/docs/source/Zh/doc/test_record/test_record_doc.rst b/docs/source/Zh/doc/test_record/test_record_doc.rst index 0df72d0..3db2b05 100644 --- a/docs/source/Zh/doc/test_record/test_record_doc.rst +++ b/docs/source/Zh/doc/test_record/test_record_doc.rst @@ -48,3 +48,5 @@ SQLite 持久化 綁定隔離的 environment 事件，greenlet 不依賴 ContextVar 繼承。 ``context.to_json()`` 輸出 v1 紀錄，包含識別、數字／null 狀態碼與結構化錯誤。 預設不擷取完整內容；async 回傳摘要不納入前次呼叫的紀錄。 + +Canonical SQLite 匯出使用 `utils.test_record.sqlite_persistence` 的 `persist_canonical_records(database_path, context)` 與 `fetch_canonical_records(database_path, run_id)`。獨立的版本化表保留舊紀錄。寫入驗證 snapshot、去除相同 ID 重複項、拒絕衝突重送，失敗時回復整批；讀取重新驗證保存的紀錄。JSON 匯出為 `context.to_json()`。 diff --git a/docs/superpowers/plans/2026-10-02-shared-record.md b/docs/superpowers/plans/2026-10-02-shared-record.md index e41f8cc..cc3777c 100644 --- a/docs/superpowers/plans/2026-10-02-shared-record.md +++ b/docs/superpowers/plans/2026-10-02-shared-record.md @@ -1,6 +1,6 @@ # Shared request record Implementation Plan -> **For agentic workers:** REQUIRED SUB-SKILL: Use superpowers:executing-plans to implement this plan task-by-task. Steps use checkbox (`- [ ]`) syntax for tracking. +> **For agentic workers:** REQUIRED SUB-SKILL: Use superpowers:executing-plans to implement this plan task-by-task. Steps use checkbox (`- [x]`) syntax for tracking. **Goal:** Publish a stdlib request-record contract and connect APITestka and LoadDensity without breaking legacy consumers. @@ -36,11 +36,11 @@ **Interfaces:** Produces `RequestRecord`, `RecordError`, `RequestRecordError`, `validate_request_record(record: Mapping[str, object]) -> RequestRecord`, `serialize_request_record(record: Mapping[str, object]) -> str` and `request_record_schema() -> dict[str, object]`. -- [ ] Write tests proving missing/version/type/outcome/latency/time/assertion errors are rejected; complete success/failure records round-trip; nested bytes/datetime/timedelta serialize safely without arbitrary-object stringification. Hand-derived fixtures assert status 500 remains 500 and the contract has no client imports. -- [ ] Run `D:/Codes/ActionCore/.venv/Scripts/python.exe -m pytest test/test_request_record.py -q`; expect missing-module failure before implementation. -- [ ] Implement the TypedDict and validators, JSON conversion and schema in the named module. Keep individual validators focused and the public normalization surface independent of transport. -- [ ] Run the new tests and `python -m pytest -q` with the ActionCore interpreter; expect all tests pass. Run static checks for the new module. -- [ ] Document the new API and commit the ActionCore stage. Do not publish or change a version by hand. +- [x] Write tests proving missing/version/type/outcome/latency/time/assertion errors are rejected; complete success/failure records round-trip; nested bytes/datetime/timedelta serialize safely without arbitrary-object stringification. Hand-derived fixtures assert status 500 remains 500 and the contract has no client imports. +- [x] Run `D:/Codes/ActionCore/.venv/Scripts/python.exe -m pytest test/test_request_record.py -q`; expect missing-module failure before implementation. +- [x] Implement the TypedDict and validators, JSON conversion and schema in the named module. Keep individual validators focused and the public normalization surface independent of transport. +- [x] Run the new tests and `python -m pytest -q` with the ActionCore interpreter; expect all tests pass. Run static checks for the new module. +- [x] Document the new API and commit the ActionCore stage. Do not publish or change a version by hand. ### Task 2: Run context and LoadDensity legacy adapters @@ -48,11 +48,11 @@ **Interfaces:** Consumes Task 1. Produces `RunContext(source='loaddensity', phase='load', engine='locust', worker_id=None)`, `RunContext.append(record: Mapping[str, object]) -> bool`, `RunContext.snapshot() -> list[RequestRecord]`, `RunContext.to_json() -> str`, `use_run_context(context)` and `get_run_context()`. Adapter: `from_legacy_record(record, context, outcome) -> RequestRecord`; legacy conversion never creates fake timestamps. -- [ ] Test two contexts remain isolated, task-local nesting restores prior contexts, identical IDs deduplicate, conflicting IDs fail, input mutations cannot alter snapshots, and LoadDensity/APITestka legacy fixtures normalize status/time/bytes correctly. -- [ ] Run `pytest test/test_record_contract.py -q` with the isolated ActionCore on PYTHONPATH; expect missing-module failure. -- [ ] Implement the context/sink and source-specific adapters. Import the new core only when a canonical operation is called; missing API raises a clear upgrade error and does not break legacy imports. -- [ ] Run the targeted suite; expect pass including malformed import positions, nullable unknown measurements and credential redaction. -- [ ] Keep this task's commit together with Task 3 because its public adapter needs a real runner consumer and user documentation. +- [x] Test two contexts remain isolated, task-local nesting restores prior contexts, identical IDs deduplicate, conflicting IDs fail, input mutations cannot alter snapshots, and LoadDensity/APITestka legacy fixtures normalize status/time/bytes correctly. +- [x] Run `pytest test/test_record_contract.py -q` with the isolated ActionCore on PYTHONPATH; expect missing-module failure. +- [x] Implement the context/sink and source-specific adapters. Import the new core only when a canonical operation is called; missing API raises a clear upgrade error and does not break legacy imports. +- [x] Run the targeted suite; expect pass including malformed import positions, nullable unknown measurements and credential redaction. +- [x] Keep this task's commit together with Task 3 because its public adapter needs a real runner consumer and user documentation. ### Task 3: Locust and async recording @@ -60,11 +60,11 @@ **Interfaces:** Consumes Task 2. Locust reads the active context; `run_async_load(..., run_context=None)` explicitly propagates the selected run to asyncio workers. Canonical records are separate from existing legacy lists. -- [ ] Test a Locust response with status 500 and falsey truthiness retains status; async success/status failure/transport failure capture distinct records with real elapsed time; consecutive runs' return summaries exclude prior records; scoped recording leaves legacy report keys unchanged. -- [ ] Run integration tests; expect failure on the currently missing context output and contaminated run summaries. -- [ ] Route measured request outcomes to the context alongside the existing legacy append. Give run_async_load per-invocation counters and propagate cancellation. Keep existing return request/failure semantics. -- [ ] Run new tests, existing asyncio/report/SLA/SQLite tests, then the full LoadDensity suite; expect pass. -- [ ] Document opt-in canonical collection and commit the LoadDensity recording stage. Record remaining schema-release coordination in progress.md rather than inventing a released version. +- [x] Test a Locust response with status 500 and falsey truthiness retains status; async success/status failure/transport failure capture distinct records with real elapsed time; consecutive runs' return summaries exclude prior records; scoped recording leaves legacy report keys unchanged. +- [x] Run integration tests; expect failure on the currently missing context output and contaminated run summaries. +- [x] Route measured request outcomes to the context alongside the existing legacy append. Give run_async_load per-invocation counters and propagate cancellation. Keep existing return request/failure semantics. +- [x] Run new tests, existing asyncio/report/SLA/SQLite tests, then the full LoadDensity suite; expect pass. +- [x] Document opt-in canonical collection and commit the LoadDensity recording stage. Record remaining schema-release coordination in progress.md rather than inventing a released version. ### Task 4: APITestka adapters and recording @@ -72,11 +72,11 @@ **Interfaces:** Same canonical context API as Task 2, with source='apitestka' and phase='functional'; context implementation is shared in ActionCore if it proves reusable rather than copied. `record_response(response_data, request, error, engine, elapsed_ms)` appends only when a context is active. -- [ ] Add tests using real local requests/httpx calls and failures; assert native return values and failure pairs remain identical while canonical records retain response information and status. Test recording disabled and context cleanup. -- [ ] Run those tests with the isolated core; expect missing-module/API failure. -- [ ] Wire a scope-aware capture helper into all three wrappers, including before-response errors and clean_record behavior. Preserve caught-error behavior and record_request_info semantics. -- [ ] Run the targeted tests, load bridge tests and full APITestka suite; expect pass. Separately run with released ActionCore to prove existing imports still work. -- [ ] Document API and commit this repository independently. +- [x] Add tests using real local requests/httpx calls and failures; assert native return values and failure pairs remain identical while canonical records retain response information and status. Test recording disabled and context cleanup. +- [x] Run those tests with the isolated core; expect missing-module/API failure. +- [x] Wire a scope-aware capture helper into all three wrappers, including before-response errors and clean_record behavior. Preserve caught-error behavior and record_request_info semantics. +- [x] Run the targeted tests, load bridge tests and full APITestka suite; expect pass. Separately run with released ActionCore to prove existing imports still work. +- [x] Document API and commit this repository independently. ### Task 5: Versioned JSON and SQLite export @@ -84,11 +84,11 @@ **Interfaces:** Consumes contexts. Produces `persist_canonical_records(database_path: str, context: RunContext) -> str`, `fetch_canonical_records(database_path: str, run_id: str) -> list[RequestRecord]`; existing tables and functions remain unchanged. -- [ ] Test empty database and existing legacy database, JSON round-trip, multiple runs, duplicate/conflicting records and rollback on invalid records. -- [ ] Run the new test; expect missing-function failure. -- [ ] Add separate versioned canonical tables and parameterized transactional writes; never migrate or overwrite legacy data implicitly. -- [ ] Run persistence and legacy regression tests; expect pass. -- [ ] Document the explicit export APIs and commit the persistence stage. +- [x] Test empty database and existing legacy database, JSON round-trip, multiple runs, duplicate/conflicting records and rollback on invalid records. +- [x] Run the new test; expect missing-function failure. +- [x] Add separate versioned canonical tables and parameterized transactional writes; never migrate or overwrite legacy data implicitly. +- [x] Run persistence and legacy regression tests; expect pass. +- [x] Document the explicit export APIs and commit the persistence stage. ### Task 6: Cross-project verification and release handoff diff --git a/docs/updates/2026-10.md b/docs/updates/2026-10.md index e482141..c1b6144 100644 --- a/docs/updates/2026-10.md +++ b/docs/updates/2026-10.md @@ -193,3 +193,7 @@ Index and query commands: [README.md](README.md). New entries go at the end. - **Tests**: 1237 passed, 2 skipped in the complete suite against the coordinated core checkout (14 new adapter/integration tests). Legacy installed-core regression: 8 passed, 2 canonical modules skipped. New source modules passed ruff E/F/W/C90/I; git diff --check passed. - **Files**: utils/test_record/{contract,run_context}.py, engine/asyncio_engine.py, wrapper/event/request_hook.py, wrapper/create_locust_env/create_locust_env.py, tests, three READMEs, Sphinx record docs and architecture.md. - **Open items**: progress.md #28 still requires APITestka integration, canonical SQLite exports and coordinated core rollout; #16 remains the full async parity work. + +## U-20261002-03 · 2026-10-02 · Persist canonical request records transactionally · #done #records + +Explicit run contexts export canonical JSON and persist to separate versioned SQLite tables. Canonical writes validate snapshots, preserve legacy tables, deduplicate retries and roll back conflicts atomically. Reads validate JSON and indexed identities. Added round-trip, multiple-run, legacy preservation, retry conflict, invalid snapshot and stored-corruption regressions. diff --git a/docs/updates/README.md b/docs/updates/README.md index fb3734a..7512d9f 100644 --- a/docs/updates/README.md +++ b/docs/updates/README.md @@ -58,6 +58,7 @@ In the same commit: delete the item from `progress.md`, add a `#done` entry here | ID | Date | Title | Tags | Batch | |---|---|---|---|---| +| U-20261002-03 | 2026-10-02 | Persist canonical request records transactionally | #done #records | [2026-10](2026-10.md) | | U-20261002-02 | 2026-10-02 | Capture canonical records in Locust and asyncio runs | #migration #records | [2026-10](2026-10.md) | | U-20261002-01 | 2026-10-02 | Testing platform design draft and outstanding work | #docs #design | [2026-10](2026-10.md) | | U-20261001-12 | 2026-10-01 | Publish jobs build with the locked setuptools | #ci #security #X-13 | [2026-10](2026-10.md) | diff --git a/je_load_density/utils/test_record/sqlite_persistence.py b/je_load_density/utils/test_record/sqlite_persistence.py index 51c3184..d6fbfd8 100644 --- a/je_load_density/utils/test_record/sqlite_persistence.py +++ b/je_load_density/utils/test_record/sqlite_persistence.py @@ -1,11 +1,16 @@ import json import sqlite3 import threading +from contextlib import closing from datetime import datetime, timezone -from typing import Any, Dict, Iterable, List, Optional +from typing import TYPE_CHECKING, Any, Dict, Iterable, List, Optional from je_load_density.utils.test_record.test_record_class import test_record_instance +if TYPE_CHECKING: + from je_action_core.request_context import RunContext + from je_action_core.request_record import RequestRecord + _SCHEMA = """ CREATE TABLE IF NOT EXISTS load_density_runs ( id INTEGER PRIMARY KEY AUTOINCREMENT, @@ -127,3 +132,83 @@ def fetch_run_records(database_path: str, run_id: int) -> Iterable[Dict[str, Any return [dict(row) for row in cursor.fetchall()] finally: connection.close() + + +_CANONICAL_TABLES = ( + "CREATE TABLE IF NOT EXISTS request_runs_v1 (run_id TEXT PRIMARY KEY, schema_version INTEGER NOT NULL " + "CHECK(schema_version = 1))", + "CREATE TABLE IF NOT EXISTS request_records_v1 (sequence INTEGER PRIMARY KEY, record_id TEXT UNIQUE NOT NULL, " + "run_id TEXT NOT NULL, record_json TEXT NOT NULL, FOREIGN KEY(run_id) REFERENCES request_runs_v1(run_id))", + "CREATE INDEX IF NOT EXISTS idx_request_records_v1_run ON request_records_v1(run_id)", +) + + +def _canonical_rows(context: "RunContext") -> list[tuple[str, str, str]]: + from je_action_core.request_record import RequestRecordError, validate_request_record + + rows = [] + for record in context.snapshot(): + validated = validate_request_record(record) + if validated["run_id"] != context.run_id: + raise RequestRecordError("run_id: snapshot contains a foreign run") + payload = json.dumps(validated, ensure_ascii=False, allow_nan=False, sort_keys=True) + rows.append((validated["record_id"], context.run_id, payload)) + return rows + + +def _ensure_canonical_schema(connection: sqlite3.Connection) -> None: + for statement in _CANONICAL_TABLES: + connection.execute(statement) + + +def _insert_canonical_row(connection: sqlite3.Connection, row: tuple[str, str, str]) -> None: + from je_action_core.request_record import RequestRecordError + + previous = connection.execute( + "SELECT record_json FROM request_records_v1 WHERE record_id = ?", (row[0],) + ).fetchone() + if previous is not None: + if previous[0] != row[2]: + raise RequestRecordError("record_id: conflicting persisted retry") + return + connection.execute( + "INSERT INTO request_records_v1(record_id, run_id, record_json) VALUES (?, ?, ?)", row + ) + + +def persist_canonical_records(database_path: str, context: "RunContext") -> str: + """Persist one explicit run atomically in separate v1 tables; retries are idempotent.""" + rows = _canonical_rows(context) + with closing(sqlite3.connect(database_path)) as connection: + connection.execute("PRAGMA foreign_keys = ON") + with connection: + connection.execute("BEGIN IMMEDIATE") + _ensure_canonical_schema(connection) + connection.execute( + "INSERT OR IGNORE INTO request_runs_v1(run_id, schema_version) VALUES (?, 1)", (context.run_id,) + ) + for row in rows: + _insert_canonical_row(connection, row) + return context.run_id + + +def fetch_canonical_records(database_path: str, run_id: str) -> list["RequestRecord"]: + """Read and revalidate an ordered canonical snapshot without changing legacy tables.""" + from je_action_core.request_record import RequestRecordError, validate_request_record + + with closing(sqlite3.connect(database_path)) as connection: + with connection: + _ensure_canonical_schema(connection) + rows = connection.execute( + "SELECT record_id, record_json FROM request_records_v1 WHERE run_id = ? ORDER BY sequence", (run_id,) + ).fetchall() + records = [] + for identifier, payload in rows: + try: + record = validate_request_record(json.loads(payload)) + except json.JSONDecodeError as error: + raise RequestRecordError("record_json: malformed stored JSON") from error + if record["run_id"] != run_id or record["record_id"] != identifier: + raise RequestRecordError("record_json: stored identity does not match its index") + records.append(record) + return records diff --git a/test/test_canonical_persistence.py b/test/test_canonical_persistence.py new file mode 100644 index 0000000..0856343 --- /dev/null +++ b/test/test_canonical_persistence.py @@ -0,0 +1,97 @@ +import sqlite3 + +import pytest + +pytest.importorskip("je_action_core.request_context", reason="coordinated ActionCore request-record release") + +from je_action_core.request_record import RequestRecordError + +from je_load_density.utils.test_record.run_context import RunContext +from je_load_density.utils.test_record.sqlite_persistence import ( + fetch_canonical_records, + fetch_run_records, + persist_canonical_records, + persist_records, +) +from je_load_density.utils.test_record.test_record_class import test_record_instance + + +def context(): + return RunContext(source="loaddensity", phase="load", engine="asyncio") + + +def capture(run, **changes): + fields = {"protocol": "http", "request_method": "GET", "request_url": "http://localhost/", + "name": "health", "status_code": 200, "start_time": 100, "end_time": 100.025, + "response_time_ms": 25, "response_length": 2, "outcome": "passed", "error": None, + "extensions": {"label": "測試", "nested": [True, 1]}} + return run.capture({**fields, **changes}) + + +def test_round_trip_multiple_runs_and_empty_run(tmp_path): + database = str(tmp_path / "records.sqlite") + first, second, empty = context(), context(), context() + first_record = capture(first) + second_record = capture(second, status_code=201) + for run in [first, second, empty]: + assert persist_canonical_records(database, run) == run.run_id + assert fetch_canonical_records(database, first.run_id) == [first_record] + assert fetch_canonical_records(database, second.run_id) == [second_record] + assert fetch_canonical_records(database, empty.run_id) == [] + + +def test_existing_legacy_database_is_preserved(tmp_path, monkeypatch): + monkeypatch.setattr(test_record_instance, "test_record_list", [ + {"Method": "GET", "test_url": "http://legacy/", "status_code": "200", "response_time_ms": 12} + ]) + monkeypatch.setattr(test_record_instance, "error_record_list", []) + database = str(tmp_path / "records.sqlite") + legacy_id = persist_records(database) + before = list(fetch_run_records(database, legacy_id)) + assert before[0]["test_url"] == "http://legacy/" + run = context() + capture(run) + persist_canonical_records(database, run) + assert list(fetch_run_records(database, legacy_id)) == before + assert len(fetch_canonical_records(database, run.run_id)) == 1 + + +def test_retry_is_idempotent_and_conflict_rolls_back_batch(tmp_path, monkeypatch): + database = str(tmp_path / "records.sqlite") + run = context() + original = capture(run) + persist_canonical_records(database, run) + persist_canonical_records(database, run) + fresh = capture(run, status_code=201) + conflicting = {**original, "extensions": {"label": "測試", "nested": [1, 1]}} + monkeypatch.setattr(run, "snapshot", lambda: [fresh, conflicting]) + with pytest.raises(RequestRecordError, match="record_id"): + persist_canonical_records(database, run) + assert fetch_canonical_records(database, run.run_id) == [original] + + +def test_invalid_or_foreign_snapshot_rejected_before_writes(tmp_path, monkeypatch): + database = str(tmp_path / "records.sqlite") + run = context() + original = capture(run) + persist_canonical_records(database, run) + fresh = capture(run, status_code=201) + monkeypatch.setattr(run, "snapshot", lambda: [fresh, {**original, "schema_version": 2}]) + with pytest.raises(RequestRecordError, match="schema_version"): + persist_canonical_records(database, run) + foreign = capture(context()) + monkeypatch.setattr(run, "snapshot", lambda: [foreign]) + with pytest.raises(RequestRecordError, match="run_id"): + persist_canonical_records(database, run) + assert fetch_canonical_records(database, run.run_id) == [original] + + +def test_corrupted_database_record_is_not_returned_as_valid(tmp_path): + database = str(tmp_path / "records.sqlite") + run = context() + capture(run) + persist_canonical_records(database, run) + with sqlite3.connect(database) as connection: + connection.execute("UPDATE request_records_v1 SET record_json = ?", ('{"schema_version": 2}',)) + with pytest.raises(RequestRecordError): + fetch_canonical_records(database, run.run_id) From 578a45991f744ece964dc1ee41a9e0760952c92e Mon Sep 17 00:00:00 2001 From: JeffreyChen Date: Fri, 2 Oct 2026 19:15:27 +0800 Subject: [PATCH 04/18] fix: scope persisted request IDs to their run --- README.md | 2 +- README/README_zh-CN.md | 2 +- README/README_zh-TW.md | 2 +- architecture.md | 2 +- docs/source/En/doc/test_record/test_record_doc.rst | 2 +- docs/source/Zh/doc/test_record/test_record_doc.rst | 2 +- docs/updates/2026-10.md | 4 ++++ docs/updates/README.md | 1 + .../utils/test_record/sqlite_persistence.py | 7 ++++--- test/test_canonical_persistence.py | 11 +++++++++++ 10 files changed, 26 insertions(+), 9 deletions(-) diff --git a/README.md b/README.md index de0465b..c46d4fd 100644 --- a/README.md +++ b/README.md @@ -781,7 +781,7 @@ Legacy single-flag form (`-e/-d/-c/--execute_str`) is still accepted for backwar ## Test Record -Canonical SQLite export uses `persist_canonical_records(database_path, context)` and `fetch_canonical_records(database_path, run_id)` from `utils.test_record.sqlite_persistence`. Separate `request_runs_v1` / `request_records_v1` tables preserve legacy runs. Writes validate snapshots, deduplicate identical IDs, reject conflicting retries and roll back the entire batch on failure. Reads revalidate stored records. JSON export is `context.to_json()`. +Canonical SQLite export uses `persist_canonical_records(database_path, context)` and `fetch_canonical_records(database_path, run_id)` from `utils.test_record.sqlite_persistence`. Separate `request_runs_v1` / `request_records_v1` tables preserve legacy runs. Writes validate snapshots, deduplicate identical IDs within each run, reject conflicting retries and roll back the entire batch on failure. Reads revalidate stored records. JSON export is `context.to_json()`. Canonical request records are available as an opt-in API with an ActionCore release that provides `je_action_core.request_context` (or the coordinated development checkout). Legacy record lists and reports diff --git a/README/README_zh-CN.md b/README/README_zh-CN.md index 564835c..471fdcd 100644 --- a/README/README_zh-CN.md +++ b/README/README_zh-CN.md @@ -781,7 +781,7 @@ python -m je_load_density serve [--host ...] # start the control socket ## 测试记录 -Canonical SQLite 导出使用 `utils.test_record.sqlite_persistence` 的 `persist_canonical_records(database_path, context)` 与 `fetch_canonical_records(database_path, run_id)`。独立的 `request_runs_v1`／`request_records_v1` 表保留旧记录。写入验证 snapshot、去除相同 ID 的重复项、拒绝冲突重送，失败时回滚整批；读取重新验证保存的记录。JSON 导出为 `context.to_json()`。 +Canonical SQLite 导出使用 `utils.test_record.sqlite_persistence` 的 `persist_canonical_records(database_path, context)` 与 `fetch_canonical_records(database_path, run_id)`。独立的 `request_runs_v1`／`request_records_v1` 表保留旧记录。写入验证 snapshot、去除同一 run 内相同 ID 的重复项、拒绝冲突重送，失败时回滚整批；读取重新验证保存的记录。JSON 导出为 `context.to_json()`。 Canonical request 记录是可选 API，需要提供 `je_action_core.request_context` 的 ActionCore 版本 或协调的开发工作树。原记录列表与报告仍支持现有依赖下限。从 diff --git a/README/README_zh-TW.md b/README/README_zh-TW.md index b33213f..81a7c67 100644 --- a/README/README_zh-TW.md +++ b/README/README_zh-TW.md @@ -781,7 +781,7 @@ python -m je_load_density serve [--host ...] # start the control socket ## 測試紀錄 -Canonical SQLite 匯出使用 `utils.test_record.sqlite_persistence` 的 `persist_canonical_records(database_path, context)` 與 `fetch_canonical_records(database_path, run_id)`。獨立的 `request_runs_v1`／`request_records_v1` 表保留舊紀錄。寫入驗證 snapshot、去除相同 ID 的重複項、拒絕衝突重送，失敗時回復整批；讀取重新驗證保存的紀錄。JSON 匯出為 `context.to_json()`。 +Canonical SQLite 匯出使用 `utils.test_record.sqlite_persistence` 的 `persist_canonical_records(database_path, context)` 與 `fetch_canonical_records(database_path, run_id)`。獨立的 `request_runs_v1`／`request_records_v1` 表保留舊紀錄。寫入驗證 snapshot、去除同一 run 內相同 ID 的重複項、拒絕衝突重送，失敗時回復整批；讀取重新驗證保存的紀錄。JSON 匯出為 `context.to_json()`。 Canonical request 紀錄為選用 API，需要提供 `je_action_core.request_context` 的 ActionCore 版本 或協調的開發工作樹。原紀錄清單與報表仍支援既有相依下限。從 diff --git a/architecture.md b/architecture.md index 816b817..d83b3d6 100644 --- a/architecture.md +++ b/architecture.md @@ -135,7 +135,7 @@ MCP `load_density.list_executor_commands` tool all read the `LD_*` names from `e ## 6. Cross-project boundaries -Canonical persistence exports `persist_canonical_records(database_path, context)` and `fetch_canonical_records(database_path, run_id)` from `utils.test_record.sqlite_persistence`. Versioned tables are separate from legacy runs; transaction rollback, JSON-aware retry conflicts and read-time schema/identity validation preserve the shared contract. +Canonical persistence exports `persist_canonical_records(database_path, context)` and `fetch_canonical_records(database_path, run_id)` from `utils.test_record.sqlite_persistence`. Versioned tables are separate from legacy runs; transaction rollback, run-scoped JSON-aware retry conflicts and read-time schema/identity validation preserve the shared contract. Canonical request records are an opt-in contract supplied by ActionCore's ``request_record``/ ``request_context`` APIs. ``utils/test_record/contract.py`` adapts legacy results and diff --git a/docs/source/En/doc/test_record/test_record_doc.rst b/docs/source/En/doc/test_record/test_record_doc.rst index 829d665..b02bbba 100644 --- a/docs/source/En/doc/test_record/test_record_doc.rst +++ b/docs/source/En/doc/test_record/test_record_doc.rst @@ -58,4 +58,4 @@ to isolated environment events so greenlets do not depend on ContextVar inherita ``context.to_json()`` exports v1 records with identities, numeric/null status and structured errors. Payload capture is disabled by default. Async summaries exclude previous invocations' records. -Canonical SQLite export uses `persist_canonical_records(database_path, context)` and `fetch_canonical_records(database_path, run_id)` from `utils.test_record.sqlite_persistence`. Separate `request_runs_v1` / `request_records_v1` tables preserve legacy runs. Writes validate snapshots, deduplicate identical IDs, reject conflicting retries and roll back the entire batch on failure. Reads revalidate stored records. JSON export is `context.to_json()`. +Canonical SQLite export uses `persist_canonical_records(database_path, context)` and `fetch_canonical_records(database_path, run_id)` from `utils.test_record.sqlite_persistence`. Separate `request_runs_v1` / `request_records_v1` tables preserve legacy runs. Writes validate snapshots, deduplicate identical IDs within each run, reject conflicting retries and roll back the entire batch on failure. Reads revalidate stored records. JSON export is `context.to_json()`. diff --git a/docs/source/Zh/doc/test_record/test_record_doc.rst b/docs/source/Zh/doc/test_record/test_record_doc.rst index 3db2b05..591c393 100644 --- a/docs/source/Zh/doc/test_record/test_record_doc.rst +++ b/docs/source/Zh/doc/test_record/test_record_doc.rst @@ -49,4 +49,4 @@ SQLite 持久化 ``context.to_json()`` 輸出 v1 紀錄，包含識別、數字／null 狀態碼與結構化錯誤。 預設不擷取完整內容；async 回傳摘要不納入前次呼叫的紀錄。 -Canonical SQLite 匯出使用 `utils.test_record.sqlite_persistence` 的 `persist_canonical_records(database_path, context)` 與 `fetch_canonical_records(database_path, run_id)`。獨立的版本化表保留舊紀錄。寫入驗證 snapshot、去除相同 ID 重複項、拒絕衝突重送，失敗時回復整批；讀取重新驗證保存的紀錄。JSON 匯出為 `context.to_json()`。 +Canonical SQLite 匯出使用 `utils.test_record.sqlite_persistence` 的 `persist_canonical_records(database_path, context)` 與 `fetch_canonical_records(database_path, run_id)`。獨立的版本化表保留舊紀錄。寫入驗證 snapshot、去除同一 run 內相同 ID 重複項、拒絕衝突重送，失敗時回復整批；讀取重新驗證保存的紀錄。JSON 匯出為 `context.to_json()`。 diff --git a/docs/updates/2026-10.md b/docs/updates/2026-10.md index c1b6144..848a0be 100644 --- a/docs/updates/2026-10.md +++ b/docs/updates/2026-10.md @@ -197,3 +197,7 @@ Index and query commands: [README.md](README.md). New entries go at the end. ## U-20261002-03 · 2026-10-02 · Persist canonical request records transactionally · #done #records Explicit run contexts export canonical JSON and persist to separate versioned SQLite tables. Canonical writes validate snapshots, preserve legacy tables, deduplicate retries and roll back conflicts atomically. Reads validate JSON and indexed identities. Added round-trip, multiple-run, legacy preservation, retry conflict, invalid snapshot and stored-corruption regressions. + +## U-20261002-04 · 2026-10-02 · Scope persisted request IDs to their run · #done #records + +SQLite deduplication now keys records by (run_id, record_id), matching the shared contract's run-local uniqueness rule. Separate runs can reuse record IDs without conflicting; same-run conflicting retries still roll back the full batch. Added a two-run regression with the same request ID. diff --git a/docs/updates/README.md b/docs/updates/README.md index 7512d9f..961fd78 100644 --- a/docs/updates/README.md +++ b/docs/updates/README.md @@ -58,6 +58,7 @@ In the same commit: delete the item from `progress.md`, add a `#done` entry here | ID | Date | Title | Tags | Batch | |---|---|---|---|---| +| U-20261002-04 | 2026-10-02 | Scope persisted request IDs to their run | #done #records | [2026-10](2026-10.md) | | U-20261002-03 | 2026-10-02 | Persist canonical request records transactionally | #done #records | [2026-10](2026-10.md) | | U-20261002-02 | 2026-10-02 | Capture canonical records in Locust and asyncio runs | #migration #records | [2026-10](2026-10.md) | | U-20261002-01 | 2026-10-02 | Testing platform design draft and outstanding work | #docs #design | [2026-10](2026-10.md) | diff --git a/je_load_density/utils/test_record/sqlite_persistence.py b/je_load_density/utils/test_record/sqlite_persistence.py index d6fbfd8..e5878d3 100644 --- a/je_load_density/utils/test_record/sqlite_persistence.py +++ b/je_load_density/utils/test_record/sqlite_persistence.py @@ -137,8 +137,9 @@ def fetch_run_records(database_path: str, run_id: int) -> Iterable[Dict[str, Any _CANONICAL_TABLES = ( "CREATE TABLE IF NOT EXISTS request_runs_v1 (run_id TEXT PRIMARY KEY, schema_version INTEGER NOT NULL " "CHECK(schema_version = 1))", - "CREATE TABLE IF NOT EXISTS request_records_v1 (sequence INTEGER PRIMARY KEY, record_id TEXT UNIQUE NOT NULL, " - "run_id TEXT NOT NULL, record_json TEXT NOT NULL, FOREIGN KEY(run_id) REFERENCES request_runs_v1(run_id))", + "CREATE TABLE IF NOT EXISTS request_records_v1 (sequence INTEGER PRIMARY KEY, record_id TEXT NOT NULL, " + "run_id TEXT NOT NULL, record_json TEXT NOT NULL, UNIQUE(run_id, record_id), " + "FOREIGN KEY(run_id) REFERENCES request_runs_v1(run_id))", "CREATE INDEX IF NOT EXISTS idx_request_records_v1_run ON request_records_v1(run_id)", ) @@ -165,7 +166,7 @@ def _insert_canonical_row(connection: sqlite3.Connection, row: tuple[str, str, s from je_action_core.request_record import RequestRecordError previous = connection.execute( - "SELECT record_json FROM request_records_v1 WHERE record_id = ?", (row[0],) + "SELECT record_json FROM request_records_v1 WHERE run_id = ? AND record_id = ?", (row[1], row[0]) ).fetchone() if previous is not None: if previous[0] != row[2]: diff --git a/test/test_canonical_persistence.py b/test/test_canonical_persistence.py index 0856343..34961e1 100644 --- a/test/test_canonical_persistence.py +++ b/test/test_canonical_persistence.py @@ -40,6 +40,17 @@ def test_round_trip_multiple_runs_and_empty_run(tmp_path): assert fetch_canonical_records(database, empty.run_id) == [] +def test_record_ids_are_unique_within_each_run(tmp_path): + database = str(tmp_path / "records.sqlite") + first, second = context(), context() + first_record = capture(first, record_id="request-1") + second_record = capture(second, record_id="request-1", status_code=201) + persist_canonical_records(database, first) + persist_canonical_records(database, second) + assert fetch_canonical_records(database, first.run_id) == [first_record] + assert fetch_canonical_records(database, second.run_id) == [second_record] + + def test_existing_legacy_database_is_preserved(tmp_path, monkeypatch): monkeypatch.setattr(test_record_instance, "test_record_list", [ {"Method": "GET", "test_url": "http://legacy/", "status_code": "200", "response_time_ms": 12} From 166a72266a839b8bbe519e2176b4630920981960 Mon Sep 17 00:00:00 2001 From: JeffreyChen Date: Fri, 2 Oct 2026 19:25:57 +0800 Subject: [PATCH 05/18] test: exercise installed wheels and propagate failed SLA status --- README.md | 6 + README/README_zh-CN.md | 6 + README/README_zh-TW.md | 6 + architecture.md | 2 + dev.toml | 1 + docs/source/En/doc/cli/cli_doc.rst | 7 + docs/source/Zh/doc/cli/cli_doc.rst | 7 + .../plans/2026-10-02-smoke-extras.md | 24 +++ docs/updates/2026-10.md | 4 + docs/updates/README.md | 1 + je_load_density/__main__.py | 28 +++- progress.md | 6 +- pyproject.toml | 1 + test/smoke/dashboard_probe.py | 32 ++++ test/smoke/server.py | 35 +++++ test/smoke/test_wheel_smoke.py | 139 ++++++++++++++++++ 16 files changed, 295 insertions(+), 10 deletions(-) create mode 100644 docs/superpowers/plans/2026-10-02-smoke-extras.md create mode 100644 test/smoke/dashboard_probe.py create mode 100644 test/smoke/server.py create mode 100644 test/smoke/test_wheel_smoke.py diff --git a/README.md b/README.md index c46d4fd..b63faba 100644 --- a/README.md +++ b/README.md @@ -779,6 +779,12 @@ python -m je_load_density serve [--host ...] # start the control socket Legacy single-flag form (`-e/-d/-c/--execute_str`) is still accepted for backwards compatibility with downstream tools. +## Smoke tests + +`run`, `run-dir`, `run-str` and legacy execute flags return a nonzero exit code when an action (including an SLA gate) fails. Actions in a file still run in order and produce their usual reports. The Python executor return format is preserved. Base installation includes httpx for the native async benchmark; HTTP/2 needs the `http2` extra. + +Run `python -m unittest discover -s test/smoke -p "test_*.py"` from a checkout. The stdlib harness starts a separate local HTTP server and subprocesses to verify real Locust/async requests, summary/JSON/JUnit files, SQLite, SLA failures, dashboard JSON/SSE and MCP initialization. Docker runs the same harness outside the source tree against an installed wheel. + ## Test Record Canonical SQLite export uses `persist_canonical_records(database_path, context)` and `fetch_canonical_records(database_path, run_id)` from `utils.test_record.sqlite_persistence`. Separate `request_runs_v1` / `request_records_v1` tables preserve legacy runs. Writes validate snapshots, deduplicate identical IDs within each run, reject conflicting retries and roll back the entire batch on failure. Reads revalidate stored records. JSON export is `context.to_json()`. diff --git a/README/README_zh-CN.md b/README/README_zh-CN.md index 471fdcd..b3b25eb 100644 --- a/README/README_zh-CN.md +++ b/README/README_zh-CN.md @@ -779,6 +779,12 @@ python -m je_load_density serve [--host ...] # start the control socket 旧式单旗标形式(`-e/-d/-c/--execute_str`)仍为与下游工具向后兼容而接受。 +## 冒烟测试 + +`run`、`run-dir`、`run-str` 与旧执行旗标在动作（包含 SLA gate）失败时返回非零 exit code。单个文件的动作仍按顺序执行并生成原有报告；Python executor 的返回格式保持兼容。基本安装包含原生 async benchmark 所需的 httpx，HTTP/2 需要 `http2` extra。 + +在 checkout 执行 `python -m unittest discover -s test/smoke -p "test_*.py"`。标准库 harness 启动独立本机 HTTP 服务及子进程，验证真实 Locust／async 请求、summary／JSON／JUnit、SQLite、SLA 失败、dashboard JSON／SSE 与 MCP 初始化。Docker 在源码树外使用同一 harness 验证已安装 wheel。 + ## 测试记录 Canonical SQLite 导出使用 `utils.test_record.sqlite_persistence` 的 `persist_canonical_records(database_path, context)` 与 `fetch_canonical_records(database_path, run_id)`。独立的 `request_runs_v1`／`request_records_v1` 表保留旧记录。写入验证 snapshot、去除同一 run 内相同 ID 的重复项、拒绝冲突重送，失败时回滚整批；读取重新验证保存的记录。JSON 导出为 `context.to_json()`。 diff --git a/README/README_zh-TW.md b/README/README_zh-TW.md index 81a7c67..47ee727 100644 --- a/README/README_zh-TW.md +++ b/README/README_zh-TW.md @@ -779,6 +779,12 @@ python -m je_load_density serve [--host ...] # start the control socket 舊式單旗標形式(`-e/-d/-c/--execute_str`)仍為與下游工具向後相容而接受。 +## 煙霧測試 + +`run`、`run-dir`、`run-str` 與舊執行旗標在動作（包含 SLA gate）失敗時回傳非零 exit code。單一檔案的動作仍依序執行並產生原有報告；Python executor 的回傳格式保持相容。基本安裝包含原生 async benchmark 所需的 httpx，HTTP/2 需要 `http2` extra。 + +在 checkout 執行 `python -m unittest discover -s test/smoke -p "test_*.py"`。標準函式庫 harness 啟動獨立本機 HTTP 服務與子行程，驗證實際 Locust／async 請求、summary／JSON／JUnit、SQLite、SLA 失敗、dashboard JSON／SSE 與 MCP 初始化。Docker 在原始碼樹外使用同一 harness 驗證已安裝 wheel。 + ## 測試紀錄 Canonical SQLite 匯出使用 `utils.test_record.sqlite_persistence` 的 `persist_canonical_records(database_path, context)` 與 `fetch_canonical_records(database_path, run_id)`。獨立的 `request_runs_v1`／`request_records_v1` 表保留舊紀錄。寫入驗證 snapshot、去除同一 run 內相同 ID 的重複項、拒絕衝突重送，失敗時回復整批；讀取重新驗證保存的紀錄。JSON 匯出為 `context.to_json()`。 diff --git a/architecture.md b/architecture.md index d83b3d6..890246c 100644 --- a/architecture.md +++ b/architecture.md @@ -135,6 +135,8 @@ MCP `load_density.list_executor_commands` tool all read the `LD_*` names from `e ## 6. Cross-project boundaries +CLI execution retains legacy flags and Python executor return shapes, but now returns a nonzero process exit code after failed actions/SLA gates. `test/smoke` runs real HTTP/report/dashboard/MCP checks in subprocesses against source or an installed wheel. Native async HTTP benchmarking requires base httpx; the `http2` extra adds HTTP/2 support. + Canonical persistence exports `persist_canonical_records(database_path, context)` and `fetch_canonical_records(database_path, run_id)` from `utils.test_record.sqlite_persistence`. Versioned tables are separate from legacy runs; transaction rollback, run-scoped JSON-aware retry conflicts and read-time schema/identity validation preserve the shared contract. Canonical request records are an opt-in contract supplied by ActionCore's ``request_record``/ diff --git a/dev.toml b/dev.toml index 2548395..eb56d7d 100644 --- a/dev.toml +++ b/dev.toml @@ -16,6 +16,7 @@ requires-python = ">=3.10" license-files = ["LICENSE"] dependencies = [ "locust", + "httpx>=0.27", "defusedxml>=0.7.1", "je_action_core>=0.0.2", "tomli>=2.0; python_version < '3.11'", diff --git a/docs/source/En/doc/cli/cli_doc.rst b/docs/source/En/doc/cli/cli_doc.rst index 8ff6d3e..21607b2 100644 --- a/docs/source/En/doc/cli/cli_doc.rst +++ b/docs/source/En/doc/cli/cli_doc.rst @@ -84,3 +84,10 @@ Legacy flags The flat ``-e/-d/-c/--execute_str`` flags from previous releases are still accepted (suppressed in ``--help``) for backwards compatibility with tools such as PyBreeze. New scripts should use the subcommands. + +Smoke tests and failure exit codes +---------------------------------- + +``run``, ``run-dir``, ``run-str`` and legacy execute flags return a nonzero exit code when an action (including an SLA gate) fails. Actions in a file still run in order and produce their usual reports. The Python executor return format is preserved. Base installation includes httpx for the native async benchmark; HTTP/2 needs the ``http2`` extra. + +Run ``python -m unittest discover -s test/smoke -p "test_*.py"`` from a checkout. The stdlib harness starts a separate local HTTP server and subprocesses to verify real Locust/async requests, summary/JSON/JUnit files, SQLite, SLA failures, dashboard JSON/SSE and MCP initialization. Docker runs the same harness outside the source tree against an installed wheel. diff --git a/docs/source/Zh/doc/cli/cli_doc.rst b/docs/source/Zh/doc/cli/cli_doc.rst index f5f6ab7..7fae0f6 100644 --- a/docs/source/Zh/doc/cli/cli_doc.rst +++ b/docs/source/Zh/doc/cli/cli_doc.rst @@ -79,3 +79,10 @@ Inline JSON（CI script 友善）:: -------- 之前版本的扁平旗標 ``-e/-d/-c/--execute_str`` 仍接受（在 ``--help`` 中隱藏），維持與 PyBreeze 等下游工具相容。新腳本應使用子指令。 + +煙霧測試與失敗結束碼 +------------------------------ + +``run``、``run-dir``、``run-str`` 與舊執行旗標在動作（包含 SLA gate）失敗時回傳非零 exit code。單一檔案的動作仍依序執行並產生原有報告；Python executor 的回傳格式保持相容。基本安裝包含原生 async benchmark 所需的 httpx，HTTP/2 需要 ``http2`` extra。 + +在 checkout 執行 ``python -m unittest discover -s test/smoke -p "test_*.py"``。標準函式庫 harness 啟動獨立本機 HTTP 服務與子行程，驗證實際 Locust／async 請求、summary／JSON／JUnit、SQLite、SLA 失敗、dashboard JSON／SSE 與 MCP 初始化。Docker 在原始碼樹外使用同一 harness 驗證已安裝 wheel。 diff --git a/docs/superpowers/plans/2026-10-02-smoke-extras.md b/docs/superpowers/plans/2026-10-02-smoke-extras.md new file mode 100644 index 0000000..3388866 --- /dev/null +++ b/docs/superpowers/plans/2026-10-02-smoke-extras.md @@ -0,0 +1,24 @@ +# Smoke tests and Docker extras implementation plan + +Execute inline in the existing isolated feature worktree. The approved roadmap and instruction to start implementation authorize these stages. + +## Task 1: Wheel smoke harness + +- [x] Create stdlib smoke tests in `test/smoke/`, with a separate local HTTP server process and bounded subprocess timeouts. +- [x] Exercise CLI action execution, real Locust requests, summary/JSON/JUnit output, SQLite and SLA pass/failure behavior. Read generated artifacts independently. +- [x] Exercise the async benchmark, dashboard JSON/SSE and MCP initialization in separate processes. Keep GUI lifecycle checks in the GUI capability probe. +- [x] Run the harness against source and an installed checkout-built wheel, repair actual failures, document and commit. + +## Task 2: Dynamic Docker installation matrix + +- [ ] Generate base/every-extra/all cells directly from `pyproject.toml`: Python 3.12 for every extra plus base 3.10/3.14 on PRs; all Python 3.10–3.14 versions on schedule. +- [ ] Build the wheel using existing locked build tools. Each cell installs that wheel plus only its selected extra in a fresh Docker image, runs pip check, CLI checks and non-skipping capability probes. +- [ ] Define probes for declared extras: create/use their relevant local SDK or framework capabilities, perform MCP handshake even without dependencies, render charts/PDF and launch Qt offscreen. Fail explicitly if a declared extra lacks a probe. +- [ ] Implement a reusable workflow consumed by stable/dev CI; keep pinned actions, minimum permissions and timeouts. Dev publication waits for these checks. +- [ ] Validate representative base/gui/charts/cloud/protocol cells locally in Docker, then complete the whole representative matrix. Repair installation/API incompatibilities rather than excluding supported cells silently. + +## Task 3: Protocol service smoke and handoff + +- [ ] Add Compose health-checked representative protocol services and distinguish service integration checks from installation probes. +- [ ] Run smoke tests and affected workflow/security tests; update all README languages, Sphinx, architecture and outstanding-work ledger. +- [ ] Commit and push completed stages. Keep GUI/async lifecycle items open if their broader redesign is needed; continue that roadmap implementation instead of claiming parity prematurely. diff --git a/docs/updates/2026-10.md b/docs/updates/2026-10.md index 848a0be..e622887 100644 --- a/docs/updates/2026-10.md +++ b/docs/updates/2026-10.md @@ -201,3 +201,7 @@ Explicit run contexts export canonical JSON and persist to separate versioned SQ ## U-20261002-04 · 2026-10-02 · Scope persisted request IDs to their run · #done #records SQLite deduplication now keys records by (run_id, record_id), matching the shared contract's run-local uniqueness rule. Separate runs can reuse record IDs without conflicting; same-run conflicting retries still roll back the full batch. Added a two-run regression with the same request ID. + +## U-20261002-05 · 2026-10-02 · Exercise installed wheels and propagate failed SLA status · #done #smoke + +Added a stdlib smoke harness that exercises local HTTP requests in isolated processes, actual CLI reports, SQLite, SLA failures, async benchmarking, dashboard JSON/SSE and MCP initialization. CLI actions now return failure status to CI while preserving Python executor returns. Clean-wheel Docker testing reproduced a missing base httpx dependency; stable/dev manifests now include it, with HTTP/2 remaining optional. Source and Python 3.12 installed-wheel Docker smoke runs each pass six tests. GUI lifecycle and the full extras matrix remain tracked separately. diff --git a/docs/updates/README.md b/docs/updates/README.md index 961fd78..29f8bc9 100644 --- a/docs/updates/README.md +++ b/docs/updates/README.md @@ -58,6 +58,7 @@ In the same commit: delete the item from `progress.md`, add a `#done` entry here | ID | Date | Title | Tags | Batch | |---|---|---|---|---| +| U-20261002-05 | 2026-10-02 | Exercise installed wheels and propagate failed SLA status | #done #smoke | [2026-10](2026-10.md) | | U-20261002-04 | 2026-10-02 | Scope persisted request IDs to their run | #done #records | [2026-10](2026-10.md) | | U-20261002-03 | 2026-10-02 | Persist canonical request records transactionally | #done #records | [2026-10](2026-10.md) | | U-20261002-02 | 2026-10-02 | Capture canonical records in Locust and asyncio runs | #migration #records | [2026-10](2026-10.md) | diff --git a/je_load_density/__main__.py b/je_load_density/__main__.py index 83786b4..0d2f022 100644 --- a/je_load_density/__main__.py +++ b/je_load_density/__main__.py @@ -4,7 +4,8 @@ from typing import List, Optional from je_load_density.utils.exception.exception_tags import argparse_get_wrong_data -from je_load_density.utils.executor.action_executor import execute_action, execute_files +from je_load_density.utils.exception.exceptions import LoadDensityTestExecuteException +from je_load_density.utils.executor.action_executor import executor from je_load_density.utils.file_process.get_dir_file_list import get_dir_files_as_list from je_load_density.utils.json.json_file.json_file import read_action_json from je_load_density.utils.project.create_project_structure import create_project_dir @@ -13,12 +14,25 @@ ) +def _execute_cli_actions(actions: object) -> None: + """Keep batch execution/reporting, then fail the CLI if an action raised.""" + records, failed = executor.collect_action_results(actions) + executor.settings.reporter.on_records(records) + if failed: + raise LoadDensityTestExecuteException(f"{len(failed)} action(s) failed") + + +def _execute_cli_files(directory: str) -> None: + for path in get_dir_files_as_list(directory): + _execute_cli_actions(read_action_json(path)) + + def _cmd_run(args: argparse.Namespace) -> None: - execute_action(read_action_json(args.file)) + _execute_cli_actions(read_action_json(args.file)) def _cmd_run_dir(args: argparse.Namespace) -> None: - execute_files(get_dir_files_as_list(args.dir)) + _execute_cli_files(args.dir) def _cmd_run_str(args: argparse.Namespace) -> None: @@ -28,9 +42,9 @@ def _cmd_run_str(args: argparse.Namespace) -> None: if isinstance(first_pass, str): payload = first_pass else: - execute_action(first_pass) + _execute_cli_actions(first_pass) return - execute_action(json.loads(payload)) + _execute_cli_actions(json.loads(payload)) def _cmd_init(args: argparse.Namespace) -> None: @@ -123,8 +137,8 @@ def _build_parser() -> argparse.ArgumentParser: def _dispatch_legacy(args: argparse.Namespace) -> bool: legacy_map = { - "execute_file": lambda value: execute_action(read_action_json(value)), - "execute_dir": lambda value: execute_files(get_dir_files_as_list(value)), + "execute_file": lambda value: _execute_cli_actions(read_action_json(value)), + "execute_dir": _execute_cli_files, "execute_str": lambda value: _cmd_run_str(argparse.Namespace(json=value)), "create_project": create_project_dir, } diff --git a/progress.md b/progress.md index b19148a..8ac4198 100644 --- a/progress.md +++ b/progress.md @@ -14,10 +14,10 @@ Cross-repo and workspace items live in `D:\Codes\progress.md` (relevant here: X- - **#19** Markdown summary report generator (`LD_generate_markdown_report`): Add `generate_markdown_report.py` under `utils/generate_report/` to render GFM summary tables (RPS, latencies, SLA breaches, error clusters) suitable for GitHub Actions `$GITHUB_STEP_SUMMARY` and GitLab MR comments. Register the command in `action_executor.py`, export in `__init__.py`, and add tests. - **#20** Pre-flight health check gate: Add a target health probe before ramping load in `start_test` / `bench`. If the target endpoint is unreachable, DNS fails, or TLS handshake fails, abort early with a descriptive error instead of spawning hundreds of workers that flood logs with connection refused errors. - **#23** [BLOCKED: two releases must ship the warning first] Flip the package gate's default to refuse packages outside the allowlist: set `self.allow_arbitrary_packages = False` in `PackageManager.__init__` (`je_load_density/utils/package_manager/package_manager_class.py`); the warning branch is je_action_core's `_check_allowed` and stays for the other projects. Update the "Package gate" paragraph in the three READMEs and the "Package Gate" section of `docs/source/{En,Zh}/doc/package_manager/package_manager_doc.rst`. The warning is first released in the version after 0.0.77 (`origin/main`); flip once two releases after that one have shipped it. Decide first how a user who only runs action files (`loaddensity`, `-e`, the socket server) allows a package without a Python host to call `executor.allow_packages(...)`. -- **#24** End-to-end smoke tests: add `test/smoke/` for installed-wheel CLI/action execution against a subprocess-local HTTP service, success and failure reports/SLA/SQLite, async, dashboard JSON/SSE and offscreen GUI execution/cancellation. Existing `test_expansion*_smoke.py` does not cover that complete lifecycle. -- **#25** Docker extras installation matrix: `.github/workflows/ci-dev.yml` and `ci-stable.yml` lack isolated installation/capability checks for base, every declared extra and all. Generate the matrix from `pyproject.toml`, test a checkout-built wheel with `pip check` and non-skipping probes, include GUI system libraries and Compose health checks, and gate publishing on required checks. Local execution needs the Docker Desktop Linux daemon running. +- **#24** Installed-wheel GUI smoke lifecycle: extend `test/smoke/` and the Docker GUI capability probe with offscreen execution and cancellation after the UI lifecycle redesign (#18). Keep client/thread cleanup observable. +- **#25** Docker extras installation matrix: `.github/workflows/ci-dev.yml` and `ci-stable.yml` lack isolated installation/capability checks for base, every declared extra and all. Generate the matrix from `pyproject.toml`, test a checkout-built wheel with `pip check` and non-skipping probes, include GUI system libraries and Compose health checks, and gate publishing on required checks. - **#26** Percentile bands: `gui/chart_panel.py` mixes RPS and latency on one axis; `utils/generate_report/generate_chart_report.py` uses `ts`/index instead of request `start_time`. Share per-window percentile aggregation across GUI/dashboard/report, show p50 with p50–p95 and p95–p99 bands, separate RPS units and handle gaps, failures and actual bucket duration. - **#27** Cloud backend failure coverage: extend `test/test_cloud_launchers.py` for invalid inputs, auth/permissions, timeout/rate limits, partial failures, Fargate `failures`, Lambda `FunctionError`/Event/warm reuse, Cloud Run refresh/error responses and ACI poller failure. Use SDK stubs/local HTTP and assert request/response semantics; no live provisioning is needed. -- **#28** Shared APITestka record format: `utils/test_record/`, `wrapper/event/request_hook.py` and `engine/asyncio_engine.py` need a versioned canonical request record and run-scoped sink, with legacy list/report compatibility and adapters for APITestka's response dicts and failure pairs. Coordinate the proposed stdlib contract in ActionCore and consumer dependency releases. Design: `docs/superpowers/specs/2026-10-02-shared-record-design.md`. +- **#28** Shared record release activation: coordinate the actual ActionCore request-record release, then activate consumer dependency floors and cross-project installed-wheel fixtures. Current opt-in APIs must retain compatibility with older core installations until that release. Design: `docs/superpowers/specs/2026-10-02-shared-record-design.md`. - **#29** Distributed worker health and recovery: `wrapper/create_locust_env/create_locust_env.py::_wait_for_workers` counts clients and starts after timeout. Add configurable ready/heartbeat/lost policy, fail or explicitly degrade on startup timeout, reassign virtual-user capacity after loss, handle reconnect/stale assignments and deduplicate results; reuse Locust heartbeat/rebalance support where possible. Target health is separately tracked by #20. - **#30** One script for functional/load/synthetic modes: extend APITestka's existing `integrations/load_density*.py` bridge and WebRunner's `utils/synthetic_monitoring/monitor.py` so common request/assertion steps are authored once and execution profiles choose functional correctness, capacity/SLA or periodic monitoring. Unsupported conversions must fail preflight and failed checks must reach monitor transitions. Depends on #28 and async parity #16; preserve existing CLI/action/summary contracts. diff --git a/pyproject.toml b/pyproject.toml index 493dc6c..f068ad4 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -15,6 +15,7 @@ requires-python = ">=3.10" license-files = ["LICENSE"] dependencies = [ "locust", + "httpx>=0.27", "defusedxml>=0.7.1", "je_action_core>=0.0.2", "tomli>=2.0; python_version < '3.11'", diff --git a/test/smoke/dashboard_probe.py b/test/smoke/dashboard_probe.py new file mode 100644 index 0000000..d7fa5e2 --- /dev/null +++ b/test/smoke/dashboard_probe.py @@ -0,0 +1,32 @@ +"""Exercise real dashboard JSON and SSE endpoints, then close the listening server.""" + +from je_load_density.utils.dashboard.live_dashboard import start_dashboard, stop_dashboard +from je_load_density.utils.test_record.test_record_class import test_record_instance + + +def main() -> None: + """Check dashboard transport and data against a measured request fixture.""" + import json + from urllib.request import urlopen + + test_record_instance.clear_records() + test_record_instance.test_record_list.append({ + "Method": "GET", "test_url": "http://localhost/ok", "name": "health", + "status_code": "200", "response_time_ms": 25, "response_length": 2, + }) + dashboard = start_dashboard(port=0, refresh_seconds=0.01) + try: + address = f"http://127.0.0.1:{dashboard._httpd.server_port}" + with urlopen(address + "/snapshot", timeout=5) as response: + snapshot = json.load(response) + with urlopen(address + "/events", timeout=5) as response: + content_type = response.headers["Content-Type"] + event = response.readline().decode("utf-8") + print(json.dumps({"snapshot": snapshot, "content_type": content_type, + "event": json.loads(event.removeprefix("data: "))})) + finally: + stop_dashboard() + + +if __name__ == "__main__": + main() diff --git a/test/smoke/server.py b/test/smoke/server.py new file mode 100644 index 0000000..00f98cf --- /dev/null +++ b/test/smoke/server.py @@ -0,0 +1,35 @@ +"""Local smoke target in a fresh process, outside either runner's monkey patches.""" + +import json +import sys +from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer +from pathlib import Path +from threading import Thread + + +class Handler(BaseHTTPRequestHandler): + def do_GET(self) -> None: + payload = json.dumps({"ok": self.path != "/fail"}).encode("utf-8") + self.send_response(500 if self.path == "/fail" else 200) + self.send_header("Content-Type", "application/json") + self.send_header("Content-Length", str(len(payload))) + self.end_headers() + self.wfile.write(payload) + + def do_POST(self) -> None: + if self.path != "/shutdown": + self.send_error(404) + return + self.send_response(200) + self.send_header("Content-Length", "0") + self.end_headers() + Thread(target=self.server.shutdown, daemon=True).start() + + def log_message(self, format: str, *args: object) -> None: + pass + + +if __name__ == "__main__": + with ThreadingHTTPServer(("127.0.0.1", 0), Handler) as server: + Path(sys.argv[1]).write_text(json.dumps({"port": server.server_port}), encoding="utf-8") + server.serve_forever() diff --git a/test/smoke/test_wheel_smoke.py b/test/smoke/test_wheel_smoke.py new file mode 100644 index 0000000..cc1a72b --- /dev/null +++ b/test/smoke/test_wheel_smoke.py @@ -0,0 +1,139 @@ +"""Stdlib smoke harness; Docker copies it outside the checkout to test the installed wheel.""" + +import json +import os +import sqlite3 +import subprocess +import sys +import tempfile +import time +import unittest +from contextlib import closing +from pathlib import Path +from urllib.error import URLError +from urllib.request import Request, urlopen +from xml.etree import ElementTree + +SMOKE_DIRECTORY = Path(__file__).resolve().parent +PROCESS_TIMEOUT_SECONDS = 45 + + +class WheelSmoke(unittest.TestCase): + @classmethod + def setUpClass(cls) -> None: + cls.temporary = tempfile.TemporaryDirectory(prefix="loaddensity-smoke-") + cls.directory = Path(cls.temporary.name) + cls.environment = os.environ.copy() + source = SMOKE_DIRECTORY.parents[1] + if (source / "je_load_density").is_dir(): + cls.environment["PYTHONPATH"] = str(source) + os.pathsep + cls.environment.get("PYTHONPATH", "") + ready = cls.directory / "ready.json" + cls.server = subprocess.Popen( + [sys.executable, str(SMOKE_DIRECTORY / "server.py"), str(ready)], + stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL, cwd=cls.directory, + ) + deadline = time.monotonic() + 10 + while not ready.exists() and cls.server.poll() is None and time.monotonic() < deadline: + time.sleep(0.05) + if not ready.exists(): + cls.tearDownClass() + raise RuntimeError("Local smoke server did not become ready") + cls.url = f"http://127.0.0.1:{json.loads(ready.read_text(encoding='utf-8'))['port']}" + + @classmethod + def tearDownClass(cls) -> None: + if hasattr(cls, "url") and cls.server.poll() is None: + try: + with urlopen(Request(cls.url + "/shutdown", method="POST"), timeout=2): + pass + except (URLError, OSError): + cls.server.terminate() + elif cls.server.poll() is None: + cls.server.terminate() + try: + cls.server.wait(timeout=5) + except subprocess.TimeoutExpired: + cls.server.kill() + cls.server.wait(timeout=5) + cls.temporary.cleanup() + + def command(self, *arguments: str, input_text: str | None = None) -> subprocess.CompletedProcess: + return subprocess.run( + [sys.executable, *arguments], input=input_text, capture_output=True, text=True, + encoding="utf-8", errors="replace", env=self.environment, cwd=self.directory, + timeout=PROCESS_TIMEOUT_SECONDS, check=False, + ) + + def actions(self, endpoint: str, *, failed_gate: bool = False) -> list[list[object]]: + definitions = [ + {"LD_start_test": {"user_detail_dict": {"user": "http_user"}, "user_count": 1, + "spawn_rate": 10, "test_time": 1, + "tasks": {"get": {"request_url": self.url + endpoint}}}}, + {"LD_generate_summary_report": {"report_name": "summary"}}, + {"LD_generate_json_report": {"json_file_name": "records"}}, + {"LD_generate_junit_report": {"report_name": "junit"}}, + {"LD_persist_records": {"database_path": "records.sqlite"}}, + {"LD_assert_sla": {"rules": [{"type": "failure_rate", "op": "lte", "value": 0}, + {"type": "requests", "op": "gte", + "value": 1000000 if failed_gate else 1}]}}, + ] + return [[name, parameters] for definition in definitions for name, parameters in definition.items()] + + def execute_actions(self, actions: list[list[object]], *, legacy: bool = False) -> subprocess.CompletedProcess: + path = self.directory / "actions.json" + path.write_text(json.dumps({"load_density": actions}), encoding="utf-8") + return self.command("-m", "je_load_density", "--execute_file" if legacy else "run", str(path)) + + def test_cli_load_reports_sqlite_and_passing_sla(self) -> None: + result = self.execute_actions(self.actions("/ok")) + self.assertEqual(result.returncode, 0, result.stdout + result.stderr) + summary = json.loads((self.directory / "summary.json").read_text(encoding="utf-8")) + self.assertGreater(summary["totals"]["requests"], 0, result.stdout + result.stderr) + self.assertEqual(summary["totals"]["failures"], 0) + self.assertIn("p95_ms", summary["latency_overall"]) + records = json.loads((self.directory / "records_success.json").read_text(encoding="utf-8")) + self.assertGreater(len(records), 0) + suite = ElementTree.parse(self.directory / "junit.xml").getroot() + self.assertGreater(int(suite.attrib["tests"]), 0) + with closing(sqlite3.connect(self.directory / "records.sqlite")) as connection: + self.assertGreater(connection.execute("SELECT COUNT(*) FROM load_density_records").fetchone()[0], 0) + + def test_legacy_cli_and_failed_sla_signal_failure(self) -> None: + result = self.execute_actions(self.actions("/fail"), legacy=True) + self.assertNotEqual(result.returncode, 0, "A failed SLA gate must fail the CLI process") + summary = json.loads((self.directory / "summary.json").read_text(encoding="utf-8")) + self.assertGreater(summary["totals"]["failures"], 0) + + def test_failed_gate_on_successful_requests_signals_failure(self) -> None: + result = self.execute_actions(self.actions("/ok", failed_gate=True)) + self.assertNotEqual(result.returncode, 0, "A failed capacity gate must fail the CLI process") + + def test_mcp_handshake(self) -> None: + request = {"jsonrpc": "2.0", "id": 1, "method": "initialize", + "params": {"protocolVersion": "2025-06-18", "capabilities": {}, + "clientInfo": {"name": "smoke", "version": "1"}}} + result = self.command("-m", "je_load_density.mcp_server", input_text=json.dumps(request) + "\n") + self.assertEqual(result.returncode, 0, result.stderr) + response = json.loads(result.stdout.splitlines()[0]) + self.assertEqual(response["id"], 1) + self.assertIn("protocolVersion", response["result"]) + + def test_async_bench_executes_real_requests(self) -> None: + result = self.command("-m", "je_load_density", "bench", self.url + "/ok", + "--users", "1", "--duration", "0.25") + self.assertEqual(result.returncode, 0, result.stdout + result.stderr) + summary = json.loads(result.stdout) + self.assertGreater(summary["totals"]["requests"], 0) + self.assertEqual(summary["totals"]["failures"], 0) + + def test_dashboard_json_and_sse(self) -> None: + result = self.command(str(SMOKE_DIRECTORY / "dashboard_probe.py")) + self.assertEqual(result.returncode, 0, result.stdout + result.stderr) + output = json.loads(result.stdout) + self.assertEqual(output["content_type"], "text/event-stream") + self.assertEqual(output["snapshot"]["totals"]["requests"], 1) + self.assertEqual(output["event"]["totals"]["requests"], 1) + + +if __name__ == "__main__": + unittest.main() From 0e755be057eae5bf684478ea4bdc9b55b092adfc Mon Sep 17 00:00:00 2001 From: JeffreyChen Date: Fri, 2 Oct 2026 19:48:02 +0800 Subject: [PATCH 06/18] ci: gate publishing on Docker extras and protocol smoke checks --- .github/workflows/ci-dev.yml | 7 +- .github/workflows/ci-stable.yml | 5 + .github/workflows/extras.yml | 87 ++++++ README.md | 2 + README/README_zh-CN.md | 2 + README/README_zh-TW.md | 2 + architecture.md | 1 + dev.toml | 4 +- docker/README.md | 33 +++ docker/ci-services.yml | 27 ++ docker/extras.Dockerfile | 17 ++ docker/services.Dockerfile | 7 + docs/source/En/doc/cli/cli_doc.rst | 3 + docs/source/Zh/doc/cli/cli_doc.rst | 3 + .../plans/2026-10-02-smoke-extras.md | 12 +- docs/updates/2026-10.md | 7 + docs/updates/README.md | 1 + .../user_template/etcd_user_template.py | 21 +- progress.md | 2 +- pyproject.toml | 4 +- scripts/extras_matrix.py | 50 ++++ test/smoke/capability_probe.py | 62 +++++ test/smoke/install_extra.py | 24 ++ test/smoke/platform_probes.py | 202 ++++++++++++++ test/smoke/protocol_probes.py | 247 ++++++++++++++++++ test/smoke/run_cell.py | 18 ++ test/smoke/service_probe.py | 86 ++++++ test/smoke/test_wheel_smoke.py | 2 +- test/test_dev_release.py | 2 +- test/test_extra_capabilities.py | 56 ++++ test/test_extras_matrix.py | 57 ++++ test/test_service_probe.py | 22 ++ test/test_user_templates_a.py | 30 ++- 33 files changed, 1085 insertions(+), 20 deletions(-) create mode 100644 .github/workflows/extras.yml create mode 100644 docker/ci-services.yml create mode 100644 docker/extras.Dockerfile create mode 100644 docker/services.Dockerfile create mode 100644 scripts/extras_matrix.py create mode 100644 test/smoke/capability_probe.py create mode 100644 test/smoke/install_extra.py create mode 100644 test/smoke/platform_probes.py create mode 100644 test/smoke/protocol_probes.py create mode 100644 test/smoke/run_cell.py create mode 100644 test/smoke/service_probe.py create mode 100644 test/test_extra_capabilities.py create mode 100644 test/test_extras_matrix.py create mode 100644 test/test_service_probe.py diff --git a/.github/workflows/ci-dev.yml b/.github/workflows/ci-dev.yml index 68dacae..f19ac62 100644 --- a/.github/workflows/ci-dev.yml +++ b/.github/workflows/ci-dev.yml @@ -12,6 +12,11 @@ permissions: contents: read jobs: + extras: + uses: ./.github/workflows/extras.yml + with: + event-name: ${{ github.event_name }} + test: runs-on: ${{ matrix.os }} timeout-minutes: 15 # about 3x the slowest recent run, at least 15 @@ -44,7 +49,7 @@ jobs: # it is still the tip of dev and ships something the newest je_load_density_dev does not. # scripts/dev_release.py picks the version from PyPI, so nothing is committed back. name: Publish je_load_density_dev to PyPI - needs: [test] + needs: [test, extras] if: github.event_name == 'push' && github.ref == 'refs/heads/dev' runs-on: ubuntu-latest timeout-minutes: 15 # about 3x the slowest recent run, at least 15 diff --git a/.github/workflows/ci-stable.yml b/.github/workflows/ci-stable.yml index 6ed4a72..1f36f84 100644 --- a/.github/workflows/ci-stable.yml +++ b/.github/workflows/ci-stable.yml @@ -12,6 +12,11 @@ permissions: contents: read jobs: + extras: + uses: ./.github/workflows/extras.yml + with: + event-name: ${{ github.event_name }} + test: runs-on: ${{ matrix.os }} timeout-minutes: 15 # about 3x the slowest recent run, at least 15 diff --git a/.github/workflows/extras.yml b/.github/workflows/extras.yml new file mode 100644 index 0000000..467e2af --- /dev/null +++ b/.github/workflows/extras.yml @@ -0,0 +1,87 @@ +name: Installed wheel extras + +on: + workflow_call: + inputs: + event-name: + required: true + type: string + +permissions: + contents: read + +jobs: + matrix: + runs-on: ubuntu-latest + timeout-minutes: 15 + outputs: + matrix: ${{ steps.matrix.outputs.matrix }} + steps: + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + persist-credentials: false + - uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 # v7.0.0 + with: + python-version: "3.12" + - id: matrix + env: + MATRIX_EVENT: ${{ inputs.event-name }} + run: python scripts/extras_matrix.py --event "$MATRIX_EVENT" + + install: + needs: [matrix] + runs-on: ubuntu-latest + timeout-minutes: 20 + strategy: + fail-fast: false + max-parallel: 8 + matrix: ${{ fromJSON(needs.matrix.outputs.matrix) }} + steps: + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + persist-credentials: false + - uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 # v7.0.0 + with: + python-version: "3.12" + - name: Build checkout wheel with locked tooling + run: | + python -m pip install --require-hashes --only-binary :all: -r .github/requirements/publish.txt + python -m build --wheel --no-isolation + - name: Install isolated extra + env: + CELL_PYTHON: ${{ matrix.python }} + CELL_EXTRA: ${{ matrix.extra }} + run: >- + docker build -f docker/extras.Dockerfile + --build-arg PYTHON_VERSION="$CELL_PYTHON" --build-arg EXTRA="$CELL_EXTRA" + -t loaddensity-extras:cell . + - name: Run capability and installed-wheel smoke checks + run: timeout 360s docker run --rm loaddensity-extras:cell + + services: + runs-on: ubuntu-latest + timeout-minutes: 15 + env: + COMPOSE_PROJECT_NAME: ld-ci-${{ github.run_id }}-${{ github.run_attempt }} + steps: + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + persist-credentials: false + - uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 # v7.0.0 + with: + python-version: "3.12" + - name: Build checkout wheel + run: | + python -m pip install --require-hashes --only-binary :all: -r .github/requirements/publish.txt + python -m build --wheel --no-isolation + docker compose -f docker/ci-services.yml build probe + - name: Wait for healthy protocol services + run: docker compose -f docker/ci-services.yml up -d --wait --wait-timeout 60 redis mosquitto + - name: Verify adapter requests and measurements + run: timeout 60s docker compose -f docker/ci-services.yml run --rm probe + - name: Show service logs after failures + if: failure() + run: docker compose -f docker/ci-services.yml logs --no-color + - name: Stop isolated services + if: always() + run: docker compose -f docker/ci-services.yml down --volumes --remove-orphans diff --git a/README.md b/README.md index b63faba..64b73de 100644 --- a/README.md +++ b/README.md @@ -785,6 +785,8 @@ Legacy single-flag form (`-e/-d/-c/--execute_str`) is still accepted for backwar Run `python -m unittest discover -s test/smoke -p "test_*.py"` from a checkout. The stdlib harness starts a separate local HTTP server and subprocesses to verify real Locust/async requests, summary/JSON/JUnit files, SQLite, SLA failures, dashboard JSON/SSE and MCP initialization. Docker runs the same harness outside the source tree against an installed wheel. +Dev and Stable CI build the checkout wheel and test base, each declared extra and all in separate Docker containers. Pull requests cover every extra on Python 3.12 plus base on 3.10/3.14; scheduled runs cover all supported Python minors. Each cell runs `pip check`, a non-skipping capability probe and the six smoke tests. A separate Compose job waits for healthy Redis/MQTT services and checks real adapter requests; SQLite is checked locally. The `etcd` extra uses `etcd3gw` (etcd v3 HTTP gateway), preserving existing steps and supporting legacy manually installed etcd3. See [Docker checks](docker/README.md). + ## Test Record Canonical SQLite export uses `persist_canonical_records(database_path, context)` and `fetch_canonical_records(database_path, run_id)` from `utils.test_record.sqlite_persistence`. Separate `request_runs_v1` / `request_records_v1` tables preserve legacy runs. Writes validate snapshots, deduplicate identical IDs within each run, reject conflicting retries and roll back the entire batch on failure. Reads revalidate stored records. JSON export is `context.to_json()`. diff --git a/README/README_zh-CN.md b/README/README_zh-CN.md index b3b25eb..1c02f0a 100644 --- a/README/README_zh-CN.md +++ b/README/README_zh-CN.md @@ -785,6 +785,8 @@ python -m je_load_density serve [--host ...] # start the control socket 在 checkout 执行 `python -m unittest discover -s test/smoke -p "test_*.py"`。标准库 harness 启动独立本机 HTTP 服务及子进程，验证真实 Locust／async 请求、summary／JSON／JUnit、SQLite、SLA 失败、dashboard JSON／SSE 与 MCP 初始化。Docker 在源码树外使用同一 harness 验证已安装 wheel。 +Dev／Stable CI 构建 checkout wheel，在独立 Docker 容器验证 base、每个声明的 extra 与 all。PR 在 Python 3.12 覆盖所有 extras，另验证 3.10／3.14 的 base；定时任务覆盖所有支持的 Python 次版本。每个 cell 执行 `pip check`、不会跳过失败的能力探测与六项烟雾测试。独立 Compose job 等待 Redis／MQTT 健康后验证真实 adapter 请求，SQLite 在本机验证。`etcd` extra 使用 `etcd3gw`（etcd v3 HTTP gateway），保留既有步骤并支持手动安装的旧 etcd3。参见 [Docker 检查](../docker/README.md)。 + ## 测试记录 Canonical SQLite 导出使用 `utils.test_record.sqlite_persistence` 的 `persist_canonical_records(database_path, context)` 与 `fetch_canonical_records(database_path, run_id)`。独立的 `request_runs_v1`／`request_records_v1` 表保留旧记录。写入验证 snapshot、去除同一 run 内相同 ID 的重复项、拒绝冲突重送，失败时回滚整批；读取重新验证保存的记录。JSON 导出为 `context.to_json()`。 diff --git a/README/README_zh-TW.md b/README/README_zh-TW.md index 47ee727..e39da65 100644 --- a/README/README_zh-TW.md +++ b/README/README_zh-TW.md @@ -785,6 +785,8 @@ python -m je_load_density serve [--host ...] # start the control socket 在 checkout 執行 `python -m unittest discover -s test/smoke -p "test_*.py"`。標準函式庫 harness 啟動獨立本機 HTTP 服務與子行程，驗證實際 Locust／async 請求、summary／JSON／JUnit、SQLite、SLA 失敗、dashboard JSON／SSE 與 MCP 初始化。Docker 在原始碼樹外使用同一 harness 驗證已安裝 wheel。 +Dev／Stable CI 建置 checkout wheel，在獨立 Docker 容器驗證 base、每個宣告的 extra 與 all。PR 在 Python 3.12 覆蓋所有 extras，另驗證 3.10／3.14 的 base；排程涵蓋所有支援的 Python 次版本。每個 cell 執行 `pip check`、不會略過失敗的能力探測與六項煙霧測試。獨立 Compose job 等待 Redis／MQTT 健康後驗證實際 adapter 請求，SQLite 在本機驗證。`etcd` extra 使用 `etcd3gw`（etcd v3 HTTP gateway），保留既有步驟並支援手動安裝的舊 etcd3。參見 [Docker 檢查](../docker/README.md)。 + ## 測試紀錄 Canonical SQLite 匯出使用 `utils.test_record.sqlite_persistence` 的 `persist_canonical_records(database_path, context)` 與 `fetch_canonical_records(database_path, run_id)`。獨立的 `request_runs_v1`／`request_records_v1` 表保留舊紀錄。寫入驗證 snapshot、去除同一 run 內相同 ID 的重複項、拒絕衝突重送，失敗時回復整批；讀取重新驗證保存的紀錄。JSON 匯出為 `context.to_json()`。 diff --git a/architecture.md b/architecture.md index 890246c..3df41c9 100644 --- a/architecture.md +++ b/architecture.md @@ -136,6 +136,7 @@ MCP `load_density.list_executor_commands` tool all read the `LD_*` names from `e ## 6. Cross-project boundaries CLI execution retains legacy flags and Python executor return shapes, but now returns a nonzero process exit code after failed actions/SLA gates. `test/smoke` runs real HTTP/report/dashboard/MCP checks in subprocesses against source or an installed wheel. Native async HTTP benchmarking requires base httpx; the `http2` extra adds HTTP/2 support. +CI derives its Docker installation matrix from declared extras, runs isolated installed-wheel capabilities and smoke checks, and gates publishing on the reusable extras workflow. Compose probes measure Redis/MQTT adapter calls after health checks. The etcd adapter prefers etcd3gw (v3 HTTP gateway) while retaining legacy etcd3 support; this avoids incompatible generated protobuf code in the old extra. Canonical persistence exports `persist_canonical_records(database_path, context)` and `fetch_canonical_records(database_path, run_id)` from `utils.test_record.sqlite_persistence`. Versioned tables are separate from legacy runs; transaction rollback, run-scoped JSON-aware retry conflicts and read-time schema/identity validation preserve the shared contract. diff --git a/dev.toml b/dev.toml index eb56d7d..e1269b4 100644 --- a/dev.toml +++ b/dev.toml @@ -97,7 +97,7 @@ thrift = ["thriftpy2>=0.4"] memcached = ["pymemcache>=4.0"] neo4j = ["neo4j>=5.15"] couchbase = ["couchbase>=4.1"] -etcd = ["etcd3>=0.12"] +etcd = ["etcd3gw>=2.4.2"] webpush = ["pywebpush>=1.14"] aws = ["boto3>=1.34"] gcp = ["google-auth>=2.27"] @@ -148,7 +148,7 @@ all = [ "pymemcache>=4.0", "neo4j>=5.15", "couchbase>=4.1", - "etcd3>=0.12", + "etcd3gw>=2.4.2", "pywebpush>=1.14", "boto3>=1.34", "google-auth>=2.27", diff --git a/docker/README.md b/docker/README.md index 1e65f6a..6577609 100644 --- a/docker/README.md +++ b/docker/README.md @@ -27,3 +27,36 @@ docker compose -f docker/docker-compose.yml down -v enough for examples, not for production load. * `mosquitto.conf` allows anonymous connections; tighten before exposing the broker. + +## Installed wheel CI checks + +The extras matrix is generated from `pyproject.toml`. Each Docker cell installs the +checkout wheel, runs `pip check`, checks its declared capability without skipping, +and runs the six base smoke tests. GUI cells install Qt system libraries and use +`QT_QPA_PLATFORM=offscreen`. These installation probes use local codecs, stubbed +SDK calls and client configuration; live protocol checks run separately. + +```bash +python -m build --wheel --no-isolation +docker build -f docker/extras.Dockerfile --build-arg EXTRA=etcd -t ld-check . +docker run --rm ld-check +``` + +Use `--build-arg PYTHON_VERSION=3.14` to select a supported Python minor. The +`etcd` extra uses `etcd3gw`, so the target etcd server must enable its v3 HTTP gateway. +Legacy manually installed `etcd3` remains a fallback when `etcd3gw` is unavailable. + +The dedicated Compose job has no exposed host ports or fixed container names. It +waits for healthy Redis/MQTT services, checks actual adapter operations and MQTT +delivery, and verifies a SQLite query. Use a unique project name for local runs: + +```bash +docker compose -p ld-check -f docker/ci-services.yml build probe +docker compose -p ld-check -f docker/ci-services.yml up -d --wait --wait-timeout 60 redis mosquitto +docker compose -p ld-check -f docker/ci-services.yml run --rm probe +docker compose -p ld-check -f docker/ci-services.yml down --volumes --remove-orphans +``` + +CI always cleans up this project, including on failure. Publishing depends on the +reusable extras workflow. Pull requests test every extra on Python 3.12 and base on +3.10/3.14; scheduled runs test every cell on 3.10–3.14. diff --git a/docker/ci-services.yml b/docker/ci-services.yml new file mode 100644 index 0000000..0b2eb18 --- /dev/null +++ b/docker/ci-services.yml @@ -0,0 +1,27 @@ +services: + redis: + image: redis:7-alpine + healthcheck: + test: ["CMD", "redis-cli", "ping"] + interval: 2s + timeout: 2s + retries: 15 + mosquitto: + image: eclipse-mosquitto:2 + volumes: + - ./mosquitto.conf:/mosquitto/config/mosquitto.conf:ro + healthcheck: + test: ["CMD", "mosquitto_pub", "-h", "127.0.0.1", "-t", "health", "-m", "ready"] + interval: 2s + timeout: 2s + retries: 15 + probe: + image: loaddensity-services:ci + build: + context: .. + dockerfile: docker/services.Dockerfile + depends_on: + redis: + condition: service_healthy + mosquitto: + condition: service_healthy diff --git a/docker/extras.Dockerfile b/docker/extras.Dockerfile new file mode 100644 index 0000000..9c96fad --- /dev/null +++ b/docker/extras.Dockerfile @@ -0,0 +1,17 @@ +ARG PYTHON_VERSION=3.12 +FROM python:${PYTHON_VERSION}-slim +COPY dist/*.whl /wheels/ +RUN --mount=type=cache,target=/root/.cache/pip,sharing=locked python -m pip install /wheels/*.whl +ARG EXTRA=base +RUN if [ "$EXTRA" = "gui" ] || [ "$EXTRA" = "all" ]; then \ + apt-get update && apt-get install -y --no-install-recommends \ + libegl1 libgl1 libopengl0 libglib2.0-0 libxkbcommon0 libdbus-1-3 \ + libxcb-cursor0 libxcb-icccm4 libxcb-keysyms1 libxcb-shape0 libxcb-xinerama0 \ + libxcb-randr0 libxcb-render-util0 libxcb-image0 fonts-dejavu-core \ + && rm -rf /var/lib/apt/lists/*; \ + fi +COPY test/smoke /checks/smoke +RUN --mount=type=cache,target=/root/.cache/pip,sharing=locked python /checks/smoke/install_extra.py "$EXTRA" +ENV LD_EXTRA=${EXTRA} QT_QPA_PLATFORM=offscreen AWS_EC2_METADATA_DISABLED=true +WORKDIR /run +CMD ["python", "/checks/smoke/run_cell.py"] diff --git a/docker/services.Dockerfile b/docker/services.Dockerfile new file mode 100644 index 0000000..f0795c5 --- /dev/null +++ b/docker/services.Dockerfile @@ -0,0 +1,7 @@ +FROM python:3.12-slim +COPY dist/*.whl /wheels/ +RUN --mount=type=cache,target=/root/.cache/pip,sharing=locked \ + for wheel in /wheels/*.whl; do python -m pip install "$wheel[redis,mqtt,sql]"; done && python -m pip check +COPY test/smoke /checks/smoke +WORKDIR /run +CMD ["python", "/checks/smoke/service_probe.py"] diff --git a/docs/source/En/doc/cli/cli_doc.rst b/docs/source/En/doc/cli/cli_doc.rst index 21607b2..bcad110 100644 --- a/docs/source/En/doc/cli/cli_doc.rst +++ b/docs/source/En/doc/cli/cli_doc.rst @@ -91,3 +91,6 @@ Smoke tests and failure exit codes ``run``, ``run-dir``, ``run-str`` and legacy execute flags return a nonzero exit code when an action (including an SLA gate) fails. Actions in a file still run in order and produce their usual reports. The Python executor return format is preserved. Base installation includes httpx for the native async benchmark; HTTP/2 needs the ``http2`` extra. Run ``python -m unittest discover -s test/smoke -p "test_*.py"`` from a checkout. The stdlib harness starts a separate local HTTP server and subprocesses to verify real Locust/async requests, summary/JSON/JUnit files, SQLite, SLA failures, dashboard JSON/SSE and MCP initialization. Docker runs the same harness outside the source tree against an installed wheel. + + +The reusable extras workflow tests an installed checkout wheel in Docker: every declared extra on Python 3.12 and base on 3.10/3.14 for pull requests; all supported Python minors for scheduled runs. Missing capabilities fail rather than skip. Separate Compose health checks exercise Redis/MQTT adapters and a local SQLite query. The etcd extra uses etcd3gw without downgrading protobuf; existing step formats and legacy etcd3 installations remain supported. diff --git a/docs/source/Zh/doc/cli/cli_doc.rst b/docs/source/Zh/doc/cli/cli_doc.rst index 7fae0f6..33e1ea1 100644 --- a/docs/source/Zh/doc/cli/cli_doc.rst +++ b/docs/source/Zh/doc/cli/cli_doc.rst @@ -86,3 +86,6 @@ Inline JSON（CI script 友善）:: ``run``、``run-dir``、``run-str`` 與舊執行旗標在動作（包含 SLA gate）失敗時回傳非零 exit code。單一檔案的動作仍依序執行並產生原有報告；Python executor 的回傳格式保持相容。基本安裝包含原生 async benchmark 所需的 httpx，HTTP/2 需要 ``http2`` extra。 在 checkout 執行 ``python -m unittest discover -s test/smoke -p "test_*.py"``。標準函式庫 harness 啟動獨立本機 HTTP 服務與子行程，驗證實際 Locust／async 請求、summary／JSON／JUnit、SQLite、SLA 失敗、dashboard JSON／SSE 與 MCP 初始化。Docker 在原始碼樹外使用同一 harness 驗證已安裝 wheel。 + + +共用 extras workflow 在 Docker 驗證已安裝的 checkout wheel：PR 在 Python 3.12 覆蓋全部宣告的 extras，另驗證 3.10／3.14 的 base；排程涵蓋所有支援的 Python 次版本。缺少能力會失敗，不會略過。獨立 Compose 健康檢查驗證 Redis／MQTT adapter 與本機 SQLite 查詢。etcd extra 使用 etcd3gw，不降級 protobuf；保留既有步驟格式與手動安裝舊 etcd3 的相容性。 diff --git a/docs/superpowers/plans/2026-10-02-smoke-extras.md b/docs/superpowers/plans/2026-10-02-smoke-extras.md index 3388866..39df0a0 100644 --- a/docs/superpowers/plans/2026-10-02-smoke-extras.md +++ b/docs/superpowers/plans/2026-10-02-smoke-extras.md @@ -11,14 +11,14 @@ Execute inline in the existing isolated feature worktree. The approved roadmap a ## Task 2: Dynamic Docker installation matrix -- [ ] Generate base/every-extra/all cells directly from `pyproject.toml`: Python 3.12 for every extra plus base 3.10/3.14 on PRs; all Python 3.10–3.14 versions on schedule. -- [ ] Build the wheel using existing locked build tools. Each cell installs that wheel plus only its selected extra in a fresh Docker image, runs pip check, CLI checks and non-skipping capability probes. -- [ ] Define probes for declared extras: create/use their relevant local SDK or framework capabilities, perform MCP handshake even without dependencies, render charts/PDF and launch Qt offscreen. Fail explicitly if a declared extra lacks a probe. -- [ ] Implement a reusable workflow consumed by stable/dev CI; keep pinned actions, minimum permissions and timeouts. Dev publication waits for these checks. +- [x] Generate base/every-extra/all cells directly from `pyproject.toml`: Python 3.12 for every extra plus base 3.10/3.14 on PRs; all Python 3.10–3.14 versions on schedule. +- [x] Build the wheel using existing locked build tools. Each cell installs that wheel plus only its selected extra in a fresh Docker image, runs pip check, CLI checks and non-skipping capability probes. +- [x] Define probes for declared extras: create/use their relevant local SDK or framework capabilities, perform MCP handshake even without dependencies, render charts/PDF and launch Qt offscreen. Fail explicitly if a declared extra lacks a probe. +- [x] Implement a reusable workflow consumed by stable/dev CI; keep pinned actions, minimum permissions and timeouts. Dev publication waits for these checks. - [ ] Validate representative base/gui/charts/cloud/protocol cells locally in Docker, then complete the whole representative matrix. Repair installation/API incompatibilities rather than excluding supported cells silently. ## Task 3: Protocol service smoke and handoff -- [ ] Add Compose health-checked representative protocol services and distinguish service integration checks from installation probes. -- [ ] Run smoke tests and affected workflow/security tests; update all README languages, Sphinx, architecture and outstanding-work ledger. +- [x] Add Compose health-checked representative protocol services and distinguish service integration checks from installation probes. +- [x] Run smoke tests and affected workflow/security tests; update all README languages, Sphinx, architecture and outstanding-work ledger. - [ ] Commit and push completed stages. Keep GUI/async lifecycle items open if their broader redesign is needed; continue that roadmap implementation instead of claiming parity prematurely. diff --git a/docs/updates/2026-10.md b/docs/updates/2026-10.md index e622887..d977f14 100644 --- a/docs/updates/2026-10.md +++ b/docs/updates/2026-10.md @@ -205,3 +205,10 @@ SQLite deduplication now keys records by (run_id, record_id), matching the share ## U-20261002-05 · 2026-10-02 · Exercise installed wheels and propagate failed SLA status · #done #smoke Added a stdlib smoke harness that exercises local HTTP requests in isolated processes, actual CLI reports, SQLite, SLA failures, async benchmarking, dashboard JSON/SSE and MCP initialization. CLI actions now return failure status to CI while preserving Python executor returns. Clean-wheel Docker testing reproduced a missing base httpx dependency; stable/dev manifests now include it, with HTTP/2 remaining optional. Source and Python 3.12 installed-wheel Docker smoke runs each pass six tests. GUI lifecycle and the full extras matrix remain tracked separately. + +## U-20261002-06 · 2026-10-02 · Gate publishing on Docker extras and protocol smoke checks · #done #ci #extras + +- Added a metadata-derived Docker matrix for base/every-extra/all, reusable Stable/Dev CI gating, isolated checkout-wheel installation with pip check, and non-skipping capability probes. +- Added a separate health-checked Compose job exercising real Redis/MQTT adapters, MQTT delivery and SQLite measurements, with unconditional project cleanup. +- Docker found incompatible generated protobuf code in etcd3; the extra now uses etcd3gw while preserving existing steps and legacy manually installed etcd3. +- Verification: full suite 1264 passed, 2 skipped (existing optional cases), 4 warnings. Ruff and workflow/document checks pass. Installed-wheel base and etcd each pass six smoke tests; Compose adapters pass. GUI/all and Python 3.10/3.14 Docker validation continues under progress #25. diff --git a/docs/updates/README.md b/docs/updates/README.md index 29f8bc9..7397ae9 100644 --- a/docs/updates/README.md +++ b/docs/updates/README.md @@ -58,6 +58,7 @@ In the same commit: delete the item from `progress.md`, add a `#done` entry here | ID | Date | Title | Tags | Batch | |---|---|---|---|---| +| U-20261002-06 | 2026-10-02 | Gate publishing on Docker extras and protocol smoke checks | #done #ci #extras | [2026-10](2026-10.md) | | U-20261002-05 | 2026-10-02 | Exercise installed wheels and propagate failed SLA status | #done #smoke | [2026-10](2026-10.md) | | U-20261002-04 | 2026-10-02 | Scope persisted request IDs to their run | #done #records | [2026-10](2026-10.md) | | U-20261002-03 | 2026-10-02 | Persist canonical request records transactionally | #done #records | [2026-10](2026-10.md) | diff --git a/je_load_density/wrapper/user_template/etcd_user_template.py b/je_load_density/wrapper/user_template/etcd_user_template.py index 7bc5784..e581219 100644 --- a/je_load_density/wrapper/user_template/etcd_user_template.py +++ b/je_load_density/wrapper/user_template/etcd_user_template.py @@ -1,5 +1,5 @@ """ -etcd user template (etcd3, lazy import). +etcd user template (etcd3gw, with legacy etcd3 compatibility; lazy import). Each task entry:: @@ -19,17 +19,22 @@ def _import_etcd3(): + try: + import etcd3gw + return etcd3gw + except ImportError: + pass try: import etcd3 except ImportError as error: raise RuntimeError( - "etcd3 is required for EtcdUser; install with: pip install etcd3" + "etcd3gw is required for EtcdUser; install with: pip install etcd3gw" ) from error return etcd3 class EtcdUserWrapper(ProtocolUserBase): - """Locust user driving etcd3 KV calls.""" + """Locust user driving etcd v3 KV calls.""" _proxy_key = "etcd_user" _request_type = "ETCD" @@ -38,9 +43,11 @@ class EtcdUserWrapper(ProtocolUserBase): def __init__(self, environment): super().__init__(environment) self._client = None + self._gateway_client = False def _connect(self, step: Dict[str, Any]) -> int: etcd3 = _import_etcd3() + self._gateway_client = etcd3.__name__ == "etcd3gw" self._client = etcd3.client( host=step.get("host", self.host), port=int(step.get("port", 2379)), @@ -59,7 +66,8 @@ def _put(self, step: Dict[str, Any]) -> int: def _get(self, step: Dict[str, Any]) -> int: if self._client is None: raise RuntimeError("etcd not connected") - value, _ = self._client.get(step["key"]) + result = self._client.get(step["key"]) + value = (result[0] if result else None) if self._gateway_client else result[0] return len(value or b"") def _delete(self, step: Dict[str, Any]) -> int: @@ -70,7 +78,10 @@ def _delete(self, step: Dict[str, Any]) -> int: def _close(self, _: Dict[str, Any]) -> int: if self._client is not None: - self._client.close() + if self._gateway_client: + self._client.session.close() + else: + self._client.close() self._client = None return 0 diff --git a/progress.md b/progress.md index 8ac4198..2d6b6a1 100644 --- a/progress.md +++ b/progress.md @@ -15,7 +15,7 @@ Cross-repo and workspace items live in `D:\Codes\progress.md` (relevant here: X- - **#20** Pre-flight health check gate: Add a target health probe before ramping load in `start_test` / `bench`. If the target endpoint is unreachable, DNS fails, or TLS handshake fails, abort early with a descriptive error instead of spawning hundreds of workers that flood logs with connection refused errors. - **#23** [BLOCKED: two releases must ship the warning first] Flip the package gate's default to refuse packages outside the allowlist: set `self.allow_arbitrary_packages = False` in `PackageManager.__init__` (`je_load_density/utils/package_manager/package_manager_class.py`); the warning branch is je_action_core's `_check_allowed` and stays for the other projects. Update the "Package gate" paragraph in the three READMEs and the "Package Gate" section of `docs/source/{En,Zh}/doc/package_manager/package_manager_doc.rst`. The warning is first released in the version after 0.0.77 (`origin/main`); flip once two releases after that one have shipped it. Decide first how a user who only runs action files (`loaddensity`, `-e`, the socket server) allows a package without a Python host to call `executor.allow_packages(...)`. - **#24** Installed-wheel GUI smoke lifecycle: extend `test/smoke/` and the Docker GUI capability probe with offscreen execution and cancellation after the UI lifecycle redesign (#18). Keep client/thread cleanup observable. -- **#25** Docker extras installation matrix: `.github/workflows/ci-dev.yml` and `ci-stable.yml` lack isolated installation/capability checks for base, every declared extra and all. Generate the matrix from `pyproject.toml`, test a checkout-built wheel with `pip check` and non-skipping probes, include GUI system libraries and Compose health checks, and gate publishing on required checks. +- **#25** Finish Docker matrix validation: base/etcd and healthy Compose Redis/MQTT/SQLite probes pass; GUI/all and Python 3.10/3.14 cells are building. Repair any remaining installation/API failures and confirm the complete Python 3.12 extras matrix in CI. - **#26** Percentile bands: `gui/chart_panel.py` mixes RPS and latency on one axis; `utils/generate_report/generate_chart_report.py` uses `ts`/index instead of request `start_time`. Share per-window percentile aggregation across GUI/dashboard/report, show p50 with p50–p95 and p95–p99 bands, separate RPS units and handle gaps, failures and actual bucket duration. - **#27** Cloud backend failure coverage: extend `test/test_cloud_launchers.py` for invalid inputs, auth/permissions, timeout/rate limits, partial failures, Fargate `failures`, Lambda `FunctionError`/Event/warm reuse, Cloud Run refresh/error responses and ACI poller failure. Use SDK stubs/local HTTP and assert request/response semantics; no live provisioning is needed. - **#28** Shared record release activation: coordinate the actual ActionCore request-record release, then activate consumer dependency floors and cross-project installed-wheel fixtures. Current opt-in APIs must retain compatibility with older core installations until that release. Design: `docs/superpowers/specs/2026-10-02-shared-record-design.md`. diff --git a/pyproject.toml b/pyproject.toml index f068ad4..7d5a93c 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -96,7 +96,7 @@ thrift = ["thriftpy2>=0.4"] memcached = ["pymemcache>=4.0"] neo4j = ["neo4j>=5.15"] couchbase = ["couchbase>=4.1"] -etcd = ["etcd3>=0.12"] +etcd = ["etcd3gw>=2.4.2"] webpush = ["pywebpush>=1.14"] aws = ["boto3>=1.34"] gcp = ["google-auth>=2.27"] @@ -147,7 +147,7 @@ all = [ "pymemcache>=4.0", "neo4j>=5.15", "couchbase>=4.1", - "etcd3>=0.12", + "etcd3gw>=2.4.2", "pywebpush>=1.14", "boto3>=1.34", "google-auth>=2.27", diff --git a/scripts/extras_matrix.py b/scripts/extras_matrix.py new file mode 100644 index 0000000..1ee7bce --- /dev/null +++ b/scripts/extras_matrix.py @@ -0,0 +1,50 @@ +"""Derive isolated Docker installation jobs from the package's declared extras.""" + +import argparse +import json +import os +import re +from pathlib import Path + +try: + import tomllib +except ModuleNotFoundError: + import tomli as tomllib + +SUPPORTED_PYTHON_VERSIONS = ("3.10", "3.11", "3.12", "3.13", "3.14") +PR_EXTRAS_PYTHON_VERSION = "3.12" + + +def build_matrix(metadata_path: Path, scheduled: bool = False) -> dict[str, list[dict[str, str]]]: + """Cover every declared extra independently; schedules cover every supported Python version.""" + with metadata_path.open("rb") as handle: + extras = tomllib.load(handle)["project"].get("optional-dependencies", {}) + for name in extras: + if name == "base" or re.fullmatch(r"[a-z0-9][a-z0-9-]*", name) is None: + raise ValueError(f"extra: invalid matrix name {name!r}") + names = ["base", *sorted(extras)] + versions = SUPPORTED_PYTHON_VERSIONS if scheduled else (PR_EXTRAS_PYTHON_VERSION,) + cells = [{"python": version, "extra": name} for version in versions for name in names] + if not scheduled: + cells.extend({"python": version, "extra": "base"} + for version in (SUPPORTED_PYTHON_VERSIONS[0], SUPPORTED_PYTHON_VERSIONS[-1])) + return {"include": cells} + + +def main() -> None: + """Emit JSON for GitHub Actions or a local matrix runner.""" + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--metadata", type=Path, default=Path("pyproject.toml")) + parser.add_argument("--event", default="pull_request") + parser.add_argument("--output", type=Path, default=os.environ.get("GITHUB_OUTPUT")) + arguments = parser.parse_args() + payload = json.dumps(build_matrix(arguments.metadata, scheduled=arguments.event == "schedule")) + if arguments.output is not None: + with arguments.output.open("a", encoding="utf-8") as handle: + handle.write(f"matrix={payload}\n") + else: + print(payload) + + +if __name__ == "__main__": + main() diff --git a/test/smoke/capability_probe.py b/test/smoke/capability_probe.py new file mode 100644 index 0000000..88b3783 --- /dev/null +++ b/test/smoke/capability_probe.py @@ -0,0 +1,62 @@ +"""Non-skipping installation probes for every declared package extra.""" + +import argparse +import traceback +from collections.abc import Callable + +import platform_probes +import protocol_probes + +PROBES: dict[str, Callable[[], None]] = { + "amqp": protocol_probes.amqp, "auth": platform_probes.auth, + "aws": platform_probes.aws, "azure": platform_probes.azure, + "cassandra": protocol_probes.cassandra, "charts": platform_probes.charts, + "cloud": platform_probes.cloud, "coap": protocol_probes.coap, + "couchbase": protocol_probes.couchbase, "datadog": platform_probes.datadog, + "elasticsearch": protocol_probes.elasticsearch, "etcd": protocol_probes.etcd, + "faker": platform_probes.faker, "gcp": platform_probes.gcp, + "grpc": protocol_probes.grpc, "gui": platform_probes.gui, + "http2": protocol_probes.http2, "http3": protocol_probes.http3, + "k8s": platform_probes.k8s, "kafka": protocol_probes.kafka, + "ldap": protocol_probes.ldap, "mcp": platform_probes.mcp, + "memcached": protocol_probes.memcached, "metrics": platform_probes.metrics, + "modbus": protocol_probes.modbus, "mongo": protocol_probes.mongo, + "mqtt": protocol_probes.mqtt, "nats": protocol_probes.nats, + "neo4j": protocol_probes.neo4j, "opcua": protocol_probes.opcua, + "opentelemetry": platform_probes.opentelemetry, "pdf": platform_probes.pdf, + "prometheus": platform_probes.prometheus, "pulsar": protocol_probes.pulsar, + "redis": protocol_probes.redis, "reliability": platform_probes.reliability, + "sftp": protocol_probes.sftp, "snmp": protocol_probes.snmp, + "sql": protocol_probes.sql, "thrift": protocol_probes.thrift, + "webpush": protocol_probes.webpush, "websocket": protocol_probes.websocket, + "yaml": platform_probes.yaml, "zmq": protocol_probes.zmq, +} + + +def run_probe(extra: str) -> None: + """Fail if a declared dependency/capability is unavailable; never turn failures into skips.""" + if extra == "base": + return # All cells run the full installed-wheel base smoke harness separately. + selected = list(PROBES) if extra == "all" else [extra] + failed = [] + for name in selected: + if name not in PROBES: + raise ValueError(f"extra: missing capability probe for {name}") + try: + PROBES[name]() + print(f"capability passed: {name}", flush=True) + except Exception: + if extra != "all": + raise + traceback.print_exc() + failed.append(name) + if failed: + raise RuntimeError(f"Failed extra capabilities: {', '.join(failed)}") + + +if __name__ == "__main__": + import je_load_density # noqa: F401 - exercise capabilities under the installed framework's runtime. + + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("extra") + run_probe(parser.parse_args().extra) diff --git a/test/smoke/install_extra.py b/test/smoke/install_extra.py new file mode 100644 index 0000000..6dd9fda --- /dev/null +++ b/test/smoke/install_extra.py @@ -0,0 +1,24 @@ +"""Install only the selected wheel extra in the Docker cell.""" + +import re +import subprocess +import sys +from pathlib import Path + + +def main() -> None: + """Install a declared extra by argument vector and check the resulting environment.""" + extra = sys.argv[1] + if re.fullmatch(r"[a-z0-9][a-z0-9-]*", extra) is None: + raise ValueError("extra: invalid installation name") + wheels = list(Path("/wheels").glob("*.whl")) + if len(wheels) != 1: + raise ValueError("wheel: expected exactly one checkout-built wheel") + requirement = str(wheels[0]) + (f"[{extra}]" if extra != "base" else "") + subprocess.run([sys.executable, "-m", "pip", "install", requirement], check=True, timeout=600) + subprocess.run([sys.executable, "-m", "pip", "check"], check=True, timeout=30) + subprocess.run([sys.executable, "-m", "je_load_density", "--help"], check=True, timeout=30) + + +if __name__ == "__main__": + main() diff --git a/test/smoke/platform_probes.py b/test/smoke/platform_probes.py new file mode 100644 index 0000000..4950752 --- /dev/null +++ b/test/smoke/platform_probes.py @@ -0,0 +1,202 @@ +"""Cloud SDK, telemetry and rendering capabilities with no provisioned external resources.""" + +import io +import json +import os +import tempfile +from pathlib import Path + + +def auth() -> None: + from cryptography.fernet import Fernet + + cipher = Fernet(Fernet.generate_key()) + if cipher.decrypt(cipher.encrypt(b"smoke")) != b"smoke": + raise RuntimeError("Cryptographic round-trip failed") + + +def aws() -> None: + import boto3 + from botocore import UNSIGNED + from botocore.config import Config + from botocore.stub import Stubber + + os.environ["AWS_EC2_METADATA_DISABLED"] = "true" + client = boto3.client("ecs", region_name="us-east-1", config=Config(signature_version=UNSIGNED)) + try: + with Stubber(client) as stub: + stub.add_response("describe_clusters", {"clusters": [], "failures": []}, {"clusters": ["smoke"]}) + if client.describe_clusters(clusters=["smoke"])["failures"]: + raise RuntimeError("AWS modeled request failed") + finally: + client.close() + + +def azure() -> None: + from azure.identity import DefaultAzureCredential + from azure.mgmt.containerinstance.models import Container, ResourceRequests, ResourceRequirements + + credential = DefaultAzureCredential() + try: + container = Container(name="smoke", image="python:3.12-slim", + resources=ResourceRequirements(requests=ResourceRequests(cpu=1, memory_in_gb=1))) + if container.as_dict()["resources"]["requests"]["cpu"] != 1: + raise RuntimeError("Azure container configuration failed") + finally: + credential.close() + + +def gcp() -> None: + from google.auth.credentials import AnonymousCredentials + from google.auth.transport.requests import AuthorizedSession + + with AuthorizedSession(AnonymousCredentials()) as session: + if not session.credentials.valid: + raise RuntimeError("Google authorized-session configuration failed") + + +def cloud() -> None: + aws() + azure() + gcp() + + +def charts() -> None: + import matplotlib + + matplotlib.use("Agg") + from matplotlib import pyplot + + figure, axes = pyplot.subplots() + try: + axes.plot([0, 1], [10, 20]) + axes.fill_between([0, 1], [10, 20], [30, 40]) + output = io.BytesIO() + figure.savefig(output, format="png") + if not output.getvalue().startswith(b"\x89PNG\r\n\x1a\n"): + raise RuntimeError("Chart rendering failed") + finally: + pyplot.close(figure) + + +def datadog() -> None: + os.environ["DD_TRACE_ENABLED"] = "false" + os.environ["DD_INSTRUMENTATION_TELEMETRY_ENABLED"] = "false" + from ddtrace.trace import tracer + + with tracer.trace("loaddensity.smoke") as span: + span.set_tag("smoke", True) + tracer.shutdown() + + +def faker() -> None: + from faker import Faker + + if not Faker().name(): + raise RuntimeError("Fake-data generation failed") + + +def gui() -> None: + os.environ["QT_QPA_PLATFORM"] = "offscreen" + from PySide6.QtWidgets import QApplication + + from je_load_density.gui.main_window import LoadDensityUI + + application = QApplication.instance() or QApplication([]) + window = LoadDensityUI() + try: + window.show() + application.processEvents() + if not window.isVisible(): + raise RuntimeError("GUI offscreen launch failed") + finally: + window.close() + application.processEvents() + + +def k8s() -> None: + import kopf + from kubernetes import client + + pod = client.V1Pod(metadata=client.V1ObjectMeta(name="smoke")) + with client.ApiClient() as api: + if api.sanitize_for_serialization(pod)["metadata"]["name"] != "smoke": + raise RuntimeError("Kubernetes model serialization failed") + if not callable(kopf.on.create): + raise RuntimeError("Kubernetes operator registration unavailable") + + +def prometheus() -> None: + from prometheus_client import CollectorRegistry, Counter, generate_latest + + registry = CollectorRegistry() + counter = Counter("loaddensity_smoke", "Smoke requests", registry=registry) + counter.inc() + if b"loaddensity_smoke_total 1.0" not in generate_latest(registry): + raise RuntimeError("Prometheus collection failed") + + +def opentelemetry() -> None: + from opentelemetry.exporter.otlp.proto.grpc.metric_exporter import OTLPMetricExporter + from opentelemetry.sdk.metrics import MeterProvider + from opentelemetry.sdk.metrics.export import InMemoryMetricReader + + reader = InMemoryMetricReader() + provider = MeterProvider(metric_readers=[reader]) + exporter = OTLPMetricExporter(endpoint="http://127.0.0.1:9", insecure=True) + try: + provider.get_meter("smoke").create_counter("requests").add(1) + data = reader.get_metrics_data().resource_metrics[0].scope_metrics[0].metrics[0].data + if data.data_points[0].value != 1: + raise RuntimeError("OpenTelemetry metric collection failed") + finally: + exporter.shutdown() + provider.shutdown() + + +def metrics() -> None: + prometheus() + opentelemetry() + + +def pdf() -> None: + from reportlab.pdfgen.canvas import Canvas + + output = io.BytesIO() + canvas = Canvas(output) + canvas.drawString(10, 10, "LoadDensity smoke") + canvas.save() + if not output.getvalue().startswith(b"%PDF-"): + raise RuntimeError("PDF rendering failed") + + +def reliability() -> None: + import psutil + + if psutil.Process().memory_info().rss <= 0: + raise RuntimeError("Process resource inspection failed") + + +def yaml() -> None: + import yaml as yaml_sdk + + expected = {"load_density": [["print", ["smoke"]]]} + with tempfile.TemporaryDirectory() as directory: + path = Path(directory) / "actions.yaml" + path.write_text(yaml_sdk.safe_dump(expected), encoding="utf-8") + if yaml_sdk.safe_load(path.read_text(encoding="utf-8")) != expected: + raise RuntimeError("YAML action round-trip failed") + + +def mcp() -> None: + import subprocess + import sys + + request = {"jsonrpc": "2.0", "id": 1, "method": "initialize", + "params": {"protocolVersion": "2025-06-18", "capabilities": {}, + "clientInfo": {"name": "extras-probe", "version": "1"}}} + response = subprocess.run([sys.executable, "-m", "je_load_density.mcp_server"], + input=json.dumps(request) + "\n", text=True, capture_output=True, + timeout=30, check=True) + if "protocolVersion" not in json.loads(response.stdout.splitlines()[0])["result"]: + raise RuntimeError("MCP handshake failed") diff --git a/test/smoke/protocol_probes.py b/test/smoke/protocol_probes.py new file mode 100644 index 0000000..c52a80a --- /dev/null +++ b/test/smoke/protocol_probes.py @@ -0,0 +1,247 @@ +"""Local protocol/codec capabilities; service integration checks run separately.""" + +import tempfile +from pathlib import Path + + +def amqp() -> None: + import pika + + if pika.ConnectionParameters(host="127.0.0.1", socket_timeout=1).host != "127.0.0.1": + raise RuntimeError("AMQP connection configuration failed") + + +def cassandra() -> None: + from cassandra.cluster import Cluster + + cluster = Cluster(contact_points=["127.0.0.1"], connect_timeout=1) + cluster.shutdown() + + +def coap() -> None: + from aiocoap import GET, Message + + message = Message(code=GET, mtype=0, mid=1, uri="coap://localhost/health") + if Message.decode(message.encode()).code != GET: + raise RuntimeError("CoAP codec round-trip failed") + + +def couchbase() -> None: + from couchbase.transcoder import RawJSONTranscoder + + encoded, flags = RawJSONTranscoder().encode_value(b'{"ok":true}') + if not encoded or not flags: + raise RuntimeError("Couchbase JSON transcoder failed") + + +def elasticsearch() -> None: + from elasticsearch import Elasticsearch + + client = Elasticsearch("http://127.0.0.1:9", request_timeout=1) + client.options(request_timeout=1) + client.close() + + +def etcd() -> None: + from je_load_density.wrapper.user_template.etcd_user_template import _import_etcd3 + + sdk = _import_etcd3() + if sdk.__name__ != "etcd3gw": + raise RuntimeError("The installed etcd extra must provide the gateway client") + client = sdk.client(host="127.0.0.1", port=9, timeout=1, api_path="/v3/") + if not client.get_url("kv/range").endswith("/v3/kv/range"): + raise RuntimeError("etcd gateway API configuration failed") + client.session.close() + + +def grpc() -> None: + import grpc as grpc_sdk + + with grpc_sdk.insecure_channel("127.0.0.1:9") as channel: + channel.unary_unary("/smoke.Service/Check") + + +def http2() -> None: + import asyncio + + import httpx + + async def configure() -> None: + async with httpx.AsyncClient(http2=True): + pass + + asyncio.run(configure()) + + +def http3() -> None: + from aioquic.quic.configuration import QuicConfiguration + from aioquic.quic.connection import QuicConnection + + connection = QuicConnection(configuration=QuicConfiguration(is_client=True, alpn_protocols=["h3"])) + connection.connect(("127.0.0.1", 443), now=0) + if not connection.datagrams_to_send(now=0): + raise RuntimeError("QUIC did not generate a client handshake") + + +def kafka() -> None: + from kafka.protocol.metadata import MetadataRequest + + if not MetadataRequest[0]([]).encode(): + raise RuntimeError("Kafka metadata request encoding failed") + + +def ldap() -> None: + from ldap3 import MOCK_SYNC, Connection, Server + + connection = Connection(Server("127.0.0.1"), client_strategy=MOCK_SYNC) + if not connection.bind(): + raise RuntimeError("LDAP mock bind failed") + connection.unbind() + + +def memcached() -> None: + from pymemcache.client.base import Client + + client = Client(("127.0.0.1", 9), connect_timeout=1, timeout=1) + client.close() + + +def modbus() -> None: + from pymodbus.client import ModbusTcpClient + from pymodbus.pdu.register_message import ReadHoldingRegistersRequest + + if not ReadHoldingRegistersRequest(address=0, count=1).encode(): + raise RuntimeError("Modbus request encoding failed") + ModbusTcpClient("127.0.0.1", port=9, timeout=1).close() + + +def mongo() -> None: + from pymongo import MongoClient + + with MongoClient("mongodb://127.0.0.1:9", connect=False) as client: + if client.codec_options.document_class is not dict: + raise RuntimeError("MongoDB document codec configuration failed") + + +def mqtt() -> None: + from paho.mqtt.client import CallbackAPIVersion, Client, topic_matches_sub + + Client(CallbackAPIVersion.VERSION2, client_id="loaddensity-smoke").user_data_set({"smoke": True}) + if not topic_matches_sub("smoke/+", "smoke/check"): + raise RuntimeError("MQTT topic matching failed") + + +def nats() -> None: + import asyncio + + from nats.aio.client import Client + + client = Client() + if client.is_connected: + raise RuntimeError("A new NATS client unexpectedly connects") + asyncio.run(client.close()) + + +def neo4j() -> None: + from neo4j import GraphDatabase + + with GraphDatabase.driver("bolt://127.0.0.1:9", auth=None, connection_timeout=1): + pass + + +def opcua() -> None: + from asyncua import ua + from asyncua.ua import ua_binary + + if not ua_binary.nodeid_to_binary(ua.NodeId(123, 2)): + raise RuntimeError("OPC UA node encoding failed") + + +def pulsar() -> None: + import pulsar as pulsar_sdk + + client = pulsar_sdk.Client("pulsar://127.0.0.1:9", operation_timeout_seconds=1) + client.close() + + +def redis() -> None: + import redis as redis_sdk + + client = redis_sdk.Redis(host="127.0.0.1", port=9, socket_timeout=1) + connection = client.connection_pool.make_connection() + try: + if b"PING" not in b"".join(connection.pack_command("PING")): + raise RuntimeError("Redis command encoding failed") + finally: + connection.disconnect() + client.close() + + +def sftp() -> None: + import paramiko + + if not paramiko.RSAKey.generate(2048).asbytes(): + raise RuntimeError("SSH key encoding failed") + + +def snmp() -> None: + from pysnmp.hlapi.v3arch.asyncio import SnmpEngine + + SnmpEngine().close_dispatcher() + + +def sql() -> None: + from sqlalchemy import create_engine, text + + engine = create_engine("sqlite://") + try: + with engine.connect() as connection: + if connection.execute(text("SELECT 1")).scalar() != 1: + raise RuntimeError("SQL local query failed") + finally: + engine.dispose() + + +def thrift() -> None: + import thriftpy2 + from thriftpy2.protocol import TBinaryProtocol + from thriftpy2.transport import TMemoryBuffer + + with tempfile.TemporaryDirectory() as directory: + path = Path(directory) / "smoke.thrift" + path.write_text("struct Check { 1: string name }", encoding="utf-8") + module = thriftpy2.load(str(path), module_name="smoke_thrift") + transport = TMemoryBuffer() + module.Check(name="smoke").write(TBinaryProtocol(transport)) + if not transport.getvalue(): + raise RuntimeError("Thrift message encoding failed") + + +def webpush() -> None: + from py_vapid import Vapid02 + from pywebpush import WebPushException + + vapid = Vapid02() + vapid.generate_keys() + headers = vapid.sign({"aud": "https://localhost", "sub": "mailto:smoke@example.invalid"}) + if "Authorization" not in headers or not isinstance(WebPushException("smoke"), Exception): + raise RuntimeError("Web Push authorization encoding failed") + + +def websocket() -> None: + from websocket import ABNF + + if len(ABNF.create_frame("smoke", ABNF.OPCODE_TEXT).format()) <= len("smoke"): + raise RuntimeError("WebSocket framing failed") + + +def zmq() -> None: + import zmq as zmq_sdk + + with zmq_sdk.Context() as context: + with context.socket(zmq_sdk.PAIR) as sender, context.socket(zmq_sdk.PAIR) as receiver: + sender.bind("inproc://loaddensity-smoke") + receiver.connect("inproc://loaddensity-smoke") + sender.send(b"smoke") + if receiver.recv() != b"smoke": + raise RuntimeError("ZeroMQ local exchange failed") diff --git a/test/smoke/run_cell.py b/test/smoke/run_cell.py new file mode 100644 index 0000000..b79a125 --- /dev/null +++ b/test/smoke/run_cell.py @@ -0,0 +1,18 @@ +"""Run installation capability checks and full wheel smoke tests with bounded child processes.""" + +import os +import subprocess +import sys +from pathlib import Path + + +def main() -> None: + """Keep capability results separate from the base runtime smoke result.""" + directory = Path(__file__).resolve().parent + subprocess.run([sys.executable, str(directory / "capability_probe.py"), os.environ.get("LD_EXTRA", "base")], + check=True, timeout=120) + subprocess.run([sys.executable, str(directory / "test_wheel_smoke.py"), "-v"], check=True, timeout=240) + + +if __name__ == "__main__": + main() diff --git a/test/smoke/service_probe.py b/test/smoke/service_probe.py new file mode 100644 index 0000000..196ddd4 --- /dev/null +++ b/test/smoke/service_probe.py @@ -0,0 +1,86 @@ +"""Exercise installed LoadDensity adapters against healthy Compose services.""" + +import os +import threading +from types import SimpleNamespace + +import je_load_density # noqa: F401 - patch before client networking imports. + + +def check_events(events: list[dict], expected: int) -> None: + if len(events) != expected: + raise RuntimeError(f"Expected {expected} measured requests, got {len(events)}") + for event in events: + if event["exception"] is not None: + raise RuntimeError(f"{event['request_type']} {event['name']} failed") from event["exception"] + if event["response_time"] < 0 or event["response_length"] < 0: + raise RuntimeError("Invalid protocol measurement") + + +def environment(): + events = [] + return SimpleNamespace(events=SimpleNamespace(request=SimpleNamespace(fire=lambda **kw: events.append(kw)))), events + + +def redis() -> None: + from je_load_density.wrapper.user_template.redis_user_template import RedisUserWrapper + + env, events = environment() + user = RedisUserWrapper(env) + user.host = f"redis://{os.environ.get('REDIS_HOST', 'redis')}:6379/0?socket_timeout=3&socket_connect_timeout=3" + try: + for step in [{"method": "set", "key": "smoke", "value": "capacity"}, + {"method": "get", "key": "smoke", "expect": "capacity"}, + {"method": "delete", "key": "smoke"}]: + user._do_step(step) + check_events(events, 3) + if events[1]["response_length"] != 8: + raise RuntimeError("Redis adapter did not measure returned bytes") + finally: + if user._client is not None: + user._client.close() + + +def mqtt() -> None: + from paho.mqtt.client import CallbackAPIVersion, Client + + from je_load_density.wrapper.user_template.mqtt_user_template import MqttUserWrapper + + broker = os.environ.get("MQTT_HOST", "mosquitto") + received = threading.Event() + subscriber = Client(CallbackAPIVersion.VERSION2) + subscriber.on_message = lambda _client, _data, message: received.set() if message.payload == b"capacity" else None + subscriber.connect(broker, 1883) + subscriber.subscribe("smoke/adapter", qos=1) + subscriber.loop_start() + env, events = environment() + user = MqttUserWrapper(env) + try: + user._do_step({"method": "publish", "broker": broker, "topic": "smoke/adapter", + "payload": "capacity", "qos": 1, "retain": True, "timeout": 5}) + check_events(events, 1) + if not received.wait(5): + raise RuntimeError("MQTT publish was not delivered to the subscriber") + finally: + user.on_stop() + subscriber.disconnect() + subscriber.loop_stop() + + +def sql() -> None: + from je_load_density.wrapper.user_template.sql_user_template import SqlUserWrapper + + env, events = environment() + user = SqlUserWrapper(env) + try: + user._do_step({"sql": "SELECT 1", "expect_rows": 1}) + check_events(events, 1) + finally: + if user._engine is not None: + user._engine.dispose() + + +if __name__ == "__main__": + for probe in (redis, mqtt, sql): + probe() + print(f"service adapter passed: {probe.__name__}") diff --git a/test/smoke/test_wheel_smoke.py b/test/smoke/test_wheel_smoke.py index cc1a72b..4799b31 100644 --- a/test/smoke/test_wheel_smoke.py +++ b/test/smoke/test_wheel_smoke.py @@ -67,7 +67,7 @@ def command(self, *arguments: str, input_text: str | None = None) -> subprocess. def actions(self, endpoint: str, *, failed_gate: bool = False) -> list[list[object]]: definitions = [ {"LD_start_test": {"user_detail_dict": {"user": "http_user"}, "user_count": 1, - "spawn_rate": 10, "test_time": 1, + "spawn_rate": 10, "test_time": 3, "tasks": {"get": {"request_url": self.url + endpoint}}}}, {"LD_generate_summary_report": {"report_name": "summary"}}, {"LD_generate_json_report": {"json_file_name": "records"}}, diff --git a/test/test_dev_release.py b/test/test_dev_release.py index c3dbe9b..b9bf27d 100644 --- a/test/test_dev_release.py +++ b/test/test_dev_release.py @@ -134,7 +134,7 @@ def _publish_job() -> str: def test_the_workflow_publishes_only_a_tested_push_to_dev(): job = _publish_job() - assert "needs: [test]" in job + assert "needs: [test, extras]" in job assert "if: github.event_name == 'push' && github.ref == 'refs/heads/dev'" in job diff --git a/test/test_extra_capabilities.py b/test/test_extra_capabilities.py new file mode 100644 index 0000000..7bef7ba --- /dev/null +++ b/test/test_extra_capabilities.py @@ -0,0 +1,56 @@ +import importlib +from pathlib import Path + +import pytest + +try: + import tomllib +except ModuleNotFoundError: + import tomli as tomllib + +ROOT = Path(__file__).resolve().parents[1] + + +def probes(monkeypatch): + monkeypatch.syspath_prepend(str(ROOT / "test/smoke")) + return importlib.import_module("capability_probe") + + +def test_every_declared_extra_has_a_non_skipping_probe(monkeypatch): + with (ROOT / "pyproject.toml").open("rb") as handle: + extras = tomllib.load(handle)["project"]["optional-dependencies"] + assert set(probes(monkeypatch).PROBES) == set(extras) - {"all"} + + +def test_unknown_extra_is_an_error(monkeypatch): + with pytest.raises(ValueError, match="extra"): + probes(monkeypatch).run_probe("unregistered") + + +def test_missing_declared_capability_is_a_failure(monkeypatch): + import builtins + + probe_module = probes(monkeypatch) + original = builtins.__import__ + + def missing_redis(name, *args, **kwargs): + if name == "redis": + raise ImportError("declared redis dependency missing") + return original(name, *args, **kwargs) + + monkeypatch.setattr(builtins, "__import__", missing_redis) + with pytest.raises(ImportError, match="redis"): + probe_module.run_probe("redis") + + +def test_all_probe_reports_failures_and_still_exercises_other_capabilities(monkeypatch): + probe_module = probes(monkeypatch) + calls = [] + + def broken(): + raise ImportError("missing dependency") + + monkeypatch.setattr(probe_module, "PROBES", {"broken": broken, "healthy": lambda: calls.append("healthy")}) + with pytest.raises(RuntimeError, match="broken"): + probe_module.run_probe("all") + assert calls == ["healthy"] diff --git a/test/test_extras_matrix.py b/test/test_extras_matrix.py new file mode 100644 index 0000000..4b6b879 --- /dev/null +++ b/test/test_extras_matrix.py @@ -0,0 +1,57 @@ +import importlib.util +import json +import subprocess +import sys +from pathlib import Path + +import pytest + +ROOT = Path(__file__).resolve().parents[1] + + +def load_matrix_module(): + specification = importlib.util.spec_from_file_location("extras_matrix", ROOT / "scripts/extras_matrix.py") + module = importlib.util.module_from_spec(specification) + specification.loader.exec_module(module) + return module + + +@pytest.fixture +def metadata(tmp_path): + path = tmp_path / "pyproject.toml" + path.write_text('[project.optional-dependencies]\nredis = ["redis"]\ngui = ["PySide6"]\nall = []\n', + encoding="utf-8") + return path + + +def test_pr_matrix_covers_each_extra_and_boundary_base_versions(metadata): + cells = load_matrix_module().build_matrix(metadata)["include"] + assert {(row["python"], row["extra"]) for row in cells} == { + ("3.10", "base"), ("3.12", "base"), ("3.14", "base"), + ("3.12", "redis"), ("3.12", "gui"), ("3.12", "all"), + } + + +def test_schedule_matrix_covers_all_supported_versions(metadata): + cells = load_matrix_module().build_matrix(metadata, scheduled=True)["include"] + assert len(cells) == 20 + assert {row["python"] for row in cells} == {"3.10", "3.11", "3.12", "3.13", "3.14"} + assert {row["extra"] for row in cells} == {"base", "redis", "gui", "all"} + + +def test_unsafe_extra_names_are_rejected_before_shell_parameters(tmp_path): + path = tmp_path / "pyproject.toml" + path.write_text('[project.optional-dependencies]\n"bad;echo x" = []\n', encoding="utf-8") + with pytest.raises(ValueError, match="extra"): + load_matrix_module().build_matrix(path) + + +def test_cli_writes_parseable_github_output(metadata, tmp_path): + destination = tmp_path / "github-output" + completed = subprocess.run([sys.executable, str(ROOT / "scripts/extras_matrix.py"), + "--metadata", str(metadata), "--output", str(destination)], + capture_output=True, text=True, timeout=10, check=False) + assert completed.returncode == 0, completed.stderr + key, value = destination.read_text(encoding="utf-8").strip().split("=", 1) + assert key == "matrix" + assert len(json.loads(value)["include"]) == 6 diff --git a/test/test_service_probe.py b/test/test_service_probe.py new file mode 100644 index 0000000..840fa28 --- /dev/null +++ b/test/test_service_probe.py @@ -0,0 +1,22 @@ +import importlib +from pathlib import Path + +import pytest + + +def load_probe(monkeypatch): + monkeypatch.syspath_prepend(str(Path(__file__).parent / "smoke")) + return importlib.import_module("service_probe") + + +def test_protocol_failure_cannot_be_reported_as_a_passing_service_probe(monkeypatch): + probe = load_probe(monkeypatch) + cause = ConnectionError("broker lost") + with pytest.raises(RuntimeError, match="MQTT publish failed") as raised: + probe.check_events([{"request_type": "MQTT", "name": "publish", "exception": cause}], 1) + assert raised.value.__cause__ is cause + + +def test_missing_protocol_measurement_is_a_probe_failure(monkeypatch): + with pytest.raises(RuntimeError, match="Expected 1"): + load_probe(monkeypatch).check_events([], 1) diff --git a/test/test_user_templates_a.py b/test/test_user_templates_a.py index a3a536f..412a223 100644 --- a/test/test_user_templates_a.py +++ b/test/test_user_templates_a.py @@ -868,6 +868,7 @@ def test_elasticsearch_unknown_method_and_missing_client(monkeypatch, env): def install_fake_etcd3(monkeypatch, stored: Optional[bytes] = b"value") -> SimpleNamespace: + block_import(monkeypatch, "etcd3gw") record = SimpleNamespace(client_kwargs=None, calls=[], closed=0) class FakeEtcdClient: @@ -927,11 +928,38 @@ def test_etcd_unknown_method_and_missing_client(monkeypatch, env): user = etcd_t.EtcdUserWrapper(env) user._do_step({"method": "watch"}) assert env.fired == [] - block_import(monkeypatch, "etcd3") + block_import(monkeypatch, "etcd3", "etcd3gw") user._do_step({"method": "connect"}) assert_failure(env, "ETCD", "connect", RuntimeError, "pip install etcd3") +@pytest.mark.parametrize("values,length", [([b"value"], 5), ([], 0)]) +def test_etcd_gateway_preserves_step_results_and_closes_session(monkeypatch, env, values, length): + calls = [] + client = SimpleNamespace( + get=lambda key: values, + put=lambda key, value: calls.append(("put", key, value)), + delete=lambda key: calls.append(("delete", key)), + session=SimpleNamespace(close=lambda: calls.append(("close",))), + ) + + def factory(**kwargs): + calls.append(("connect", kwargs)) + return client + + monkeypatch.setitem(sys.modules, "etcd3gw", make_module("etcd3gw", client=factory)) + block_import(monkeypatch, "etcd3") + user = etcd_t.EtcdUserWrapper(env) + for step in [{"method": "connect"}, {"method": "put", "key": "k", "value": "v"}, + {"method": "get", "key": "k"}, {"method": "delete", "key": "k"}, {"method": "close"}]: + user._do_step(step) + assert all(event["exception"] is None for event in env.fired) + assert env.fired[2]["response_length"] == length + assert calls == [("connect", {"host": "127.0.0.1", "port": 2379}), + ("put", "k", "v"), ("delete", "k"), ("close",)] + assert user._client is None + + # --------------------------------------------------------------------------- # FCM (stdlib urllib) # --------------------------------------------------------------------------- From 2460ffedf83acac4ad571353dd14b08fe2405d54 Mon Sep 17 00:00:00 2001 From: JeffreyChen Date: Fri, 2 Oct 2026 19:53:18 +0800 Subject: [PATCH 07/18] fix: align CI installs and isolate smoke container permissions --- dev_requirements.txt | 1 + docker/extras.Dockerfile | 5 +++-- docker/services.Dockerfile | 6 ++++-- docs/updates/2026-10.md | 7 +++++++ docs/updates/README.md | 1 + requirements.txt | 3 ++- scripts/extras_matrix.py | 4 ++-- test/smoke/install_extra.py | 3 ++- test/test_extras_matrix.py | 11 +++++++---- 9 files changed, 29 insertions(+), 12 deletions(-) diff --git a/dev_requirements.txt b/dev_requirements.txt index 0a953e3..39d9107 100644 --- a/dev_requirements.txt +++ b/dev_requirements.txt @@ -1,4 +1,5 @@ je_load_density_dev +httpx>=0.27 defusedxml>=0.7.1 je_action_core>=0.0.2 sphinx diff --git a/docker/extras.Dockerfile b/docker/extras.Dockerfile index 9c96fad..7c4e738 100644 --- a/docker/extras.Dockerfile +++ b/docker/extras.Dockerfile @@ -1,7 +1,7 @@ ARG PYTHON_VERSION=3.12 FROM python:${PYTHON_VERSION}-slim COPY dist/*.whl /wheels/ -RUN --mount=type=cache,target=/root/.cache/pip,sharing=locked python -m pip install /wheels/*.whl +RUN --mount=type=cache,target=/root/.cache/pip,sharing=locked python -m pip install --only-binary :all: /wheels/*.whl ARG EXTRA=base RUN if [ "$EXTRA" = "gui" ] || [ "$EXTRA" = "all" ]; then \ apt-get update && apt-get install -y --no-install-recommends \ @@ -13,5 +13,6 @@ RUN if [ "$EXTRA" = "gui" ] || [ "$EXTRA" = "all" ]; then \ COPY test/smoke /checks/smoke RUN --mount=type=cache,target=/root/.cache/pip,sharing=locked python /checks/smoke/install_extra.py "$EXTRA" ENV LD_EXTRA=${EXTRA} QT_QPA_PLATFORM=offscreen AWS_EC2_METADATA_DISABLED=true -WORKDIR /run +USER 65534:65534 +WORKDIR /tmp CMD ["python", "/checks/smoke/run_cell.py"] diff --git a/docker/services.Dockerfile b/docker/services.Dockerfile index f0795c5..85923ba 100644 --- a/docker/services.Dockerfile +++ b/docker/services.Dockerfile @@ -1,7 +1,9 @@ FROM python:3.12-slim COPY dist/*.whl /wheels/ RUN --mount=type=cache,target=/root/.cache/pip,sharing=locked \ - for wheel in /wheels/*.whl; do python -m pip install "$wheel[redis,mqtt,sql]"; done && python -m pip check + for wheel in /wheels/*.whl; do python -m pip install --only-binary :all: "$wheel[redis,mqtt,sql]"; done && \ + python -m pip check COPY test/smoke /checks/smoke -WORKDIR /run +USER 65534:65534 +WORKDIR /tmp CMD ["python", "/checks/smoke/service_probe.py"] diff --git a/docs/updates/2026-10.md b/docs/updates/2026-10.md index d977f14..c02d772 100644 --- a/docs/updates/2026-10.md +++ b/docs/updates/2026-10.md @@ -212,3 +212,10 @@ Added a stdlib smoke harness that exercises local HTTP requests in isolated proc - Added a separate health-checked Compose job exercising real Redis/MQTT adapters, MQTT delivery and SQLite measurements, with unconditional project cleanup. - Docker found incompatible generated protobuf code in etcd3; the extra now uses etcd3gw while preserving existing steps and legacy manually installed etcd3. - Verification: full suite 1264 passed, 2 skipped (existing optional cases), 4 warnings. Ruff and workflow/document checks pass. Installed-wheel base and etcd each pass six smoke tests; Compose adapters pass. GUI/all and Python 3.10/3.14 Docker validation continues under progress #25. + +## U-20261002-07 · 2026-10-02 · Align CI install paths and isolate smoke container permissions · #done #ci + +- CI feedback exposed the requirements-file install path missing mandatory httpx; synchronized stable/dev requirements with package metadata. +- The matrix CLI now reads only its checkout metadata regardless of working directory, and a regression test verifies a neighboring pyproject cannot redirect it. +- Docker smoke runs as uid/gid 65534, with dependency installation restricted to binary distributions. +- Verification: 41 focused matrix/workflow/probe tests pass; updated non-root base Docker cell passes six smoke tests; non-root Redis/MQTT/SQLite Compose probes pass and clean up. Original full matrix stage: 1264 passed, 2 existing optional skips. diff --git a/docs/updates/README.md b/docs/updates/README.md index 7397ae9..3b6a8a6 100644 --- a/docs/updates/README.md +++ b/docs/updates/README.md @@ -58,6 +58,7 @@ In the same commit: delete the item from `progress.md`, add a `#done` entry here | ID | Date | Title | Tags | Batch | |---|---|---|---|---| +| U-20261002-07 | 2026-10-02 | Align CI install paths and isolate smoke container permissions | #done #ci | [2026-10](2026-10.md) | | U-20261002-06 | 2026-10-02 | Gate publishing on Docker extras and protocol smoke checks | #done #ci #extras | [2026-10](2026-10.md) | | U-20261002-05 | 2026-10-02 | Exercise installed wheels and propagate failed SLA status | #done #smoke | [2026-10](2026-10.md) | | U-20261002-04 | 2026-10-02 | Scope persisted request IDs to their run | #done #records | [2026-10](2026-10.md) | diff --git a/requirements.txt b/requirements.txt index 3be9183..2baebbe 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,4 +1,5 @@ je_load_density +httpx>=0.27 defusedxml>=0.7.1 je_action_core>=0.0.2 -pytest \ No newline at end of file +pytest diff --git a/scripts/extras_matrix.py b/scripts/extras_matrix.py index 1ee7bce..eda774a 100644 --- a/scripts/extras_matrix.py +++ b/scripts/extras_matrix.py @@ -34,11 +34,11 @@ def build_matrix(metadata_path: Path, scheduled: bool = False) -> dict[str, list def main() -> None: """Emit JSON for GitHub Actions or a local matrix runner.""" parser = argparse.ArgumentParser(description=__doc__) - parser.add_argument("--metadata", type=Path, default=Path("pyproject.toml")) parser.add_argument("--event", default="pull_request") parser.add_argument("--output", type=Path, default=os.environ.get("GITHUB_OUTPUT")) arguments = parser.parse_args() - payload = json.dumps(build_matrix(arguments.metadata, scheduled=arguments.event == "schedule")) + metadata = Path(__file__).resolve().parents[1] / "pyproject.toml" + payload = json.dumps(build_matrix(metadata, scheduled=arguments.event == "schedule")) if arguments.output is not None: with arguments.output.open("a", encoding="utf-8") as handle: handle.write(f"matrix={payload}\n") diff --git a/test/smoke/install_extra.py b/test/smoke/install_extra.py index 6dd9fda..4c42639 100644 --- a/test/smoke/install_extra.py +++ b/test/smoke/install_extra.py @@ -15,7 +15,8 @@ def main() -> None: if len(wheels) != 1: raise ValueError("wheel: expected exactly one checkout-built wheel") requirement = str(wheels[0]) + (f"[{extra}]" if extra != "base" else "") - subprocess.run([sys.executable, "-m", "pip", "install", requirement], check=True, timeout=600) + subprocess.run([sys.executable, "-m", "pip", "install", "--only-binary", ":all:", requirement], + check=True, timeout=600) subprocess.run([sys.executable, "-m", "pip", "check"], check=True, timeout=30) subprocess.run([sys.executable, "-m", "je_load_density", "--help"], check=True, timeout=30) diff --git a/test/test_extras_matrix.py b/test/test_extras_matrix.py index 4b6b879..2a20664 100644 --- a/test/test_extras_matrix.py +++ b/test/test_extras_matrix.py @@ -46,12 +46,15 @@ def test_unsafe_extra_names_are_rejected_before_shell_parameters(tmp_path): load_matrix_module().build_matrix(path) -def test_cli_writes_parseable_github_output(metadata, tmp_path): +def test_cli_writes_checkout_matrix_independently_of_current_directory(tmp_path): destination = tmp_path / "github-output" + (tmp_path / "pyproject.toml").write_text('[project.optional-dependencies]\nattacker = []\n', encoding="utf-8") completed = subprocess.run([sys.executable, str(ROOT / "scripts/extras_matrix.py"), - "--metadata", str(metadata), "--output", str(destination)], - capture_output=True, text=True, timeout=10, check=False) + "--output", str(destination)], + cwd=tmp_path, capture_output=True, text=True, timeout=10, check=False) assert completed.returncode == 0, completed.stderr key, value = destination.read_text(encoding="utf-8").strip().split("=", 1) assert key == "matrix" - assert len(json.loads(value)["include"]) == 6 + cells = json.loads(value)["include"] + assert any(cell["extra"] == "gui" for cell in cells) + assert not any(cell["extra"] == "attacker" for cell in cells) From 695e01f3f1d9bc2c58d391d457157acb4526c21a Mon Sep 17 00:00:00 2001 From: JeffreyChen Date: Fri, 2 Oct 2026 20:12:44 +0800 Subject: [PATCH 08/18] fix: repair Qt Web Push and NATS installation probes --- docker/README.md | 3 +++ docker/extras.Dockerfile | 2 +- docs/updates/2026-10.md | 7 +++++++ docs/updates/README.md | 1 + test/smoke/install_extra.py | 4 +++- test/smoke/protocol_probes.py | 8 +++----- 6 files changed, 18 insertions(+), 7 deletions(-) diff --git a/docker/README.md b/docker/README.md index 6577609..28fddcc 100644 --- a/docker/README.md +++ b/docker/README.md @@ -35,6 +35,9 @@ checkout wheel, runs `pip check`, checks its declared capability without skippin and runs the six base smoke tests. GUI cells install Qt system libraries and use `QT_QPA_PLATFORM=offscreen`. These installation probes use local codecs, stubbed SDK calls and client configuration; live protocol checks run separately. +Dependencies require wheels except `http-ece`, whose upstream distribution is a +source archive used by Web Push. That explicit exception is built only in the +isolated image. Runtime probes run as uid/gid 65534. ```bash python -m build --wheel --no-isolation diff --git a/docker/extras.Dockerfile b/docker/extras.Dockerfile index 7c4e738..e7c0d95 100644 --- a/docker/extras.Dockerfile +++ b/docker/extras.Dockerfile @@ -5,7 +5,7 @@ RUN --mount=type=cache,target=/root/.cache/pip,sharing=locked python -m pip inst ARG EXTRA=base RUN if [ "$EXTRA" = "gui" ] || [ "$EXTRA" = "all" ]; then \ apt-get update && apt-get install -y --no-install-recommends \ - libegl1 libgl1 libopengl0 libglib2.0-0 libxkbcommon0 libdbus-1-3 \ + libegl1 libgl1 libopengl0 libglib2.0-0 libxkbcommon0 libdbus-1-3 libfontconfig1 \ libxcb-cursor0 libxcb-icccm4 libxcb-keysyms1 libxcb-shape0 libxcb-xinerama0 \ libxcb-randr0 libxcb-render-util0 libxcb-image0 fonts-dejavu-core \ && rm -rf /var/lib/apt/lists/*; \ diff --git a/docs/updates/2026-10.md b/docs/updates/2026-10.md index c02d772..9da77dc 100644 --- a/docs/updates/2026-10.md +++ b/docs/updates/2026-10.md @@ -219,3 +219,10 @@ Added a stdlib smoke harness that exercises local HTTP requests in isolated proc - The matrix CLI now reads only its checkout metadata regardless of working directory, and a regression test verifies a neighboring pyproject cannot redirect it. - Docker smoke runs as uid/gid 65534, with dependency installation restricted to binary distributions. - Verification: 41 focused matrix/workflow/probe tests pass; updated non-root base Docker cell passes six smoke tests; non-root Redis/MQTT/SQLite Compose probes pass and clean up. Original full matrix stage: 1264 passed, 2 existing optional skips. + +## U-20261002-08 · 2026-10-02 · Repair Qt Web Push and NATS installation probes · #done #ci #extras + +- Full CI matrix identified missing Qt fontconfig, the source-only http-ece dependency of Web Push, and a NATS probe closing an unconnected client. +- Added libfontconfig1 for GUI/all cells. Kept wheel-only dependencies with a narrowly named http-ece source exception; runtime stays unprivileged. +- NATS now exercises its public unique request/reply inbox generator without invoking connection-only cleanup. +- Verification: installed Docker NATS and Web Push probes pass; their environment passes pip check. Forty-four of the forty-eight initial cells passed; these identified failures remain required and are rerun, not excluded. GUI/fontconfig and all coexistence await the updated CI run. diff --git a/docs/updates/README.md b/docs/updates/README.md index 3b6a8a6..7039092 100644 --- a/docs/updates/README.md +++ b/docs/updates/README.md @@ -58,6 +58,7 @@ In the same commit: delete the item from `progress.md`, add a `#done` entry here | ID | Date | Title | Tags | Batch | |---|---|---|---|---| +| U-20261002-08 | 2026-10-02 | Repair Qt Web Push and NATS installation probes | #done #ci #extras | [2026-10](2026-10.md) | | U-20261002-07 | 2026-10-02 | Align CI install paths and isolate smoke container permissions | #done #ci | [2026-10](2026-10.md) | | U-20261002-06 | 2026-10-02 | Gate publishing on Docker extras and protocol smoke checks | #done #ci #extras | [2026-10](2026-10.md) | | U-20261002-05 | 2026-10-02 | Exercise installed wheels and propagate failed SLA status | #done #smoke | [2026-10](2026-10.md) | diff --git a/test/smoke/install_extra.py b/test/smoke/install_extra.py index 4c42639..d7b2bf5 100644 --- a/test/smoke/install_extra.py +++ b/test/smoke/install_extra.py @@ -15,7 +15,9 @@ def main() -> None: if len(wheels) != 1: raise ValueError("wheel: expected exactly one checkout-built wheel") requirement = str(wheels[0]) + (f"[{extra}]" if extra != "base" else "") - subprocess.run([sys.executable, "-m", "pip", "install", "--only-binary", ":all:", requirement], + # pywebpush's http-ece dependency publishes source archives; all other dependencies require wheels. + subprocess.run([sys.executable, "-m", "pip", "install", "--only-binary", ":all:", + "--no-binary", "http-ece", requirement], check=True, timeout=600) subprocess.run([sys.executable, "-m", "pip", "check"], check=True, timeout=30) subprocess.run([sys.executable, "-m", "je_load_density", "--help"], check=True, timeout=30) diff --git a/test/smoke/protocol_probes.py b/test/smoke/protocol_probes.py index c52a80a..7b54cba 100644 --- a/test/smoke/protocol_probes.py +++ b/test/smoke/protocol_probes.py @@ -132,14 +132,12 @@ def mqtt() -> None: def nats() -> None: - import asyncio - from nats.aio.client import Client client = Client() - if client.is_connected: - raise RuntimeError("A new NATS client unexpectedly connects") - asyncio.run(client.close()) + first, second = client.new_inbox(), client.new_inbox() + if not first.startswith("_INBOX.") or first == second: + raise RuntimeError("NATS request/reply inbox generation failed") def neo4j() -> None: From 547bec275ec08674c1dd1e0643a3d00e6254864d Mon Sep 17 00:00:00 2001 From: JeffreyChen Date: Fri, 2 Oct 2026 20:14:38 +0800 Subject: [PATCH 09/18] fix: validate cloud launch outcomes and retain partial failures --- README.md | 2 + README/README_zh-CN.md | 2 + README/README_zh-TW.md | 2 + architecture.md | 2 + .../En/doc/distributed/distributed_doc.rst | 17 + .../Zh/doc/distributed/distributed_doc.rst | 11 + .../plans/2026-10-02-cloud-backends.md | 75 +++ docs/updates/2026-10.md | 7 + docs/updates/README.md | 1 + je_load_density/cloud/__init__.py | 5 + je_load_density/cloud/_validation.py | 69 +++ je_load_density/cloud/aws_fargate.py | 80 ++- je_load_density/cloud/aws_lambda.py | 116 ++++- je_load_density/cloud/azure_aci.py | 80 ++- je_load_density/cloud/gcp_cloud_run.py | 88 +++- progress.md | 1 - test/test_cloud_contracts.py | 491 ++++++++++++++++++ test/test_cloud_launchers.py | 20 +- test/test_cloud_sdk_contracts.py | 180 +++++++ 19 files changed, 1191 insertions(+), 58 deletions(-) create mode 100644 docs/superpowers/plans/2026-10-02-cloud-backends.md create mode 100644 je_load_density/cloud/_validation.py create mode 100644 test/test_cloud_contracts.py create mode 100644 test/test_cloud_sdk_contracts.py diff --git a/README.md b/README.md index 64b73de..e93d7cb 100644 --- a/README.md +++ b/README.md @@ -1082,6 +1082,8 @@ Added in the 2026-05 expansion. Each one is imported lazily and needs only its o Options: `--method`, `--body`, `--http2`, `--max-in-flight`. - **Cloud workers** (`aws`, `gcp`, `azure` or `cloud` extras): `cloud.aws_fargate.launch_fargate_workers`, `cloud.aws_lambda.invoke_lambda_workers` (with `lambda_worker_handler` as the function entry), `cloud.azure_aci.launch_aci_workers` and `cloud.gcp_cloud_run.run_cloud_run_job` start remote workers for a distributed run. + +Cloud launchers validate counts/resources before contacting providers. `cloud.CloudLaunchError` retains prior accepted responses, failed worker indices and available failed response details; provider exceptions remain chained. Fargate rejects partial/malformed submissions. Lambda distinguishes execution success from Event acceptance and DryRun validation, closes payload streams and preserves FunctionError payloads. Cloud Run refreshes credentials per call; configure `parallelism` on the deployed Job, since per-run overrides support `task_count` but reject `parallelism`. ACI waits for provisioning and returns unique `name`, `status="Succeeded"` and `resource_id`. Launchers do not roll back accepted resources or retry launch requests. - **Chaos helpers**: `utils.chaos.toxiproxy` adds and removes latency or bandwidth toxics on a Toxiproxy instance (`install_latency`, `install_bandwidth`, `reset_all`); `utils.chaos.chaos_mesh` builds and applies Chaos Mesh manifests (`build_network_delay`, `apply_manifest`, `delete_manifest`). - **Stub server**: `utils.stub_server.start_stub_server` / `stop_stub_server` serve canned responses so a scenario can run against a fake backend. It serves from a thread, which works beside Locust's gevent users; for the asyncio engine start it in a separate process, because a server thread in the engine's own process never gets scheduled. - **More report formats** next to the seven above: Allure, cost, CycloneDX, Excel, latency histogram, PDF (`pdf` extra), SARIF and a service map, one `generate_*_report.py` module each under `utils/generate_report/`. diff --git a/README/README_zh-CN.md b/README/README_zh-CN.md index 1c02f0a..01d22c9 100644 --- a/README/README_zh-CN.md +++ b/README/README_zh-CN.md @@ -1080,6 +1080,8 @@ diagnostics。以 `npm install && npm run package` 构建,再安装 选项:`--method`、`--body`、`--http2`、`--max-in-flight`。 - **Cloud workers**(`aws`、`gcp`、`azure` 或 `cloud` extras):`cloud.aws_fargate.launch_fargate_workers`、`cloud.aws_lambda.invoke_lambda_workers`(以 `lambda_worker_handler` 作为函数入口)、`cloud.azure_aci.launch_aci_workers` 与 `cloud.gcp_cloud_run.run_cloud_run_job` 为分布式跑法启动远端 worker。 + +云端 launcher 在联系 provider 前验证数量／资源参数。`cloud.CloudLaunchError` 保留先前接受的响应、失败 worker 索引及可取得的失败响应，并链接 provider 异常。Fargate 拒绝部分失败／格式错误的提交。Lambda 区分执行成功、Event 接受与 DryRun 验证，关闭 payload stream 并保留 FunctionError payload。Cloud Run 每次刷新 credentials；`parallelism` 请设置于部署的 Job，每次执行可覆盖 `task_count`，但拒绝 `parallelism`。ACI 等待 provisioning 并返回唯一 `name`、`status="Succeeded"` 与 `resource_id`。Launcher 不回滚已接受的资源，也不重试启动请求。 - **Chaos 辅助工具**:`utils.chaos.toxiproxy` 在 Toxiproxy 实例上新增与移除 latency 或 bandwidth toxic(`install_latency`、`install_bandwidth`、`reset_all`);`utils.chaos.chaos_mesh` 构建并套用 Chaos Mesh manifest(`build_network_delay`、`apply_manifest`、`delete_manifest`)。 - **Stub server**:`utils.stub_server.start_stub_server` / `stop_stub_server` 供应罐头响应,让场景能对一个假后端执行。它从一个线程供应,可与 Locust 的 gevent 用户并存;对 asyncio 引擎则要在独立进程启动它,因为引擎自己进程里的服务器线程永远得不到调度。 - **更多报告格式**,在上述七种之外:Allure、cost、CycloneDX、Excel、latency histogram、PDF(`pdf` extra)、SARIF 与一张 service map,各自对应 `utils/generate_report/` 下一个 `generate_*_report.py` 模块。 diff --git a/README/README_zh-TW.md b/README/README_zh-TW.md index e39da65..5dcd3e9 100644 --- a/README/README_zh-TW.md +++ b/README/README_zh-TW.md @@ -1080,6 +1080,8 @@ diagnostics。以 `npm install && npm run package` 建置,再安裝 選項:`--method`、`--body`、`--http2`、`--max-in-flight`。 - **Cloud workers**(`aws`、`gcp`、`azure` 或 `cloud` extras):`cloud.aws_fargate.launch_fargate_workers`、`cloud.aws_lambda.invoke_lambda_workers`(以 `lambda_worker_handler` 作為函式進入點)、`cloud.azure_aci.launch_aci_workers` 與 `cloud.gcp_cloud_run.run_cloud_run_job` 為分散式跑法啟動遠端 worker。 + +雲端 launcher 在聯絡 provider 前驗證數量／資源參數。`cloud.CloudLaunchError` 保留先前接受的回應、失敗 worker 索引及可取得的失敗回應，並串接 provider 例外。Fargate 拒絕部分失敗／格式錯誤的提交。Lambda 區分執行成功、Event 接受與 DryRun 驗證，關閉 payload stream 並保留 FunctionError payload。Cloud Run 每次刷新 credentials；`parallelism` 請設定於部署的 Job，每次執行可覆寫 `task_count`，但拒絕 `parallelism`。ACI 等待 provisioning 並回傳唯一 `name`、`status="Succeeded"` 與 `resource_id`。Launcher 不回滾已接受的資源，也不重試啟動請求。 - **Chaos 輔助工具**:`utils.chaos.toxiproxy` 在 Toxiproxy 執行個體上新增與移除 latency 或 bandwidth toxic(`install_latency`、`install_bandwidth`、`reset_all`);`utils.chaos.chaos_mesh` 建構並套用 Chaos Mesh manifest(`build_network_delay`、`apply_manifest`、`delete_manifest`)。 - **Stub server**:`utils.stub_server.start_stub_server` / `stop_stub_server` 供應罐頭回應,讓情境能對一個假後端執行。它從一個執行緒供應,可與 Locust 的 gevent 使用者並存;對 asyncio 引擎則要在獨立行程啟動它,因為引擎自己行程裡的伺服器執行緒永遠得不到排程。 - **更多報告格式**,在上述七種之外:Allure、cost、CycloneDX、Excel、latency histogram、PDF(`pdf` extra)、SARIF 與一張 service map,各自對應 `utils/generate_report/` 下一個 `generate_*_report.py` 模組。 diff --git a/architecture.md b/architecture.md index 3df41c9..9ed2fc3 100644 --- a/architecture.md +++ b/architecture.md @@ -194,6 +194,8 @@ the published dependency floor. The old action-executor record contract is separ APITestka, FileAutomation, AutoControlGUI and TestPioneer register no builtins at all (workspace `progress.md` X-12). +Cloud launchers preflight counts/resources and distinguish accepted submissions from successful execution/provisioning. Public cloud.CloudLaunchError retains accepted worker responses and failure indices. Cloud Run per-run parallelism is rejected (configure the deployed Job); ACI waits its poller and returns unique names/resource IDs. No adapter retries or rolls back launches. + ## 7. Design constraints - SOLID, composition over inheritance, patterns only where they reduce complexity (CLAUDE.md diff --git a/docs/source/En/doc/distributed/distributed_doc.rst b/docs/source/En/doc/distributed/distributed_doc.rst index da8edc5..66a1c54 100644 --- a/docs/source/En/doc/distributed/distributed_doc.rst +++ b/docs/source/En/doc/distributed/distributed_doc.rst @@ -66,3 +66,20 @@ Tips on master and workers — the master broadcasts the user class name. * If you parameterise tasks with ``${csv.X.col}``, register the same CSV files on every worker (they don't share state). + +Cloud launch contracts +---------------------- + +Cloud adapters validate worker/resource settings before contacting providers. +``je_load_density.cloud.CloudLaunchError`` preserves prior accepted responses, +failed worker indices and failed response details; provider exceptions remain chained. +Fargate rejects partial and malformed submissions. Lambda distinguishes successful +execution from Event acceptance and DryRun validation, retains FunctionError payloads, +and closes payload streams. ACI waits for provisioning and returns unique ``name``, +``status="Succeeded"`` and ``resource_id``. Cloud Run refreshes credentials per call; +configure parallelism on the deployed Job, since per-run overrides accept +``task_count`` and reject ``parallelism``. Launchers do not retry or roll back launches. + +Mandatory contract tests use controlled SDK doubles and local HTTP. Optional cloud +extra tests also verify official SDK serialization/stubs; they create no resources. +Accepted launch responses do not prove worker readiness or successful load completion. diff --git a/docs/source/Zh/doc/distributed/distributed_doc.rst b/docs/source/Zh/doc/distributed/distributed_doc.rst index 9795b46..1808b55 100644 --- a/docs/source/Zh/doc/distributed/distributed_doc.rst +++ b/docs/source/Zh/doc/distributed/distributed_doc.rst @@ -4,6 +4,17 @@ 概觀 ---- +雲端 adapter 在聯絡 provider 前驗證 worker／資源設定。 +``je_load_density.cloud.CloudLaunchError`` 保留先前接受的回應、失敗 worker 索引與失敗回應細節， +並串接 provider 例外。Fargate 拒絕部分失敗或格式錯誤的提交。Lambda 區分執行成功、Event 接受與 +DryRun 驗證，保留 FunctionError payload 並關閉 payload stream。ACI 等待 provisioning， +回傳唯一 ``name``、``status="Succeeded"`` 與 ``resource_id``。 +Cloud Run 每次刷新 credentials；parallelism 請設定於部署的 Job，執行覆寫支援 ``task_count``， +但拒絕 ``parallelism``。Launcher 不重試啟動，也不回滾已接受的資源。 + +必要契約測試使用可控 SDK double 與本機 HTTP；選用 cloud extra 測試另驗證官方 SDK serializer／stub， +不建立真實資源。接受啟動不代表 worker 已就緒或負載執行成功。 + LoadDensity 透過 ``start_test`` / ``prepare_env`` 的 ``runner_mode`` 參數開放 Locust 的分散式 runner。三種模式： * ``local`` — 單一程序（預設）。 diff --git a/docs/superpowers/plans/2026-10-02-cloud-backends.md b/docs/superpowers/plans/2026-10-02-cloud-backends.md new file mode 100644 index 0000000..3507518 --- /dev/null +++ b/docs/superpowers/plans/2026-10-02-cloud-backends.md @@ -0,0 +1,75 @@ +# Cloud Backend Completeness Implementation Plan + +> **For agentic workers:** REQUIRED SUB-SKILL: Use superpowers:executing-plans to implement this plan task-by-task. Steps use checkbox (`- [ ]`) syntax for tracking. + +**Goal:** Validate cloud launcher inputs and report service, partial-launch and malformed-response failures without creating real resources. + +**Architecture:** Keep the four public launchers and lazy optional SDK imports. Share preflight validation and a RuntimeError subtype carrying successful/accepted responses and failed worker indices. No adapter retry is added. + +**Tech Stack:** Python 3.10+, pytest, controllable SDK doubles, urllib, local HTTP. + +**Spec:** `docs/superpowers/specs/2026-10-02-testing-platform-roadmap.md`, 雲端後端測試. + +## Global Constraints + +- 合約測試使用可控 SDK stub／本機 HTTP，不申請真實雲端資源。 +- 測試 SDK 呼叫參數與結果判定，不能只驗證 fake 回傳值原封不動傳出。 +- 對可能已被服務接受的啟動請求必須使用服務支援的冪等識別，不能盲目重送。 +- Preserve public launcher signatures and successful response shapes where API-correct. +- Root worker owns README/Sphinx/architecture/log updates and commits. + +## Review Focus + +- bool, fractional and zero worker counts must fail before SDK calls. +- Previously accepted workers remain inspectable after another worker fails. +- A Lambda HTTP success containing FunctionError must fail; Event means accepted only. +- Cloud Run run overrides do not support parallelism: reject it before authentication. +- ACI group names differ across launches and failed polling cannot report success. + +### Task 1: AWS worker contracts + +**Files:** Modify `je_load_density/cloud/aws_fargate.py`, `aws_lambda.py`; create `_validation.py`; test `test/test_cloud_contracts.py`, `test/test_cloud_launchers.py`. + +**Interfaces:** `CloudLaunchError(RuntimeError)` exposes `responses`, `failed_workers`, `backend`; successes keep Fargate raw responses and Lambda worker-ordered dictionaries. + +- [x] Write failing tests for invalid counts/network/env; Fargate failures/malformed tasks; Lambda FunctionError/status/non-JSON/Event and mixed results. Assert index/count overrides, service call count, accepted responses, error causes and closed payload streams. +- [x] Run `python -m pytest test/test_cloud_contracts.py -q`; observe contract failures. +- [x] Implement positive integer/string/env validation, SDK error chaining, strict response judgement and isolated reference handler behavior. +- [x] Run AWS contract and existing launcher tests; expect all AWS tests passing. + +### Task 2: Cloud Run REST contract + +**Files:** Modify `je_load_density/cloud/gcp_cloud_run.py`; test `test/test_cloud_contracts.py`. + +**Interfaces:** `run_cloud_run_job(...) -> Dict[str, Any]` returns a valid Operation; non-None `parallelism` raises ValueError because jobs.run does not support that override. + +- [x] Write failing tests for unsupported parallelism, invalid task_count/timeout/overrides, empty token, bad Operation/error JSON. Test refresh, permission/timeout/throttle/server errors with stubs and local HTTP; assert POST/auth/payload/timeout and no retry. +- [x] Run Cloud Run tests and observe failures. +- [x] Validate supported overrides; refresh token per call; reject empty token and malformed/error operations. +- [x] Run Cloud Run tests; expect passing. + +### Task 3: Azure operation lifecycle + +**Files:** Modify `je_load_density/cloud/azure_aci.py`; test `test/test_cloud_contracts.py`, `test/test_cloud_launchers.py`. + +**Interfaces:** `launch_aci_workers(...) -> List[Dict[str, Any]]` waits for provisioning, returns unique name/status/resource_id; exceptions retain prior responses and failed indices. + +- [x] Write failing tests for positive finite cpu/memory, DNS prefix limits, unique names between runs, poller failure/malformed result, credentials/permissions/timeout/throttle and partial provisioning. Assert model fields and create calls. +- [x] Run ACI tests and observe failures. +- [x] Add unique invocation suffix, await poller.result, validate successful resource, chain SDK errors with partial evidence. +- [x] Run `python -m pytest test/test_cloud_launchers.py test/test_cloud_contracts.py -q`; expect passing; root runs whole suite after integration. + +## Additional SDK verification + +`test/test_cloud_sdk_contracts.py` uses botocore Stubber/StreamingBody, official AWS and Google exception classes, and a local Azure HttpTransport implementing a successful PUT response. It checks the real Azure serializer and long-running operation result. These twelve tests require the corresponding cloud extras; the mandatory contract tests do not skip when SDKs are absent. + +Red evidence: initial contracts 87 failed / 5 passed; expanded boundaries 13 failed / 98 passed; missing public error export 1 failed / 1 passed; missing Lambda decoded failure payload and ACI attempted name 2 failed. + +Green evidence: cloud contracts and official SDK verification 141 passed. Production complexity maximum 10, function length maximum 73 lines, no lines over 120 characters. Final whole-suite evidence is recorded by the coordinating worker in the update log. + +## Execution rulings + +- Execution and design were already approved before dispatch; continue without another approval gate. +- SDK-specific exceptions are caught at service boundaries only; unexpected programming exceptions propagate. +- Cloud Run parallelism migration: configure parallelism on the deployed Job; use task_count for the run override. +- Commits and shared documentation are performed by the coordinating worker. diff --git a/docs/updates/2026-10.md b/docs/updates/2026-10.md index 9da77dc..8a00234 100644 --- a/docs/updates/2026-10.md +++ b/docs/updates/2026-10.md @@ -226,3 +226,10 @@ Added a stdlib smoke harness that exercises local HTTP requests in isolated proc - Added libfontconfig1 for GUI/all cells. Kept wheel-only dependencies with a narrowly named http-ece source exception; runtime stays unprivileged. - NATS now exercises its public unique request/reply inbox generator without invoking connection-only cleanup. - Verification: installed Docker NATS and Web Push probes pass; their environment passes pip check. Forty-four of the forty-eight initial cells passed; these identified failures remain required and are rerun, not excluded. GUI/fontconfig and all coexistence await the updated CI run. + +## U-20261002-09 · 2026-10-02 · Validate cloud launch outcomes and preserve partial failures · #done #cloud + +- Expanded Fargate/Lambda/Cloud Run/ACI validation and failure contracts. Public CloudLaunchError preserves accepted responses, failed worker indices and failed payloads while chaining SDK errors. +- Corrected unsupported Cloud Run parallelism overrides, distinguished Lambda execution/acceptance/validation, closed payload streams, and awaited ACI provisioning with unique worker names. +- Reviewer malformed-stream regression now validates callable read/close methods without swallowing SDK-internal TypeError. +- Verification: 129 mandatory cloud tests and 12 isolated official SDK contract tests passed (141 total), no real resources. Root cloud/distributed focused run 160 passed; whole-suite snapshot after fixes 1428 passed, 14 optional skips. Ruff/complexity and scoped whitespace checks pass. GUI/chart work remains separate and continues. diff --git a/docs/updates/README.md b/docs/updates/README.md index 7039092..13e1e8d 100644 --- a/docs/updates/README.md +++ b/docs/updates/README.md @@ -58,6 +58,7 @@ In the same commit: delete the item from `progress.md`, add a `#done` entry here | ID | Date | Title | Tags | Batch | |---|---|---|---|---| +| U-20261002-09 | 2026-10-02 | Validate cloud launch outcomes and preserve partial failures | #done #cloud | [2026-10](2026-10.md) | | U-20261002-08 | 2026-10-02 | Repair Qt Web Push and NATS installation probes | #done #ci #extras | [2026-10](2026-10.md) | | U-20261002-07 | 2026-10-02 | Align CI install paths and isolate smoke container permissions | #done #ci | [2026-10](2026-10.md) | | U-20261002-06 | 2026-10-02 | Gate publishing on Docker extras and protocol smoke checks | #done #ci #extras | [2026-10](2026-10.md) | diff --git a/je_load_density/cloud/__init__.py b/je_load_density/cloud/__init__.py index e69de29..d671471 100644 --- a/je_load_density/cloud/__init__.py +++ b/je_load_density/cloud/__init__.py @@ -0,0 +1,5 @@ +"""Cloud worker launchers and inspectable partial-launch errors.""" + +from ._validation import CloudLaunchError + +__all__ = ["CloudLaunchError"] diff --git a/je_load_density/cloud/_validation.py b/je_load_density/cloud/_validation.py new file mode 100644 index 0000000..be8e1c4 --- /dev/null +++ b/je_load_density/cloud/_validation.py @@ -0,0 +1,69 @@ +"""Shared preflight validation and partial launch evidence for cloud adapters.""" + +import math +from typing import Any, Dict, List, Optional, Tuple, Type + + +class CloudLaunchError(RuntimeError): + """A failed launch with prior accepted responses; never implies rollback. + + ``failed_workers`` contains worker indices. ``responses`` contains successful + or accepted responses in worker order, so callers can inspect existing work + before deciding whether to launch replacements. ``response`` holds the failed + service response when available. SDK errors are preserved as exception causes. + """ + + def __init__(self, backend: str, failed_workers: List[int], responses: List[Dict[str, Any]], + message: str, response: Optional[Dict[str, Any]] = None) -> None: + super().__init__(f"{backend}: {message}; failed workers {failed_workers}") + self.backend = backend + self.failed_workers = list(failed_workers) + self.responses = list(responses) + self.response = response + + +def positive_integer(value: int, name: str) -> None: + """Require an actual positive integer, excluding booleans.""" + if isinstance(value, bool) or not isinstance(value, int) or value <= 0: + raise ValueError(f"{name} must be a positive integer") + + +def positive_number(value: float, name: str) -> None: + """Require a positive finite numeric resource or timeout value.""" + if isinstance(value, bool) or not isinstance(value, (int, float)): + raise ValueError(f"{name} must be a positive finite number") + if not math.isfinite(value) or value <= 0: + raise ValueError(f"{name} must be a positive finite number") + + +def required_string(value: str, name: str) -> None: + """Require a nonblank string without embedded null characters.""" + if not isinstance(value, str) or not value.strip() or "\x00" in value: + raise ValueError(f"{name} must be a nonempty string") + + +def validate_environment(environment: Optional[Dict[str, str]]) -> None: + """Validate string environment values and reserved worker identity names.""" + if environment is None: + return + if not isinstance(environment, dict): + raise ValueError("overrides_env must be a dictionary of strings") + for name, value in environment.items(): + required_string(name, "environment name") + if name in {"LD_WORKER_INDEX", "LD_WORKER_COUNT"}: + raise ValueError(f"{name} is reserved for worker identity") + if not isinstance(value, str) or "\x00" in value: + raise ValueError("environment values must be strings without null characters") + + +def sdk_error_types(provider: str) -> Tuple[Type[Exception], ...]: + """Return optional SDK base errors without requiring SDKs for module import.""" + try: + if provider == "aws": + from botocore.exceptions import BotoCoreError, ClientError + return (BotoCoreError, ClientError, OSError) + from azure.core.exceptions import AzureError + return (AzureError, OSError) + except ImportError: + # Lightweight SDK doubles need only the standard transport boundary. + return (OSError,) diff --git a/je_load_density/cloud/aws_fargate.py b/je_load_density/cloud/aws_fargate.py index 5203af8..7ab47d3 100644 --- a/je_load_density/cloud/aws_fargate.py +++ b/je_load_density/cloud/aws_fargate.py @@ -8,6 +8,8 @@ from typing import Any, Dict, List, Optional +from ._validation import CloudLaunchError, positive_integer, required_string, sdk_error_types, validate_environment + def _import_boto3(): try: @@ -20,6 +22,34 @@ def _import_boto3(): return boto3 +def _validate_network(subnets: List[str], security_groups: Optional[List[str]], assign_public_ip: bool) -> None: + if not isinstance(subnets, list) or not 1 <= len(subnets) <= 16: + raise ValueError("subnets must contain 1 to 16 subnet IDs") + if security_groups is not None: + if not isinstance(security_groups, list) or len(security_groups) > 5: + raise ValueError("security_groups must contain at most 5 IDs") + for subnet in subnets: + required_string(subnet, "subnet ID") + for group in security_groups or []: + required_string(group, "security group ID") + if not isinstance(assign_public_ip, bool): + raise ValueError("assign_public_ip must be a boolean") + + +def _judge_response(response: Dict[str, Any], index: int, responses: List[Dict[str, Any]]) -> None: + if not isinstance(response, dict): + raise CloudLaunchError("fargate", [index], responses, "malformed run_task response") + tasks = response.get("tasks") + failures = response.get("failures", []) + valid_tasks = isinstance(tasks, list) and len(tasks) == 1 + if valid_tasks: + arn = tasks[0].get("taskArn") if isinstance(tasks[0], dict) else None + valid_tasks = isinstance(arn, str) and bool(arn.strip()) + if failures or not valid_tasks or not isinstance(failures, list): + accepted = responses + ([response] if valid_tasks else []) + raise CloudLaunchError("fargate", [index], accepted, "run_task failed or malformed", response) + + def launch_fargate_workers( cluster: str, task_definition: str, @@ -31,9 +61,23 @@ def launch_fargate_workers( container_name: str = "loaddensity", overrides_env: Optional[Dict[str, str]] = None, ) -> List[Dict[str, Any]]: - """Launch ``workers`` ECS tasks on Fargate. Returns the run_task responses.""" + """Launch positive ``workers`` tasks; raise CloudLaunchError on partial failure. + + Accepted task responses remain on the exception. No launch is retried or + rolled back; a returned task ARN confirms submission, not worker readiness. + """ + positive_integer(workers, "workers") + for value, name in [(cluster, "cluster"), (task_definition, "task_definition"), + (container_name, "container_name")]: + required_string(value, name) + _validate_network(subnets, security_groups, assign_public_ip) + validate_environment(overrides_env) boto3 = _import_boto3() - client = boto3.client("ecs", region_name=region_name) + service_errors = sdk_error_types("aws") + try: + client = boto3.client("ecs", region_name=region_name) + except service_errors as error: + raise CloudLaunchError("fargate", list(range(workers)), [], "client initialization failed") from error env = [{"name": k, "value": v} for k, v in (overrides_env or {}).items()] network = { "awsvpcConfiguration": { @@ -44,19 +88,23 @@ def launch_fargate_workers( } responses: List[Dict[str, Any]] = [] for index in range(workers): - response = client.run_task( - cluster=cluster, - taskDefinition=task_definition, - launchType="FARGATE", - count=1, - networkConfiguration=network, - overrides={"containerOverrides": [{ - "name": container_name, - "environment": env + [ - {"name": "LD_WORKER_INDEX", "value": str(index)}, - {"name": "LD_WORKER_COUNT", "value": str(workers)}, - ], - }]}, - ) + try: + response = client.run_task( + cluster=cluster, + taskDefinition=task_definition, + launchType="FARGATE", + count=1, + networkConfiguration=network, + overrides={"containerOverrides": [{ + "name": container_name, + "environment": env + [ + {"name": "LD_WORKER_INDEX", "value": str(index)}, + {"name": "LD_WORKER_COUNT", "value": str(workers)}, + ], + }]}, + ) + except service_errors as error: + raise CloudLaunchError("fargate", [index], responses, "run_task failed") from error + _judge_response(response, index, responses) responses.append(response) return responses diff --git a/je_load_density/cloud/aws_lambda.py b/je_load_density/cloud/aws_lambda.py index 96e05cd..149c901 100644 --- a/je_load_density/cloud/aws_lambda.py +++ b/je_load_density/cloud/aws_lambda.py @@ -19,9 +19,11 @@ """ import json -from concurrent.futures import ThreadPoolExecutor +from concurrent.futures import Future, ThreadPoolExecutor from typing import Any, Dict, List, Optional +from ._validation import CloudLaunchError, positive_integer, required_string, sdk_error_types + def _import_boto3(): try: @@ -34,6 +36,68 @@ def _import_boto3(): return boto3 +def _consume_payload(payload: Any, read_body: bool) -> Any: + close = getattr(payload, "close", None) + try: + if hasattr(payload, "close") and not callable(close): + raise ValueError("malformed Lambda payload close method") + read = getattr(payload, "read", None) + if hasattr(payload, "read") and not callable(read): + raise ValueError("malformed Lambda payload read method") + return read() if read_body and callable(read) else payload + finally: + if callable(close): + close() + + +def _decode_payload(response: Dict[str, Any]) -> Dict[str, Any]: + body = _consume_payload(response.get("Payload", b""), read_body=True) + if not isinstance(body, (bytes, str)): + raise ValueError("malformed Lambda payload") + text = body.decode("utf-8") if isinstance(body, bytes) else body + if not text.strip(): + raise ValueError("empty Lambda worker payload") + try: + decoded = json.loads(text) + except json.JSONDecodeError: + return {"raw": text} + if not isinstance(decoded, dict): + raise ValueError("Lambda worker payload must be an object") + return decoded + + +def _judge_response(response: Dict[str, Any], index: int, invocation_type: str) -> Dict[str, Any]: + if not isinstance(response, dict): + raise ValueError("malformed Lambda response") + expected_status = {"RequestResponse": 200, "Event": 202, "DryRun": 204}[invocation_type] + payload = _decode_payload(response) if invocation_type == "RequestResponse" else {} + if invocation_type != "RequestResponse": + _consume_payload(response.get("Payload"), read_body=False) + if response.get("StatusCode") != expected_status or "FunctionError" in response: + failed_response = {**response, "worker_payload": payload} + raise CloudLaunchError("lambda", [index], [], "invocation failed or returned FunctionError", failed_response) + if invocation_type == "RequestResponse": + return payload + return {"status": "accepted" if invocation_type == "Event" else "validated", + "worker_index": index, "StatusCode": expected_status} + + +def _collect_results(futures: List[Future]) -> List[Dict[str, Any]]: + results = [] + errors = [] + for future in futures: + try: + results.append(future.result()) + except CloudLaunchError as error: + errors.append(error) + if errors: + failed_workers = [index for error in errors for index in error.failed_workers] + error = CloudLaunchError("lambda", failed_workers, results, "one or more invocations failed", + errors[0].response) + raise error from (errors[0].__cause__ or errors[0]) + return results + + def invoke_lambda_workers( function_name: str, workers: int, @@ -41,35 +105,49 @@ def invoke_lambda_workers( region_name: Optional[str] = None, invocation_type: str = "RequestResponse", ) -> List[Dict[str, Any]]: - """Invoke ``workers`` Lambda functions in parallel and collect results.""" + """Invoke positive ``workers`` and return worker-ordered results. + + FunctionError and malformed responses raise CloudLaunchError with other + successful results. Event returns acceptance only; DryRun returns validation. + Non-JSON text remains available as ``raw``. No invocation is retried. + """ + positive_integer(workers, "workers") + required_string(function_name, "function_name") + if invocation_type not in {"RequestResponse", "Event", "DryRun"}: + raise ValueError("invocation_type must be RequestResponse, Event or DryRun") + if not isinstance(payload_template, dict): + raise ValueError("payload_template must be a dictionary") + # Serialize before creating clients or submitting any work. + template = json.dumps(payload_template) boto3 = _import_boto3() - client = boto3.client("lambda", region_name=region_name) - results: List[Dict[str, Any]] = [] + service_errors = sdk_error_types("aws") + try: + client = boto3.client("lambda", region_name=region_name) + except service_errors as error: + raise CloudLaunchError("lambda", list(range(workers)), [], "client initialization failed") from error def _invoke(index: int) -> Dict[str, Any]: - payload = dict(payload_template) + payload = json.loads(template) payload["worker_index"] = index payload["worker_count"] = workers - response = client.invoke( - FunctionName=function_name, - InvocationType=invocation_type, - Payload=json.dumps(payload).encode("utf-8"), - ) - body_bytes = response.get("Payload", b"").read() if hasattr( - response.get("Payload"), "read" - ) else response.get("Payload", b"") - body_text = body_bytes.decode("utf-8") if body_bytes else "{}" try: - return json.loads(body_text) - except json.JSONDecodeError: - return {"raw": body_text} + response = client.invoke( + FunctionName=function_name, InvocationType=invocation_type, + Payload=json.dumps(payload).encode("utf-8"), + ) + except service_errors as error: + raise CloudLaunchError("lambda", [index], [], "invoke failed") from error + try: + return _judge_response(response, index, invocation_type) + except (ValueError,) + service_errors as error: + raise CloudLaunchError("lambda", [index], [], "invalid invocation response", + response if isinstance(response, dict) else None) from error # Results come back in worker order (results[i] is worker i), not completion order, so a # caller can tell which slice each summary belongs to. with ThreadPoolExecutor(max_workers=max(workers, 1)) as pool: futures = [pool.submit(_invoke, i) for i in range(workers)] - results.extend(future.result() for future in futures) - return results + return _collect_results(futures) def lambda_worker_handler(event: Dict[str, Any], _context: Any) -> Dict[str, Any]: diff --git a/je_load_density/cloud/azure_aci.py b/je_load_density/cloud/azure_aci.py index a1edc9e..033aa2b 100644 --- a/je_load_density/cloud/azure_aci.py +++ b/je_load_density/cloud/azure_aci.py @@ -5,7 +5,18 @@ ``azure-identity`` + ``azure-mgmt-containerinstance`` (both lazy). """ +import re from typing import Any, Dict, List, Optional +from uuid import uuid4 + +from ._validation import ( + CloudLaunchError, + positive_integer, + positive_number, + required_string, + sdk_error_types, + validate_environment, +) def _import_azure(): @@ -51,10 +62,28 @@ def launch_aci_workers( overrides_env: Optional[Dict[str, str]] = None, name_prefix: str = "loaddensity-worker", ) -> List[Dict[str, Any]]: - """Create ``workers`` ACI container groups. Returns the create responses.""" + """Provision positive ``workers`` groups with unique names per launch. + + Wait for each SDK poller to complete and return name/status/resource_id. + CloudLaunchError preserves earlier provisioned responses on partial failure. + No create request is retried and existing groups are never rolled back. + """ + positive_integer(workers, "workers") + positive_number(cpu, "cpu") + positive_number(memory_gb, "memory_gb") + for value, name in [(subscription_id, "subscription_id"), (resource_group, "resource_group"), + (location, "location"), (image, "image")]: + required_string(value, name) + _validate_prefix(name_prefix, workers) + validate_environment(overrides_env) + _validate_environment_names(overrides_env) az = _import_azure() - credential = az["DefaultAzureCredential"]() - client = az["Client"](credential, subscription_id) + service_errors = sdk_error_types("azure") + try: + credential = az["DefaultAzureCredential"]() + client = az["Client"](credential, subscription_id) + except service_errors as error: + raise CloudLaunchError("aci", list(range(workers)), [], "client initialization failed") from error base_env = [ az["EnvironmentVariable"](name=k, value=v) @@ -65,13 +94,15 @@ def launch_aci_workers( ) responses: List[Dict[str, Any]] = [] + invocation_id = uuid4().hex[:12] for index in range(workers): + name = f"{name_prefix}-{invocation_id}-{index}" env = base_env + [ az["EnvironmentVariable"](name="LD_WORKER_INDEX", value=str(index)), az["EnvironmentVariable"](name="LD_WORKER_COUNT", value=str(workers)), ] container = az["Container"]( - name=f"{name_prefix}-{index}", + name=name, image=image, resources=requirements, environment_variables=env, @@ -82,11 +113,38 @@ def launch_aci_workers( os_type=az["OSType"].linux, restart_policy=az["RestartPolicy"].never, ) - poller = client.container_groups.begin_create_or_update( - resource_group_name=resource_group, - container_group_name=f"{name_prefix}-{index}", - container_group=group, - ) - responses.append({"name": f"{name_prefix}-{index}", "status": "submitted", - "poller_id": id(poller)}) + try: + poller = client.container_groups.begin_create_or_update( + resource_group_name=resource_group, container_group_name=name, container_group=group) + if not callable(getattr(poller, "result", None)): + raise CloudLaunchError("aci", [index], responses, "malformed provisioning poller", {"name": name}) + result = poller.result() + except service_errors as error: + raise CloudLaunchError("aci", [index], responses, "provisioning failed", + {"name": name}) from error + responses.append(_judge_provisioning(result, name, index, responses)) return responses + + +def _validate_prefix(name_prefix: str, workers: int) -> None: + required_string(name_prefix, "name_prefix") + if not re.fullmatch(r"[a-z0-9](?:[a-z0-9-]*[a-z0-9])?", name_prefix): + raise ValueError("name_prefix must contain lowercase letters, digits and internal hyphens") + if len(name_prefix) + 14 + len(str(workers - 1)) > 63: + raise ValueError("name_prefix is invalid or too long for unique worker names") + + +def _validate_environment_names(environment: Optional[Dict[str, str]]) -> None: + for name in environment or {}: + if len(name) > 63 or not re.fullmatch(r"[a-zA-Z0-9](?:[a-zA-Z0-9_]*[a-zA-Z0-9])?", name): + raise ValueError("ACI environment names require 1 to 63 alphanumeric/internal underscore characters") + + +def _judge_provisioning(result: Any, name: str, index: int, + responses: List[Dict[str, Any]]) -> Dict[str, Any]: + resource_id = getattr(result, "id", None) + state = getattr(result, "provisioning_state", None) + if not isinstance(resource_id, str) or not resource_id or state != "Succeeded": + raise CloudLaunchError("aci", [index], responses, "provisioning failed or malformed", + {"name": name, "status": state, "resource_id": resource_id}) + return {"name": name, "status": state, "resource_id": resource_id} diff --git a/je_load_density/cloud/gcp_cloud_run.py b/je_load_density/cloud/gcp_cloud_run.py index 9c2f208..53d6961 100644 --- a/je_load_density/cloud/gcp_cloud_run.py +++ b/je_load_density/cloud/gcp_cloud_run.py @@ -11,6 +11,8 @@ import urllib.request from typing import Any, Dict, List, Optional +from ._validation import CloudLaunchError, positive_integer, positive_number, required_string + def _import_google_auth(): try: @@ -28,9 +30,75 @@ def _bearer_token(scopes: List[str]) -> str: google_auth, gauth_requests = _import_google_auth() credentials, _project = google_auth.default(scopes=scopes) credentials.refresh(gauth_requests.Request()) + if not isinstance(credentials.token, str) or not credentials.token.strip(): + raise RuntimeError("Cloud Run credential refresh returned an empty token") return credentials.token +def _validate_overrides(container_overrides: Optional[List[Dict[str, Any]]]) -> None: + if container_overrides is None: + return + if not isinstance(container_overrides, list): + raise ValueError("container_overrides must be a list") + for override in container_overrides: + if not isinstance(override, dict) or not override: + raise ValueError("each container override must be a nonempty object") + if set(override) - {"name", "args", "env", "clearArgs"}: + raise ValueError("unsupported container override field") + _validate_container_override(override) + + +def _validate_container_override(override: Dict[str, Any]) -> None: + if "name" in override: + required_string(override["name"], "container name") + if "clearArgs" in override and not isinstance(override["clearArgs"], bool): + raise ValueError("clearArgs must be a boolean") + if "args" in override: + args = override["args"] + if not isinstance(args, list) or any(not isinstance(arg, str) for arg in args): + raise ValueError("container args must be a list of strings") + if "env" in override: + _validate_env(override["env"]) + + +def _validate_env(environment: List[Dict[str, str]]) -> None: + if not isinstance(environment, list): + raise ValueError("container env must be a list") + for variable in environment: + if not isinstance(variable, dict) or set(variable) - {"name", "value", "valueSource"}: + raise ValueError("invalid container environment variable") + required_string(variable.get("name"), "environment name") + if "value" in variable and not isinstance(variable["value"], str): + raise ValueError("environment value must be a string") + if "valueSource" in variable: + if "value" in variable: + raise ValueError("environment value and valueSource are mutually exclusive") + _validate_value_source(variable["valueSource"]) + + +def _validate_value_source(source: Dict[str, Any]) -> None: + if not isinstance(source, dict) or set(source) != {"secretKeyRef"}: + raise ValueError("valueSource must contain secretKeyRef") + secret = source["secretKeyRef"] + if not isinstance(secret, dict) or set(secret) - {"secret", "version"}: + raise ValueError("secretKeyRef must be an object with secret and optional version") + required_string(secret.get("secret"), "secret name") + if "version" in secret: + required_string(secret["version"], "secret version") + + +def _judge_operation(body: bytes) -> Dict[str, Any]: + try: + response = json.loads(body.decode("utf-8")) + except (UnicodeError, json.JSONDecodeError) as error: + raise CloudLaunchError("cloud_run", [0], [], "malformed operation JSON") from error + if not isinstance(response, dict) or not isinstance(response.get("name"), str): + raise CloudLaunchError("cloud_run", [0], [], "malformed operation") + if not response["name"].strip() or "error" in response: + raise CloudLaunchError("cloud_run", [0], [], "operation failed or malformed", response) + return response + + def run_cloud_run_job( project: str, region: str, @@ -40,7 +108,21 @@ def run_cloud_run_job( container_overrides: Optional[List[Dict[str, Any]]] = None, timeout: float = 30.0, ) -> Dict[str, Any]: - """Trigger a Cloud Run Job execution. Returns the API response JSON.""" + """Trigger a Job and return its Operation; submission does not imply completion. + + jobs.run supports task_count and container overrides. Set parallelism on the + deployed Job; specifying it here raises ValueError before authentication. + Credentials refresh each call. HTTP/authentication errors propagate and no + potentially accepted POST is retried. + """ + for value, name in [(project, "project"), (region, "region"), (job_name, "job_name")]: + required_string(value, name) + if parallelism is not None: + raise ValueError("parallelism is not a jobs.run override; configure it on the deployed Job") + if task_count is not None: + positive_integer(task_count, "task_count") + positive_number(timeout, "timeout") + _validate_overrides(container_overrides) token = _bearer_token(["https://www.googleapis.com/auth/cloud-platform"]) # Each name is one path segment; quoting keeps a stray "/" or "?" from addressing another resource. project, region, job_name = (urllib.parse.quote(str(part), safe="") for part in (project, region, job_name)) @@ -49,8 +131,6 @@ def run_cloud_run_job( f"{region}/jobs/{job_name}:run" ) body: Dict[str, Any] = {"overrides": {}} - if parallelism is not None: - body["overrides"]["parallelism"] = parallelism if task_count is not None: body["overrides"]["taskCount"] = task_count if container_overrides: @@ -62,4 +142,4 @@ def run_cloud_run_job( method="POST", ) with urllib.request.urlopen(request, timeout=timeout) as response: # noqa: S310 # nosec B310 - return json.loads(response.read().decode("utf-8")) + return _judge_operation(response.read()) diff --git a/progress.md b/progress.md index 2d6b6a1..585db72 100644 --- a/progress.md +++ b/progress.md @@ -17,7 +17,6 @@ Cross-repo and workspace items live in `D:\Codes\progress.md` (relevant here: X- - **#24** Installed-wheel GUI smoke lifecycle: extend `test/smoke/` and the Docker GUI capability probe with offscreen execution and cancellation after the UI lifecycle redesign (#18). Keep client/thread cleanup observable. - **#25** Finish Docker matrix validation: base/etcd and healthy Compose Redis/MQTT/SQLite probes pass; GUI/all and Python 3.10/3.14 cells are building. Repair any remaining installation/API failures and confirm the complete Python 3.12 extras matrix in CI. - **#26** Percentile bands: `gui/chart_panel.py` mixes RPS and latency on one axis; `utils/generate_report/generate_chart_report.py` uses `ts`/index instead of request `start_time`. Share per-window percentile aggregation across GUI/dashboard/report, show p50 with p50–p95 and p95–p99 bands, separate RPS units and handle gaps, failures and actual bucket duration. -- **#27** Cloud backend failure coverage: extend `test/test_cloud_launchers.py` for invalid inputs, auth/permissions, timeout/rate limits, partial failures, Fargate `failures`, Lambda `FunctionError`/Event/warm reuse, Cloud Run refresh/error responses and ACI poller failure. Use SDK stubs/local HTTP and assert request/response semantics; no live provisioning is needed. - **#28** Shared record release activation: coordinate the actual ActionCore request-record release, then activate consumer dependency floors and cross-project installed-wheel fixtures. Current opt-in APIs must retain compatibility with older core installations until that release. Design: `docs/superpowers/specs/2026-10-02-shared-record-design.md`. - **#29** Distributed worker health and recovery: `wrapper/create_locust_env/create_locust_env.py::_wait_for_workers` counts clients and starts after timeout. Add configurable ready/heartbeat/lost policy, fail or explicitly degrade on startup timeout, reassign virtual-user capacity after loss, handle reconnect/stale assignments and deduplicate results; reuse Locust heartbeat/rebalance support where possible. Target health is separately tracked by #20. - **#30** One script for functional/load/synthetic modes: extend APITestka's existing `integrations/load_density*.py` bridge and WebRunner's `utils/synthetic_monitoring/monitor.py` so common request/assertion steps are authored once and execution profiles choose functional correctness, capacity/SLA or periodic monitoring. Unsupported conversions must fail preflight and failed checks must reach monitor transitions. Depends on #28 and async parity #16; preserve existing CLI/action/summary contracts. diff --git a/test/test_cloud_contracts.py b/test/test_cloud_contracts.py new file mode 100644 index 0000000..1424a7a --- /dev/null +++ b/test/test_cloud_contracts.py @@ -0,0 +1,491 @@ +"""Cloud boundary contracts: no credentials or remote resources are used.""" + +import io +import json +from http.server import BaseHTTPRequestHandler, HTTPServer +from threading import Thread +from types import SimpleNamespace +from urllib.error import HTTPError + +import pytest + +from je_load_density.cloud import aws_fargate, aws_lambda, azure_aci, gcp_cloud_run + + +def test_cloud_launch_error_has_public_import(): + import je_load_density.cloud as cloud + error_type = getattr(cloud, "CloudLaunchError", None) + assert error_type is not None and issubclass(error_type, RuntimeError) + + +def test_lambda_warm_handler_clears_records_after_failed_invocation(monkeypatch): + import je_load_density.engine.asyncio_engine as engine + from je_load_density.utils.test_record.test_record_class import test_record_instance + calls = [] + async def run(tasks, users, duration_seconds, http2): + calls.append((tasks, users, duration_seconds, http2)) + assert test_record_instance.test_record_list == [] + assert test_record_instance.error_record_list == [] + test_record_instance.error_record_list.append({"name": "old-failure"}) + if len(calls) == 1: + raise ValueError("controlled invocation failure") + return {"requests": 1, "failures": 1} + monkeypatch.setattr(engine, "run_async_load", run) + event = {"url": "https://target.invalid", "users": 2, "duration": 0.5, "http2": True} + try: + with pytest.raises(ValueError, match="controlled"): + aws_lambda.lambda_worker_handler(event, None) + assert aws_lambda.lambda_worker_handler(event, None) == {"requests": 1, "failures": 1} + assert calls == [([{"method": "get", "request_url": "https://target.invalid"}], 2, 0.5, True)] * 2 + assert event == {"url": "https://target.invalid", "users": 2, "duration": 0.5, "http2": True} + finally: + test_record_instance.clear_records() + + +class ServiceFailure(OSError): + """Controllable SDK transport failure at the external service boundary.""" + + +class BotoStub: + def __init__(self, operation): + self.operation = operation + self.calls = [] + + def client(self, service, region_name=None): + self.calls.append((service, region_name)) + return SimpleNamespace(run_task=self.operation, invoke=self.operation) + + +def aws_stub(monkeypatch, module, operation): + stub = BotoStub(operation) + monkeypatch.setattr(module, "_import_boto3", lambda: stub) + return stub + + +@pytest.mark.parametrize("count", [0, -1, True, 1.5, "2"]) +@pytest.mark.parametrize("backend", ["fargate", "lambda", "aci"]) +def test_invalid_worker_counts_fail_before_sdk_import(monkeypatch, count, backend): + def forbidden(): + pytest.fail("invalid configuration reached SDK import") + module = {"fargate": aws_fargate, "lambda": aws_lambda, "aci": azure_aci}[backend] + monkeypatch.setattr(module, "_import_azure" if backend == "aci" else "_import_boto3", forbidden) + with pytest.raises(ValueError, match="workers"): + if backend == "fargate": + module.launch_fargate_workers("c", "t", count, ["s"]) + elif backend == "lambda": + module.invoke_lambda_workers("f", count, {}) + else: + module.launch_aci_workers("sub", "rg", "region", "image", count) + + +@pytest.mark.parametrize("kwargs", [ + {"subnets": []}, {"subnets": [""]}, {"subnets": ["s"] * 17}, + {"security_groups": ["g"] * 6}, {"overrides_env": {"LD_WORKER_INDEX": "99"}}, + {"overrides_env": {"N": 1}}, {"cluster": ""}, {"assign_public_ip": "yes"}, +]) +def test_fargate_invalid_resources_fail_before_sdk(monkeypatch, kwargs): + monkeypatch.setattr(aws_fargate, "_import_boto3", lambda: pytest.fail("SDK reached")) + arguments = dict(cluster="c", task_definition="t", workers=1, subnets=["s"]) + arguments.update(kwargs) + with pytest.raises(ValueError): + aws_fargate.launch_fargate_workers(**arguments) + + +@pytest.mark.parametrize("bad", [ + {"tasks": [], "failures": [{"reason": "RESOURCE:CPU"}]}, + {"tasks": []}, {"tasks": [{}]}, {"tasks": "bad"}, None, + {"tasks": [{"taskArn": 123}]}, {"tasks": [{"taskArn": " "}]}, +]) +def test_fargate_failed_or_malformed_launch_retains_prior_tasks(monkeypatch, bad): + calls = [] + def run(**kwargs): + calls.append(kwargs) + return {"tasks": [{"taskArn": "arn:first"}], "failures": []} if len(calls) == 1 else bad + aws_stub(monkeypatch, aws_fargate, run) + with pytest.raises(RuntimeError) as caught: + aws_fargate.launch_fargate_workers("cluster", "definition", 3, ["s"]) + assert caught.value.failed_workers == [1] + assert caught.value.responses[0]["tasks"][0]["taskArn"] == "arn:first" + assert len(calls) == 2 + assert calls[1]["count"] == 1 + assert calls[1]["overrides"]["containerOverrides"][0]["environment"][0]["value"] == "1" + + +def test_fargate_response_with_tasks_and_failures_preserves_accepted_task(monkeypatch): + aws_stub(monkeypatch, aws_fargate, lambda **kwargs: { + "tasks": [{"taskArn": "arn:accepted"}], "failures": [{"reason": "partial"}]}) + with pytest.raises(RuntimeError) as caught: + aws_fargate.launch_fargate_workers("c", "t", 1, ["s"]) + assert caught.value.responses[0]["tasks"][0]["taskArn"] == "arn:accepted" + assert caught.value.failed_workers == [0] + + +@pytest.mark.parametrize("module", [aws_fargate, aws_lambda]) +@pytest.mark.parametrize("reason", ["credentials", "permission denied", "timeout", "throttled", "service error"]) +def test_aws_service_errors_preserve_cause_without_resending(monkeypatch, module, reason): + calls = [] + error = ServiceFailure(reason) + def fail(**kwargs): + calls.append(kwargs) + raise error + aws_stub(monkeypatch, module, fail) + with pytest.raises(RuntimeError) as caught: + if module is aws_fargate: + module.launch_fargate_workers("c", "t", 1, ["s"]) + else: + module.invoke_lambda_workers("f", 1, {}) + assert caught.value.__cause__ is error + assert caught.value.failed_workers == [0] + assert caught.value.responses == [] + assert len(calls) == 1 + + +@pytest.mark.parametrize("response", [ + {"StatusCode": 200, "FunctionError": "Unhandled", "Payload": b'{"errorMessage":"bad"}'}, + {"StatusCode": 500, "Payload": b'{}'}, {"Payload": b'{}'}, + {"StatusCode": 200, "Payload": b'[]'}, {"StatusCode": 200, "Payload": b'null'}, + {"StatusCode": 200, "Payload": b'\xff'}, None, + {"StatusCode": 200}, {"StatusCode": 200, "Payload": b""}, + {"StatusCode": 200, "FunctionError": "", "Payload": b"{}"}, +]) +def test_lambda_bad_responses_fail_with_worker_identity(monkeypatch, response): + aws_stub(monkeypatch, aws_lambda, lambda **kwargs: response) + with pytest.raises(RuntimeError) as caught: + aws_lambda.invoke_lambda_workers("f", 1, {}) + assert caught.value.failed_workers == [0] + + +def test_lambda_event_reports_acceptance_without_execution_success(monkeypatch): + calls = [] + def invoke(**kwargs): + calls.append(kwargs) + return {"StatusCode": 202, "Payload": io.BytesIO(b"")} + aws_stub(monkeypatch, aws_lambda, invoke) + template = {"tasks": [{"url": "https://example.invalid"}]} + result = aws_lambda.invoke_lambda_workers("f", 2, template, invocation_type="Event") + assert result == [{"status": "accepted", "worker_index": 0, "StatusCode": 202}, + {"status": "accepted", "worker_index": 1, "StatusCode": 202}] + assert all(call["FunctionName"] == "f" and call["InvocationType"] == "Event" for call in calls) + assert sorted(json.loads(call["Payload"])["worker_index"] for call in calls) == [0, 1] + assert all(json.loads(call["Payload"])["worker_count"] == 2 for call in calls) + assert template == {"tasks": [{"url": "https://example.invalid"}]} + + +def test_lambda_partial_failure_collects_other_results_and_closes_streams(monkeypatch): + streams = [] + calls = [] + def invoke(**kwargs): + index = json.loads(kwargs["Payload"])["worker_index"] + calls.append(index) + stream = io.BytesIO(json.dumps({"worker": index}).encode()) + streams.append(stream) + response = {"StatusCode": 200, "Payload": stream} + if index == 1: + response["FunctionError"] = "Handled" + return response + aws_stub(monkeypatch, aws_lambda, invoke) + with pytest.raises(RuntimeError) as caught: + aws_lambda.invoke_lambda_workers("f", 3, {}) + assert caught.value.responses == [{"worker": 0}, {"worker": 2}] + assert caught.value.failed_workers == [1] + assert caught.value.response["worker_payload"] == {"worker": 1} + assert sorted(calls) == [0, 1, 2] + assert all(stream.closed for stream in streams) + + +def test_lambda_stream_timeout_retains_other_results_and_closes_payload(monkeypatch): + error = ServiceFailure("read timeout") + class Stream(io.BytesIO): + def read(self): + raise error + stream = Stream() + def invoke(**kwargs): + index = json.loads(kwargs["Payload"])["worker_index"] + return {"StatusCode": 200, "Payload": stream if index else b'{"worker":0}'} + aws_stub(monkeypatch, aws_lambda, invoke) + with pytest.raises(RuntimeError) as caught: + aws_lambda.invoke_lambda_workers("f", 2, {}) + assert caught.value.responses == [{"worker": 0}] + assert caught.value.failed_workers == [1] + assert caught.value.__cause__ is error + assert stream.closed + + +@pytest.mark.parametrize("invocation,status", [("RequestResponse", 200), ("Event", 202), ("DryRun", 204)]) +@pytest.mark.parametrize("malformed_attribute", ["read", "close"]) +def test_lambda_noncallable_payload_methods_retain_all_other_worker_results( + monkeypatch, invocation, status, malformed_attribute): + calls = [] + successful_streams = [] + malformed_closed = [] + bad = SimpleNamespace(read=lambda: b"{}", close=lambda: malformed_closed.append(True)) + setattr(bad, malformed_attribute, 123) + def invoke(**kwargs): + index = json.loads(kwargs["Payload"])["worker_index"] + calls.append(index) + if index == 1: + return {"StatusCode": status, "Payload": bad} + body = json.dumps({"worker": index}).encode() if invocation == "RequestResponse" else b"" + stream = io.BytesIO(body) + successful_streams.append(stream) + return {"StatusCode": status, "Payload": stream} + aws_stub(monkeypatch, aws_lambda, invoke) + with pytest.raises(RuntimeError) as caught: + aws_lambda.invoke_lambda_workers("f", 3, {}, invocation_type=invocation) + if invocation == "RequestResponse": + assert caught.value.responses == [{"worker": 0}, {"worker": 2}] + else: + state = "accepted" if invocation == "Event" else "validated" + assert caught.value.responses == [{"status": state, "worker_index": 0, "StatusCode": status}, + {"status": state, "worker_index": 2, "StatusCode": status}] + assert caught.value.failed_workers == [1] + assert isinstance(caught.value.__cause__, ValueError) + assert sorted(calls) == [0, 1, 2] + assert all(stream.closed for stream in successful_streams) + assert malformed_closed == ([True] if malformed_attribute == "read" else []) + + +def test_lambda_sdk_internal_type_error_propagates_without_reclassification(monkeypatch): + error = TypeError("controlled SDK internal defect") + calls = [] + def invoke(**kwargs): + calls.append(kwargs) + raise error + aws_stub(monkeypatch, aws_lambda, invoke) + with pytest.raises(TypeError) as caught: + aws_lambda.invoke_lambda_workers("f", 1, {}) + assert caught.value is error + assert len(calls) == 1 + + +@pytest.mark.parametrize("invocation,status", [("Event", 200), ("DryRun", 200)]) +def test_lambda_acceptance_modes_reject_wrong_status(monkeypatch, invocation, status): + aws_stub(monkeypatch, aws_lambda, lambda **kwargs: {"StatusCode": status}) + with pytest.raises(RuntimeError) as caught: + aws_lambda.invoke_lambda_workers("f", 1, {}, invocation_type=invocation) + assert caught.value.failed_workers == [0] + + +def test_lambda_dry_run_reports_validation(monkeypatch): + aws_stub(monkeypatch, aws_lambda, lambda **kwargs: {"StatusCode": 204}) + assert aws_lambda.invoke_lambda_workers("f", 1, {}, invocation_type="DryRun") == [ + {"status": "validated", "worker_index": 0, "StatusCode": 204}] + + +@pytest.mark.parametrize("kwargs", [ + {"invocation_type": "invalid"}, {"payload_template": []}, {"function_name": ""}, +]) +def test_lambda_invalid_configuration_fails_before_sdk(monkeypatch, kwargs): + monkeypatch.setattr(aws_lambda, "_import_boto3", lambda: pytest.fail("SDK reached")) + arguments = dict(function_name="f", workers=1, payload_template={}) + arguments.update(kwargs) + with pytest.raises(ValueError): + aws_lambda.invoke_lambda_workers(**arguments) + + +@pytest.mark.parametrize("kwargs", [ + {"parallelism": 2}, {"task_count": 0}, {"task_count": True}, {"task_count": 1.5}, + {"timeout": 0}, {"timeout": float("nan")}, {"timeout": float("inf")}, + {"container_overrides": [{}]}, {"container_overrides": [{"name": "c", "parallelism": 2}]}, + {"container_overrides": [{"name": "c", "args": [1]}]}, {"project": ""}, + {"container_overrides": [{"env": [{"name": "N", "value": "x", "valueSource": {}}]}]}, + {"container_overrides": [{"env": [{"name": "N", "valueSource": "invalid"}]}]}, + {"container_overrides": [{"env": [{"name": "N", "valueSource": {"unknown": "bad"}}]}]}, +]) +def test_cloud_run_invalid_configuration_fails_before_auth(monkeypatch, kwargs): + monkeypatch.setattr(gcp_cloud_run, "_bearer_token", lambda scopes: pytest.fail("auth reached")) + arguments = dict(project="p", region="r", job_name="j") + arguments.update(kwargs) + with pytest.raises(ValueError): + gcp_cloud_run.run_cloud_run_job(**arguments) + + +@pytest.mark.parametrize("body", [b'[]', b'null', b'{}', b'not json', b'\xff', + b'{"name":"operations/x","error":{"code":13}}']) +def test_cloud_run_bad_operation_is_failure(monkeypatch, body): + monkeypatch.setattr(gcp_cloud_run, "_bearer_token", lambda scopes: "token") + monkeypatch.setattr(gcp_cloud_run.urllib.request, "urlopen", lambda req, timeout: io.BytesIO(body)) + with pytest.raises(RuntimeError): + gcp_cloud_run.run_cloud_run_job("p", "r", "j") + + +def test_cloud_run_refreshes_credentials_for_each_run(monkeypatch): + refreshed = [] + credentials = SimpleNamespace(token=None) + def refresh(request): + refreshed.append(request) + credentials.token = "token-" + str(len(refreshed)) + credentials.refresh = refresh + auth = SimpleNamespace(default=lambda scopes: (credentials, "project")) + transport = SimpleNamespace(Request=lambda: "refresh-request") + monkeypatch.setattr(gcp_cloud_run, "_import_google_auth", lambda: (auth, transport)) + headers = [] + def open_response(request, timeout): + headers.append(request.get_header("Authorization")) + assert timeout == 30.0 + assert json.loads(request.data) == {"overrides": {"taskCount": 3}} + return io.BytesIO(b'{"name":"operations/x"}') + monkeypatch.setattr(gcp_cloud_run.urllib.request, "urlopen", open_response) + gcp_cloud_run.run_cloud_run_job("p", "r", "j", task_count=3) + gcp_cloud_run.run_cloud_run_job("p", "r", "j", task_count=3) + assert headers == ["Bearer token-1", "Bearer token-2"] + assert refreshed == ["refresh-request", "refresh-request"] + + +def test_cloud_run_empty_refreshed_token_fails_before_http(monkeypatch): + credentials = SimpleNamespace(token=None, refresh=lambda request: None) + monkeypatch.setattr(gcp_cloud_run, "_import_google_auth", lambda: ( + SimpleNamespace(default=lambda scopes: (credentials, "p")), SimpleNamespace(Request=lambda: None))) + monkeypatch.setattr(gcp_cloud_run.urllib.request, "urlopen", lambda *args, **kwargs: pytest.fail("HTTP reached")) + with pytest.raises(RuntimeError, match="token"): + gcp_cloud_run.run_cloud_run_job("p", "r", "j") + + +@pytest.mark.parametrize("stage", ["default", "refresh", "http"]) +def test_cloud_run_credentials_and_timeout_errors_propagate_once(monkeypatch, stage): + error = ServiceFailure(stage) + calls = [] + def fail(*args, **kwargs): + calls.append(stage) + raise error + credentials = SimpleNamespace(token="token", refresh=fail if stage == "refresh" else lambda req: None) + auth = SimpleNamespace(default=fail if stage == "default" else lambda scopes: (credentials, "p")) + monkeypatch.setattr(gcp_cloud_run, "_import_google_auth", lambda: (auth, SimpleNamespace(Request=lambda: None))) + monkeypatch.setattr(gcp_cloud_run.urllib.request, "urlopen", fail if stage == "http" else + lambda *args, **kwargs: pytest.fail("authentication failure reached HTTP")) + with pytest.raises(ServiceFailure) as caught: + gcp_cloud_run.run_cloud_run_job("p", "r", "j") + assert caught.value is error + assert calls == [stage] + + +@pytest.mark.parametrize("status", [401, 403, 429, 500]) +def test_cloud_run_real_local_http_errors_propagate_without_retry(monkeypatch, status): + requests = [] + class Handler(BaseHTTPRequestHandler): + def do_POST(self): + requests.append((self.path, self.headers["Authorization"], + json.loads(self.rfile.read(int(self.headers["Content-Length"]))))) + self.send_response(status) + self.end_headers() + self.wfile.write(b'{"error":{"message":"denied"}}') + def log_message(self, *args): + pass + server = HTTPServer(("127.0.0.1", 0), Handler) + thread = Thread(target=server.serve_forever, daemon=True) + thread.start() + original_open = gcp_cloud_run.urllib.request.urlopen + def local_open(request, timeout): + request.full_url = f"http://127.0.0.1:{server.server_port}/job:run" + return original_open(request, timeout=timeout) + monkeypatch.setattr(gcp_cloud_run, "_bearer_token", lambda scopes: "local-token") + monkeypatch.setattr(gcp_cloud_run.urllib.request, "urlopen", local_open) + try: + with pytest.raises(HTTPError) as caught: + gcp_cloud_run.run_cloud_run_job("p", "r", "j", task_count=2) + assert caught.value.code == status + caught.value.close() + assert requests == [("/job:run", "Bearer local-token", {"overrides": {"taskCount": 2}})] + finally: + server.shutdown() + server.server_close() + thread.join(timeout=2) + + +class Model: + def __init__(self, **kwargs): + self.__dict__.update(kwargs) + + +def aci_stub(monkeypatch, create): + models = {name: Model for name in ["Container", "ContainerGroup", "EnvironmentVariable", + "ResourceRequests", "ResourceRequirements"]} + models.update(DefaultAzureCredential=lambda: "credential", + Client=lambda credential, subscription: SimpleNamespace( + container_groups=SimpleNamespace(begin_create_or_update=create)), + RestartPolicy=SimpleNamespace(never="Never"), OSType=SimpleNamespace(linux="Linux")) + monkeypatch.setattr(azure_aci, "_import_azure", lambda: models) + + +@pytest.mark.parametrize("kwargs", [ + {"cpu": 0}, {"cpu": True}, {"cpu": float("inf")}, {"memory_gb": -1}, + {"memory_gb": float("nan")}, {"name_prefix": "Bad Prefix"}, {"name_prefix": "x" * 64}, + {"overrides_env": {"LD_WORKER_COUNT": "9"}}, {"image": ""}, + {"overrides_env": {"BAD NAME": "x"}}, {"overrides_env": {"NAME_": "x"}}, + {"overrides_env": {"N" * 64: "x"}}, +]) +def test_aci_invalid_resource_parameters_fail_before_sdk(monkeypatch, kwargs): + monkeypatch.setattr(azure_aci, "_import_azure", lambda: pytest.fail("SDK reached")) + with pytest.raises(ValueError): + azure_aci.launch_aci_workers("sub", "rg", "region", kwargs.pop("image", "image"), 1, **kwargs) + + +def test_aci_names_are_unique_across_launches_and_provisioning_is_checked(monkeypatch): + calls = [] + polled = [] + def create(**kwargs): + calls.append(kwargs) + def result(): + polled.append(kwargs["container_group_name"]) + return SimpleNamespace(id="resource/" + kwargs["container_group_name"], provisioning_state="Succeeded") + return SimpleNamespace(result=result) + aci_stub(monkeypatch, create) + first = azure_aci.launch_aci_workers("sub", "rg", "region", "image", 2, cpu=2, memory_gb=4) + second = azure_aci.launch_aci_workers("sub", "rg", "region", "image", 2) + names = [entry["name"] for entry in first + second] + assert len(set(names)) == 4 + assert polled == names + assert all(entry["status"] == "Succeeded" and entry["resource_id"] for entry in first + second) + group = calls[1]["container_group"] + assert group.containers[0].resources.requests.cpu == 2 + assert group.containers[0].resources.requests.memory_in_gb == 4 + assert group.location == "region" and group.restart_policy == "Never" + assert calls[1]["resource_group_name"] == "rg" + assert {item.name: item.value for item in group.containers[0].environment_variables} == { + "LD_WORKER_INDEX": "1", "LD_WORKER_COUNT": "2"} + + +def test_aci_prefix_allows_internal_consecutive_hyphens(monkeypatch): + aci_stub(monkeypatch, lambda **kwargs: SimpleNamespace(result=lambda: SimpleNamespace( + id="id", provisioning_state="Succeeded"))) + responses = azure_aci.launch_aci_workers("sub", "rg", "region", "image", 1, name_prefix="a--b") + assert responses[0]["name"].startswith("a--b-") + + +@pytest.mark.parametrize("reason", ["credentials", "permission", "timeout", "throttle", "service"]) +def test_aci_poller_failure_retains_prior_success_and_never_resubmits(monkeypatch, reason): + calls = [] + error = ServiceFailure(reason) + def create(**kwargs): + calls.append(kwargs) + index = len(calls) - 1 + def result(): + if index == 1: + raise error + return SimpleNamespace(id="id/first", provisioning_state="Succeeded") + return SimpleNamespace(result=result) + aci_stub(monkeypatch, create) + with pytest.raises(RuntimeError) as caught: + azure_aci.launch_aci_workers("sub", "rg", "region", "image", 3) + assert caught.value.__cause__ is error + assert caught.value.failed_workers == [1] + assert caught.value.responses[0]["resource_id"] == "id/first" + assert len(calls) == 2 + + +@pytest.mark.parametrize("result", [None, SimpleNamespace(id="id", provisioning_state="Failed"), + SimpleNamespace(id=None, provisioning_state="Succeeded")]) +def test_aci_malformed_or_failed_provisioning_is_not_success(monkeypatch, result): + aci_stub(monkeypatch, lambda **kwargs: SimpleNamespace(result=lambda: result)) + with pytest.raises(RuntimeError) as caught: + azure_aci.launch_aci_workers("sub", "rg", "region", "image", 1) + assert caught.value.failed_workers == [0] + + +def test_aci_malformed_poller_retains_attempted_resource_name(monkeypatch): + calls = [] + aci_stub(monkeypatch, lambda **kwargs: calls.append(kwargs) or None) + with pytest.raises(RuntimeError) as caught: + azure_aci.launch_aci_workers("sub", "rg", "region", "image", 1) + assert caught.value.response["name"] == calls[0]["container_group_name"] + assert len(calls) == 1 diff --git a/test/test_cloud_launchers.py b/test/test_cloud_launchers.py index bc44b51..a7699ae 100644 --- a/test/test_cloud_launchers.py +++ b/test/test_cloud_launchers.py @@ -27,7 +27,8 @@ def client(self, service, region_name=None): def test_fargate_runs_one_task_per_worker_with_its_index(monkeypatch): calls = [] - ecs = SimpleNamespace(run_task=lambda **kwargs: calls.append(kwargs) or {"tasks": [{}]}) + ecs = SimpleNamespace(run_task=lambda **kwargs: calls.append(kwargs) or { + "tasks": [{"taskArn": "arn:aws:ecs:region:account:task/cluster/task-id"}], "failures": []}) boto3 = _FakeBoto3(ecs) monkeypatch.setattr(aws_fargate, "_import_boto3", lambda: boto3) responses = aws_fargate.launch_fargate_workers( @@ -48,7 +49,7 @@ def invoke(FunctionName, InvocationType, Payload): # noqa: N803 - boto3 keyword payload = json.loads(Payload) # Later workers answer first, so completion order is the reverse of worker order. time.sleep(0.05 * (3 - payload["worker_index"])) - return {"Payload": io.BytesIO(json.dumps({"worker": payload["worker_index"], + return {"StatusCode": 200, "Payload": io.BytesIO(json.dumps({"worker": payload["worker_index"], "of": payload["worker_count"], "url": payload["url"]}).encode("utf-8"))} @@ -59,7 +60,7 @@ def invoke(FunctionName, InvocationType, Payload): # noqa: N803 - boto3 keyword def test_lambda_non_json_payload_is_returned_raw(monkeypatch): - fake = SimpleNamespace(invoke=lambda **_kwargs: {"Payload": b"not json"}) + fake = SimpleNamespace(invoke=lambda **_kwargs: {"StatusCode": 200, "Payload": b"not json"}) monkeypatch.setattr(aws_lambda, "_import_boto3", lambda: _FakeBoto3(fake)) assert aws_lambda.invoke_lambda_workers("f", workers=1, payload_template={}) == [{"raw": "not json"}] @@ -98,11 +99,11 @@ def fake_urlopen(request, timeout): monkeypatch.setattr(gcp_cloud_run, "_bearer_token", lambda _scopes: "tok") monkeypatch.setattr(gcp_cloud_run.urllib.request, "urlopen", fake_urlopen) - result = gcp_cloud_run.run_cloud_run_job("proj", "asia-east1", "ld/job?x", parallelism=4, task_count=8) + result = gcp_cloud_run.run_cloud_run_job("proj", "asia-east1", "ld/job?x", task_count=8) assert result == {"name": "operations/1"} assert sent["url"] == "https://run.googleapis.com/v2/projects/proj/locations/asia-east1/jobs/ld%2Fjob%3Fx:run" assert sent["headers"]["Authorization"] == "Bearer tok" - assert sent["body"] == {"overrides": {"parallelism": 4, "taskCount": 8}} + assert sent["body"] == {"overrides": {"taskCount": 8}} def test_aci_creates_one_group_per_worker(monkeypatch): @@ -112,7 +113,9 @@ def test_aci_creates_one_group_per_worker(monkeypatch): def record(**kwargs): with lock: created.append(kwargs) - return object() + return SimpleNamespace(result=lambda: SimpleNamespace( + id="/subscriptions/sub/resourceGroups/rg/containerGroups/" + kwargs["container_group_name"], + provisioning_state="Succeeded")) class _Model: def __init__(self, **kwargs): @@ -129,7 +132,10 @@ def __init__(self, **kwargs): monkeypatch.setattr(azure_aci, "_import_azure", lambda: fake) responses = azure_aci.launch_aci_workers("sub", "rg", "eastasia", "ld:latest", workers=2, overrides_env={"TARGET": "https://x"}) - assert [response["name"] for response in responses] == ["loaddensity-worker-0", "loaddensity-worker-1"] + assert responses[0]["name"].startswith("loaddensity-worker-") + assert responses[0]["name"].endswith("-0") and responses[1]["name"].endswith("-1") + assert responses[0]["name"] != responses[1]["name"] + assert all(response["status"] == "Succeeded" for response in responses) group = created[1]["container_group"] assert created[1]["resource_group_name"] == "rg" assert group.restart_policy == "Never" and group.os_type == "Linux" diff --git a/test/test_cloud_sdk_contracts.py b/test/test_cloud_sdk_contracts.py new file mode 100644 index 0000000..1129893 --- /dev/null +++ b/test/test_cloud_sdk_contracts.py @@ -0,0 +1,180 @@ +"""Optional extra-specific contracts against official SDK serializers/stubs. + +The dependency-free cloud contracts run on every installation. These tests also +run when the relevant cloud extra is installed; no SDK request reaches a cloud. +""" + +import io +import json +from types import SimpleNamespace + +import pytest + +from je_load_density.cloud import aws_fargate, aws_lambda, azure_aci, gcp_cloud_run + + +def aws_client(service): + boto3 = pytest.importorskip("boto3", reason="official AWS contract requires aws/cloud extra") + return boto3.client(service, region_name="us-east-1", aws_access_key_id="test-key", + aws_secret_access_key="test-secret", endpoint_url="https://aws.invalid") + + +def test_official_ecs_stubber_validates_payload_and_reports_failures(monkeypatch): + client = aws_client("ecs") + from botocore.stub import Stubber + monkeypatch.setattr(aws_fargate, "_import_boto3", lambda: SimpleNamespace(client=lambda *a, **k: client)) + expected = { + "cluster": "cluster", "taskDefinition": "definition", "launchType": "FARGATE", "count": 1, + "networkConfiguration": {"awsvpcConfiguration": { + "subnets": ["subnet-1"], "securityGroups": ["sg-1"], "assignPublicIp": "ENABLED"}}, + "overrides": {"containerOverrides": [{"name": "worker", "environment": [ + {"name": "TARGET", "value": "https://target.invalid"}, + {"name": "LD_WORKER_INDEX", "value": "0"}, {"name": "LD_WORKER_COUNT", "value": "1"}]}]}, + } + with Stubber(client) as stub: + stub.add_response("run_task", {"tasks": [], "failures": [{"reason": "RESOURCE:CPU"}]}, expected) + with pytest.raises(RuntimeError) as caught: + aws_fargate.launch_fargate_workers("cluster", "definition", 1, ["subnet-1"], ["sg-1"], + True, container_name="worker", + overrides_env={"TARGET": "https://target.invalid"}) + assert caught.value.failed_workers == [0] + assert caught.value.response["failures"][0]["reason"] == "RESOURCE:CPU" + stub.assert_no_pending_responses() + + +@pytest.mark.parametrize("code", ["AccessDeniedException", "TooManyRequestsException", "ServiceException"]) +def test_official_lambda_service_errors_retain_client_error(monkeypatch, code): + client = aws_client("lambda") + from botocore.exceptions import ClientError + from botocore.stub import Stubber + monkeypatch.setattr(aws_lambda, "_import_boto3", lambda: SimpleNamespace(client=lambda *a, **k: client)) + expected = {"FunctionName": "worker", "InvocationType": "RequestResponse", + "Payload": b'{"worker_index": 0, "worker_count": 1}'} + with Stubber(client) as stub: + stub.add_client_error("invoke", service_error_code=code, service_message="controlled", + http_status_code={"AccessDeniedException": 403, "TooManyRequestsException": 429, + "ServiceException": 500}[code], expected_params=expected) + with pytest.raises(RuntimeError) as caught: + aws_lambda.invoke_lambda_workers("worker", 1, {}) + assert isinstance(caught.value.__cause__, ClientError) + assert caught.value.__cause__.response["Error"]["Code"] == code + assert caught.value.responses == [] + stub.assert_no_pending_responses() + + +@pytest.mark.parametrize("failure", ["credentials", "timeout"]) +def test_official_aws_credential_and_timeout_exceptions_are_chained(monkeypatch, failure): + client = aws_client("lambda") + from botocore.exceptions import NoCredentialsError, ReadTimeoutError + error = NoCredentialsError() if failure == "credentials" else ReadTimeoutError(endpoint_url="https://aws.invalid") + def fail(**kwargs): + raise error + monkeypatch.setattr(client, "invoke", fail) + monkeypatch.setattr(aws_lambda, "_import_boto3", lambda: SimpleNamespace(client=lambda *a, **k: client)) + with pytest.raises(RuntimeError) as caught: + aws_lambda.invoke_lambda_workers("worker", 1, {}) + assert caught.value.__cause__ is error + assert caught.value.failed_workers == [0] + + +def test_official_lambda_streamingbody_closes_on_function_error(monkeypatch): + client = aws_client("lambda") + from botocore.response import StreamingBody + from botocore.stub import Stubber + body = b'{"errorMessage":"controlled function error"}' + raw = io.BytesIO(body) + stream = StreamingBody(raw, len(body)) + monkeypatch.setattr(aws_lambda, "_import_boto3", lambda: SimpleNamespace(client=lambda *a, **k: client)) + with Stubber(client) as stub: + stub.add_response("invoke", {"StatusCode": 200, "FunctionError": "Unhandled", "Payload": stream}, + {"FunctionName": "worker", "InvocationType": "RequestResponse", + "Payload": b'{"worker_index": 0, "worker_count": 1}'}) + with pytest.raises(RuntimeError) as caught: + aws_lambda.invoke_lambda_workers("worker", 1, {}) + assert caught.value.response["FunctionError"] == "Unhandled" + assert raw.closed + stub.assert_no_pending_responses() + + +def test_official_azure_serialization_and_poller_result_without_network(monkeypatch): + pytest.importorskip("azure.identity", reason="official Azure contract requires azure/cloud extra") + pytest.importorskip("azure.mgmt.containerinstance", reason="official Azure contract requires azure/cloud extra") + from azure.core.credentials import AccessToken + from azure.core.pipeline.transport import HttpResponse, HttpTransport + from azure.mgmt.containerinstance import ContainerInstanceManagementClient + + requests = [] + class Response(HttpResponse): + def __init__(self, request): + super().__init__(request, None) + self.status_code = 201 + self.headers = {"Content-Type": "application/json"} + def body(self): + resource_id = self.request.url.split("?")[0].removeprefix("https://management.azure.com") + return json.dumps({"id": resource_id, "name": resource_id.rsplit("/", 1)[-1], "location": "eastus", + "properties": {"provisioningState": "Succeeded", "osType": "Linux", + "containers": []}}).encode() + class Transport(HttpTransport): + def open(self): + pass + def close(self): + pass + def __enter__(self): + return self + def __exit__(self, *args): + pass + def send(self, request, **kwargs): + requests.append(request) + return Response(request) + credential = SimpleNamespace(get_token=lambda *args, **kwargs: AccessToken("stub-token", 9999999999)) + official_models = azure_aci._import_azure() + official_models["DefaultAzureCredential"] = lambda: credential + official_models["Client"] = lambda cred, sub: ContainerInstanceManagementClient( + cred, sub, transport=Transport(), retry_total=0) + monkeypatch.setattr(azure_aci, "_import_azure", lambda: official_models) + result = azure_aci.launch_aci_workers("sub", "rg", "eastus", "image:v1", 1, cpu=2, memory_gb=3, + overrides_env={"TARGET": "https://target.invalid"}) + assert result[0]["status"] == "Succeeded" + assert result[0]["resource_id"].endswith("/containerGroups/" + result[0]["name"]) + assert len(requests) == 1 and requests[0].method == "PUT" + resource_path = f"/resourceGroups/rg/providers/Microsoft.ContainerInstance/containerGroups/{result[0]['name']}?" + assert resource_path in requests[0].url + payload = json.loads(requests[0].body) + assert payload["location"] == "eastus" + assert payload["properties"]["restartPolicy"] == "Never" + assert payload["properties"]["containers"][0]["properties"]["resources"]["requests"] == { + "cpu": 2.0, "memoryInGB": 3.0} + assert requests[0].headers["Authorization"] == "Bearer stub-token" + + +@pytest.mark.parametrize("failure", ["credentials", "permission", "timeout"]) +def test_official_azure_errors_are_chained(monkeypatch, failure): + pytest.importorskip("azure.core", reason="official Azure errors require azure/cloud extra") + from azure.core.exceptions import ClientAuthenticationError, HttpResponseError, ServiceRequestError + errors = {"credentials": ClientAuthenticationError, "permission": HttpResponseError, "timeout": ServiceRequestError} + error = errors[failure]("controlled") + def create(**kwargs): + return SimpleNamespace(result=lambda: fail()) + def fail(): + raise error + from test.test_cloud_contracts import aci_stub + aci_stub(monkeypatch, create) + with pytest.raises(RuntimeError) as caught: + azure_aci.launch_aci_workers("sub", "rg", "eastus", "image", 1) + assert caught.value.__cause__ is error + assert caught.value.failed_workers == [0] + + +def test_official_google_refresh_error_prevents_http(monkeypatch): + pytest.importorskip("google.auth", reason="official Google errors require gcp/cloud extra") + from google.auth.exceptions import RefreshError + error = RefreshError("controlled refresh failure") + def refresh(request): + raise error + credentials = SimpleNamespace(token=None, refresh=refresh) + monkeypatch.setattr(gcp_cloud_run, "_import_google_auth", lambda: ( + SimpleNamespace(default=lambda scopes: (credentials, "p")), SimpleNamespace(Request=lambda: None))) + monkeypatch.setattr(gcp_cloud_run.urllib.request, "urlopen", lambda *args, **kwargs: pytest.fail("HTTP reached")) + with pytest.raises(RefreshError) as caught: + gcp_cloud_run.run_cloud_run_job("p", "r", "j") + assert caught.value is error From b6582db759477b0213195f0a5408db1590648383 Mon Sep 17 00:00:00 2001 From: JeffreyChen Date: Fri, 2 Oct 2026 20:24:54 +0800 Subject: [PATCH 10/18] feat: monitor worker health and rebalance distributed load --- README.md | 18 +- README/README_zh-CN.md | 15 +- README/README_zh-TW.md | 15 +- architecture.md | 9 + .../En/doc/distributed/distributed_doc.rst | 19 +- .../Zh/doc/distributed/distributed_doc.rst | 15 +- .../plans/2026-10-02-distributed-health.md | 55 +++ docs/updates/2026-10.md | 7 + docs/updates/README.md | 1 + .../create_locust_env/create_locust_env.py | 248 +++++++++--- je_load_density/wrapper/distributed_config.py | 75 ++++ je_load_density/wrapper/distributed_health.py | 149 +++++++ .../wrapper/start_wrapper/start_test.py | 115 +++--- progress.md | 2 +- test/test_distributed_health.py | 374 ++++++++++++++++++ 15 files changed, 1003 insertions(+), 114 deletions(-) create mode 100644 docs/superpowers/plans/2026-10-02-distributed-health.md create mode 100644 je_load_density/wrapper/distributed_config.py create mode 100644 je_load_density/wrapper/distributed_health.py create mode 100644 test/test_distributed_health.py diff --git a/README.md b/README.md index e93d7cb..4078c22 100644 --- a/README.md +++ b/README.md @@ -88,7 +88,7 @@ LoadDensity (`je_load_density`) started as a Locust wrapper and grew into a full - **Live web dashboard.** `start_dashboard()` boots a stdlib HTTP + SSE server that streams running RPS / avg / p95 / failure counts to any browser, per-name table included. - **Slack + Teams notifiers.** Block Kit + MessageCard summary posters keyed off the build_summary output (`LD_post_slack_summary`, `LD_post_teams_summary`). - **Assertions + extractors.** `status_code`, `contains`, `not_contains`, `json_path`, `header` assertions run under Locust's `catch_response`; extractors with sources `json_path` / `header` / `status_code` write back into the parameter resolver. -- **Distributed runners.** `runner_mode="master"` / `"worker"` for cross-machine load with the same `start_test` API; master waits up to 60 s for the configured worker count before ramping. +- **Distributed runners.** `runner_mode="master"` / `"worker"` with a configurable healthy-worker startup gate, native heartbeat monitoring and virtual-user rebalancing after worker loss. - **Six importers.** HAR (browser traffic), Postman v2.1 collections, OpenAPI 3.x specs, standalone cURL commands, **k6 scripts**, and **JMeter JMX** plans — each converts to action JSON or a single task ready for `LD_start_test`. - **Auth helpers.** Stdlib OAuth2 client (`client_credentials` / `password` / `refresh` with token cache), JWT signer (HS256/384/512 + RS256/384/512), AWS SigV4 request signer, plus mTLS client-cert support on every HTTP user template via `task["cert"]`. - **Persistent records.** Optional SQLite sink with `runs` / `records` / `metadata` schema, indexed for cross-run regression checks; works against an empty file out of the box. @@ -669,7 +669,21 @@ start_test( ) ``` -The master waits up to 60 s for `expected_workers` workers to register before starting the load ramp. +The master waits for healthy ready workers before ramping. Defaults are +`worker_startup_timeout=60`, `worker_heartbeat_interval=5`, `worker_lost_timeout=15` +seconds and `worker_startup_policy="fail"`. An unmet worker count raises +`TimeoutError` after cleanup. Explicit `"degraded"` policy permits a shortfall, +but at least one ready worker is required, including when `expected_workers=0`. +Use matching heartbeat settings on every node; loss detection follows interval ticks. +Locust rebalances virtual-user capacity after loss/reconnection. All workers lost +terminates the run; master results include `distributed_health`, observed capacity +and affected worker IDs. Stateful journeys may restart; requests are not replayed. +Finite-work leases and canonical worker-record aggregation remain pending. + +`on_environment(env)` runs before startup in the execution thread; +`stop_requested()` cooperatively cancels startup, ramp-up or execution. Callback +errors propagate after cleanup. `prepare_env` owns runner/UI/RPC/auxiliary tasks; +direct `create_env` callers must call `cleanup_env(env)` when finished. ## HAR Record / Replay diff --git a/README/README_zh-CN.md b/README/README_zh-CN.md index 01d22c9..2bced03 100644 --- a/README/README_zh-CN.md +++ b/README/README_zh-CN.md @@ -88,7 +88,7 @@ LoadDensity(`je_load_density`)从 Locust 封装起步,逐步成长为完整的 - **实时 web dashboard。** `start_dashboard()` 会启动一个 stdlib HTTP + SSE 服务器,把运行中的 RPS / avg / p95 / failure 计数流式推送到任意浏览器,并附上 per-name 表格。 - **Slack + Teams 通知。** 以 build_summary 的输出为基础的 Block Kit + MessageCard 摘要投递器(`LD_post_slack_summary`、`LD_post_teams_summary`)。 - **断言 + 提取。** `status_code`、`contains`、`not_contains`、`json_path`、`header` 断言在 Locust 的 `catch_response` 下执行;来源为 `json_path` / `header` / `status_code` 的提取器会把值写回参数解析器。 -- **分布式 runner。** `runner_mode="master"` / `"worker"` 以同一套 `start_test` API 进行跨机负载;master 会先等待配置的 worker 数量最多 60 秒,再开始 ramp。 +- **分布式 runner。** `runner_mode="master"` / `"worker"` 提供可配置的健康 worker 启动门槛、原生 heartbeat 监测与失联后的虚拟用户负载重新分配。 - **六种导入器。** HAR(浏览器流量)、Postman v2.1 collection、OpenAPI 3.x spec、独立的 cURL 命令、**k6 脚本**,以及 **JMeter JMX** plan — 每一种都能转成动作 JSON 或一个可直接喂给 `LD_start_test` 的 task。 - **Auth 辅助工具。** stdlib OAuth2 client(`client_credentials` / `password` / `refresh`,含 token cache)、JWT 签名器(HS256/384/512 + RS256/384/512)、AWS SigV4 请求签名器,再加上每个 HTTP 用户模板都能通过 `task["cert"]` 支持 mTLS client-cert。 - **持久化记录。** 可选的 SQLite sink,采用 `runs` / `records` / `metadata` schema 并建立索引以便跨次回归检查;开箱即可对空文件运作。 @@ -669,7 +669,18 @@ start_test( ) ``` -master 会等待最多 60 秒,让 `expected_workers` 个 worker 完成注册,再开始负载 ramp。 +master 在 ramp 前等待健康且 ready 的 worker。默认秒数为 +`worker_startup_timeout=60`、`worker_heartbeat_interval=5`、`worker_lost_timeout=15`， +启动策略为 `worker_startup_policy="fail"`；人数不足时清理资源后抛出 `TimeoutError`。 +明确选择 `"degraded"` 可接受不足的人数，但至少要有一个 ready worker， +即使 `expected_workers=0` 也适用。每个节点必须使用相同 heartbeat 设置；失联检测依 interval tick 判定。 +Locust 在失联／重连后重新分配虚拟用户；所有 worker 失联时终止执行。 +master 结果包含 `distributed_health`、观测容量与受影响 worker ID。 +有状态流程可能重新开始；不重放请求。有限工作租约与 canonical worker record 汇总仍待实现。 + +`on_environment(env)` 在执行线程、启动前调用；`stop_requested()` 可协作取消启动、ramp 或执行， +callback 错误在清理后传回。`prepare_env` 负责 runner／UI／RPC／辅助 task 的资源生命周期； +直接调用 `create_env` 的用户须在完成后调用 `cleanup_env(env)`。 ## HAR 录制/重放 diff --git a/README/README_zh-TW.md b/README/README_zh-TW.md index 5dcd3e9..ccd7448 100644 --- a/README/README_zh-TW.md +++ b/README/README_zh-TW.md @@ -88,7 +88,7 @@ LoadDensity(`je_load_density`)從 Locust 封裝起家,逐步成長為完整的 - **即時 web dashboard。** `start_dashboard()` 會啟動一個 stdlib HTTP + SSE 伺服器,把執行中的 RPS / avg / p95 / failure 計數串流到任意瀏覽器,並附上 per-name 表格。 - **Slack + Teams 通知。** 以 build_summary 的輸出為基礎的 Block Kit + MessageCard 摘要張貼器(`LD_post_slack_summary`、`LD_post_teams_summary`)。 - **斷言 + 擷取。** `status_code`、`contains`、`not_contains`、`json_path`、`header` 斷言在 Locust 的 `catch_response` 下執行;來源為 `json_path` / `header` / `status_code` 的擷取器會把值寫回參數解析器。 -- **分散式 runner。** `runner_mode="master"` / `"worker"` 以同一套 `start_test` API 進行跨機負載;master 會先等待設定的 worker 數量最多 60 秒,再開始 ramp。 +- **分散式 runner。** `runner_mode="master"` / `"worker"` 提供可設定的健康 worker 啟動門檻、原生 heartbeat 監測與失聯後的虛擬使用者負載重新分配。 - **六種匯入器。** HAR(瀏覽器流量)、Postman v2.1 collection、OpenAPI 3.x spec、獨立的 cURL 指令、**k6 腳本**,以及 **JMeter JMX** plan — 每一種都能轉成動作 JSON 或一個可直接餵給 `LD_start_test` 的 task。 - **Auth 輔助工具。** stdlib OAuth2 client(`client_credentials` / `password` / `refresh`,含 token cache)、JWT 簽章器(HS256/384/512 + RS256/384/512)、AWS SigV4 請求簽章器,再加上每個 HTTP 使用者模板都能透過 `task["cert"]` 支援 mTLS client-cert。 - **持久化紀錄。** 選用的 SQLite sink,採 `runs` / `records` / `metadata` schema 並建立索引以利跨次回歸檢查;開箱即可對空檔運作。 @@ -669,7 +669,18 @@ start_test( ) ``` -master 會等待最多 60 秒,讓 `expected_workers` 個 worker 完成註冊,再開始負載 ramp。 +master 在 ramp 前等待健康且 ready 的 worker。預設秒數為 +`worker_startup_timeout=60`、`worker_heartbeat_interval=5`、`worker_lost_timeout=15`， +啟動策略為 `worker_startup_policy="fail"`；人數不足時清理資源後拋出 `TimeoutError`。 +明確選擇 `"degraded"` 可接受不足的人數，但至少要有一個 ready worker， +即使 `expected_workers=0` 也適用。每個節點必須使用相同 heartbeat 設定；失聯偵測依 interval tick 判定。 +Locust 在失聯／重連後重新分配虛擬使用者；所有 worker 失聯時終止執行。 +master 結果包含 `distributed_health`、觀測容量與受影響 worker ID。 +有狀態流程可能重新開始；不重放請求。有限工作租約與 canonical worker record 彙整仍待實作。 + +`on_environment(env)` 在執行執行緒、啟動前呼叫；`stop_requested()` 可協作取消啟動、ramp 或執行， +callback 錯誤在清理後傳回。`prepare_env` 負責 runner／UI／RPC／輔助 task 的資源生命週期； +直接呼叫 `create_env` 的使用者須在完成後呼叫 `cleanup_env(env)`。 ## HAR 錄製/重放 diff --git a/architecture.md b/architecture.md index 9ed2fc3..e32d681 100644 --- a/architecture.md +++ b/architecture.md @@ -135,6 +135,15 @@ MCP `load_density.list_executor_commands` tool all read the `LD_*` names from `e ## 6. Cross-project boundaries +Locust master/worker runs use scoped native heartbeat settings and rebalancing. +Startup defaults to failing an unmet healthy ready-worker count; explicit degraded +policy still requires one worker. Master results add `distributed_health`; this is +infrastructure health, separate from target request outcomes. `prepare_env` owns +runner/UI/RPC/auxiliary cleanup, including callback failures during ramp-up; +direct `create_env` callers own `cleanup_env`. Lifecycle callbacks are +`on_environment(env)` and `stop_requested()`. No finite-work leases, request replay +or canonical worker-record aggregation are exposed. + CLI execution retains legacy flags and Python executor return shapes, but now returns a nonzero process exit code after failed actions/SLA gates. `test/smoke` runs real HTTP/report/dashboard/MCP checks in subprocesses against source or an installed wheel. Native async HTTP benchmarking requires base httpx; the `http2` extra adds HTTP/2 support. CI derives its Docker installation matrix from declared extras, runs isolated installed-wheel capabilities and smoke checks, and gates publishing on the reusable extras workflow. Compose probes measure Redis/MQTT adapter calls after health checks. The etcd adapter prefers etcd3gw (v3 HTTP gateway) while retaining legacy etcd3 support; this avoids incompatible generated protobuf code in the old extra. diff --git a/docs/source/En/doc/distributed/distributed_doc.rst b/docs/source/En/doc/distributed/distributed_doc.rst index 66a1c54..961fd18 100644 --- a/docs/source/En/doc/distributed/distributed_doc.rst +++ b/docs/source/En/doc/distributed/distributed_doc.rst @@ -33,9 +33,22 @@ Master tasks=[...], ) -The master waits up to 60 s for ``expected_workers`` workers to join -before starting the load ramp. If only N workers (N < expected) join, -it logs a warning and starts anyway. +The master waits for healthy ready workers. Configure ``worker_startup_timeout`` +(default 60 s), ``worker_heartbeat_interval`` (5 s), ``worker_lost_timeout`` (15 s) +and ``worker_startup_policy`` (``"fail"``). An unmet count raises ``TimeoutError`` +after cleanup. Explicit ``"degraded"`` policy permits a shortfall but requires at +least one ready worker, including when ``expected_workers=0``. + +All nodes must use matching heartbeat settings; native detection follows interval +ticks. Locust rebalances virtual-user capacity after loss/reconnection and all +workers lost terminates the run. Master results include ``distributed_health``, +observed capacity and affected IDs. Stateful journeys may restart; requests are +not replayed. Finite-work leases and canonical worker-record aggregation remain pending. + +``on_environment(env)`` runs before startup in the execution thread; +``stop_requested()`` cooperatively cancels startup, ramp-up or execution. Callback +errors propagate after cleanup. ``prepare_env`` owns runner/UI/RPC/auxiliary tasks; +direct ``create_env`` callers must call ``cleanup_env(env)`` when finished. Worker ------ diff --git a/docs/source/Zh/doc/distributed/distributed_doc.rst b/docs/source/Zh/doc/distributed/distributed_doc.rst index 1808b55..abc42c3 100644 --- a/docs/source/Zh/doc/distributed/distributed_doc.rst +++ b/docs/source/Zh/doc/distributed/distributed_doc.rst @@ -41,7 +41,20 @@ Master tasks=[...], ) -Master 在開始 ramp 前最多等待 60 秒，等待 ``expected_workers`` 個 worker 加入。若僅 N（N < expected）人加入，會記錄警告並照常啟動。 +Master 在 ramp 前等待健康且 ready 的 worker。設定包含 ``worker_startup_timeout`` +（預設 60 秒）、``worker_heartbeat_interval``（5 秒）、``worker_lost_timeout``（15 秒） +與 ``worker_startup_policy``（``"fail"``）。人數不足時清理資源後拋出 ``TimeoutError``。 +明確選擇 ``"degraded"`` 可接受不足的人數，但至少須有一個 ready worker， +即使 ``expected_workers=0`` 也適用。 + +每個節點須使用相同 heartbeat 設定；原生失聯偵測依 interval tick 判定。 +Locust 在失聯／重連後重新分配虛擬使用者，所有 worker 失聯時終止執行。 +master 結果包含 ``distributed_health``、觀測容量與受影響 ID。 +有狀態流程可能重新開始；不重放請求。有限工作租約與 canonical worker record 彙整仍待實作。 + +``on_environment(env)`` 在執行執行緒、啟動前呼叫；``stop_requested()`` 可協作取消啟動、ramp 或執行， +callback 錯誤在清理後传回。``prepare_env`` 負責 runner／UI／RPC／輔助 task 的資源生命週期； +直接呼叫 ``create_env`` 的使用者須在完成後呼叫 ``cleanup_env(env)``。 Worker ------ diff --git a/docs/superpowers/plans/2026-10-02-distributed-health.md b/docs/superpowers/plans/2026-10-02-distributed-health.md new file mode 100644 index 0000000..0e0f44f --- /dev/null +++ b/docs/superpowers/plans/2026-10-02-distributed-health.md @@ -0,0 +1,55 @@ +# Distributed Health Implementation Plan + +> **For agentic workers:** Use superpowers:executing-plans to implement this approved plan task by task. + +**Goal:** Fail unhealthy startup, observe worker lifecycle, and reuse Locust redistribution with deterministic cleanup. + +**Architecture:** Configure Locust native heartbeat constants in a scoped process lease and enable native rebalancing. Observe public event hooks and runner client state; do not introduce a heartbeat sender or user allocator. Own auxiliary greenlets and RPC cleanup at the environment boundary. + +**Tech Stack:** Python 3.10+, gevent, installed Locust 2.43.4; official stable source inspected at 2.46.6. + +**Spec:** ../specs/2026-10-02-testing-platform-roadmap.md, distributed health section. + +## Constraints + +- Defaults: worker_startup_timeout=60, worker_heartbeat_interval=5, worker_lost_timeout=15; finite positive values, lost timeout greater than interval. +- Only healthy ready workers satisfy expected_workers. Default startup policy fails and cleans up; explicit degraded policy requires at least one healthy ready worker. +- Redistribute virtual-user capacity using Locust at the existing target and spawn rate; never replay HTTP requests or migrate sessions. +- Locust heartbeat constants are process scoped. Matching concurrent configurations share a lease; conflicting configurations fail before creating a runner. Restore previous constants after final cleanup. +- Finite work shards are not supported by this wrapper. Do not advertise exactly-once requests, finite-work leases, or request-result deduplication. +- Root owns shared README, Sphinx, architecture, progress, update logs, and commits. + +## Tasks + +- [x] Add failing tests for configuration validation, readiness, startup timeout/degraded policy, and scoped settings. +- [x] Add native configuration scope and observable worker lifecycle; make the startup gate fail by default. +- [x] Add failing native-message tests for loss, reconnection, all lost, capacity shortfall, ramp, shapes, and cleanup. +- [x] Enable native rebalancing and preserve target/ramp while exposing health and affected worker journeys. +- [x] Add cooperative stop_requested and on_environment hooks; test callback order and cancellation cleanup. +- [x] Run focused tests and report verification evidence and limits to the root for shared documentation. + +## References + +- https://docs.locust.io/en/stable/_modules/locust/runners.html +- https://docs.locust.io/en/stable/api.html#event-hooks +- https://docs.locust.io/en/stable/configuration.html + +## Timing and capacity limits + +Native heartbeat checks occur on interval ticks; lost detection is quantized to that interval. All nodes in a distributed run must use matching timing settings. Missing/reconnected workers may interrupt stateful journeys; rebalanced users begin fresh sessions. Health distinguishes worker availability from target HTTP health. Native worker CPU warnings and reported user counts expose inadequate capacity; workers do not declare a hard user-capacity limit. Finite work assignment and canonical distributed request aggregation remain separate future work. + +## Delivered API + +`start_test` and `prepare_env` accept the worker settings through existing keyword arguments. Master results include `distributed_health`; an active master exposes `env.distributed_health.snapshot()` with worker states, target and reported users, affected worker identifiers, session continuity disclosure, and request replay disabled. A lost worker remains observable until native Locust removal or master cleanup; reconnection uses native dispatch bookkeeping. + +`on_environment(env)` runs in the calling execution thread before readiness/start. `stop_requested()` is checked during startup and polled every 50ms during execution. `prepare_env` closes owned runner, telemetry/timer/cancellation greenlets, UI, and RPC resources on return/error. Direct `create_env` callers own the returned environment and must call `cleanup_env(env)`; cleanup is idempotent. Shape factories are instantiated before environment construction and native headless completion terminates the run. + +## Verification evidence + +- Red: seven initial configuration/scope tests failed against previous behavior; cancellation timed out because its callback was ignored. +- Red: native-state and startup cancellation tests caught premature loss classification and starting after cancellation; shape and cleanup tests caught shape factory misuse, a completion hang, lost degraded status, and UI-stop errors interrupting transport cleanup. +- Green: `test/test_distributed_health.py`: 31 passed, including a real loopback RPC worker plus controlled RPC delivery for deterministic loss/recovery/ramp assertions and callback failures during ramp-up. +- Full suite: `python -m pytest test/ -q` with the project interpreter and ActionCore worktree on `PYTHONPATH`: 1411 passed, 2 skipped, 4 existing warnings in 77.06s. Root performs final verification after concurrent UI/cloud work. +- Owned modules compile and `git diff --check` reports no whitespace errors. + +No finite-work assignment option is exposed. Native virtual-user redistribution does not deduplicate HTTP side effects or aggregate canonical request records across workers. Those capabilities require a separate explicit distributed record/shard contract. diff --git a/docs/updates/2026-10.md b/docs/updates/2026-10.md index 8a00234..976ff98 100644 --- a/docs/updates/2026-10.md +++ b/docs/updates/2026-10.md @@ -233,3 +233,10 @@ Added a stdlib smoke harness that exercises local HTTP requests in isolated proc - Corrected unsupported Cloud Run parallelism overrides, distinguished Lambda execution/acceptance/validation, closed payload streams, and awaited ACI provisioning with unique worker names. - Reviewer malformed-stream regression now validates callable read/close methods without swallowing SDK-internal TypeError. - Verification: 129 mandatory cloud tests and 12 isolated official SDK contract tests passed (141 total), no real resources. Root cloud/distributed focused run 160 passed; whole-suite snapshot after fixes 1428 passed, 14 optional skips. Ruff/complexity and scoped whitespace checks pass. GUI/chart work remains separate and continues. + +## U-20261002-10 · 2026-10-02 · Monitor worker health and rebalance native load · #done #distributed + +- Healthy ready workers gate startup; default timeout fails after cleanup, explicit degraded policy still requires one worker. +- Scoped matching native heartbeat settings and Locust rebalancing retain target/ramp, report observed capacity and affected worker IDs, and terminate after all workers are lost. No request replay, finite shard leases or canonical distributed aggregation is advertised. +- Cooperative lifecycle callbacks cancel readiness/ramp/running work; callback errors propagate after cleanup. Native runner/UI/RPC/auxiliary resources close idempotently. +- Verification: 31 native/control/RPC health tests plus 5 documentation-count tests passed. Independent review and Ruff E/F/W/C90/I pass; lifecycle helpers remain within complexity limits. Remaining distributed record/shard scope retained in progress.md #29. diff --git a/docs/updates/README.md b/docs/updates/README.md index 13e1e8d..d7821c8 100644 --- a/docs/updates/README.md +++ b/docs/updates/README.md @@ -58,6 +58,7 @@ In the same commit: delete the item from `progress.md`, add a `#done` entry here | ID | Date | Title | Tags | Batch | |---|---|---|---|---| +| U-20261002-10 | 2026-10-02 | Monitor worker health and rebalance native load | #done #distributed | [2026-10](2026-10.md) | | U-20261002-09 | 2026-10-02 | Validate cloud launch outcomes and preserve partial failures | #done #cloud | [2026-10](2026-10.md) | | U-20261002-08 | 2026-10-02 | Repair Qt Web Push and NATS installation probes | #done #ci #extras | [2026-10](2026-10.md) | | U-20261002-07 | 2026-10-02 | Align CI install paths and isolate smoke container permissions | #done #ci | [2026-10](2026-10.md) | diff --git a/je_load_density/wrapper/create_locust_env/create_locust_env.py b/je_load_density/wrapper/create_locust_env/create_locust_env.py index 5494243..e83afbc 100644 --- a/je_load_density/wrapper/create_locust_env/create_locust_env.py +++ b/je_load_density/wrapper/create_locust_env/create_locust_env.py @@ -1,20 +1,25 @@ from functools import partial -from typing import Any, Dict, List, Optional +from time import monotonic +from typing import Any, Dict, Optional import gevent +from gevent.pool import Group from locust import User, events +from locust.argument_parser import get_parser from locust.env import Environment from locust.event import Events from locust.log import setup_logging from locust.stats import stats_history, stats_printer from je_load_density.utils.logging.loggin_instance import load_density_logger +from je_load_density.wrapper.distributed_config import DistributedConfig, NativeHeartbeatScope +from je_load_density.wrapper.distributed_health import DistributedHealth setup_logging("INFO", None) def prepare_env( - user_class: List[User], + user_class: type[User], user_count: int = 50, spawn_rate: int = 10, test_time: Optional[int] = 60, @@ -23,21 +28,28 @@ def prepare_env( load_shape: Optional[str] = None, shape_config: Optional[Dict[str, Any]] = None, **kwargs, -): +) -> Environment: """ - 啟動 Locust 環境，支援 local / master / worker 三種模式。 - Prepare a Locust environment in local, master, or worker mode. + 啟動 Locust 環境。Prepare a Locust environment in local, master, or worker mode. - Distributed-mode fields (``master_bind_host`` / ``master_bind_port`` / - ``master_host`` / ``master_port`` / ``expected_workers``) are read - from ``**kwargs`` so the signature stays within the project's - public-API parameter budget. + Distributed-mode and worker-health settings are read from ``**kwargs``. + + Worker timing defaults are startup=60, heartbeat=5 and lost=15 seconds. Startup + fails and cleans up unless ``worker_startup_policy="degraded"`` permits + fewer healthy ready workers; at least one is always required. All nodes + must use matching heartbeat settings. Native detection uses interval ticks. + ``on_environment(env)`` runs before the ready gate in the execution thread. + ``stop_requested()`` cooperatively cancels startup or running work. + This function owns and closes the environment's resources before returning. + Callback errors propagate to the caller after cleanup, including during ramp-up. """ master_bind_host = kwargs.pop("master_bind_host", "*") master_bind_port = kwargs.pop("master_bind_port", 5557) master_host = kwargs.pop("master_host", "127.0.0.1") master_port = kwargs.pop("master_port", 5557) - expected_workers = kwargs.pop("expected_workers", 0) + config = DistributedConfig.from_options(kwargs) + stop_requested = kwargs.pop("stop_requested", None) + on_environment = kwargs.pop("on_environment", None) run_context = kwargs.pop("run_context", None) if run_context is None: from je_load_density.utils.test_record.contract import get_optional_run_context @@ -52,34 +64,64 @@ def prepare_env( env = create_env(user_class, runner_mode=runner_mode, master_bind_host=master_bind_host, master_bind_port=master_bind_port, master_host=master_host, master_port=master_port, - load_shape=load_shape, shape_config=shape_config, run_context=run_context) - - if runner_mode == "worker": + load_shape=load_shape, shape_config=shape_config, run_context=run_context, + **kwargs) + try: + if not _prepare_start(env, runner_mode, config, stop_requested, on_environment): + return env + if runner_mode != "worker": + if not _start_controller_load(env, user_count, spawn_rate, web_ui_dict, test_time): + return env env.runner.greenlet.join() - return env + _raise_cancellation_error(env) + finally: + cleanup_env(env) + return env - if runner_mode == "master" and expected_workers > 0: - _wait_for_workers(env, expected_workers) - env.runner.start(user_count, spawn_rate=spawn_rate) +def _prepare_start(env: Environment, runner_mode: str, config: DistributedConfig, + stop_requested, on_environment) -> bool: + """Notify the caller, satisfy master readiness, then start cancellation observation.""" + if on_environment is not None: + on_environment(env) + if stop_requested is not None and stop_requested(): + return False + if runner_mode == "master": + _wait_for_workers(env, max(1, config.expected_workers), timeout=config.startup_timeout, + policy=config.startup_policy, stop_requested=stop_requested) + if stop_requested is not None and stop_requested(): + return False + if stop_requested is not None: + env.load_density_tasks.spawn(_watch_cancel, env, stop_requested) + return True + +def _start_controller_load(env: Environment, user_count: int, spawn_rate: int, + web_ui_dict: Optional[Dict[str, Any]], test_time: Optional[int]) -> bool: + """Start a controller's native shape or owned ramp before optional UI and timer setup.""" + if env.shape_class is not None: + env.runner.start_shape() + else: + env.startup_task = env.load_density_tasks.spawn(env.runner.start, user_count, spawn_rate=spawn_rate) + env.startup_task.get() + _raise_cancellation_error(env) + if env.cancellation_requested: + return False if web_ui_dict is not None: env.create_web_ui(web_ui_dict.get("host", "127.0.0.1"), web_ui_dict.get("port", 8089)) - if test_time is not None: - gevent.spawn_later(test_time, env.runner.quit) + env.load_density_tasks.add(gevent.spawn_later(test_time, env.runner.quit)) + return True - env.runner.greenlet.join() - if web_ui_dict is not None and getattr(env, "web_ui", None) is not None: - env.web_ui.stop() - - return env +def _raise_cancellation_error(env: Environment) -> None: + if env.cancellation_error is not None: + raise env.cancellation_error def create_env( - user_class: List[User], - another_event: events = events, + user_class: type[User], + another_event: Events = events, runner_mode: str = "local", master_bind_host: str = "*", master_bind_port: int = 5557, @@ -88,14 +130,22 @@ def create_env( load_shape: Optional[str] = None, shape_config: Optional[Dict[str, Any]] = None, **kwargs, -): +) -> Environment: """ 建立 Locust Environment 並依模式建立 runner。 Create Locust Environment and build the matching runner. + + Call ``cleanup_env`` when the caller has finished with the environment. + Master health is available as ``env.distributed_health.snapshot()``; + request health and worker health are separate. Distributed environments + in the same process must share heartbeat timing settings. """ load_density_logger.info( f"create_env mode={runner_mode}, user_class={user_class}, another_event={another_event}" ) + if runner_mode not in ("local", "master", "worker"): + raise ValueError("runner_mode must be local, master or worker") + config = DistributedConfig.from_options(kwargs) shape_class = _resolve_shape(load_shape, shape_config) run_context = kwargs.pop("run_context", None) if run_context is not None: @@ -104,19 +154,40 @@ def create_env( raise ValueError("a canonical run context requires an isolated default event environment") another_event = Events() another_event.request.add_listener(partial(request_hook, record_run=run_context)) + if another_event is events and runner_mode != "local": + # Shared global event hooks would let one master's quit stop another run. + another_event = Events() + from je_load_density.wrapper.event.request_hook import request_hook + another_event.request.add_listener(request_hook) + options = None + scope = None + if runner_mode != "local" or shape_class is not None: + options = get_parser(default_config_files=[]).parse_args(args=[]) + options.enable_rebalancing = True + options.headless = True env = Environment(user_classes=[user_class], events=another_event, - shape_class=shape_class) - - if runner_mode == "master": - env.create_master_runner(master_bind_host=master_bind_host, master_bind_port=master_bind_port) - elif runner_mode == "worker": - env.create_worker_runner(master_host=master_host, master_port=master_port) - else: - env.create_local_runner() - - if runner_mode != "worker": - gevent.spawn(stats_printer(env.stats)) - gevent.spawn(stats_history, env.runner) + shape_class=shape_class, parsed_options=options) + if runner_mode != "local": + scope = NativeHeartbeatScope(config) + env.load_density_tasks = Group() + env.native_heartbeat_scope = scope + env.load_density_closed = False + env.cancellation_error = None + env.cancellation_requested = False + env.startup_task = None + runner_created = False + try: + _create_runner(env, runner_mode, master_bind_host, master_bind_port, master_host, master_port) + if runner_mode == "master": + env.distributed_health = DistributedHealth(env, config) + env.load_density_tasks.spawn(env.distributed_health.monitor) + if runner_mode != "worker": + env.load_density_tasks.spawn(stats_printer(env.stats)) + env.load_density_tasks.spawn(stats_history, env.runner) + runner_created = True + finally: + if not runner_created: + cleanup_env(env) return env @@ -124,20 +195,93 @@ def _resolve_shape(load_shape: Optional[str], shape_config: Optional[Dict[str, A if not load_shape: return None from je_load_density.utils.load_shapes.shapes import build_load_shape - return build_load_shape(load_shape, shape_config or {}) + return build_load_shape(load_shape, shape_config or {})() -def _wait_for_workers(env, expected_workers: int, timeout: float = 60.0) -> None: - """ - 等待指定數量的 worker 加入後再開始壓測。 - """ - deadline = gevent.time.time() + timeout - while gevent.time.time() < deadline: - workers = getattr(env.runner, "clients", None) - connected = len(workers) if workers is not None else 0 +def _create_runner(env, mode, bind_host, bind_port, host, port) -> None: + if mode == "master": + env.create_master_runner(master_bind_host=bind_host, master_bind_port=bind_port) + elif mode == "worker": + env.create_worker_runner(master_host=host, master_port=port) + else: + env.create_local_runner() + + +def _wait_for_workers(env, expected_workers: int, timeout: float = 60.0, + policy: str = "fail", stop_requested=None) -> None: + """Wait for healthy ready workers; fail closed unless degraded startup is explicit.""" + deadline = monotonic() + timeout + while True: + connected = env.distributed_health.ready_count if connected >= expected_workers: return - gevent.sleep(0.5) - load_density_logger.warning( - f"only {connected}/{expected_workers} workers joined within {timeout}s; starting anyway" - ) + if stop_requested is not None and stop_requested(): + return + remaining = deadline - monotonic() + if remaining <= 0: + break + gevent.sleep(min(0.1, remaining)) + message = f"only {connected}/{expected_workers} healthy ready workers joined within {timeout}s" + if policy == "degraded" and connected: + env.distributed_health.degraded_startup(expected_workers, message) + load_density_logger.warning(message) + return + env.distributed_health.fail(message) + raise TimeoutError(message) + + +def _watch_cancel(env, stop_requested) -> None: + while env.runner.greenlet: + try: + should_stop = stop_requested() + except Exception as error: + # User callbacks can raise any Exception; propagate it in the caller's + # execution thread after the native runner is stopped and cleaned up. + env.cancellation_error = error + health = getattr(env, "distributed_health", None) + if health is not None: + health.fail("stop_requested callback failed") + _stop_owned_run(env) + return + if should_stop: + _stop_owned_run(env) + return + gevent.sleep(0.05) + + +def _stop_owned_run(env: Environment) -> None: + env.cancellation_requested = True + if env.startup_task is not None and not env.startup_task.dead: + env.startup_task.kill(block=True) + env.runner.quit() + + +def cleanup_env(env: Environment) -> None: + """Close owned runner, UI, auxiliary greenlets, RPC transport, and timing scope once.""" + if getattr(env, "load_density_closed", False): + return + env.load_density_closed = True + health = getattr(env, "distributed_health", None) + if health is not None: + health.close() + try: + if env.runner is not None and env.runner.greenlet: + env.runner.quit() + finally: + try: + env.load_density_tasks.kill(block=True) + if env.web_ui is not None: + env.web_ui.stop() + finally: + _close_distributed_resources(env) + + +def _close_distributed_resources(env: Environment) -> None: + try: + if env.runner is not None: + transport = getattr(env.runner, "server", None) or getattr(env.runner, "client", None) + if transport is not None: + transport.close(linger=0) + finally: + if env.native_heartbeat_scope is not None: + env.native_heartbeat_scope.close() diff --git a/je_load_density/wrapper/distributed_config.py b/je_load_density/wrapper/distributed_config.py new file mode 100644 index 0000000..902e45e --- /dev/null +++ b/je_load_density/wrapper/distributed_config.py @@ -0,0 +1,75 @@ +"""Validated distributed policy and scoped Locust process heartbeat settings.""" +import math +from dataclasses import dataclass +from threading import Lock +from typing import Mapping + +from locust import runners + + +@dataclass(frozen=True) +class DistributedConfig: + """Worker health policy; heartbeat timeout is quantized to native interval ticks.""" + + startup_timeout: float = 60.0 + heartbeat_interval: float = 5.0 + lost_timeout: float = 15.0 + startup_policy: str = "fail" + expected_workers: int = 0 + + @classmethod + def from_options(cls, options: Mapping[str, object]) -> "DistributedConfig": + """Validate settings before constructing or binding a runner.""" + times = [options.get(name, default) for name, default in ( + ("worker_startup_timeout", 60), ("worker_heartbeat_interval", 5), ("worker_lost_timeout", 15))] + if any(isinstance(value, bool) or not isinstance(value, (int, float)) for value in times): + raise ValueError("worker timeouts and heartbeat interval must be positive finite numbers") + if any(not math.isfinite(value) or value <= 0 for value in times): + raise ValueError("worker timeouts and heartbeat interval must be positive finite numbers") + if times[2] <= times[1]: + raise ValueError("worker_lost_timeout must be greater than worker_heartbeat_interval") + policy = options.get("worker_startup_policy", "fail") + if policy not in ("fail", "degraded"): + raise ValueError("worker_startup_policy must be fail or degraded") + count = options.get("expected_workers", 0) + if isinstance(count, bool) or not isinstance(count, int) or count < 0: + raise ValueError("expected_workers must be a nonnegative integer") + return cls(*map(float, times), policy, count) + + +class NativeHeartbeatScope: + """Share matching native timing settings and restore them after the last runner closes. + + Locust exposes heartbeat constants at module scope, not per environment. Configurations + cannot differ while distributed environments coexist within one process. + """ + + _lock = Lock() + _active = 0 + _timing = None + _previous = None + + def __init__(self, config: DistributedConfig) -> None: + self._released = False + timing = (config.heartbeat_interval, max(0, math.ceil(config.lost_timeout / config.heartbeat_interval) - 2)) + with self._lock: + if self._active and self._timing != timing: + raise ValueError("conflicting Locust heartbeat settings in the same process") + cls = type(self) + if not cls._active: + cls._previous = (runners.HEARTBEAT_INTERVAL, runners.HEARTBEAT_LIVENESS) + runners.HEARTBEAT_INTERVAL, runners.HEARTBEAT_LIVENESS = timing + cls._timing = timing + cls._active += 1 + + def close(self) -> None: + """Release once; a second cleanup leaves another environment's settings intact.""" + with self._lock: + if self._released: + return + self._released = True + cls = type(self) + cls._active -= 1 + if not cls._active: + runners.HEARTBEAT_INTERVAL, runners.HEARTBEAT_LIVENESS = cls._previous + cls._timing = cls._previous = None diff --git a/je_load_density/wrapper/distributed_health.py b/je_load_density/wrapper/distributed_health.py new file mode 100644 index 0000000..eae2235 --- /dev/null +++ b/je_load_density/wrapper/distributed_health.py @@ -0,0 +1,149 @@ +"""Observe Locust worker health without sending heartbeats or assigning virtual users.""" +from enum import Enum + +import gevent +from locust.env import Environment +from locust.runners import WorkerNode + +from .distributed_config import DistributedConfig + + +class WorkerState(str, Enum): + """Public worker lifecycle states.""" + + CONNECTING = "connecting" + READY = "ready" + RUNNING = "running" + LOST = "lost" + STOPPED = "stopped" + + +class DistributedHealth: + """Observe native missing/reconnection state and disclose affected stateful journeys.""" + + def __init__(self, environment: Environment, config: DistributedConfig) -> None: + self.environment = environment + self.config = config + self.status = "connecting" + self.reason: str | None = None + self.workers: dict[str, WorkerState] = {} + self.affected_workers: set[str] = set() + self._started = False + self._closed = False + self._required_workers = config.expected_workers + self._listeners = [ + (environment.events.worker_connect, self._connecting), + (environment.events.test_start, self._start), + (environment.events.test_stop, self._stop), + ] + for hook, listener in self._listeners: + hook.add_listener(listener) + + @property + def ready_count(self) -> int: + """Count only ready clients with an unexpired native heartbeat lease.""" + return sum(client.state == "ready" and client.heartbeat >= 0 + for client in self.environment.runner.clients.all) + + def _connecting(self, client_id: str, **_kwargs) -> None: + self.workers[client_id] = WorkerState.CONNECTING + + def _start(self, **_kwargs) -> None: + self._started = True + if self.status != "degraded": + self.status = "running" + + def _stop(self, **_kwargs) -> None: + self.refresh() + if self.status not in ("failed", "degraded"): + self.status = "stopped" + + def degraded_startup(self, expected_workers: int, reason: str) -> None: + """Retain an explicitly accepted startup shortfall in the final report.""" + self._required_workers = expected_workers + self.status = "degraded" + self.reason = reason + + def fail(self, reason: str) -> None: + """Record an infrastructure failure independently of target request errors.""" + self.status = "failed" + self.reason = reason + self.environment.process_exit_code = 1 + + def refresh(self) -> None: + """Read native state; preserve loss history while allowing native reconnection.""" + if self._closed: + return + runner = self.environment.runner + present = set() + healthy = 0 + for client in runner.clients.all: + present.add(client.id) + state = self._worker_state(client) + self.workers[client.id] = state + if state == WorkerState.LOST: + self.affected_workers.add(client.id) + healthy += state in (WorkerState.READY, WorkerState.RUNNING) + for worker_id in self.workers.keys() - present: + self.workers[worker_id] = WorkerState.STOPPED + if self._started: + self.affected_workers.add(worker_id) + self._update_status(healthy) + + @staticmethod + def _worker_state(client: WorkerNode) -> WorkerState: + if client.state == "missing": + return WorkerState.LOST + if client.state in ("running", "spawning"): + return WorkerState.RUNNING + if client.state == "ready": + return WorkerState.READY + return WorkerState.STOPPED + + def _update_status(self, healthy: int) -> None: + runner = self.environment.runner + if self.status in ("failed", "stopped"): + return + if not self._started: + self.status = "ready" if self.ready_count else "connecting" + return + if not healthy: + self.fail("all workers lost or stopped") + return + insufficient = runner.user_count < runner.target_user_count and runner.spawning_completed + if healthy < self._required_workers or insufficient or runner.worker_cpu_warning_emitted: + self.status = "degraded" + self.reason = "worker availability or observed capacity below the requested load" + else: + self.status = "running" + self.reason = None + + def monitor(self) -> None: + """Poll native state only; quit when no worker can deliver the load.""" + while not self._closed: + self.refresh() + if self.status == "failed": + self.environment.runner.quit() + return + gevent.sleep(min(self.config.heartbeat_interval, 0.1)) + + def snapshot(self) -> dict[str, object]: + """Return serializable health, capacity and session-continuity disclosure.""" + self.refresh() + runner = self.environment.runner + return {"status": self.status, "reason": self.reason, + "workers": {key: state.value for key, state in self.workers.items()}, + "target_users": runner.target_user_count, "reported_users": runner.user_count, + "affected_workers": sorted(self.affected_workers), + "session_continuity": "not guaranteed", "request_replay": False} + + def close(self) -> None: + """Detach health listeners while preserving the final run health snapshot.""" + self.refresh() + if self.status not in ("failed", "degraded"): + self.status = "stopped" + self._closed = True + for worker_id in self.workers: + self.workers[worker_id] = WorkerState.STOPPED + for hook, listener in self._listeners: + hook.remove_listener(listener) diff --git a/je_load_density/wrapper/start_wrapper/start_test.py b/je_load_density/wrapper/start_wrapper/start_test.py index 01c2ed3..44433d7 100644 --- a/je_load_density/wrapper/start_wrapper/start_test.py +++ b/je_load_density/wrapper/start_wrapper/start_test.py @@ -6,6 +6,10 @@ AmqpUserWrapper, set_wrapper_amqp_user, ) +from je_load_density.wrapper.user_template.apns_user_template import ( + ApnsUserWrapper, + set_wrapper_apns_user, +) from je_load_density.wrapper.user_template.async_http_user_template import ( AsyncHttpUserWrapper, set_wrapper_async_http_user, @@ -18,14 +22,30 @@ CoapUserWrapper, set_wrapper_coap_user, ) +from je_load_density.wrapper.user_template.consul_user_template import ( + ConsulUserWrapper, + set_wrapper_consul_user, +) +from je_load_density.wrapper.user_template.couchbase_user_template import ( + CouchbaseUserWrapper, + set_wrapper_couchbase_user, +) from je_load_density.wrapper.user_template.elasticsearch_user_template import ( ElasticsearchUserWrapper, set_wrapper_elasticsearch_user, ) +from je_load_density.wrapper.user_template.etcd_user_template import ( + EtcdUserWrapper, + set_wrapper_etcd_user, +) from je_load_density.wrapper.user_template.fast_http_user_template import ( FastHttpUserWrapper, set_wrapper_fasthttp_user, ) +from je_load_density.wrapper.user_template.fcm_user_template import ( + FcmUserWrapper, + set_wrapper_fcm_user, +) from je_load_density.wrapper.user_template.ftp_user_template import ( FtpUserWrapper, set_wrapper_ftp_user, @@ -58,6 +78,18 @@ KafkaUserWrapper, set_wrapper_kafka_user, ) +from je_load_density.wrapper.user_template.ldap_user_template import ( + LdapUserWrapper, + set_wrapper_ldap_user, +) +from je_load_density.wrapper.user_template.memcached_user_template import ( + MemcachedUserWrapper, + set_wrapper_memcached_user, +) +from je_load_density.wrapper.user_template.modbus_user_template import ( + ModbusUserWrapper, + set_wrapper_modbus_user, +) from je_load_density.wrapper.user_template.mongo_user_template import ( MongoUserWrapper, set_wrapper_mongo_user, @@ -70,6 +102,14 @@ NatsUserWrapper, set_wrapper_nats_user, ) +from je_load_density.wrapper.user_template.neo4j_user_template import ( + Neo4jUserWrapper, + set_wrapper_neo4j_user, +) +from je_load_density.wrapper.user_template.opcua_user_template import ( + OpcuaUserWrapper, + set_wrapper_opcua_user, +) from je_load_density.wrapper.user_template.pulsar_user_template import ( PulsarUserWrapper, set_wrapper_pulsar_user, @@ -86,6 +126,14 @@ SmtpUserWrapper, set_wrapper_smtp_user, ) +from je_load_density.wrapper.user_template.snmp_user_template import ( + SnmpUserWrapper, + set_wrapper_snmp_user, +) +from je_load_density.wrapper.user_template.soap_user_template import ( + SoapUserWrapper, + set_wrapper_soap_user, +) from je_load_density.wrapper.user_template.socket_user_template import ( SocketUserWrapper, set_wrapper_socket_user, @@ -98,54 +146,6 @@ SseUserWrapper, set_wrapper_sse_user, ) -from je_load_density.wrapper.user_template.apns_user_template import ( - ApnsUserWrapper, - set_wrapper_apns_user, -) -from je_load_density.wrapper.user_template.consul_user_template import ( - ConsulUserWrapper, - set_wrapper_consul_user, -) -from je_load_density.wrapper.user_template.couchbase_user_template import ( - CouchbaseUserWrapper, - set_wrapper_couchbase_user, -) -from je_load_density.wrapper.user_template.etcd_user_template import ( - EtcdUserWrapper, - set_wrapper_etcd_user, -) -from je_load_density.wrapper.user_template.fcm_user_template import ( - FcmUserWrapper, - set_wrapper_fcm_user, -) -from je_load_density.wrapper.user_template.ldap_user_template import ( - LdapUserWrapper, - set_wrapper_ldap_user, -) -from je_load_density.wrapper.user_template.memcached_user_template import ( - MemcachedUserWrapper, - set_wrapper_memcached_user, -) -from je_load_density.wrapper.user_template.modbus_user_template import ( - ModbusUserWrapper, - set_wrapper_modbus_user, -) -from je_load_density.wrapper.user_template.neo4j_user_template import ( - Neo4jUserWrapper, - set_wrapper_neo4j_user, -) -from je_load_density.wrapper.user_template.opcua_user_template import ( - OpcuaUserWrapper, - set_wrapper_opcua_user, -) -from je_load_density.wrapper.user_template.snmp_user_template import ( - SnmpUserWrapper, - set_wrapper_snmp_user, -) -from je_load_density.wrapper.user_template.soap_user_template import ( - SoapUserWrapper, - set_wrapper_soap_user, -) from je_load_density.wrapper.user_template.thrift_user_template import ( ThriftUserWrapper, set_wrapper_thrift_user, @@ -167,7 +167,6 @@ set_wrapper_zmq_user, ) - _USER_REGISTRY: Dict[str, Dict[str, Any]] = { "fast_http_user": {"actually_user": FastHttpUserWrapper, "init": set_wrapper_fasthttp_user}, "http_user": {"actually_user": HttpUserWrapper, "init": set_wrapper_http_user}, @@ -245,6 +244,10 @@ def _pop_distributed_config(kwargs: Dict[str, Any]) -> Dict[str, Any]: "master_host": kwargs.pop("master_host", "127.0.0.1"), "master_port": kwargs.pop("master_port", 5557), "expected_workers": kwargs.pop("expected_workers", 0), + "worker_startup_timeout": kwargs.pop("worker_startup_timeout", 60), + "worker_heartbeat_interval": kwargs.pop("worker_heartbeat_interval", 5), + "worker_lost_timeout": kwargs.pop("worker_lost_timeout", 15), + "worker_startup_policy": kwargs.pop("worker_startup_policy", "fail"), } @@ -267,6 +270,13 @@ def start_test( ``master_host`` / ``master_port`` / ``expected_workers``) are accepted via ``**kwargs`` so the signature stays under the public API parameter budget. + + Worker timing defaults are startup=60, heartbeat=5, lost=15 seconds. + ``worker_startup_policy="fail"`` rejects startup shortfalls; explicitly + selecting ``"degraded"`` allows fewer workers while requiring at least one. + ``stop_requested`` and ``on_environment`` are cooperative lifecycle callbacks. + Master results include ``distributed_health`` and affected worker journeys; + native rebalancing recreates capacity without replaying individual requests. """ distributed = _pop_distributed_config(kwargs) load_density_logger.info( @@ -285,7 +295,7 @@ def start_test( init_function(user_detail_dict, **kwargs) - prepare_env( + env = prepare_env( user_class=actually_user, user_count=user_count, spawn_rate=spawn_rate, @@ -298,7 +308,7 @@ def start_test( **kwargs, ) - return { + result = { "user_detail": user_detail_dict, "user_count": user_count, "spawn_rate": spawn_rate, @@ -306,3 +316,6 @@ def start_test( "web_ui": web_ui_dict, "runner_mode": runner_mode, } + if getattr(env, "distributed_health", None) is not None: + result["distributed_health"] = env.distributed_health.snapshot() + return result diff --git a/progress.md b/progress.md index 585db72..85bcc18 100644 --- a/progress.md +++ b/progress.md @@ -18,5 +18,5 @@ Cross-repo and workspace items live in `D:\Codes\progress.md` (relevant here: X- - **#25** Finish Docker matrix validation: base/etcd and healthy Compose Redis/MQTT/SQLite probes pass; GUI/all and Python 3.10/3.14 cells are building. Repair any remaining installation/API failures and confirm the complete Python 3.12 extras matrix in CI. - **#26** Percentile bands: `gui/chart_panel.py` mixes RPS and latency on one axis; `utils/generate_report/generate_chart_report.py` uses `ts`/index instead of request `start_time`. Share per-window percentile aggregation across GUI/dashboard/report, show p50 with p50–p95 and p95–p99 bands, separate RPS units and handle gaps, failures and actual bucket duration. - **#28** Shared record release activation: coordinate the actual ActionCore request-record release, then activate consumer dependency floors and cross-project installed-wheel fixtures. Current opt-in APIs must retain compatibility with older core installations until that release. Design: `docs/superpowers/specs/2026-10-02-shared-record-design.md`. -- **#29** Distributed worker health and recovery: `wrapper/create_locust_env/create_locust_env.py::_wait_for_workers` counts clients and starts after timeout. Add configurable ready/heartbeat/lost policy, fail or explicitly degrade on startup timeout, reassign virtual-user capacity after loss, handle reconnect/stale assignments and deduplicate results; reuse Locust heartbeat/rebalance support where possible. Target health is separately tracked by #20. +- **#29** Distributed canonical record aggregation and finite-work recovery: health/startup/native virtual-user rebalancing is implemented. Add an explicit master ingestion contract that preserves source worker identity, bounded worker batches/final flush and run-scoped retry deduplication. ActionCore `RunContext.append` currently requires matching worker identity, so aggregation cannot silently rewrite it. Define lease/reassignment semantics only when introducing actual finite-work shards; native ongoing load does not replay HTTP side effects or migrate sessions. Target health is separately tracked by #20. - **#30** One script for functional/load/synthetic modes: extend APITestka's existing `integrations/load_density*.py` bridge and WebRunner's `utils/synthetic_monitoring/monitor.py` so common request/assertion steps are authored once and execution profiles choose functional correctness, capacity/SLA or periodic monitoring. Unsupported conversions must fail preflight and failed checks must reach monitor transitions. Depends on #28 and async parity #16; preserve existing CLI/action/summary contracts. diff --git a/test/test_distributed_health.py b/test/test_distributed_health.py new file mode 100644 index 0000000..f8a9349 --- /dev/null +++ b/test/test_distributed_health.py @@ -0,0 +1,374 @@ +"""Distributed lifecycle contract, with native runners and controlled RPC messages.""" +from types import SimpleNamespace + +import gevent +import pytest +from locust import User, runners, task +from locust.event import Events +from locust.rpc import Message + +from je_load_density.wrapper.create_locust_env import create_locust_env as wrapper + + +class IdleUser(User): + @task + def idle(self): + gevent.sleep(0.01) + + +@pytest.fixture +def master(): + env = wrapper.create_env(IdleUser, another_event=Events(), runner_mode="master", master_bind_port=0, + worker_heartbeat_interval=0.02, worker_lost_timeout=0.06) + yield env + wrapper.cleanup_env(env) + + +@pytest.mark.parametrize("options", [ + {"worker_startup_timeout": 0}, {"worker_heartbeat_interval": -1}, + {"worker_lost_timeout": float("nan")}, {"worker_lost_timeout": 5}, + {"worker_startup_policy": "anyway"}, {"expected_workers": -1}, +]) +def test_invalid_distributed_settings_rejected_before_runner_creation(options): + with pytest.raises(ValueError): + env = wrapper.create_env(IdleUser, runner_mode="master", master_bind_port=0, **options) + env.runner.quit() + + +def test_ready_gate_excludes_missing_running_and_expired_workers(master): + master.runner.clients["ready"] = runners.WorkerNode("ready", heartbeat_liveness=1) + master.runner.clients["lost"] = runners.WorkerNode("lost", state="missing") + master.runner.clients["running"] = runners.WorkerNode("running", state="running") + master.runner.clients["expired"] = runners.WorkerNode("expired", heartbeat_liveness=-1) + assert master.distributed_health.ready_count == 1 + with pytest.raises(TimeoutError): + wrapper._wait_for_workers(master, 2, timeout=0.01) + + +def test_native_settings_restore_after_last_environment_cleanup(): + original = (runners.HEARTBEAT_INTERVAL, runners.HEARTBEAT_LIVENESS) + env = wrapper.create_env(IdleUser, another_event=Events(), runner_mode="master", master_bind_port=0) + try: + assert runners.HEARTBEAT_INTERVAL == 5 + assert env.runner.rebalancing_enabled() + with pytest.raises(ValueError, match="conflicting"): + wrapper.create_env(IdleUser, runner_mode="master", master_bind_port=0, worker_heartbeat_interval=2) + finally: + wrapper.cleanup_env(env) + assert (runners.HEARTBEAT_INTERVAL, runners.HEARTBEAT_LIVENESS) == original + + +def test_prepare_env_startup_timeout_never_starts_and_cleans_runner(monkeypatch, master): + monkeypatch.setattr(wrapper, "create_env", lambda *_args, **_kwargs: master) + with pytest.raises(TimeoutError): + wrapper.prepare_env(IdleUser, runner_mode="master", expected_workers=1, worker_startup_timeout=0.01) + assert master.runner.target_user_count == 0 + assert len(master.runner.greenlet) == 0 + assert master.distributed_health.snapshot()["status"] == "failed" + assert master.runner.server.socket.closed + + +def test_explicit_degraded_startup_requires_at_least_one_worker(monkeypatch, master): + monkeypatch.setattr(wrapper, "create_env", lambda *_args, **_kwargs: master) + with pytest.raises(TimeoutError): + wrapper.prepare_env(IdleUser, runner_mode="master", expected_workers=2, worker_startup_timeout=0.01, + worker_startup_policy="degraded") + assert master.runner.target_user_count == 0 + + +def test_cancel_callback_follows_environment_setup_and_cleans_greenlets(): + seen = [] + def created(env): + assert env.runner is not None + seen.append(env) + with gevent.Timeout(1): + env = wrapper.prepare_env(IdleUser, user_count=1, test_time=None, on_environment=created, + stop_requested=lambda: bool(seen)) + assert seen == [env] + assert len(env.runner.greenlet) == 0 + assert len(env.runner.user_greenlets) == 0 + assert len(env.load_density_tasks) == 0 + + +def _controlled_workers(env, monkeypatch, worker_ids=("first", "second")): + """Keep native runner/dispatcher real; replace only RPC wire delivery.""" + sent = [] + active = set(worker_ids) + def deliver(message): + if message.type == "spawn": + sent.append((gevent.time.monotonic(), message.node_id, message.data["user_classes_count"])) + env.runner.handle_message(message.node_id, Message("spawning_complete", { + "user_classes_count": message.data["user_classes_count"]}, message.node_id)) + elif message.type == "stop": + env.runner.clients[message.node_id].user_classes_count = {} + monkeypatch.setattr(env.runner.server, "send_to_client", deliver) + for worker_id in worker_ids: + env.runner.handle_message(worker_id, Message("client_ready", -1, worker_id)) + def beat(): + while True: + for worker_id in list(active): + if worker_id in env.runner.clients: + state = env.runner.clients[worker_id].state + env.runner.handle_message(worker_id, Message("heartbeat", { + "state": state, "current_cpu_usage": 0}, worker_id)) + gevent.sleep(0.005) + env.load_density_tasks.spawn(beat) + return sent, active + + +def _until(predicate, timeout=2): + with gevent.Timeout(timeout): + while not predicate(): + gevent.sleep(0.005) + + +def test_health_waits_for_native_missing_before_marking_worker_lost(master): + master.runner.clients["pending"] = runners.WorkerNode("pending", state="running", heartbeat_liveness=-1) + master.distributed_health.refresh() + assert master.distributed_health.snapshot()["workers"]["pending"] == "running" + + +def test_native_network_loss_rebalances_capacity_at_original_target_and_rate(master, monkeypatch): + sent, active = _controlled_workers(master, monkeypatch) + master.runner.start(4, spawn_rate=4) + active.remove("second") + _until(lambda: master.runner.clients["second"].state == "missing") + _until(lambda: master.runner.clients["first"].user_count == 4) + snapshot = master.distributed_health.snapshot() + assert snapshot["target_users"] == snapshot["reported_users"] == 4 + assert snapshot["affected_workers"] == ["second"] + assert snapshot["request_replay"] is False + assert master.runner.spawn_rate == 4 + assert sent[-1][1:] == ("first", {"IdleUser": 4}) + + +def test_native_late_heartbeat_rejoins_without_double_assignment(master, monkeypatch): + _, active = _controlled_workers(master, monkeypatch) + master.runner.start(4, spawn_rate=100) + active.remove("second") + _until(lambda: master.runner.clients["second"].state == "missing") + master.runner.handle_message("second", Message("heartbeat", { + "state": "running", "current_cpu_usage": 0}, "second")) + active.add("second") + _until(lambda: master.runner.user_count == 4) + assert sum(client.user_count for client in master.runner.clients.all) == 4 + assert len(master.runner.clients) == 2 + assert master.distributed_health.snapshot()["workers"]["second"] == "running" + + +def test_native_duplicate_ready_keeps_one_worker_assignment(master, monkeypatch): + _controlled_workers(master, monkeypatch) + master.runner.start(4, spawn_rate=100) + master.runner.handle_message("second", Message("client_ready", -1, "second")) + assert len(master.runner.clients) == 2 + assert master.runner.user_count == 4 + + +def test_native_worker_quit_redistributes_users(master, monkeypatch): + _, active = _controlled_workers(master, monkeypatch) + master.runner.start(4, spawn_rate=100) + active.remove("second") + master.runner.handle_message("second", Message("quit", None, "second")) + assert master.runner.user_count == 4 + assert master.distributed_health.snapshot()["affected_workers"] == ["second"] + + +def test_all_workers_lost_marks_failed_and_terminates_master(master, monkeypatch): + _, active = _controlled_workers(master, monkeypatch) + master.runner.start(2, spawn_rate=100) + active.clear() + _until(lambda: not master.runner.greenlet) + assert master.distributed_health.snapshot()["status"] == "failed" + assert master.process_exit_code == 1 + assert set(master.distributed_health.snapshot()["affected_workers"]) == {"first", "second"} + + +def test_native_loss_during_ramp_retains_total_and_spawn_constraint(master, monkeypatch): + sent, active = _controlled_workers(master, monkeypatch) + ramp = gevent.spawn(master.runner.start, 6, spawn_rate=2) + _until(lambda: master.runner.user_count >= 2) + active.remove("second") + ramp.get(timeout=5) + assert master.runner.user_count == 6 + assert master.runner.target_user_count == 6 + assert master.runner.spawn_rate == 2 + assert sent[-1][0] - sent[0][0] >= 1.8 + + +def test_observed_capacity_shortfall_is_degraded(master, monkeypatch): + _controlled_workers(master, monkeypatch) + master.runner.start(4, spawn_rate=100) + master.runner.clients["second"].user_classes_count = {"IdleUser": 0} + assert master.distributed_health.snapshot()["status"] == "degraded" + master.runner.clients["second"].user_classes_count = {"IdleUser": 2} + master.runner.worker_cpu_warning_emitted = True + assert master.distributed_health.snapshot()["status"] == "degraded" + + +def test_cancel_during_startup_does_not_start_users(monkeypatch, master): + monkeypatch.setattr(wrapper, "create_env", lambda *_args, **_kwargs: master) + stop = iter((False, True, True)) + wrapper.prepare_env(IdleUser, runner_mode="master", expected_workers=1, + stop_requested=lambda: next(stop, True)) + assert master.runner.target_user_count == 0 + + +def test_callback_error_cleans_transport_and_settings(monkeypatch, master): + monkeypatch.setattr(wrapper, "create_env", lambda *_args, **_kwargs: master) + def rejected(_env): + raise RuntimeError("callback failed") + with pytest.raises(RuntimeError, match="callback failed"): + wrapper.prepare_env(IdleUser, on_environment=rejected) + assert master.runner.server.socket.closed + assert len(master.load_density_tasks) == 0 + + +def test_shape_changes_native_target_and_finishes_without_hanging(monkeypatch): + seen = [] + def created(env): + _controlled_workers(env, monkeypatch) + shape_time = iter((0.0, 1.0, 2.0)) + monkeypatch.setattr(env.shape_class, "get_run_time", lambda: next(shape_time)) + env.events.spawning_complete.add_listener(lambda **_kw: seen.append(env.runner.target_user_count)) + with gevent.Timeout(5): + env = wrapper.prepare_env(IdleUser, runner_mode="master", expected_workers=2, test_time=None, + worker_heartbeat_interval=0.02, worker_lost_timeout=0.06, + load_shape="stages", shape_config={"stages": [ + {"duration": 1, "users": 2, "spawn_rate": 100}, + {"duration": 1, "users": 4, "spawn_rate": 100}]}, + on_environment=created) + assert seen[:2] == [2, 4] + assert len(env.runner.greenlet) == 0 + assert env.runner.server.socket.closed + + +def test_degraded_startup_reports_shortfall_after_cleanup(monkeypatch, master): + monkeypatch.setattr(wrapper, "create_env", lambda *_args, **_kwargs: master) + _controlled_workers(master, monkeypatch, worker_ids=("first",)) + env = wrapper.prepare_env(IdleUser, runner_mode="master", expected_workers=2, user_count=2, + spawn_rate=100, worker_startup_timeout=0.01, + worker_startup_policy="degraded", test_time=0.01) + assert env.distributed_health.snapshot()["status"] == "degraded" + + +def test_cancel_stops_running_local_users_and_timer(): + seen = [] + with gevent.Timeout(3): + env = wrapper.prepare_env(IdleUser, user_count=2, test_time=60, on_environment=seen.append, + stop_requested=lambda: bool(seen and seen[0].runner.user_count)) + assert len(env.runner.user_greenlets) == 0 + assert len(env.load_density_tasks) == 0 + + +def test_matching_environment_scope_restores_only_after_last_close(): + original = (runners.HEARTBEAT_INTERVAL, runners.HEARTBEAT_LIVENESS) + first = wrapper.create_env(IdleUser, runner_mode="master", master_bind_port=0) + second = wrapper.create_env(IdleUser, runner_mode="master", master_bind_port=0) + try: + wrapper.cleanup_env(first) + assert runners.HEARTBEAT_INTERVAL == 5 + wrapper.cleanup_env(first) + assert runners.HEARTBEAT_INTERVAL == 5 + finally: + wrapper.cleanup_env(second) + assert (runners.HEARTBEAT_INTERVAL, runners.HEARTBEAT_LIVENESS) == original + + +def test_real_rpc_worker_connects_runs_and_closes_both_transports(master): + worker = wrapper.create_env(IdleUser, another_event=Events(), runner_mode="worker", + master_host="127.0.0.1", master_port=master.runner.server.port, + worker_heartbeat_interval=0.02, worker_lost_timeout=0.06) + try: + _until(lambda: master.distributed_health.ready_count == 1) + master.runner.start(2, spawn_rate=100) + _until(lambda: worker.runner.user_count == 2) + assert master.distributed_health.snapshot()["reported_users"] == 2 + finally: + wrapper.cleanup_env(worker) + assert worker.runner.client.socket.closed + assert len(worker.runner.user_greenlets) == 0 + assert len(worker.runner.greenlet) == 0 + + +def test_cleanup_ui_error_still_closes_transport_and_restores_settings(): + original = (runners.HEARTBEAT_INTERVAL, runners.HEARTBEAT_LIVENESS) + env = wrapper.create_env(IdleUser, runner_mode="master", master_bind_port=0) + def broken_stop(): + raise RuntimeError("UI stop failed") + env.web_ui = SimpleNamespace(stop=broken_stop) + with pytest.raises(RuntimeError, match="UI stop failed"): + wrapper.cleanup_env(env) + assert env.runner.server.socket.closed + assert (runners.HEARTBEAT_INTERVAL, runners.HEARTBEAT_LIVENESS) == original + + +def test_local_shape_completes_and_releases_tasks(monkeypatch): + def created(env): + shape_time = iter((0.0, 1.0)) + monkeypatch.setattr(env.shape_class, "get_run_time", lambda: next(shape_time)) + with gevent.Timeout(3): + env = wrapper.prepare_env(IdleUser, test_time=None, load_shape="stages", + shape_config={"stages": [{"duration": 1, "users": 1, "spawn_rate": 100}]}, + on_environment=created) + assert len(env.runner.greenlet) == 0 + assert len(env.load_density_tasks) == 0 + + +def test_running_stop_callback_error_propagates_and_cleans_resources(): + seen = [] + calls = 0 + def broken_stop(): + nonlocal calls + calls += 1 + if calls > 2: + raise RuntimeError("stop callback failed") + return False + with gevent.Timeout(1): + with pytest.raises(RuntimeError, match="stop callback failed"): + wrapper.prepare_env(IdleUser, user_count=2, test_time=None, + on_environment=seen.append, stop_requested=broken_stop) + assert len(seen[0].runner.greenlet) == 0 + assert len(seen[0].runner.user_greenlets) == 0 + assert len(seen[0].load_density_tasks) == 0 + + +def test_master_stop_callback_error_propagates_and_restores_settings(monkeypatch): + original = (runners.HEARTBEAT_INTERVAL, runners.HEARTBEAT_LIVENESS) + seen = [] + calls = 0 + def created(env): + seen.append(env) + _controlled_workers(env, monkeypatch) + def broken_stop(): + nonlocal calls + calls += 1 + if calls > 2: + raise RuntimeError("master stop callback failed") + return False + with gevent.Timeout(2): + with pytest.raises(RuntimeError, match="master stop callback failed"): + wrapper.prepare_env(IdleUser, runner_mode="master", expected_workers=2, + user_count=2, spawn_rate=100, test_time=None, + on_environment=created, stop_requested=broken_stop) + assert seen[0].runner.server.socket.closed + assert len(seen[0].load_density_tasks) == 0 + assert (runners.HEARTBEAT_INTERVAL, runners.HEARTBEAT_LIVENESS) == original + + +def test_master_stop_callback_error_interrupts_slow_ramp(monkeypatch): + seen = [] + def created(env): + seen.append(env) + _controlled_workers(env, monkeypatch) + def broken_stop(): + if seen and seen[0].runner.user_count >= 1: + raise RuntimeError("ramp callback failed") + return False + with gevent.Timeout(2): + with pytest.raises(RuntimeError, match="ramp callback failed"): + wrapper.prepare_env(IdleUser, runner_mode="master", expected_workers=2, + user_count=10, spawn_rate=1, test_time=None, + on_environment=created, stop_requested=broken_stop) + assert len(seen[0].runner.greenlet) == 0 + assert len(seen[0].load_density_tasks) == 0 + assert seen[0].runner.server.socket.closed From 82912d59c60fe2cd4c65c8ffbf1ee40c90579ba8 Mon Sep 17 00:00:00 2001 From: JeffreyChen Date: Fri, 2 Oct 2026 20:29:14 +0800 Subject: [PATCH 11/18] feat: share percentile bands and redesign live dashboard --- README.md | 21 +- README/README_zh-CN.md | 18 +- README/README_zh-TW.md | 18 +- architecture.md | 9 + docker/extras.Dockerfile | 3 +- .../generate_report/generate_report_doc.rst | 16 ++ .../doc/live_dashboard/live_dashboard_doc.rst | 10 + .../generate_report/generate_report_doc.rst | 14 ++ .../doc/live_dashboard/live_dashboard_doc.rst | 7 + .../plans/2026-10-02-ui-percentile-bands.md | 46 ++++ docs/updates/2026-10.md | 9 + docs/updates/README.md | 1 + je_load_density/gui/chart_panel.py | 182 +++++++++------- .../utils/dashboard/dashboard_html.py | 136 ++++++++++++ .../utils/dashboard/live_dashboard.py | 201 +++++++++--------- .../generate_report/generate_chart_report.py | 53 ++--- .../generate_summary_report.py | 14 +- .../utils/test_record/window_statistics.py | 121 +++++++++++ .../wrapper/user_template/_common.py | 7 +- .../user_template/async_http_user_template.py | 3 +- .../user_template/grpc_user_template.py | 3 +- .../user_template/kafka_user_template.py | 3 +- .../user_template/mongo_user_template.py | 3 +- .../user_template/mqtt_user_template.py | 7 +- .../user_template/redis_user_template.py | 3 +- .../user_template/socket_user_template.py | 7 +- .../user_template/sql_user_template.py | 3 +- .../user_template/sse_user_template.py | 7 +- .../user_template/websocket_user_template.py | 7 +- progress.md | 5 +- test/smoke/platform_probes.py | 29 +-- test/test_chart_report.py | 47 ++-- test/test_dashboard_ui.py | 52 +++++ test/test_gui_bands.py | 50 +++++ test/test_live_dashboard.py | 56 ++++- test/test_protocol_event_timing.py | 41 ++++ test/test_window_statistics.py | 60 ++++++ 37 files changed, 995 insertions(+), 277 deletions(-) create mode 100644 docs/superpowers/plans/2026-10-02-ui-percentile-bands.md create mode 100644 je_load_density/utils/dashboard/dashboard_html.py create mode 100644 je_load_density/utils/test_record/window_statistics.py create mode 100644 test/test_dashboard_ui.py create mode 100644 test/test_gui_bands.py create mode 100644 test/test_protocol_event_timing.py create mode 100644 test/test_window_statistics.py diff --git a/README.md b/README.md index 4078c22..3fc25c3 100644 --- a/README.md +++ b/README.md @@ -85,7 +85,7 @@ LoadDensity (`je_load_density`) started as a Locust wrapper and grew into a full - **SLA gates + regression diff.** `LD_assert_sla` fails CI when latency / failure-rate / request-count rules breach; `LD_diff_runs` compares two SQLite-persisted runs and flags per-name regressions over a tolerance. - **Seven report formats.** HTML, JSON, XML, CSV, JUnit XML, percentile-summary JSON, plus optional matplotlib **chart reports** (`latency-over-time` + `RPS-over-time` PNGs via `[charts]` extra). - **Four live exporters.** Prometheus HTTP endpoint, InfluxDB line-protocol UDP/HTTP sink, OpenTelemetry OTLP gRPC exporter, **Datadog DogStatsD UDP** sink — all lazily imported and gated by the matching install extra. -- **Live web dashboard.** `start_dashboard()` boots a stdlib HTTP + SSE server that streams running RPS / avg / p95 / failure counts to any browser, per-name table included. +- **Live web dashboard.** Responsive metric cards, separate latency/RPS charts with percentile bands, SSE connection status and a per-name table, served by `start_dashboard()`. - **Slack + Teams notifiers.** Block Kit + MessageCard summary posters keyed off the build_summary output (`LD_post_slack_summary`, `LD_post_teams_summary`). - **Assertions + extractors.** `status_code`, `contains`, `not_contains`, `json_path`, `header` assertions run under Locust's `catch_response`; extractors with sources `json_path` / `header` / `status_code` write back into the parameter resolver. - **Distributed runners.** `runner_mode="master"` / `"worker"` with a configurable healthy-worker startup gate, native heartbeat monitoring and virtual-user rebalancing after worker loss. @@ -685,6 +685,25 @@ Finite-work leases and canonical worker-record aggregation remain pending. errors propagate after cleanup. `prepare_env` owns runner/UI/RPC/auxiliary tasks; direct `create_env` callers must call `cleanup_env(env)` when finished. +## Percentile charts and dashboard + +Qt, browser and `[charts]` PNG reports share request-start time buckets that include +successful and failed requests. Charts show a p50 line, p50–p95 and p95–p99 bands, +with RPS on a separate chart. Empty or unmeasured latency windows are gaps; timed +requests still count toward throughput. Invalid/negative/nonfinite latency is +excluded from latency statistics without dropping request counts. + +Live charts retain the newest 120 one-second buckets. PNG reports retain up to +10,000 buckets by default; `generate_chart_report(..., bucket_size_seconds=1.0, +max_buckets=10000)` controls the limits. Partial buckets use actual duration for +RPS. Window percentiles use `round(p / 100 * (n - 1))` (Python ties-to-even); +overall summary/card percentiles preserve linear interpolation. Report filenames, +return keys and existing dashboard snapshot keys remain compatible; +`latency_windows` adds bounded chart data. SSE clients can stream while other +clients fetch snapshots, and stopping the dashboard closes the stream. + +The desktop GUI controls and execution lifecycle are still tracked separately. + ## HAR Record / Replay ```python diff --git a/README/README_zh-CN.md b/README/README_zh-CN.md index 2bced03..fa29eb1 100644 --- a/README/README_zh-CN.md +++ b/README/README_zh-CN.md @@ -85,7 +85,7 @@ LoadDensity(`je_load_density`)从 Locust 封装起步,逐步成长为完整的 - **SLA gate + 回归 diff。** 当 latency / failure-rate / request-count 规则破线时,`LD_assert_sla` 会让 CI 失败;`LD_diff_runs` 比对两个持久化到 SQLite 的 run,并标出超过容忍范围的 per-name 回归。 - **七种报告格式。** HTML、JSON、XML、CSV、JUnit XML、百分位摘要 JSON,再加上可选的 matplotlib **chart 报告**(通过 `[charts]` extra 生成 `latency-over-time` 与 `RPS-over-time` PNG)。 - **四种实时 exporter。** Prometheus HTTP 端点、InfluxDB line-protocol UDP/HTTP sink、OpenTelemetry OTLP gRPC exporter、**Datadog DogStatsD UDP** sink — 全部延迟导入,并由对应的安装 extra 控制。 -- **实时 web dashboard。** `start_dashboard()` 会启动一个 stdlib HTTP + SSE 服务器,把运行中的 RPS / avg / p95 / failure 计数流式推送到任意浏览器,并附上 per-name 表格。 +- **实时 web dashboard。** `start_dashboard()` 提供响应式指标卡片、分开的延迟／RPS 百分位数带状图、SSE 连接状态与 per-name 表格。 - **Slack + Teams 通知。** 以 build_summary 的输出为基础的 Block Kit + MessageCard 摘要投递器(`LD_post_slack_summary`、`LD_post_teams_summary`)。 - **断言 + 提取。** `status_code`、`contains`、`not_contains`、`json_path`、`header` 断言在 Locust 的 `catch_response` 下执行;来源为 `json_path` / `header` / `status_code` 的提取器会把值写回参数解析器。 - **分布式 runner。** `runner_mode="master"` / `"worker"` 提供可配置的健康 worker 启动门槛、原生 heartbeat 监测与失联后的虚拟用户负载重新分配。 @@ -682,6 +682,22 @@ master 结果包含 `distributed_health`、观测容量与受影响 worker ID。 callback 错误在清理后传回。`prepare_env` 负责 runner／UI／RPC／辅助 task 的资源生命周期； 直接调用 `create_env` 的用户须在完成后调用 `cleanup_env(env)`。 +## 百分位数图表与 Dashboard + +Qt、浏览器与 `[charts]` PNG 报告共用以请求开始时间分桶的统计，纳入成功与失败请求。 +延迟图显示 p50 线、p50–p95 与 p95–p99 带状区，RPS 使用独立图表。 +空窗或没有延迟测量的时间桶显示断点；有时间戳的请求仍计入 throughput。 +无效、负值或非有限延迟不列入延迟统计，但不删除请求计数。 + +实时图保留最新 120 个一秒桶；PNG 默认保留最多 10,000 桶， +可用 `generate_chart_report(..., bucket_size_seconds=1.0, max_buckets=10000)` 设置。 +部分时间桶依实际长度计算 RPS。时间窗百分位数使用 `round(p / 100 * (n - 1))` +（Python ties-to-even）；整体 summary／卡片保留线性插值。 +报告文件名、返回键与原有 dashboard snapshot 键保持兼容， +新增 `latency_windows` 提供有上限的图表数据。SSE 流不阻挡其他 snapshot 请求，停止 dashboard 时关闭流。 + +桌面 GUI 的控制界面与执行生命周期仍另列待办。 + ## HAR 录制/重放 ```python diff --git a/README/README_zh-TW.md b/README/README_zh-TW.md index ccd7448..d92fd7a 100644 --- a/README/README_zh-TW.md +++ b/README/README_zh-TW.md @@ -85,7 +85,7 @@ LoadDensity(`je_load_density`)從 Locust 封裝起家,逐步成長為完整的 - **SLA gate + 回歸 diff。** 當 latency / failure-rate / request-count 規則破線時,`LD_assert_sla` 會讓 CI 失敗;`LD_diff_runs` 比對兩個持久化到 SQLite 的 run,並標出超過容忍範圍的 per-name 回歸。 - **七種報告格式。** HTML、JSON、XML、CSV、JUnit XML、百分位摘要 JSON,再加上選用的 matplotlib **chart 報告**(透過 `[charts]` extra 產生 `latency-over-time` 與 `RPS-over-time` PNG)。 - **四種即時 exporter。** Prometheus HTTP 端點、InfluxDB line-protocol UDP/HTTP sink、OpenTelemetry OTLP gRPC exporter、**Datadog DogStatsD UDP** sink — 全部延遲匯入,並由對應的安裝 extra 控制。 -- **即時 web dashboard。** `start_dashboard()` 會啟動一個 stdlib HTTP + SSE 伺服器,把執行中的 RPS / avg / p95 / failure 計數串流到任意瀏覽器,並附上 per-name 表格。 +- **即時 web dashboard。** `start_dashboard()` 提供響應式指標卡片、分開的延遲／RPS 百分位數帶狀圖、SSE 連線狀態與 per-name 表格。 - **Slack + Teams 通知。** 以 build_summary 的輸出為基礎的 Block Kit + MessageCard 摘要張貼器(`LD_post_slack_summary`、`LD_post_teams_summary`)。 - **斷言 + 擷取。** `status_code`、`contains`、`not_contains`、`json_path`、`header` 斷言在 Locust 的 `catch_response` 下執行;來源為 `json_path` / `header` / `status_code` 的擷取器會把值寫回參數解析器。 - **分散式 runner。** `runner_mode="master"` / `"worker"` 提供可設定的健康 worker 啟動門檻、原生 heartbeat 監測與失聯後的虛擬使用者負載重新分配。 @@ -682,6 +682,22 @@ master 結果包含 `distributed_health`、觀測容量與受影響 worker ID。 callback 錯誤在清理後傳回。`prepare_env` 負責 runner／UI／RPC／輔助 task 的資源生命週期； 直接呼叫 `create_env` 的使用者須在完成後呼叫 `cleanup_env(env)`。 +## 百分位數圖表與 Dashboard + +Qt、瀏覽器與 `[charts]` PNG 報告共用以請求開始時間分桶的統計，納入成功與失敗請求。 +延遲圖顯示 p50 線、p50–p95 與 p95–p99 帶狀區，RPS 使用獨立圖表。 +空窗或沒有延遲量測的時間桶顯示斷點；有時間戳的請求仍計入 throughput。 +無效、負值或非有限延遲不列入延遲統計，但不刪除請求計數。 + +即時圖保留最新 120 個一秒桶；PNG 預設保留最多 10,000 桶， +可用 `generate_chart_report(..., bucket_size_seconds=1.0, max_buckets=10000)` 設定。 +部分時間桶依實際長度計算 RPS。時間窗百分位數使用 `round(p / 100 * (n - 1))` +（Python ties-to-even）；整體 summary／卡片保留線性插值。 +報告檔名、回傳鍵與原有 dashboard snapshot 鍵維持相容， +新增 `latency_windows` 提供有上限的圖表資料。SSE 串流不阻擋其他 snapshot 請求，停止 dashboard 時關閉串流。 + +桌面 GUI 的控制介面與執行生命週期仍另列待辦。 + ## HAR 錄製/重放 ```python diff --git a/architecture.md b/architecture.md index e32d681..ae2a81c 100644 --- a/architecture.md +++ b/architecture.md @@ -135,6 +135,15 @@ MCP `load_density.list_executor_commands` tool all read the `LD_*` names from `e ## 6. Cross-project boundaries +`utils.test_record.window_statistics` supplies shared request-start buckets for +Qt, browser and PNG percentile bands. It merges success/failure samples, leaves +unmeasured windows null and counts timed requests using actual bucket duration. +Window percentiles use rounded order statistics; existing aggregate summary keys +and interpolation remain compatible. Dashboard snapshots add `latency_windows` +and retain legacy keys. Live charts bound history to 120 buckets; offline defaults +to 10,000. Protocol events convert monotonic duration clocks to epoch starts. +The responsive dashboard uses SVG/text nodes and a threaded SSE server. + Locust master/worker runs use scoped native heartbeat settings and rebalancing. Startup defaults to failing an unmet healthy ready-worker count; explicit degraded policy still requires one worker. Master results add `distributed_health`; this is diff --git a/docker/extras.Dockerfile b/docker/extras.Dockerfile index e7c0d95..b673f95 100644 --- a/docker/extras.Dockerfile +++ b/docker/extras.Dockerfile @@ -13,6 +13,7 @@ RUN if [ "$EXTRA" = "gui" ] || [ "$EXTRA" = "all" ]; then \ COPY test/smoke /checks/smoke RUN --mount=type=cache,target=/root/.cache/pip,sharing=locked python /checks/smoke/install_extra.py "$EXTRA" ENV LD_EXTRA=${EXTRA} QT_QPA_PLATFORM=offscreen AWS_EC2_METADATA_DISABLED=true -USER 65534:65534 +RUN useradd --create-home --uid 10001 --user-group --shell /usr/sbin/nologin ld-smoke +USER 10001:10001 WORKDIR /tmp CMD ["python", "/checks/smoke/run_cell.py"] diff --git a/docs/source/En/doc/generate_report/generate_report_doc.rst b/docs/source/En/doc/generate_report/generate_report_doc.rst index 27f0d14..91e31a5 100644 --- a/docs/source/En/doc/generate_report/generate_report_doc.rst +++ b/docs/source/En/doc/generate_report/generate_report_doc.rst @@ -75,6 +75,22 @@ The summary contains totals, per-name counts, min / max / mean / percentile (p50 / p90 / p95 / p99) latencies, and an overall block. Useful for charting and regression checks across runs. +Percentile band charts +---------------------- + +Install the ``charts`` extra, then call ``generate_chart_report(report_name, +bucket_size_seconds=1.0, max_buckets=10000)``. Existing ``-latency.png`` / +``-rps.png`` filenames and ``latency`` / ``rps`` return keys are preserved. +Request-start buckets merge success/failure samples, plot a p50 line and +p50–p95 / p95–p99 bands, and calculate RPS using actual bucket duration. +Missing/invalid latency leaves a gap without discarding timed request counts. +The default history limit retains the newest 10,000 buckets. + +Window percentiles select index ``round(p / 100 * (n - 1))`` with Python +ties-to-even rounding. Overall summary percentiles retain linear interpolation. +Latency statistics exclude negative/nonfinite/invalid measurements; totals retain +all requests. Qt and browser live charts use the same helper and retain 120 buckets. + Action JSON ----------- diff --git a/docs/source/En/doc/live_dashboard/live_dashboard_doc.rst b/docs/source/En/doc/live_dashboard/live_dashboard_doc.rst index 1ca1ef3..e1ca5cc 100644 --- a/docs/source/En/doc/live_dashboard/live_dashboard_doc.rst +++ b/docs/source/En/doc/live_dashboard/live_dashboard_doc.rst @@ -8,6 +8,16 @@ A stdlib-only HTTP server with a Server-Sent Events stream that pushes running stats from ``test_record_instance`` to any browser. Use it for demos, war rooms, and on-call. +The responsive page shows metric cards, SSE connection status, separate latency +and RPS charts, p50–p95 / p95–p99 bands and text-only request-group names. +``latency_windows`` extends the existing snapshot keys with the newest 120 +one-second buckets, including failed requests. Empty latency windows stay +disconnected; partial buckets use actual duration for RPS. Window percentiles +use rounded order statistics; overall cards preserve summary interpolation. +The request-group table retains the legacy successful-request statistics and +shows up to 100 groups. Streaming clients do not block snapshots; stop closes +streams and the server thread. + Endpoints --------- diff --git a/docs/source/Zh/doc/generate_report/generate_report_doc.rst b/docs/source/Zh/doc/generate_report/generate_report_doc.rst index 4f9f536..585d44a 100644 --- a/docs/source/Zh/doc/generate_report/generate_report_doc.rst +++ b/docs/source/Zh/doc/generate_report/generate_report_doc.rst @@ -66,6 +66,20 @@ Summary（百分位） 包含 totals、per-name 計數、min / max / mean / 百分位（p50 / p90 / p95 / p99）延遲與整體區塊。便於繪圖與跨次回歸檢查。 +百分位數帶狀圖 +-------------- + +安裝 ``charts`` extra 後呼叫 ``generate_chart_report(report_name, +bucket_size_seconds=1.0, max_buckets=10000)``。 +原有 ``-latency.png``／``-rps.png`` 檔名與 ``latency``／``rps`` 回傳鍵保持相容。 +依請求開始時間分桶，合併成功／失敗樣本，繪製 p50 線與 p50–p95／p95–p99 帶狀區， +依實際桶長計算 RPS。沒有或無效的延遲顯示斷點，有時間戳的請求仍計數。 +預設保留最新 10,000 個桶。 + +時間窗百分位數選取 ``round(p / 100 * (n - 1))`` 索引（Python ties-to-even）； +整體 summary 百分位數保留線性插值。延遲統計排除負值、非有限或無效量測， +請求總數仍包含所有請求。Qt 與瀏覽器即時圖共用統計 helper，保留 120 桶。 + 動作 JSON --------- diff --git a/docs/source/Zh/doc/live_dashboard/live_dashboard_doc.rst b/docs/source/Zh/doc/live_dashboard/live_dashboard_doc.rst index 2eb2fae..f405fb9 100644 --- a/docs/source/Zh/doc/live_dashboard/live_dashboard_doc.rst +++ b/docs/source/Zh/doc/live_dashboard/live_dashboard_doc.rst @@ -1,6 +1,13 @@ 即時 Dashboard ============== +響應式頁面顯示指標卡片、SSE 連線狀態、分開的延遲／RPS 圖表、 +p50–p95／p95–p99 帶狀區與以純文字呈現的請求群組名稱。 +``latency_windows`` 在原有 snapshot 鍵之外提供最新 120 個一秒桶，包含失敗請求。 +空窗保持斷點，部分桶依實際長度計算 RPS。時間窗百分位數使用取整的順序統計； +整體卡片保留 summary 插值。請求群組表維持原有成功請求統計，最多顯示 100 群組。 +串流不阻擋 snapshot；停止時關閉串流與伺服器執行緒。 + 概觀 ---- diff --git a/docs/superpowers/plans/2026-10-02-ui-percentile-bands.md b/docs/superpowers/plans/2026-10-02-ui-percentile-bands.md new file mode 100644 index 0000000..2b048a2 --- /dev/null +++ b/docs/superpowers/plans/2026-10-02-ui-percentile-bands.md @@ -0,0 +1,46 @@ +# UI and percentile band implementation plan + +Execute in the existing isolated feature worktree under the approved roadmap. + +## Shared statistical windows + +- [x] Add a pure statistics helper that merges success/failure samples, prefers epoch start_time, sorts observations, and creates bounded time buckets. Exclude missing/invalid latency measurements, retain request counts, and represent empty latency windows as null. +- [x] Use the approved rounded-order-statistic rule for p50/p95/p99 window charts. Preserve the existing summary report's linear interpolation and keys for consumers; explicitly document this existing aggregate/window distinction. +- [x] Verify unsorted records, failure samples, zero timestamps, gaps, singleton windows, non-unit bucket durations and bounded history. + +## Offline and live charts + +- [x] Render a p50 line, p50–p95 and p95–p99 fill_between bands for offline reports, with a separate RPS chart. Preserve report filenames and return keys. +- [x] Replace mixed Qt axes with separate latency/RPS charts and Qt area series; bound histories and leave gaps disconnected. +- [x] Extend dashboard JSON/SSE with bounded chart windows, responsive SVG bands, failure rate and percentile cards; preserve existing snapshot keys and render dynamic names with text nodes. + +## GUI lifecycle and installed wheel smoke + +- [ ] Inspect and redesign existing GUI into settings/run controls, metrics/charts and results/logs/history. Preserve language and script loading. +- [ ] Make Start/Stop states explicit; background execution emits Qt signals only, failures/completion retain results, cancellation cleans up runner/client/thread. +- [ ] Extend offscreen installed-wheel GUI probe with real execution and cancellation; verify Qt/widget lifecycle, dashboard SSE concurrency and safe dynamic content. +- [x] Run focused and full checks for this batch, update three README languages/Sphinx/architecture and outstanding ledger, review, commit and push. + +## User-requested stopping boundary + +On 2026-10-02 the user requested finishing the current batch, committing it and +removing completed progress items, then stopping. The shared bands, responsive +browser dashboard and existing native worker-health work are the current batch. +Full desktop controls/Start-Stop/action-file redesign and execution/cancellation +installed-wheel GUI smoke remain in progress.md #18/#24. Async engine parity and +shared functional/load/synthetic scripts are also deferred; no new implementation +for those domains was started after the stopping request. + +The browser UI was verified in headless Edge at desktop (1440px) and mobile +(390px) widths with real SSE, disconnected latency gaps, text-only malicious +request names, no JavaScript errors and no horizontal overflow. The installed +charts probe now renders actual LoadDensity report PNGs. Protocol event start +timestamps are converted from monotonic duration clocks to epoch time. Summary +latency filtering and scaled means preserve finite dashboard JSON and counts. + +Final batch verification: 1442 passed, 14 optional skips and four existing +warnings; Ruff E/F/W/C90/I passed. Six subprocess smoke tests passed both from +the checkout and from the rebuilt installed wheel in a network-disabled Docker +container. The Docker runtime user now has a real writable home, addressing +qt-material theme generation under an unprivileged account. Full extras CI +verification remains tracked until the updated matrix finishes. diff --git a/docs/updates/2026-10.md b/docs/updates/2026-10.md index 976ff98..8adea98 100644 --- a/docs/updates/2026-10.md +++ b/docs/updates/2026-10.md @@ -240,3 +240,12 @@ Added a stdlib smoke harness that exercises local HTTP requests in isolated proc - Scoped matching native heartbeat settings and Locust rebalancing retain target/ramp, report observed capacity and affected worker IDs, and terminate after all workers are lost. No request replay, finite shard leases or canonical distributed aggregation is advertised. - Cooperative lifecycle callbacks cancel readiness/ramp/running work; callback errors propagate after cleanup. Native runner/UI/RPC/auxiliary resources close idempotently. - Verification: 31 native/control/RPC health tests plus 5 documentation-count tests passed. Independent review and Ruff E/F/W/C90/I pass; lifecycle helpers remain within complexity limits. Remaining distributed record/shard scope retained in progress.md #29. + +## U-20261002-11 · 2026-10-02 · Share percentile bands and redesign live dashboard · #done #charts #ui #26 + +- Closed progress.md #26: shared request-start windows merge success/failure samples, retain timed counts with invalid latency, use actual bucket duration and preserve gaps. Live history is120 buckets; offline default10000. Window order statistics retain the approved rounding; aggregate summary interpolation/keys stay compatible. +- Responsive browser metrics/SVG bands and separate Qt latency/RPS charts now share the helper. SSE no longer blocks snapshots and stops streams during shutdown. Dynamic request names use text nodes. +- Corrected protocol start timestamps from duration clocks to epoch starts. Invalid/huge finite latency regressions retain counts and valid JSON through filtering and scaled means. +- Installed chart capability now exercises actual report PNG generation. Unprivileged Docker cells have a real writable home for qt-material resources; updated full extras validation remains pending CI. +- Verification: full suite1442 passed/14 optional skips/four existing warnings; six checkout smoke tests and six rebuilt installed-wheel Docker smoke tests passed; Ruff E/F/W/C90/I and whitespace checks pass. Real Edge desktop/mobile/SSE/XSS/gap/responsiveness checks passed. Independent review's numeric edge was reproduced and fixed. +- User requested stopping after this batch. Full desktop Start/Stop/action-file redesign, async parity, canonical worker aggregation and shared functional/load/synthetic scripts remain explicitly outstanding; no implementation started after that stopping request. diff --git a/docs/updates/README.md b/docs/updates/README.md index d7821c8..e71a1db 100644 --- a/docs/updates/README.md +++ b/docs/updates/README.md @@ -58,6 +58,7 @@ In the same commit: delete the item from `progress.md`, add a `#done` entry here | ID | Date | Title | Tags | Batch | |---|---|---|---|---| +| U-20261002-11 | 2026-10-02 | Share percentile bands and redesign live dashboard | #done #charts #ui #26 | [2026-10](2026-10.md) | | U-20261002-10 | 2026-10-02 | Monitor worker health and rebalance native load | #done #distributed | [2026-10](2026-10.md) | | U-20261002-09 | 2026-10-02 | Validate cloud launch outcomes and preserve partial failures | #done #cloud | [2026-10](2026-10.md) | | U-20261002-08 | 2026-10-02 | Repair Qt Web Push and NATS installation probes | #done #ci #extras | [2026-10](2026-10.md) | diff --git a/je_load_density/gui/chart_panel.py b/je_load_density/gui/chart_panel.py index f94d3b3..a99910e 100644 --- a/je_load_density/gui/chart_panel.py +++ b/je_load_density/gui/chart_panel.py @@ -1,99 +1,133 @@ -""" -Live RPS / latency chart widget. - -Drop into the existing PySide6 GUI alongside ``StatsPanel``. Uses -``QtCharts`` (ships with PySide6) so no extra dependency. -""" +"""Bounded latency bands and a separate request-rate chart for Qt.""" +import math import time -from collections import deque -from typing import Deque, Tuple +from itertools import chain -from PySide6.QtCharts import QChart, QChartView, QLineSeries, QValueAxis +from PySide6.QtCharts import QAreaSeries, QChart, QChartView, QLineSeries, QValueAxis from PySide6.QtCore import QPointF, Qt, QTimer -from PySide6.QtGui import QPainter +from PySide6.QtGui import QColor, QPainter from PySide6.QtWidgets import QVBoxLayout, QWidget from je_load_density.utils.test_record.test_record_class import test_record_instance - +from je_load_density.utils.test_record.window_statistics import latency_windows _HISTORY_SECONDS = 120 -def _percentile(values, pct: float) -> float: - if not values: - return 0.0 - ordered = sorted(values) - index = max(0, min(len(ordered) - 1, int(round(pct / 100.0 * (len(ordered) - 1))))) - return float(ordered[index]) +def _chart(title: str, unit: str): + chart = QChart() + chart.setTitle(title) + chart.legend().setAlignment(Qt.AlignBottom) + x, y = QValueAxis(), QValueAxis() + x.setTitleText("elapsed seconds") + x.setRange(0, _HISTORY_SECONDS) + y.setTitleText(unit) + y.setRange(0, 100) + chart.addAxis(x, Qt.AlignBottom) + chart.addAxis(y, Qt.AlignLeft) + view = QChartView(chart) + view.setRenderHint(QPainter.Antialiasing) + return chart, view, x, y + + +def _segments(windows): + segment = [] + for window in windows: + if window["p50_ms"] is None: + if segment: + yield segment + segment = [] + else: + segment.append(window) + if segment: + yield segment class LiveChartPanel(QWidget): - """Two-series line chart (RPS + p95 latency).""" + """p50 line and p50–p95/p95–p99 bands; gaps remain disconnected.""" def __init__(self, parent=None) -> None: super().__init__(parent) - self._buffer: Deque[Tuple[float, int, float]] = deque(maxlen=_HISTORY_SECONDS) - self._last_count = 0 - self._start_ts = time.monotonic() - + self._latency_chart, self._chart_view, self._latency_axis_x, self._latency_axis_y = _chart( + "Latency · p50 / p95 / p99", "milliseconds") + self._rps_chart, self._rps_view, self._rps_axis_x, self._rps_axis_y = _chart("Throughput", "requests / second") + self._rps_chart.legend().hide() self._rps_series = QLineSeries() self._rps_series.setName("RPS") - self._latency_series = QLineSeries() - self._latency_series.setName("p95 ms") - - chart = QChart() - chart.addSeries(self._rps_series) - chart.addSeries(self._latency_series) - chart.setTitle("LoadDensity live") - chart.legend().setAlignment(Qt.AlignBottom) - - axis_x = QValueAxis() - axis_x.setTitleText("seconds") - axis_x.setRange(0, _HISTORY_SECONDS) - chart.addAxis(axis_x, Qt.AlignBottom) - self._rps_series.attachAxis(axis_x) - self._latency_series.attachAxis(axis_x) - - axis_y = QValueAxis() - axis_y.setTitleText("value") - axis_y.setRange(0, 100) - chart.addAxis(axis_y, Qt.AlignLeft) - self._rps_series.attachAxis(axis_y) - self._latency_series.attachAxis(axis_y) - - self._chart_view = QChartView(chart) - self._chart_view.setRenderHint(QPainter.Antialiasing) - - layout = QVBoxLayout() - layout.addWidget(self._chart_view) - self.setLayout(layout) - + self._rps_chart.addSeries(self._rps_series) + self._rps_series.attachAxis(self._rps_axis_x) + self._rps_series.attachAxis(self._rps_axis_y) + self._latency_lines = [] + self._band_series = [] + self.reset_history() + layout = QVBoxLayout(self) + layout.setContentsMargins(0, 0, 0, 0) + layout.addWidget(self._chart_view, 2) + layout.addWidget(self._rps_view, 1) + self._chart_view.setMinimumHeight(220) + self._rps_view.setMinimumHeight(160) self._timer = QTimer(self) self._timer.setInterval(1000) self._timer.timeout.connect(self.refresh) self._timer.start() - def refresh(self) -> None: - records = ( - list(test_record_instance.test_record_list) - + list(test_record_instance.error_record_list) - ) - total = len(records) - delta = max(total - self._last_count, 0) - self._last_count = total - - latencies = [ - float(record.get("response_time_ms")) - for record in records[-200:] - if record.get("response_time_ms") is not None - ] - p95 = _percentile(latencies, 95) - now = time.monotonic() - self._start_ts - self._buffer.append((now, delta, p95)) - + def reset_history(self, start_time: float | None = None) -> None: + self._start_epoch = math.floor(time.time()) if start_time is None else start_time self._rps_series.clear() - self._latency_series.clear() - for sample in self._buffer: - self._rps_series.append(QPointF(sample[0], sample[1])) - self._latency_series.append(QPointF(sample[0], sample[2])) + self._latency_chart.removeAllSeries() + self._latency_lines.clear() + self._band_series.clear() + + def _add_latency(self, series) -> None: + self._latency_chart.addSeries(series) + series.attachAxis(self._latency_axis_x) + series.attachAxis(self._latency_axis_y) + + def _line(self, segment, key: str): + series = QLineSeries() + series.replace([QPointF(item["start_time"] - self._start_epoch, item[key]) for item in segment]) + return series + + def _band(self, segment, lower: str, upper: str, color: str): + low, high = self._line(segment, lower), self._line(segment, upper) + area = QAreaSeries(high, low) + # Area boundaries stay caller-owned; parent them explicitly for refresh cleanup. + low.setParent(area) + high.setParent(area) + area.setName(f"{lower.split('_')[0]}–{upper.split('_')[0]}") + area.setColor(QColor(color)) + area.setBorderColor(QColor("transparent")) + self._add_latency(area) + self._band_series.append(area) + + def _render_latency(self, windows) -> None: + self._latency_chart.removeAllSeries() + self._latency_lines.clear() + self._band_series.clear() + for index, segment in enumerate(_segments(windows)): + self._band(segment, "p50_ms", "p95_ms", "#664fc3f7") + self._band(segment, "p95_ms", "p99_ms", "#33a78bfa") + line = self._line(segment, "p50_ms") + line.setName("p50") + line.setColor(QColor("#1591d0")) + line.setPointsVisible(True) + self._add_latency(line) + self._latency_lines.append(line) + if index: + for series in (line, *self._band_series[-2:]): + for marker in self._latency_chart.legend().markers(series): + marker.setVisible(False) + + def refresh(self) -> None: + now = max(time.time(), self._start_epoch) + records = chain(test_record_instance.test_record_list, test_record_instance.error_record_list) + windows = latency_windows(records, start=self._start_epoch, end=now, max_buckets=_HISTORY_SECONDS) + self._rps_series.replace([QPointF(item["start_time"] - self._start_epoch, item["rps"]) for item in windows]) + self._render_latency(windows) + left = windows[0]["start_time"] - self._start_epoch if windows else 0 + right = max(left + 1, now - self._start_epoch) + for axis in (self._latency_axis_x, self._rps_axis_x): + axis.setRange(left, right) + self._latency_axis_y.setRange(0, max(1, max((item["p99_ms"] or 0 for item in windows), default=0) * 1.15)) + self._rps_axis_y.setRange(0, max(1, max((item["rps"] for item in windows), default=0) * 1.15)) diff --git a/je_load_density/utils/dashboard/dashboard_html.py b/je_load_density/utils/dashboard/dashboard_html.py new file mode 100644 index 0000000..85d5946 --- /dev/null +++ b/je_load_density/utils/dashboard/dashboard_html.py @@ -0,0 +1,136 @@ +"""Responsive telemetry page with bounded SVG bands and text-only dynamic names.""" + +HTML = """ + +LoadDensity Live
+
LOADDENSITY / OBSERVABILITY

Live performance

+Connecting…
+

Capacity, latency and failures from the current record stream.

+
+
Requests
0
+
Recent throughput
+
0 req/s
+
Failure rate
0%
+
Overall p50
+
0 ms
+
Overall p95
+
0 ms
+
Overall p99
+
0 ms
+
Failures
0
+
+
+

Latency distribution · milliseconds

+ +
p50p50–p95p95–p99
+

Throughput · requests / second

+ +
Each bucket uses its actual duration. Empty latency windows remain disconnected.
+
+

Request groups

+ +
NameCountMean msp95 ms
+
Group statistics cover successful requests. Showing up to 100 groups.
+

Recent one-second windows include successful and failed requests. Empty windows are gaps. +

+
+""" diff --git a/je_load_density/utils/dashboard/live_dashboard.py b/je_load_density/utils/dashboard/live_dashboard.py index 02841e9..4a4d42d 100644 --- a/je_load_density/utils/dashboard/live_dashboard.py +++ b/je_load_density/utils/dashboard/live_dashboard.py @@ -16,53 +16,20 @@ import json import threading import time -from http.server import BaseHTTPRequestHandler, HTTPServer -from typing import Any, Dict, List, Optional - - -_HTML = """ -LoadDensity Live - - -

LoadDensity Live

-
-
Total
0
-
RPS
0
-
Avg ms
0
-
P95 ms
0
-
- -
NameCountMeanP95
- - -""" +from functools import partial +from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer +from itertools import chain +from typing import Any, Dict, Optional + +from je_load_density.utils.dashboard.dashboard_html import HTML +from je_load_density.utils.test_record.window_statistics import ( + finite_number, + latency_mean, + latency_windows, + sample_time, +) + +_HTML = HTML def snapshot_metrics(window_seconds: float = 10.0) -> Dict[str, Any]: @@ -87,13 +54,18 @@ def snapshot_metrics(window_seconds: float = 10.0) -> Dict[str, Any]: "p99_ms": 0, "max_ms": 0, "count": 0}, "per_name": {}} - records: List[Dict[str, Any]] = list(test_record_instance.test_record_list) - cutoff = time.time() - window_seconds - recent = [r for r in records if r.get("ts") is None or float(r["ts"]) >= cutoff] - latencies = [float(r.get("response_time_ms") or 0) for r in recent - if r.get("response_time_ms") is not None] - rps = (len(recent) / window_seconds) if window_seconds > 0 else 0.0 - avg_ms = (sum(latencies) / len(latencies)) if latencies else 0.0 + duration = finite_number(window_seconds) + if duration is None or duration <= 0: + raise ValueError("window_seconds: expected a finite positive number") + now = time.time() + records = chain(test_record_instance.test_record_list, test_record_instance.error_record_list) + windows = latency_windows(records, start=int(now) - 120, end=now, max_buckets=120) + recent = [record for record in chain(test_record_instance.test_record_list, test_record_instance.error_record_list) + if (timestamp := sample_time(record)) is not None and now - duration <= timestamp <= now] + latencies = [latency for record in recent + if (latency := finite_number(record.get("response_time_ms"))) is not None and latency >= 0] + rps = len(recent) / duration + avg_ms = latency_mean(latencies) return { "totals": summary["totals"], @@ -101,7 +73,8 @@ def snapshot_metrics(window_seconds: float = 10.0) -> Dict[str, Any]: "per_name": summary["per_name"], "rps": rps, "avg_ms": avg_ms, - "ts": time.time(), + "ts": now, + "latency_windows": windows, } @@ -113,71 +86,89 @@ def __init__(self, host: str = "127.0.0.1", port: int = 8765, self.port = port self.refresh_seconds = refresh_seconds self.window_seconds = window_seconds - self._httpd: Optional[HTTPServer] = None + self._httpd: Optional[ThreadingHTTPServer] = None self._thread: Optional[threading.Thread] = None + self._stopping = threading.Event() def start(self) -> None: - handler = _build_handler(self.refresh_seconds, self.window_seconds) - self._httpd = HTTPServer((self.host, self.port), handler) + if self._httpd is not None: + raise RuntimeError("dashboard is already running") + for name in ("refresh_seconds", "window_seconds"): + value = finite_number(getattr(self, name)) + if value is None or value <= 0: + raise ValueError(f"{name}: expected a finite positive number") + self._stopping.clear() + handler = _build_handler(self.refresh_seconds, self.window_seconds, self._stopping) + self._httpd = ThreadingHTTPServer((self.host, self.port), handler) self._thread = threading.Thread(target=self._httpd.serve_forever, daemon=True) self._thread.start() def stop(self) -> None: + self._stopping.set() if self._httpd is not None: self._httpd.shutdown() self._httpd.server_close() self._httpd = None + if self._thread is not None: + self._thread.join(timeout=2) self._thread = None -def _build_handler(refresh_seconds: float, window_seconds: float): - - class _Handler(BaseHTTPRequestHandler): - # pylint: disable=redefined-builtin - def log_message(self, format, *args): # noqa: A002 — match stdlib signature - # Silence stdlib request-log chatter; the dashboard already - # surfaces stats through SSE. - del format, args - - def _write(self, status: int, body: bytes, content_type: str, - extra_headers: Optional[Dict[str, str]] = None) -> None: - self.send_response(status) - self.send_header("Content-Type", content_type) - self.send_header("Content-Length", str(len(body))) - for key, value in (extra_headers or {}).items(): - self.send_header(key, value) - self.end_headers() - self.wfile.write(body) - - def do_GET(self): # noqa: N802 (BaseHTTPRequestHandler protocol) - if self.path == "/" or self.path == "/index.html": - self._write(200, _HTML.encode("utf-8"), "text/html; charset=utf-8") - return - if self.path == "/snapshot": - payload = json.dumps(snapshot_metrics(window_seconds)).encode("utf-8") - self._write(200, payload, "application/json") - return - if self.path == "/events": - self._serve_sse() - return - self._write(404, b"not found", "text/plain") - - def _serve_sse(self) -> None: - self.send_response(200) - self.send_header("Content-Type", "text/event-stream") - self.send_header("Cache-Control", "no-cache") - self.end_headers() - try: - while True: - payload = json.dumps(snapshot_metrics(window_seconds)) - self.wfile.write(f"data: {payload}\n\n".encode("utf-8")) - self.wfile.flush() - time.sleep(refresh_seconds) - except (BrokenPipeError, ConnectionResetError): - return - - return _Handler +def _build_handler(refresh_seconds: float, window_seconds: float, stopping: threading.Event): + return partial(_DashboardHandler, refresh_seconds=refresh_seconds, window_seconds=window_seconds, stopping=stopping) + + +class _DashboardHandler(BaseHTTPRequestHandler): + def __init__(self, *args, refresh_seconds, window_seconds, stopping): + self.refresh_seconds = refresh_seconds + self.window_seconds = window_seconds + self.stopping = stopping + super().__init__(*args) + + # pylint: disable=redefined-builtin + def log_message(self, format, *args): # noqa: A002 — match stdlib signature + # Silence stdlib request-log chatter; the dashboard already + # surfaces stats through SSE. + del format, args + + def _write(self, status: int, body: bytes, content_type: str, + extra_headers: Optional[Dict[str, str]] = None) -> None: + self.send_response(status) + self.send_header("Content-Type", content_type) + self.send_header("Content-Length", str(len(body))) + for key, value in (extra_headers or {}).items(): + self.send_header(key, value) + self.end_headers() + self.wfile.write(body) + + def do_GET(self): # noqa: N802 (BaseHTTPRequestHandler protocol) + if self.path == "/" or self.path == "/index.html": + self._write(200, _HTML.encode("utf-8"), "text/html; charset=utf-8") + return + if self.path == "/snapshot": + payload = json.dumps(snapshot_metrics(self.window_seconds)).encode("utf-8") + self._write(200, payload, "application/json") + return + if self.path == "/events": + self._serve_sse() + return + self._write(404, b"not found", "text/plain") + + def _serve_sse(self) -> None: + self.send_response(200) + self.send_header("Content-Type", "text/event-stream") + self.send_header("Cache-Control", "no-cache") + self.end_headers() + try: + while not self.stopping.is_set(): + payload = json.dumps(snapshot_metrics(self.window_seconds)) + self.wfile.write(f"data: {payload}\n\n".encode("utf-8")) + self.wfile.flush() + self.stopping.wait(self.refresh_seconds) + except (BrokenPipeError, ConnectionResetError): + return + _SERVER: Optional[LiveDashboardServer] = None diff --git a/je_load_density/utils/generate_report/generate_chart_report.py b/je_load_density/utils/generate_report/generate_chart_report.py index a49b0d0..60b9406 100644 --- a/je_load_density/utils/generate_report/generate_chart_report.py +++ b/je_load_density/utils/generate_report/generate_chart_report.py @@ -3,7 +3,7 @@ Two charts are produced from ``test_record_instance``: -* Latency over time (scatter, per request) +* Latency over time (p50 line, p50–p95 and p95–p99 bands) * Rolling RPS (line, 1-second buckets) matplotlib is imported lazily. Install with ``pip install matplotlib`` @@ -11,9 +11,11 @@ """ import os -from typing import Dict, List, Tuple +from itertools import chain +from typing import Dict from je_load_density.utils.test_record.test_record_class import test_record_instance +from je_load_density.utils.test_record.window_statistics import latency_windows class ChartDependencyError(RuntimeError): @@ -33,37 +35,19 @@ def _require_matplotlib(): return plt -def _collect_points() -> Tuple[List[float], List[float]]: - timestamps: List[float] = [] - latencies: List[float] = [] - for index, record in enumerate(test_record_instance.test_record_list): - latency = record.get("response_time_ms") - if latency is None: - continue - timestamps.append(float(record.get("ts") or index)) - latencies.append(float(latency)) - return timestamps, latencies - - -def _bucket_rps(timestamps: List[float], bucket_size: float = 1.0) -> Tuple[List[float], List[int]]: - if not timestamps: - return [], [] - start = min(timestamps) - buckets: Dict[int, int] = {} - for ts in timestamps: - idx = int((ts - start) // bucket_size) - buckets[idx] = buckets.get(idx, 0) + 1 - xs = sorted(buckets.keys()) - return [start + x * bucket_size for x in xs], [buckets[x] for x in xs] - - -def _render_latency_chart(plt, output_path: str, timestamps, latencies) -> str: +def _render_latency_chart(plt, output_path: str, windows) -> str: fig, ax = plt.subplots(figsize=(10, 4)) - ax.scatter(timestamps, latencies, s=8, alpha=0.6) - ax.set_xlabel("Request index (or timestamp)") + xs = [item["start_time"] for item in windows] + bands = {key: [item[key] if item[key] is not None else float("nan") for item in windows] + for key in ("p50_ms", "p95_ms", "p99_ms")} + ax.fill_between(xs, bands["p50_ms"], bands["p95_ms"], alpha=0.35, label="p50–p95") + ax.fill_between(xs, bands["p95_ms"], bands["p99_ms"], alpha=0.2, label="p95–p99") + ax.plot(xs, bands["p50_ms"], marker=".", label="p50") + ax.set_xlabel("Request start (epoch seconds)") ax.set_ylabel("Response time (ms)") ax.set_title("LoadDensity latency over time") ax.grid(True, alpha=0.3) + ax.legend() fig.tight_layout() fig.savefig(output_path) plt.close(fig) @@ -86,6 +70,7 @@ def _render_rps_chart(plt, output_path: str, xs, counts) -> str: def generate_chart_report( report_name: str = "loaddensity-charts", bucket_size_seconds: float = 1.0, + max_buckets: int = 10000, ) -> Dict[str, str]: """ Render latency + RPS charts. Writes @@ -93,12 +78,14 @@ def generate_chart_report( returns ``{"latency": path, "rps": path}``. """ plt = _require_matplotlib() - timestamps, latencies = _collect_points() - if not latencies: + records = chain(test_record_instance.test_record_list, test_record_instance.error_record_list) + windows = latency_windows(records, bucket_size_seconds=bucket_size_seconds, max_buckets=max_buckets) + if not any(item["p50_ms"] is not None for item in windows): raise ChartDependencyError("no records to render charts") latency_path = _render_latency_chart(plt, f"{report_name}-latency.png", - timestamps, latencies) - rps_xs, rps_counts = _bucket_rps(timestamps, bucket_size_seconds) + windows) + rps_xs = [item["start_time"] for item in windows] + rps_counts = [item["rps"] for item in windows] rps_path = _render_rps_chart(plt, f"{report_name}-rps.png", rps_xs, rps_counts) return {"latency": latency_path, "rps": rps_path} diff --git a/je_load_density/utils/generate_report/generate_summary_report.py b/je_load_density/utils/generate_report/generate_summary_report.py index 81ba28f..a3cff02 100644 --- a/je_load_density/utils/generate_report/generate_summary_report.py +++ b/je_load_density/utils/generate_report/generate_summary_report.py @@ -1,9 +1,9 @@ import json -import statistics import sys from typing import Any, Dict, Iterable, List, Optional from je_load_density.utils.test_record.test_record_class import test_record_instance +from je_load_density.utils.test_record.window_statistics import finite_number, latency_mean def _percentile(values: List[float], pct: float) -> float: @@ -23,10 +23,10 @@ def _by_name(records: Iterable[Dict[str, Any]]) -> Dict[str, List[float]]: grouped: Dict[str, List[float]] = {} for record in records: key = str(record.get("name") or record.get("test_url") or "unknown") - latency = record.get("response_time_ms") - if latency is None: + latency = finite_number(record.get("response_time_ms")) + if latency is None or latency < 0: continue - grouped.setdefault(key, []).append(float(latency)) + grouped.setdefault(key, []).append(latency) return grouped @@ -40,9 +40,9 @@ def build_summary() -> Dict[str, Any]: failures = test_record_instance.error_record_list all_latencies: List[float] = [ - float(r.get("response_time_ms")) + latency for r in (*success, *failures) - if r.get("response_time_ms") is not None + if (latency := finite_number(r.get("response_time_ms"))) is not None and latency >= 0 ] grouped = _by_name(success) @@ -52,7 +52,7 @@ def build_summary() -> Dict[str, Any]: "count": len(values), "min_ms": float(min(values)), "max_ms": float(max(values)), - "mean_ms": float(statistics.fmean(values)), + "mean_ms": latency_mean(values), "p50_ms": _percentile(values, 50), "p90_ms": _percentile(values, 90), "p95_ms": _percentile(values, 95), diff --git a/je_load_density/utils/test_record/window_statistics.py b/je_load_density/utils/test_record/window_statistics.py new file mode 100644 index 0000000..924a3e6 --- /dev/null +++ b/je_load_density/utils/test_record/window_statistics.py @@ -0,0 +1,121 @@ +"""Shared, bounded time windows for latency bands and request rates.""" + +import math +from collections.abc import Iterable, Mapping +from typing import Any, TypedDict + + +class LatencyWindow(TypedDict): + start_time: float + duration_seconds: float + count: int + rps: float + p50_ms: float | None + p95_ms: float | None + p99_ms: float | None + + +def finite_number(value: Any) -> float | None: + """Unknown, non-finite or boolean measurements never become zero latency.""" + if value is None or isinstance(value, bool): + return None + try: + result = float(value) + except (TypeError, ValueError, OverflowError): + return None + return result if math.isfinite(result) else None + + +def latency_mean(values: list[float]) -> float: + """Average finite nonnegative measurements without overflowing their sum.""" + scale = max(values, default=0.0) + if not scale: + return 0.0 + return (math.fsum(value / scale for value in values) / len(values)) * scale + + +def sample_time(record: Mapping[str, Any]) -> float | None: + """Prefer request start epoch; support canonical records and legacy ts fields.""" + for key in ("start_time", "started_at", "ts"): + value = finite_number(record.get(key)) + if value is not None: + return value + return None + + +def percentile(values: list[float], percent: float) -> float | None: + """Window order statistic, using Python's ties-to-even rounded index.""" + if not values: + return None + ordered = sorted(values) + index = int(round(percent / 100 * (len(ordered) - 1))) + return ordered[max(0, min(index, len(ordered) - 1))] + + +def _bounds(samples, start, end, duration: float, limit: int) -> tuple[float, float]: + if start is None: + start = math.floor(samples[0][0] / duration) * duration if samples else 0.0 + if end is None: + end = (math.floor(samples[-1][0] / duration) + 1) * duration if samples else start + start, end = finite_number(start), finite_number(end) + if start is None or end is None or end < start: + raise ValueError("window: finite end must be greater than or equal to start") + if not math.isfinite((end - start) / duration): + raise ValueError("window: time range is too large") + buckets = math.ceil((end - start) / duration) + if buckets > limit: + start += (buckets - limit) * duration + return start, end + + +def _window(start: float, duration: float, count: int, values: list[float]) -> LatencyWindow: + return {"start_time": start, "duration_seconds": duration, "count": count, "rps": count / duration, + "p50_ms": percentile(values, 50), "p95_ms": percentile(values, 95), "p99_ms": percentile(values, 99)} + + +def _optional_boundary(value) -> float | None: + if value is None: + return None + number = finite_number(value) + if number is None: + raise ValueError("window: expected finite timestamps") + return number + + +def latency_windows(records: Iterable[Mapping[str, Any]], *, start: float | None = None, + end: float | None = None, bucket_size_seconds: float = 1.0, + max_buckets: int = 120) -> list[LatencyWindow]: + """Count all timed requests, compute measured latencies, and retain the newest buckets. + + Buckets are [start, end). Missing latency leaves a null band; empty buckets have + zero RPS. The final partial bucket uses its actual duration. Callers pass both + success and failure samples, never aggregate worker percentiles. + """ + duration = finite_number(bucket_size_seconds) + if duration is None or duration <= 0: + raise ValueError("bucket_size_seconds: expected a finite positive number") + if isinstance(max_buckets, bool) or not isinstance(max_buckets, int) or max_buckets <= 0: + raise ValueError("max_buckets: expected a positive integer") + start, end = _optional_boundary(start), _optional_boundary(end) + if start is not None and end is not None: + start, end = _bounds([], start, end, duration, max_buckets) + samples = _samples(records, start, end) + start, end = _bounds(samples, start, end, duration, max_buckets) + groups: dict[int, tuple[int, list[float]]] = {} + for timestamp, _index, record in samples: + if not start <= timestamp < end: + continue + bucket = int((timestamp - start) // duration) + count, values = groups.setdefault(bucket, (0, [])) + groups[bucket] = (count + 1, values) + latency = finite_number(record.get("elapsed_ms", record.get("response_time_ms"))) + if latency is not None and latency >= 0: + values.append(latency) + return [_window(start + index * duration, min(duration, end - start - index * duration), + *groups.get(index, (0, []))) for index in range(math.ceil((end - start) / duration))] + + +def _samples(records, start, end): + return sorted((timestamp, index, record) for index, record in enumerate(records) + if (timestamp := sample_time(record)) is not None + and (start is None or timestamp >= start) and (end is None or timestamp < end)) diff --git a/je_load_density/wrapper/user_template/_common.py b/je_load_density/wrapper/user_template/_common.py index a883479..1a07f6a 100644 --- a/je_load_density/wrapper/user_template/_common.py +++ b/je_load_density/wrapper/user_template/_common.py @@ -61,6 +61,11 @@ def run_template_coroutine(coro: Coroutine[Any, Any, T]) -> T: loop.close() +def request_start_epoch(monotonic_start: float) -> float: + """Convert a duration clock start to the wall-clock epoch used by request charts.""" + return time.time() - (time.monotonic() - monotonic_start) + + def fire_request_event( environment: Any, request_type: str, @@ -79,7 +84,7 @@ def fire_request_event( context={}, url=name, response=None, - start_time=start, + start_time=request_start_epoch(start), ) diff --git a/je_load_density/wrapper/user_template/async_http_user_template.py b/je_load_density/wrapper/user_template/async_http_user_template.py index ca3f07d..b082e6a 100644 --- a/je_load_density/wrapper/user_template/async_http_user_template.py +++ b/je_load_density/wrapper/user_template/async_http_user_template.py @@ -22,6 +22,7 @@ register_variables, ) from je_load_density.wrapper.proxy.proxy_user import locust_wrapper_proxy +from je_load_density.wrapper.user_template._common import request_start_epoch def set_wrapper_async_http_user(user_detail_dict: Dict[str, Any], **kwargs) -> type: @@ -79,7 +80,7 @@ def _fire(self, name: str, start: float, length: int, exception: Exception = Non context={}, url=name, response=None, - start_time=start, + start_time=request_start_epoch(start), ) def _check_assertions(self, response, step: Dict[str, Any]) -> Optional[str]: diff --git a/je_load_density/wrapper/user_template/grpc_user_template.py b/je_load_density/wrapper/user_template/grpc_user_template.py index c846a36..b30e1df 100644 --- a/je_load_density/wrapper/user_template/grpc_user_template.py +++ b/je_load_density/wrapper/user_template/grpc_user_template.py @@ -12,6 +12,7 @@ register_variables, ) from je_load_density.wrapper.proxy.proxy_user import locust_wrapper_proxy +from je_load_density.wrapper.user_template._common import request_start_epoch def set_wrapper_grpc_user(user_detail_dict: Dict[str, Any], **kwargs) -> type: @@ -106,7 +107,7 @@ def _fire(self, name: str, target: str, start: float, length: int, exception: Ex context={}, url=target, response=None, - start_time=start, + start_time=request_start_epoch(start), ) def _do_step(self, raw_task: Dict[str, Any]) -> None: diff --git a/je_load_density/wrapper/user_template/kafka_user_template.py b/je_load_density/wrapper/user_template/kafka_user_template.py index f1a4858..856ec6c 100644 --- a/je_load_density/wrapper/user_template/kafka_user_template.py +++ b/je_load_density/wrapper/user_template/kafka_user_template.py @@ -23,6 +23,7 @@ register_variables, ) from je_load_density.wrapper.proxy.proxy_user import locust_wrapper_proxy +from je_load_density.wrapper.user_template._common import request_start_epoch def set_wrapper_kafka_user(user_detail_dict: Dict[str, Any], **kwargs) -> type: @@ -99,7 +100,7 @@ def _fire(self, name: str, start: float, length: int, exception: Exception = Non context={}, url=name, response=None, - start_time=start, + start_time=request_start_epoch(start), ) def _do_produce(self, step: Dict[str, Any]) -> int: diff --git a/je_load_density/wrapper/user_template/mongo_user_template.py b/je_load_density/wrapper/user_template/mongo_user_template.py index 12dbaad..196c027 100644 --- a/je_load_density/wrapper/user_template/mongo_user_template.py +++ b/je_load_density/wrapper/user_template/mongo_user_template.py @@ -27,6 +27,7 @@ register_variables, ) from je_load_density.wrapper.proxy.proxy_user import locust_wrapper_proxy +from je_load_density.wrapper.user_template._common import request_start_epoch def set_wrapper_mongo_user(user_detail_dict: Dict[str, Any], **kwargs) -> type: @@ -79,7 +80,7 @@ def _fire(self, name: str, start: float, length: int, exception: Exception = Non context={}, url=name, response=None, - start_time=start, + start_time=request_start_epoch(start), ) def _coerce(self, value: Any) -> int: diff --git a/je_load_density/wrapper/user_template/mqtt_user_template.py b/je_load_density/wrapper/user_template/mqtt_user_template.py index 234bfa4..a18c635 100644 --- a/je_load_density/wrapper/user_template/mqtt_user_template.py +++ b/je_load_density/wrapper/user_template/mqtt_user_template.py @@ -11,7 +11,10 @@ register_variables, ) from je_load_density.wrapper.proxy.proxy_user import locust_wrapper_proxy -from je_load_density.wrapper.user_template._common import default_host +from je_load_density.wrapper.user_template._common import ( + default_host, + request_start_epoch, +) def set_wrapper_mqtt_user(user_detail_dict: Dict[str, Any], **kwargs) -> type: @@ -101,7 +104,7 @@ def _fire(self, name: str, broker: str, start: float, length: int, exception: Ex context={}, url=broker, response=None, - start_time=start, + start_time=request_start_epoch(start), ) def _do_step(self, raw_task: Dict[str, Any]) -> None: diff --git a/je_load_density/wrapper/user_template/redis_user_template.py b/je_load_density/wrapper/user_template/redis_user_template.py index f2b7deb..70798cf 100644 --- a/je_load_density/wrapper/user_template/redis_user_template.py +++ b/je_load_density/wrapper/user_template/redis_user_template.py @@ -22,6 +22,7 @@ register_variables, ) from je_load_density.wrapper.proxy.proxy_user import locust_wrapper_proxy +from je_load_density.wrapper.user_template._common import request_start_epoch def set_wrapper_redis_user(user_detail_dict: Dict[str, Any], **kwargs) -> type: @@ -67,7 +68,7 @@ def _fire(self, name: str, start: float, length: int, exception: Exception = Non context={}, url=name, response=None, - start_time=start, + start_time=request_start_epoch(start), ) def _coerce(self, value: Any) -> int: diff --git a/je_load_density/wrapper/user_template/socket_user_template.py b/je_load_density/wrapper/user_template/socket_user_template.py index 2f253d2..930ac36 100644 --- a/je_load_density/wrapper/user_template/socket_user_template.py +++ b/je_load_density/wrapper/user_template/socket_user_template.py @@ -11,7 +11,10 @@ register_variables, ) from je_load_density.wrapper.proxy.proxy_user import locust_wrapper_proxy -from je_load_density.wrapper.user_template._common import default_host +from je_load_density.wrapper.user_template._common import ( + default_host, + request_start_epoch, +) def set_wrapper_socket_user(user_detail_dict: Dict[str, Any], **kwargs) -> type: @@ -64,7 +67,7 @@ def _fire( context={}, url=target, response=None, - start_time=start, + start_time=request_start_epoch(start), ) @staticmethod diff --git a/je_load_density/wrapper/user_template/sql_user_template.py b/je_load_density/wrapper/user_template/sql_user_template.py index 8df9eb8..bb82c98 100644 --- a/je_load_density/wrapper/user_template/sql_user_template.py +++ b/je_load_density/wrapper/user_template/sql_user_template.py @@ -22,6 +22,7 @@ register_variables, ) from je_load_density.wrapper.proxy.proxy_user import locust_wrapper_proxy +from je_load_density.wrapper.user_template._common import request_start_epoch def set_wrapper_sql_user(user_detail_dict: Dict[str, Any], **kwargs) -> type: @@ -69,7 +70,7 @@ def _fire(self, name: str, start: float, length: int, exception: Exception = Non context={}, url=name, response=None, - start_time=start, + start_time=request_start_epoch(start), ) def _execute(self, sql: str, params: Optional[Dict[str, Any]]) -> int: diff --git a/je_load_density/wrapper/user_template/sse_user_template.py b/je_load_density/wrapper/user_template/sse_user_template.py index c9bbe1e..a46ae1b 100644 --- a/je_load_density/wrapper/user_template/sse_user_template.py +++ b/je_load_density/wrapper/user_template/sse_user_template.py @@ -24,7 +24,10 @@ register_variables, ) from je_load_density.wrapper.proxy.proxy_user import locust_wrapper_proxy -from je_load_density.wrapper.user_template._common import default_host +from je_load_density.wrapper.user_template._common import ( + default_host, + request_start_epoch, +) def set_wrapper_sse_user(user_detail_dict: Dict[str, Any], **kwargs) -> type: @@ -87,7 +90,7 @@ def _fire(self, name: str, start: float, length: int, exception: Exception = Non context={}, url=self._url, response=None, - start_time=start, + start_time=request_start_epoch(start), ) def _wait_for(self, expect: Optional[str], timeout: float) -> int: diff --git a/je_load_density/wrapper/user_template/websocket_user_template.py b/je_load_density/wrapper/user_template/websocket_user_template.py index d89cfcb..dcb70d8 100644 --- a/je_load_density/wrapper/user_template/websocket_user_template.py +++ b/je_load_density/wrapper/user_template/websocket_user_template.py @@ -10,7 +10,10 @@ register_variables, ) from je_load_density.wrapper.proxy.proxy_user import locust_wrapper_proxy -from je_load_density.wrapper.user_template._common import default_host +from je_load_density.wrapper.user_template._common import ( + default_host, + request_start_epoch, +) def set_wrapper_websocket_user(user_detail_dict: Dict[str, Any], **kwargs) -> type: @@ -85,7 +88,7 @@ def _fire(self, name: str, start: float, length: int, exception: Exception = Non context={}, url=self._url, response=None, - start_time=start, + start_time=request_start_epoch(start), ) def _do_step(self, raw_task: Dict[str, Any]) -> None: diff --git a/progress.md b/progress.md index 85bcc18..0b5d54a 100644 --- a/progress.md +++ b/progress.md @@ -10,13 +10,12 @@ Cross-repo and workspace items live in `D:\Codes\progress.md` (relevant here: X- - **#15** Virtual user session context & parameter isolation: `parameter_resolver` (`je_load_density/utils/parameterization/parameter_resolver.py`) stores variables in a single global dictionary. When multiple concurrent users extract values (e.g. auth tokens, session IDs) via `task["extract"]`, they overwrite each other. Introduce session-scoped / user-local context support (`${session.VAR}` or per-user local resolution) so concurrent user journeys maintain isolated state. - **#16** Asyncio engine as a first-class runner: `je_load_density/engine/asyncio_engine.py` is not dispatched by `wrapper/start_wrapper/start_test.py`, and package initialisation imports Locust even for async callers. Add engine selection across Python/action/CLI/GUI, lazy Locust loading, request kwargs and all assertions/extractors, scenario/retry/pacing/load shapes, per-user sessions (#15), run isolation, cancellation, reports/persistence/exporters and local/master/worker parity. Track protocol capabilities explicitly; unsupported options must fail preflight. Deliver HTTP parity first without closing full parity prematurely. Design: `docs/superpowers/specs/2026-10-02-testing-platform-roadmap.md`. - **#17** Mixed workload multi-user support: `start_test` and `prepare_env` (`wrapper/start_wrapper/start_test.py`, `wrapper/create_locust_env/create_locust_env.py`) only instantiate a single user class from `_USER_REGISTRY`. Extend `user_detail_dict` to accept a list of user types with weights (`users: [{"user": "http_user", "weight": 8}, {"user": "websocket_user", "weight": 2}]`) and pass them to Locust's `Environment(user_classes=[...])` for realistic multi-protocol simulation. -- **#18** Redesign GUI and web dashboard: `je_load_density/gui/main_widget.py` lacks cancellation and action-file execution; `utils/dashboard/live_dashboard.py` lacks the shared run-state/chart view. Add script/target/engine/load controls, Start/Stop lifecycle, metric cards and result/log/history views, responsive dashboard layout and translations. Use the run lifecycle from #16 and chart data from #26. Design: `docs/superpowers/specs/2026-10-02-testing-platform-roadmap.md`. +- **#18** Finish desktop GUI redesign and shared run-state views: add action-file/target/engine/load controls, explicit Start/Stop lifecycle, retained result/log/history views and translations. Connect native cooperative callbacks and the async lifecycle from #16; expose run-state/engine/users/target context on the dashboard. The responsive browser metrics/bands and separate Qt charts are implemented. Design: `docs/superpowers/specs/2026-10-02-testing-platform-roadmap.md`. - **#19** Markdown summary report generator (`LD_generate_markdown_report`): Add `generate_markdown_report.py` under `utils/generate_report/` to render GFM summary tables (RPS, latencies, SLA breaches, error clusters) suitable for GitHub Actions `$GITHUB_STEP_SUMMARY` and GitLab MR comments. Register the command in `action_executor.py`, export in `__init__.py`, and add tests. - **#20** Pre-flight health check gate: Add a target health probe before ramping load in `start_test` / `bench`. If the target endpoint is unreachable, DNS fails, or TLS handshake fails, abort early with a descriptive error instead of spawning hundreds of workers that flood logs with connection refused errors. - **#23** [BLOCKED: two releases must ship the warning first] Flip the package gate's default to refuse packages outside the allowlist: set `self.allow_arbitrary_packages = False` in `PackageManager.__init__` (`je_load_density/utils/package_manager/package_manager_class.py`); the warning branch is je_action_core's `_check_allowed` and stays for the other projects. Update the "Package gate" paragraph in the three READMEs and the "Package Gate" section of `docs/source/{En,Zh}/doc/package_manager/package_manager_doc.rst`. The warning is first released in the version after 0.0.77 (`origin/main`); flip once two releases after that one have shipped it. Decide first how a user who only runs action files (`loaddensity`, `-e`, the socket server) allows a package without a Python host to call `executor.allow_packages(...)`. - **#24** Installed-wheel GUI smoke lifecycle: extend `test/smoke/` and the Docker GUI capability probe with offscreen execution and cancellation after the UI lifecycle redesign (#18). Keep client/thread cleanup observable. -- **#25** Finish Docker matrix validation: base/etcd and healthy Compose Redis/MQTT/SQLite probes pass; GUI/all and Python 3.10/3.14 cells are building. Repair any remaining installation/API failures and confirm the complete Python 3.12 extras matrix in CI. -- **#26** Percentile bands: `gui/chart_panel.py` mixes RPS and latency on one axis; `utils/generate_report/generate_chart_report.py` uses `ts`/index instead of request `start_time`. Share per-window percentile aggregation across GUI/dashboard/report, show p50 with p50–p95 and p95–p99 bands, separate RPS units and handle gaps, failures and actual bucket duration. +- **#25** Finish Docker matrix validation: base Python 3.10/3.12/3.14, etcd, NATS/Web Push, cloud and healthy Compose Redis/MQTT/SQLite probes pass. GUI/all runtime theme generation required a real writable unprivileged home; the fixed rebuilt base passes six smoke checks. Confirm GUI/all and the complete updated extras matrix in CI; Docker Hub token-fetch transport failures are infrastructure failures, not skipped capabilities. - **#28** Shared record release activation: coordinate the actual ActionCore request-record release, then activate consumer dependency floors and cross-project installed-wheel fixtures. Current opt-in APIs must retain compatibility with older core installations until that release. Design: `docs/superpowers/specs/2026-10-02-shared-record-design.md`. - **#29** Distributed canonical record aggregation and finite-work recovery: health/startup/native virtual-user rebalancing is implemented. Add an explicit master ingestion contract that preserves source worker identity, bounded worker batches/final flush and run-scoped retry deduplication. ActionCore `RunContext.append` currently requires matching worker identity, so aggregation cannot silently rewrite it. Define lease/reassignment semantics only when introducing actual finite-work shards; native ongoing load does not replay HTTP side effects or migrate sessions. Target health is separately tracked by #20. - **#30** One script for functional/load/synthetic modes: extend APITestka's existing `integrations/load_density*.py` bridge and WebRunner's `utils/synthetic_monitoring/monitor.py` so common request/assertion steps are authored once and execution profiles choose functional correctness, capacity/SLA or periodic monitoring. Unsupported conversions must fail preflight and failed checks must reach monitor transitions. Depends on #28 and async parity #16; preserve existing CLI/action/summary contracts. diff --git a/test/smoke/platform_probes.py b/test/smoke/platform_probes.py index 4950752..de36e73 100644 --- a/test/smoke/platform_probes.py +++ b/test/smoke/platform_probes.py @@ -62,21 +62,26 @@ def cloud() -> None: def charts() -> None: - import matplotlib + from je_load_density.utils.generate_report.generate_chart_report import generate_chart_report + from je_load_density.utils.test_record.test_record_class import test_record_instance - matplotlib.use("Agg") - from matplotlib import pyplot - - figure, axes = pyplot.subplots() + success = list(test_record_instance.test_record_list) + failures = list(test_record_instance.error_record_list) try: - axes.plot([0, 1], [10, 20]) - axes.fill_between([0, 1], [10, 20], [30, 40]) - output = io.BytesIO() - figure.savefig(output, format="png") - if not output.getvalue().startswith(b"\x89PNG\r\n\x1a\n"): - raise RuntimeError("Chart rendering failed") + test_record_instance.clear_records() + test_record_instance.test_record_list.append({"start_time": 10.1, "response_time_ms": 10}) + test_record_instance.error_record_list.append({"start_time": 12.1, "response_time_ms": 50}) + with tempfile.TemporaryDirectory() as directory: + outputs = generate_chart_report(str(Path(directory) / "smoke")) + if set(outputs) != {"latency", "rps"}: + raise RuntimeError("Chart report paths missing") + for path in outputs.values(): + if not Path(path).read_bytes().startswith(b"\x89PNG\r\n\x1a\n"): + raise RuntimeError("Chart rendering failed") finally: - pyplot.close(figure) + test_record_instance.clear_records() + test_record_instance.test_record_list.extend(success) + test_record_instance.error_record_list.extend(failures) def datadog() -> None: diff --git a/test/test_chart_report.py b/test/test_chart_report.py index c872535..68fe87a 100644 --- a/test/test_chart_report.py +++ b/test/test_chart_report.py @@ -2,8 +2,6 @@ from je_load_density.utils.generate_report.generate_chart_report import ( ChartDependencyError, - _bucket_rps, - _collect_points, generate_chart_report, ) from je_load_density.utils.test_record.test_record_class import test_record_instance @@ -20,27 +18,6 @@ def _push_records(latencies): }) -def test_collect_points_skips_records_without_latency(): - test_record_instance.test_record_list.clear() - test_record_instance.error_record_list.clear() - test_record_instance.test_record_list.append({"Method": "GET"}) - test_record_instance.test_record_list.append({"response_time_ms": 12.0, "ts": 1.0}) - timestamps, latencies = _collect_points() - assert latencies == [12.0] - assert timestamps == [1.0] - - -def test_bucket_rps_returns_counts_per_bucket(): - xs, counts = _bucket_rps([0.0, 0.5, 1.5, 2.1], bucket_size=1.0) - assert counts == [2, 1, 1] - assert xs[0] == pytest.approx(0.0) - assert xs[1] == pytest.approx(1.0) - - -def test_bucket_rps_empty(): - assert _bucket_rps([]) == ([], []) - - def test_generate_chart_report_writes_png(tmp_path, monkeypatch): pytest.importorskip("matplotlib") monkeypatch.chdir(tmp_path) @@ -57,3 +34,27 @@ def test_generate_chart_report_raises_when_no_records(tmp_path, monkeypatch): test_record_instance.error_record_list.clear() with pytest.raises(ChartDependencyError): generate_chart_report("charts") + + +def test_latency_bands_include_failures_and_leave_empty_seconds_disconnected(tmp_path, monkeypatch): + pytest.importorskip("matplotlib") + from matplotlib.axes import Axes + + _push_records([10]) + test_record_instance.error_record_list.append({"start_time": 2.1, "response_time_ms": 80, "error": "500"}) + bands = [] + original = Axes.fill_between + + def capture(axis, x, low, high, **kwargs): + bands.append((list(x), list(low), list(high))) + return original(axis, x, low, high, **kwargs) + + monkeypatch.setattr(Axes, "fill_between", capture) + paths = generate_chart_report(str(tmp_path / "bands")) + assert len(bands) == 2 + assert bands[0][1][0] == 10 + assert bands[0][1][2] == 80 + assert bands[0][1][1] != bands[0][1][1] # NaN prevents connecting the empty bucket. + for path in paths.values(): + with open(path, "rb") as image: + assert image.read(8) == b"\x89PNG\r\n\x1a\n" diff --git a/test/test_dashboard_ui.py b/test/test_dashboard_ui.py new file mode 100644 index 0000000..3edc621 --- /dev/null +++ b/test/test_dashboard_ui.py @@ -0,0 +1,52 @@ +import json +import re +import shutil +import subprocess + +import pytest + +from je_load_density.utils.dashboard.live_dashboard import _HTML + + +def test_dashboard_renders_bands_and_untrusted_names_as_text(): + node = shutil.which("node") + if node is None: + pytest.skip("Node is required for browser-script DOM contract checks") + script = re.search(r"", _HTML, re.DOTALL).group(1) + harness = r""" +const vm = require('node:vm'); +const input = JSON.parse(require('node:fs').readFileSync(0, 'utf8')); +class Element { + constructor(tag='div') { this.tag=tag; this.children=[]; this.attributes={}; this.textContent=''; } + set innerHTML(_) { throw Error('Dynamic innerHTML is forbidden'); } + appendChild(child) { this.children.push(child); return child; } + replaceChildren(...items) { this.children=items; } + setAttribute(name, value) { this.attributes[name]=String(value); } +} +const nodes={}; +const document={getElementById:id=>(nodes[id]??=new Element()), + createElement:tag=>new Element(tag), createElementNS:(_,tag)=>new Element(tag)}; +let source; +class EventSource { constructor() { source=this; } } +const context=vm.createContext({document,EventSource,Date,console}); +vm.runInContext(input.script,context); +source.onopen(); +source.onmessage({data:JSON.stringify(input.data)}); +console.log(JSON.stringify({name:nodes.tbody.children[0].children[0].textContent, + bands:nodes.latency.children.filter(n=>n.attributes['data-band']).length, + status:nodes.connection.textContent, failure:nodes.failures.textContent})); +""" + name = '' + window = {"duration_seconds": 1, "count": 1, "rps": 1, "p50_ms": 10, "p95_ms": 20, "p99_ms": 30} + data = {"totals": {"requests": 3, "failures": 1, "failure_rate": 1 / 3}, + "rps": 0.3, "avg_ms": 20, "ts": 100, + "latency_overall": {"p50_ms": 10, "p95_ms": 20, "p99_ms": 30}, + "per_name": {name: {"count": 3, "mean_ms": 20, "p95_ms": 20}}, + "latency_windows": [{**window, "start_time": 0}, + {**window, "start_time": 1, "p50_ms": None, "p95_ms": None, "p99_ms": None}, + {**window, "start_time": 2}]} + completed = subprocess.run([node, "-e", harness], input=json.dumps({"script": script, "data": data}), + text=True, encoding="utf-8", capture_output=True, timeout=10, check=False) + assert completed.returncode == 0, completed.stderr + output = json.loads(completed.stdout) + assert output == {"name": name, "bands": 4, "status": "Connected", "failure": "1"} diff --git a/test/test_gui_bands.py b/test/test_gui_bands.py new file mode 100644 index 0000000..35a5417 --- /dev/null +++ b/test/test_gui_bands.py @@ -0,0 +1,50 @@ +import os + +import pytest + +os.environ.setdefault("QT_QPA_PLATFORM", "offscreen") +pytest.importorskip("PySide6.QtWidgets") + +from PySide6.QtWidgets import QApplication # noqa: E402 + +from je_load_density.gui import chart_panel # noqa: E402 +from je_load_density.utils.test_record.test_record_class import test_record_instance # noqa: E402 + + +@pytest.fixture +def panel(): + app = QApplication.instance() or QApplication([]) + widget = chart_panel.LiveChartPanel() + widget._timer.stop() + test_record_instance.clear_records() + yield widget + widget.close() + widget.deleteLater() + app.processEvents() + test_record_instance.clear_records() + + +def test_qt_bands_include_failed_samples_and_do_not_connect_latency_gaps(panel, monkeypatch): + panel.reset_history(start_time=0) + monkeypatch.setattr(chart_panel.time, "time", lambda: 3.0) + test_record_instance.test_record_list.extend([ + {"start_time": 0.1, "response_time_ms": 10}, {"start_time": 0.2, "response_time_ms": 30}]) + test_record_instance.error_record_list.append({"start_time": 2.1, "response_time_ms": 80}) + panel.refresh() + assert panel._rps_series.count() == 3 + assert panel._rps_series.at(1).y() == 0 + assert len(panel._latency_lines) == 2 + assert [line.at(0).y() for line in panel._latency_lines] == [10, 80] + assert len(panel._band_series) == 4 + assert panel._band_series[0].upperSeries().at(0).y() == 30 + assert panel._rps_chart is not panel._latency_chart + + +def test_qt_history_and_visible_axes_follow_a_long_run(panel, monkeypatch): + panel.reset_history(start_time=0) + monkeypatch.setattr(chart_panel.time, "time", lambda: 1000.0) + test_record_instance.test_record_list.append({"start_time": 999.1, "response_time_ms": 20000}) + panel.refresh() + assert panel._rps_series.count() == 120 + assert panel._latency_axis_y.max() >= 20000 + assert panel._latency_axis_x.min() >= 880 diff --git a/test/test_live_dashboard.py b/test/test_live_dashboard.py index 3b1373b..a2f111e 100644 --- a/test/test_live_dashboard.py +++ b/test/test_live_dashboard.py @@ -1,6 +1,6 @@ import json -import urllib.request import time +import urllib.request import pytest @@ -55,3 +55,57 @@ def test_dashboard_serves_html_and_snapshot(): assert payload["avg_ms"] == pytest.approx(10.0) finally: stop_dashboard() + + +def test_dashboard_windows_and_rate_include_failures_and_use_request_start(monkeypatch): + from je_load_density.utils.dashboard import live_dashboard + + monkeypatch.setattr(live_dashboard.time, "time", lambda: 100.0) + test_record_instance.clear_records() + test_record_instance.test_record_list.append({"start_time": 90.2, "response_time_ms": 10}) + test_record_instance.error_record_list.append({"start_time": 99.2, "response_time_ms": 50, "error": "500"}) + snapshot = snapshot_metrics() + assert snapshot["rps"] == pytest.approx(0.2) + assert snapshot["avg_ms"] == 30 + assert len(snapshot["latency_windows"]) == 120 + assert snapshot["latency_windows"][-1]["p95_ms"] == 50 + assert snapshot["latency_windows"][-2]["p50_ms"] is None + + +def test_sse_client_does_not_block_snapshot_or_shutdown(): + _populate([5]) + server = start_dashboard(port=0, refresh_seconds=0.05) + root = f"http://127.0.0.1:{server._httpd.server_port}" + stream = urllib.request.urlopen(root + "/events", timeout=2) + try: + assert stream.readline().startswith(b"data: ") + with urllib.request.urlopen(root + "/snapshot", timeout=2) as response: + assert json.loads(response.read())["totals"]["requests"] == 1 + thread = server._thread + before = time.monotonic() + stop_dashboard() + assert time.monotonic() - before < 1.5 + assert not thread.is_alive() + finally: + stream.close() + stop_dashboard() + + +def test_invalid_latency_keeps_request_counts_and_json_finite(): + _populate([10, float("nan"), float("inf"), -2, "unknown", True]) + snapshot = snapshot_metrics() + assert snapshot["totals"]["requests"] == 6 + assert snapshot["latency_overall"]["count"] == 1 + assert snapshot["latency_overall"]["p95_ms"] == 10 + assert snapshot["per_name"]["/x"]["count"] == 1 + assert snapshot["rps"] == pytest.approx(0.6) + json.dumps(snapshot, allow_nan=False) + + +def test_large_finite_latencies_keep_finite_mean_and_request_counts(): + _populate([1e308, 1e308]) + snapshot = snapshot_metrics() + assert snapshot["totals"]["requests"] == 2 + assert snapshot["avg_ms"] == 1e308 + assert snapshot["per_name"]["/x"]["mean_ms"] == 1e308 + json.dumps(snapshot, allow_nan=False) diff --git a/test/test_protocol_event_timing.py b/test/test_protocol_event_timing.py new file mode 100644 index 0000000..b66491f --- /dev/null +++ b/test/test_protocol_event_timing.py @@ -0,0 +1,41 @@ +"""Protocol events must provide epoch starts for shared report/dashboard windows.""" + +import importlib +from types import SimpleNamespace + +import pytest + +from je_load_density.wrapper.user_template import _common + + +@pytest.mark.parametrize("module_name,class_name,args", [ + ("async_http", "AsyncHttpUserWrapper", ("health", 42, 3)), + ("grpc", "GrpcUserWrapper", ("health", "host", 42, 3)), + ("kafka", "KafkaUserWrapper", ("health", 42, 3)), + ("mongo", "MongoUserWrapper", ("health", 42, 3)), + ("mqtt", "MqttUserWrapper", ("health", "host", 42, 3)), + ("redis", "RedisUserWrapper", ("health", 42, 3)), + ("socket", "SocketUserWrapper", ("health", "host", "tcp", 42, 3)), + ("sql", "SqlUserWrapper", ("health", 42, 3)), + ("sse", "SseUserWrapper", ("health", 42, 3)), + ("websocket", "WebSocketUserWrapper", ("health", 42, 3)), +]) +def test_protocol_events_report_epoch_start_and_monotonic_duration(monkeypatch, module_name, class_name, args): + module = importlib.import_module(f"je_load_density.wrapper.user_template.{module_name}_user_template") + clock = SimpleNamespace(time=lambda: 1000, monotonic=lambda: 44) + monkeypatch.setattr(module, "time", clock) + monkeypatch.setattr(_common, "time", clock) + events = [] + environment = SimpleNamespace(events=SimpleNamespace(request=SimpleNamespace(fire=lambda **kw: events.append(kw)))) + getattr(module, class_name)._fire(SimpleNamespace(environment=environment, _url="host"), *args) + assert events[0]["start_time"] == 998 + assert events[0]["response_time"] == 2000 + + +def test_common_protocol_event_reports_epoch_start(monkeypatch): + monkeypatch.setattr(_common, "time", SimpleNamespace(time=lambda: 1000, monotonic=lambda: 44)) + events = [] + environment = SimpleNamespace(events=SimpleNamespace(request=SimpleNamespace(fire=lambda **kw: events.append(kw)))) + _common.fire_request_event(environment, "SMTP", "health", 42) + assert events[0]["start_time"] == 998 + assert events[0]["response_time"] == 2000 diff --git a/test/test_window_statistics.py b/test/test_window_statistics.py new file mode 100644 index 0000000..e31bc29 --- /dev/null +++ b/test/test_window_statistics.py @@ -0,0 +1,60 @@ +import math + +import pytest + +from je_load_density.utils.test_record.window_statistics import latency_windows + + +def test_windows_merge_failed_and_successful_measurements_in_timestamp_order(): + records = [{"start_time": 12.1, "response_time_ms": 90, "error": "HTTP 500"}, + {"start_time": 10.2, "response_time_ms": 10}, + {"start_time": 10.8, "response_time_ms": 30}] + windows = latency_windows(records, start=10, end=13) + assert windows == [ + {"start_time": 10.0, "duration_seconds": 1.0, "count": 2, "rps": 2.0, + "p50_ms": 10.0, "p95_ms": 30.0, "p99_ms": 30.0}, + {"start_time": 11.0, "duration_seconds": 1.0, "count": 0, "rps": 0.0, + "p50_ms": None, "p95_ms": None, "p99_ms": None}, + {"start_time": 12.0, "duration_seconds": 1.0, "count": 1, "rps": 1.0, + "p50_ms": 90.0, "p95_ms": 90.0, "p99_ms": 90.0}, + ] + + +def test_epoch_zero_is_preserved_and_start_time_takes_precedence_over_ts(): + windows = latency_windows([{"start_time": 0, "ts": 50, "response_time_ms": 0}], start=0, end=1) + assert windows[0]["p50_ms"] == 0 + assert windows[0]["count"] == 1 + + +def test_canonical_records_and_actual_partial_bucket_duration(): + windows = latency_windows([{"started_at": 0.1, "elapsed_ms": 20}, {"started_at": 1.1, "elapsed_ms": 40}], + start=0, end=1.5) + assert [item["rps"] for item in windows] == [1, 2] + assert windows[1]["duration_seconds"] == 0.5 + assert windows[1]["p99_ms"] == 40 + + +def test_history_is_bounded_without_merging_old_samples_into_percentiles(): + windows = latency_windows([{"ts": value, "response_time_ms": value} for value in range(1000)], + start=0, end=1000, max_buckets=3) + assert [item["start_time"] for item in windows] == [997, 998, 999] + assert [item["p50_ms"] for item in windows] == [997, 998, 999] + + +def test_missing_and_invalid_latencies_count_requests_but_leave_a_gap(): + records = [{"ts": 0.1}, {"ts": 0.2, "response_time_ms": math.nan}, + {"ts": 0.3, "response_time_ms": -1}, {"ts": 0.4, "response_time_ms": "bad"}] + result = latency_windows(records, start=0, end=1)[0] + assert result["count"] == 4 + assert result["p95_ms"] is None + + +@pytest.mark.parametrize("kwargs", [{"bucket_size_seconds": 0}, {"max_buckets": 0}, + {"start": 2, "end": 1}, {"end": math.inf}]) +def test_invalid_windows_fail_before_rendering(kwargs): + with pytest.raises(ValueError): + latency_windows([], **kwargs) + + +def test_empty_input_produces_no_artificial_time_series(): + assert latency_windows([]) == [] From 67249edf754d4ddb5237d1de297b95d5ea09d6d8 Mon Sep 17 00:00:00 2001 From: JeffreyChen Date: Sat, 3 Oct 2026 08:28:32 +0800 Subject: [PATCH 12/18] fix: resolve quality findings and verify extras matrix --- docker/extras.Dockerfile | 6 +- docker/services.Dockerfile | 2 +- .../plans/2026-10-02-smoke-extras.md | 8 ++- .../plans/2026-10-02-ui-percentile-bands.md | 4 +- docs/updates/2026-10.md | 8 +++ docs/updates/README.md | 1 + je_load_density/cloud/_validation.py | 11 ++-- je_load_density/cloud/azure_aci.py | 2 +- je_load_density/wrapper/event/request_hook.py | 57 +++++++------------ .../user_template/etcd_user_template.py | 5 +- progress.md | 1 - test/smoke/server.py | 2 +- test/test_chart_report.py | 4 +- test/test_cloud_contracts.py | 49 +++++++++++----- test/test_cloud_launchers.py | 6 +- test/test_cloud_sdk_contracts.py | 3 +- test/test_distributed_health.py | 25 +++++--- test/test_extra_capabilities.py | 3 +- test/test_extras_matrix.py | 3 +- test/test_record_contract.py | 3 +- test/test_service_probe.py | 3 +- 21 files changed, 126 insertions(+), 80 deletions(-) diff --git a/docker/extras.Dockerfile b/docker/extras.Dockerfile index b673f95..0246526 100644 --- a/docker/extras.Dockerfile +++ b/docker/extras.Dockerfile @@ -5,9 +5,9 @@ RUN --mount=type=cache,target=/root/.cache/pip,sharing=locked python -m pip inst ARG EXTRA=base RUN if [ "$EXTRA" = "gui" ] || [ "$EXTRA" = "all" ]; then \ apt-get update && apt-get install -y --no-install-recommends \ - libegl1 libgl1 libopengl0 libglib2.0-0 libxkbcommon0 libdbus-1-3 libfontconfig1 \ - libxcb-cursor0 libxcb-icccm4 libxcb-keysyms1 libxcb-shape0 libxcb-xinerama0 \ - libxcb-randr0 libxcb-render-util0 libxcb-image0 fonts-dejavu-core \ + fonts-dejavu-core libdbus-1-3 libegl1 libfontconfig1 libgl1 libglib2.0-0 libopengl0 \ + libxcb-cursor0 libxcb-icccm4 libxcb-image0 libxcb-keysyms1 libxcb-randr0 \ + libxcb-render-util0 libxcb-shape0 libxcb-xinerama0 libxkbcommon0 \ && rm -rf /var/lib/apt/lists/*; \ fi COPY test/smoke /checks/smoke diff --git a/docker/services.Dockerfile b/docker/services.Dockerfile index 85923ba..b256690 100644 --- a/docker/services.Dockerfile +++ b/docker/services.Dockerfile @@ -1,7 +1,7 @@ FROM python:3.12-slim COPY dist/*.whl /wheels/ RUN --mount=type=cache,target=/root/.cache/pip,sharing=locked \ - for wheel in /wheels/*.whl; do python -m pip install --only-binary :all: "$wheel[redis,mqtt,sql]"; done && \ + for wheel in /wheels/*.whl; do python -m pip install --only-binary :all: "${wheel}[redis,mqtt,sql]"; done && \ python -m pip check COPY test/smoke /checks/smoke USER 65534:65534 diff --git a/docs/superpowers/plans/2026-10-02-smoke-extras.md b/docs/superpowers/plans/2026-10-02-smoke-extras.md index 39df0a0..49cc7c5 100644 --- a/docs/superpowers/plans/2026-10-02-smoke-extras.md +++ b/docs/superpowers/plans/2026-10-02-smoke-extras.md @@ -15,7 +15,13 @@ Execute inline in the existing isolated feature worktree. The approved roadmap a - [x] Build the wheel using existing locked build tools. Each cell installs that wheel plus only its selected extra in a fresh Docker image, runs pip check, CLI checks and non-skipping capability probes. - [x] Define probes for declared extras: create/use their relevant local SDK or framework capabilities, perform MCP handshake even without dependencies, render charts/PDF and launch Qt offscreen. Fail explicitly if a declared extra lacks a probe. - [x] Implement a reusable workflow consumed by stable/dev CI; keep pinned actions, minimum permissions and timeouts. Dev publication waits for these checks. -- [ ] Validate representative base/gui/charts/cloud/protocol cells locally in Docker, then complete the whole representative matrix. Repair installation/API incompatibilities rather than excluding supported cells silently. +- [x] Validate representative base/cloud/protocol cells locally in Docker, then complete the whole representative matrix including GUI/charts/all in CI. Repair installation/API incompatibilities rather than excluding supported cells silently. + +2026-10-03 verification: all 48 Docker installation cells, ten Windows/Ubuntu +Python3.10–3.14 unit-test jobs and the healthy Compose service job passed for +head82912d5 (Actions run37006974442). GUI/all offscreen launch uses the writable +home of the unprivileged account. Full GUI execution/cancellation smoke remains +separately tracked in progress.md #24 until the desktop lifecycle redesign. ## Task 3: Protocol service smoke and handoff diff --git a/docs/superpowers/plans/2026-10-02-ui-percentile-bands.md b/docs/superpowers/plans/2026-10-02-ui-percentile-bands.md index 2b048a2..3fde89d 100644 --- a/docs/superpowers/plans/2026-10-02-ui-percentile-bands.md +++ b/docs/superpowers/plans/2026-10-02-ui-percentile-bands.md @@ -42,5 +42,5 @@ Final batch verification: 1442 passed, 14 optional skips and four existing warnings; Ruff E/F/W/C90/I passed. Six subprocess smoke tests passed both from the checkout and from the rebuilt installed wheel in a network-disabled Docker container. The Docker runtime user now has a real writable home, addressing -qt-material theme generation under an unprivileged account. Full extras CI -verification remains tracked until the updated matrix finishes. +qt-material theme generation under an unprivileged account. On 2026-10-03 all +48 updated extras cells and the Compose service job passed (run37006974442). diff --git a/docs/updates/2026-10.md b/docs/updates/2026-10.md index 8adea98..8c6bf96 100644 --- a/docs/updates/2026-10.md +++ b/docs/updates/2026-10.md @@ -249,3 +249,11 @@ Added a stdlib smoke harness that exercises local HTTP requests in isolated proc - Installed chart capability now exercises actual report PNG generation. Unprivileged Docker cells have a real writable home for qt-material resources; updated full extras validation remains pending CI. - Verification: full suite1442 passed/14 optional skips/four existing warnings; six checkout smoke tests and six rebuilt installed-wheel Docker smoke tests passed; Ruff E/F/W/C90/I and whitespace checks pass. Real Edge desktop/mobile/SSE/XSS/gap/responsiveness checks passed. Independent review's numeric edge was reproduced and fixed. - User requested stopping after this batch. Full desktop Start/Stop/action-file redesign, async parity, canonical worker aggregation and shared functional/load/synthetic scripts remain explicitly outstanding; no implementation started after that stopping request. + +## U-20261003-01 · 2026-10-03 · Verify extras matrix and resolve quality findings · #done #ci #quality #25 + +- Closed progress.md #25 after Actions run37006974442: all48 Docker installed-wheel extras cells,10 Windows/Ubuntu Python3.10-3.14 unit jobs and healthy Compose Redis/MQTT/SQLite service checks passed. GUI/all cells pass with an unprivileged writable home. +- Queried SonarCloud and Codacy APIs. Corrected reported test assertion/raises structure, variable-length exception tuple return, ASCII-preserving ACI regex, request-hook complexity, etcd nested conditional, package ordering and two concrete smoke/Docker style findings. +- Refactored hook metadata construction while preserving legacy success-only fields, falsey HTTP-error responses, key order and canonical binding. Differential characterization108 combinations matched old/new records. +- Verification:205 focused integration/contract/doc tests passed;560 report/protocol/matrix tests passed with one existing optional skip;139 cloud mandatory and12 isolated official SDK tests passed. Bounded Ruff E/F/W/C90/I and diffcheck pass. Remote quality analysis reruns after commit. +- Remaining Codacy security audits concern fixed argument-vector subprocesses, parameterized test imports and locally generated XML in a stdlib-only harness; no shell is used and no external XML is accepted. No dependency/scan-rule bypass was introduced to suppress these audits. diff --git a/docs/updates/README.md b/docs/updates/README.md index e71a1db..e611a8d 100644 --- a/docs/updates/README.md +++ b/docs/updates/README.md @@ -58,6 +58,7 @@ In the same commit: delete the item from `progress.md`, add a `#done` entry here | ID | Date | Title | Tags | Batch | |---|---|---|---|---| +| U-20261003-01 | 2026-10-03 | Verify extras matrix and resolve quality findings | #done #ci #quality #25 | [2026-10](2026-10.md) | | U-20261002-11 | 2026-10-02 | Share percentile bands and redesign live dashboard | #done #charts #ui #26 | [2026-10](2026-10.md) | | U-20261002-10 | 2026-10-02 | Monitor worker health and rebalance native load | #done #distributed | [2026-10](2026-10.md) | | U-20261002-09 | 2026-10-02 | Validate cloud launch outcomes and preserve partial failures | #done #cloud | [2026-10](2026-10.md) | diff --git a/je_load_density/cloud/_validation.py b/je_load_density/cloud/_validation.py index be8e1c4..6080441 100644 --- a/je_load_density/cloud/_validation.py +++ b/je_load_density/cloud/_validation.py @@ -58,12 +58,15 @@ def validate_environment(environment: Optional[Dict[str, str]]) -> None: def sdk_error_types(provider: str) -> Tuple[Type[Exception], ...]: """Return optional SDK base errors without requiring SDKs for module import.""" + sdk_errors: List[Type[Exception]] try: if provider == "aws": from botocore.exceptions import BotoCoreError, ClientError - return (BotoCoreError, ClientError, OSError) - from azure.core.exceptions import AzureError - return (AzureError, OSError) + sdk_errors = [BotoCoreError, ClientError] + else: + from azure.core.exceptions import AzureError + sdk_errors = [AzureError] except ImportError: # Lightweight SDK doubles need only the standard transport boundary. - return (OSError,) + sdk_errors = [] + return tuple(sdk_errors + [OSError]) diff --git a/je_load_density/cloud/azure_aci.py b/je_load_density/cloud/azure_aci.py index 033aa2b..21f0f02 100644 --- a/je_load_density/cloud/azure_aci.py +++ b/je_load_density/cloud/azure_aci.py @@ -136,7 +136,7 @@ def _validate_prefix(name_prefix: str, workers: int) -> None: def _validate_environment_names(environment: Optional[Dict[str, str]]) -> None: for name in environment or {}: - if len(name) > 63 or not re.fullmatch(r"[a-zA-Z0-9](?:[a-zA-Z0-9_]*[a-zA-Z0-9])?", name): + if len(name) > 63 or not re.fullmatch(r"[a-zA-Z0-9](?:\w*[a-zA-Z0-9])?", name, flags=re.ASCII): raise ValueError("ACI environment names require 1 to 63 alphanumeric/internal underscore characters") diff --git a/je_load_density/wrapper/event/request_hook.py b/je_load_density/wrapper/event/request_hook.py index 9333966..3ae49ed 100644 --- a/je_load_density/wrapper/event/request_hook.py +++ b/je_load_density/wrapper/event/request_hook.py @@ -15,6 +15,19 @@ def _epoch_seconds(start_time): return float(start_time) if start_time is not None else None +def _response_fields(response, successful: bool) -> dict[str, object]: + """Preserve response metadata, including HTTP error responses whose truth value is false.""" + if response is None: + fields = {"status_code": None, "text": None} + if successful: + fields.update(content=None, headers=None) + return fields + fields = {"status_code": str(response.status_code), "text": str(response.text)} + if successful: + fields.update(content=str(response.content), headers=str(response.headers)) + return fields + + @events.request.add_listener def request_hook( start_time, @@ -33,42 +46,14 @@ def request_hook( 將每個 request 的結果紀錄到 test_record_instance """ - if exception is None: - # 成功紀錄 (Success record) - test_record_instance.test_record_list.append( - { - "Method": str(request_type), - "test_url": str(url), - "name": str(name), - "status_code": str(response.status_code) if response is not None else None, - "text": str(response.text) if response is not None else None, - "content": str(response.content) if response is not None else None, - "headers": str(response.headers) if response is not None else None, - "response_time_ms": float(response_time or 0), - "response_length": int(response_length or 0), - "error": None, - "start_time": _epoch_seconds(start_time), - } - ) - else: - # 失敗紀錄 (Failure record) - test_record_instance.error_record_list.append( - { - "Method": str(request_type), - "test_url": str(url), - "name": str(name), - "status_code": str(response.status_code) if response is not None else None, - "text": str(response.text) if response is not None else None, - "response_time_ms": float(response_time or 0), - "response_length": int(response_length or 0), - "error": str(exception), - "start_time": _epoch_seconds(start_time), - } - ) - - outcome = "passed" if exception is None else "failed" - records = test_record_instance.test_record_list if exception is None else test_record_instance.error_record_list - entry = records[-1] + successful = exception is None + entry = {"Method": str(request_type), "test_url": str(url), "name": str(name)} + entry.update(_response_fields(response, successful)) + entry.update(response_time_ms=float(response_time or 0), response_length=int(response_length or 0), + error=None if successful else str(exception), start_time=_epoch_seconds(start_time)) + records = test_record_instance.test_record_list if successful else test_record_instance.error_record_list + records.append(entry) + outcome = "passed" if successful else "failed" selected_run = kwargs.get("record_run") if selected_run is not None: from_legacy_record(entry, selected_run, outcome) diff --git a/je_load_density/wrapper/user_template/etcd_user_template.py b/je_load_density/wrapper/user_template/etcd_user_template.py index e581219..5916855 100644 --- a/je_load_density/wrapper/user_template/etcd_user_template.py +++ b/je_load_density/wrapper/user_template/etcd_user_template.py @@ -67,7 +67,10 @@ def _get(self, step: Dict[str, Any]) -> int: if self._client is None: raise RuntimeError("etcd not connected") result = self._client.get(step["key"]) - value = (result[0] if result else None) if self._gateway_client else result[0] + if self._gateway_client: + value = result[0] if result else None + else: + value = result[0] return len(value or b"") def _delete(self, step: Dict[str, Any]) -> int: diff --git a/progress.md b/progress.md index 0b5d54a..7747706 100644 --- a/progress.md +++ b/progress.md @@ -15,7 +15,6 @@ Cross-repo and workspace items live in `D:\Codes\progress.md` (relevant here: X- - **#20** Pre-flight health check gate: Add a target health probe before ramping load in `start_test` / `bench`. If the target endpoint is unreachable, DNS fails, or TLS handshake fails, abort early with a descriptive error instead of spawning hundreds of workers that flood logs with connection refused errors. - **#23** [BLOCKED: two releases must ship the warning first] Flip the package gate's default to refuse packages outside the allowlist: set `self.allow_arbitrary_packages = False` in `PackageManager.__init__` (`je_load_density/utils/package_manager/package_manager_class.py`); the warning branch is je_action_core's `_check_allowed` and stays for the other projects. Update the "Package gate" paragraph in the three READMEs and the "Package Gate" section of `docs/source/{En,Zh}/doc/package_manager/package_manager_doc.rst`. The warning is first released in the version after 0.0.77 (`origin/main`); flip once two releases after that one have shipped it. Decide first how a user who only runs action files (`loaddensity`, `-e`, the socket server) allows a package without a Python host to call `executor.allow_packages(...)`. - **#24** Installed-wheel GUI smoke lifecycle: extend `test/smoke/` and the Docker GUI capability probe with offscreen execution and cancellation after the UI lifecycle redesign (#18). Keep client/thread cleanup observable. -- **#25** Finish Docker matrix validation: base Python 3.10/3.12/3.14, etcd, NATS/Web Push, cloud and healthy Compose Redis/MQTT/SQLite probes pass. GUI/all runtime theme generation required a real writable unprivileged home; the fixed rebuilt base passes six smoke checks. Confirm GUI/all and the complete updated extras matrix in CI; Docker Hub token-fetch transport failures are infrastructure failures, not skipped capabilities. - **#28** Shared record release activation: coordinate the actual ActionCore request-record release, then activate consumer dependency floors and cross-project installed-wheel fixtures. Current opt-in APIs must retain compatibility with older core installations until that release. Design: `docs/superpowers/specs/2026-10-02-shared-record-design.md`. - **#29** Distributed canonical record aggregation and finite-work recovery: health/startup/native virtual-user rebalancing is implemented. Add an explicit master ingestion contract that preserves source worker identity, bounded worker batches/final flush and run-scoped retry deduplication. ActionCore `RunContext.append` currently requires matching worker identity, so aggregation cannot silently rewrite it. Define lease/reassignment semantics only when introducing actual finite-work shards; native ongoing load does not replay HTTP side effects or migrate sessions. Target health is separately tracked by #20. - **#30** One script for functional/load/synthetic modes: extend APITestka's existing `integrations/load_density*.py` bridge and WebRunner's `utils/synthetic_monitoring/monitor.py` so common request/assertion steps are authored once and execution profiles choose functional correctness, capacity/SLA or periodic monitoring. Unsupported conversions must fail preflight and failed checks must reach monitor transitions. Depends on #28 and async parity #16; preserve existing CLI/action/summary contracts. diff --git a/test/smoke/server.py b/test/smoke/server.py index 00f98cf..3bec794 100644 --- a/test/smoke/server.py +++ b/test/smoke/server.py @@ -25,7 +25,7 @@ def do_POST(self) -> None: self.end_headers() Thread(target=self.server.shutdown, daemon=True).start() - def log_message(self, format: str, *args: object) -> None: + def log_message(self, _format: str, *_args: object) -> None: pass diff --git a/test/test_chart_report.py b/test/test_chart_report.py index 68fe87a..3af8b12 100644 --- a/test/test_chart_report.py +++ b/test/test_chart_report.py @@ -1,3 +1,5 @@ +import math + import pytest from je_load_density.utils.generate_report.generate_chart_report import ( @@ -54,7 +56,7 @@ def capture(axis, x, low, high, **kwargs): assert len(bands) == 2 assert bands[0][1][0] == 10 assert bands[0][1][2] == 80 - assert bands[0][1][1] != bands[0][1][1] # NaN prevents connecting the empty bucket. + assert math.isnan(bands[0][1][1]) # NaN prevents connecting the empty bucket. for path in paths.values(): with open(path, "rb") as image: assert image.read(8) == b"\x89PNG\r\n\x1a\n" diff --git a/test/test_cloud_contracts.py b/test/test_cloud_contracts.py index 1424a7a..c51a06b 100644 --- a/test/test_cloud_contracts.py +++ b/test/test_cloud_contracts.py @@ -15,7 +15,8 @@ def test_cloud_launch_error_has_public_import(): import je_load_density.cloud as cloud error_type = getattr(cloud, "CloudLaunchError", None) - assert error_type is not None and issubclass(error_type, RuntimeError) + assert error_type is not None + assert issubclass(error_type, RuntimeError) def test_lambda_warm_handler_clears_records_after_failed_invocation(monkeypatch): @@ -69,13 +70,13 @@ def forbidden(): pytest.fail("invalid configuration reached SDK import") module = {"fargate": aws_fargate, "lambda": aws_lambda, "aci": azure_aci}[backend] monkeypatch.setattr(module, "_import_azure" if backend == "aci" else "_import_boto3", forbidden) + launch, arguments = { + "fargate": (aws_fargate.launch_fargate_workers, ("c", "t", count, ["s"])), + "lambda": (aws_lambda.invoke_lambda_workers, ("f", count, {})), + "aci": (azure_aci.launch_aci_workers, ("sub", "rg", "region", "image", count)), + }[backend] with pytest.raises(ValueError, match="workers"): - if backend == "fargate": - module.launch_fargate_workers("c", "t", count, ["s"]) - elif backend == "lambda": - module.invoke_lambda_workers("f", count, {}) - else: - module.launch_aci_workers("sub", "rg", "region", "image", count) + launch(*arguments) @pytest.mark.parametrize("kwargs", [ @@ -129,11 +130,12 @@ def fail(**kwargs): calls.append(kwargs) raise error aws_stub(monkeypatch, module, fail) + if module is aws_fargate: + launch, arguments = aws_fargate.launch_fargate_workers, ("c", "t", 1, ["s"]) + else: + launch, arguments = aws_lambda.invoke_lambda_workers, ("f", 1, {}) with pytest.raises(RuntimeError) as caught: - if module is aws_fargate: - module.launch_fargate_workers("c", "t", 1, ["s"]) - else: - module.invoke_lambda_workers("f", 1, {}) + launch(*arguments) assert caught.value.__cause__ is error assert caught.value.failed_workers == [0] assert caught.value.responses == [] @@ -413,11 +415,31 @@ def aci_stub(monkeypatch, create): {"overrides_env": {"LD_WORKER_COUNT": "9"}}, {"image": ""}, {"overrides_env": {"BAD NAME": "x"}}, {"overrides_env": {"NAME_": "x"}}, {"overrides_env": {"N" * 64: "x"}}, + {"overrides_env": {"NéM": "x"}}, {"overrides_env": {"N中M": "x"}}, + {"overrides_env": {"N١M": "x"}}, {"overrides_env": {"NKM": "x"}}, + {"overrides_env": {"éN": "x"}}, {"overrides_env": {"_NAME": "x"}}, ]) def test_aci_invalid_resource_parameters_fail_before_sdk(monkeypatch, kwargs): monkeypatch.setattr(azure_aci, "_import_azure", lambda: pytest.fail("SDK reached")) + arguments = dict(kwargs) + image = arguments.pop("image", "image") with pytest.raises(ValueError): - azure_aci.launch_aci_workers("sub", "rg", "region", kwargs.pop("image", "image"), 1, **kwargs) + azure_aci.launch_aci_workers("sub", "rg", "region", image, 1, **arguments) + + +@pytest.mark.parametrize("environment_name", ["N", "7", "TARGET_1", "A__B"]) +def test_aci_ascii_environment_names_are_supported(monkeypatch, environment_name): + created = [] + def create(**kwargs): + created.append(kwargs) + return SimpleNamespace(result=lambda: SimpleNamespace(id="resource/worker", provisioning_state="Succeeded")) + aci_stub(monkeypatch, create) + result = azure_aci.launch_aci_workers("sub", "rg", "region", "image", 1, + overrides_env={environment_name: "value"}) + assert result[0]["status"] == "Succeeded" + variables = created[0]["container_group"].containers[0].environment_variables + environment = {entry.name: entry.value for entry in variables} + assert environment == {environment_name: "value", "LD_WORKER_INDEX": "0", "LD_WORKER_COUNT": "1"} def test_aci_names_are_unique_across_launches_and_provisioning_is_checked(monkeypatch): @@ -439,7 +461,8 @@ def result(): group = calls[1]["container_group"] assert group.containers[0].resources.requests.cpu == 2 assert group.containers[0].resources.requests.memory_in_gb == 4 - assert group.location == "region" and group.restart_policy == "Never" + assert group.location == "region" + assert group.restart_policy == "Never" assert calls[1]["resource_group_name"] == "rg" assert {item.name: item.value for item in group.containers[0].environment_variables} == { "LD_WORKER_INDEX": "1", "LD_WORKER_COUNT": "2"} diff --git a/test/test_cloud_launchers.py b/test/test_cloud_launchers.py index a7699ae..e4fad2a 100644 --- a/test/test_cloud_launchers.py +++ b/test/test_cloud_launchers.py @@ -133,12 +133,14 @@ def __init__(self, **kwargs): responses = azure_aci.launch_aci_workers("sub", "rg", "eastasia", "ld:latest", workers=2, overrides_env={"TARGET": "https://x"}) assert responses[0]["name"].startswith("loaddensity-worker-") - assert responses[0]["name"].endswith("-0") and responses[1]["name"].endswith("-1") + assert responses[0]["name"].endswith("-0") + assert responses[1]["name"].endswith("-1") assert responses[0]["name"] != responses[1]["name"] assert all(response["status"] == "Succeeded" for response in responses) group = created[1]["container_group"] assert created[1]["resource_group_name"] == "rg" - assert group.restart_policy == "Never" and group.os_type == "Linux" + assert group.restart_policy == "Never" + assert group.os_type == "Linux" env = {item.name: item.value for item in group.containers[0].environment_variables} assert env == {"TARGET": "https://x", "LD_WORKER_INDEX": "1", "LD_WORKER_COUNT": "2"} diff --git a/test/test_cloud_sdk_contracts.py b/test/test_cloud_sdk_contracts.py index 1129893..9b05727 100644 --- a/test/test_cloud_sdk_contracts.py +++ b/test/test_cloud_sdk_contracts.py @@ -136,7 +136,8 @@ def send(self, request, **kwargs): overrides_env={"TARGET": "https://target.invalid"}) assert result[0]["status"] == "Succeeded" assert result[0]["resource_id"].endswith("/containerGroups/" + result[0]["name"]) - assert len(requests) == 1 and requests[0].method == "PUT" + assert len(requests) == 1 + assert requests[0].method == "PUT" resource_path = f"/resourceGroups/rg/providers/Microsoft.ContainerInstance/containerGroups/{result[0]['name']}?" assert resource_path in requests[0].url payload = json.loads(requests[0].body) diff --git a/test/test_distributed_health.py b/test/test_distributed_health.py index f8a9349..29ee9e9 100644 --- a/test/test_distributed_health.py +++ b/test/test_distributed_health.py @@ -30,9 +30,13 @@ def master(): {"worker_startup_policy": "anyway"}, {"expected_workers": -1}, ]) def test_invalid_distributed_settings_rejected_before_runner_creation(options): - with pytest.raises(ValueError): - env = wrapper.create_env(IdleUser, runner_mode="master", master_bind_port=0, **options) - env.runner.quit() + env = None + try: + with pytest.raises(ValueError): + env = wrapper.create_env(IdleUser, runner_mode="master", master_bind_port=0, **options) + finally: + if env is not None: + wrapper.cleanup_env(env) def test_ready_gate_excludes_missing_running_and_expired_workers(master): @@ -55,7 +59,8 @@ def test_native_settings_restore_after_last_environment_cleanup(): wrapper.create_env(IdleUser, runner_mode="master", master_bind_port=0, worker_heartbeat_interval=2) finally: wrapper.cleanup_env(env) - assert (runners.HEARTBEAT_INTERVAL, runners.HEARTBEAT_LIVENESS) == original + current_timing = runners.HEARTBEAT_INTERVAL, runners.HEARTBEAT_LIVENESS + assert current_timing == original def test_prepare_env_startup_timeout_never_starts_and_cleans_runner(monkeypatch, master): @@ -135,7 +140,8 @@ def test_native_network_loss_rebalances_capacity_at_original_target_and_rate(mas _until(lambda: master.runner.clients["second"].state == "missing") _until(lambda: master.runner.clients["first"].user_count == 4) snapshot = master.distributed_health.snapshot() - assert snapshot["target_users"] == snapshot["reported_users"] == 4 + assert snapshot["target_users"] == 4 + assert snapshot["reported_users"] == 4 assert snapshot["affected_workers"] == ["second"] assert snapshot["request_replay"] is False assert master.runner.spawn_rate == 4 @@ -271,7 +277,8 @@ def test_matching_environment_scope_restores_only_after_last_close(): assert runners.HEARTBEAT_INTERVAL == 5 finally: wrapper.cleanup_env(second) - assert (runners.HEARTBEAT_INTERVAL, runners.HEARTBEAT_LIVENESS) == original + current_timing = runners.HEARTBEAT_INTERVAL, runners.HEARTBEAT_LIVENESS + assert current_timing == original def test_real_rpc_worker_connects_runs_and_closes_both_transports(master): @@ -299,7 +306,8 @@ def broken_stop(): with pytest.raises(RuntimeError, match="UI stop failed"): wrapper.cleanup_env(env) assert env.runner.server.socket.closed - assert (runners.HEARTBEAT_INTERVAL, runners.HEARTBEAT_LIVENESS) == original + current_timing = runners.HEARTBEAT_INTERVAL, runners.HEARTBEAT_LIVENESS + assert current_timing == original def test_local_shape_completes_and_releases_tasks(monkeypatch): @@ -352,7 +360,8 @@ def broken_stop(): on_environment=created, stop_requested=broken_stop) assert seen[0].runner.server.socket.closed assert len(seen[0].load_density_tasks) == 0 - assert (runners.HEARTBEAT_INTERVAL, runners.HEARTBEAT_LIVENESS) == original + current_timing = runners.HEARTBEAT_INTERVAL, runners.HEARTBEAT_LIVENESS + assert current_timing == original def test_master_stop_callback_error_interrupts_slow_ramp(monkeypatch): diff --git a/test/test_extra_capabilities.py b/test/test_extra_capabilities.py index 7bef7ba..c2c0c53 100644 --- a/test/test_extra_capabilities.py +++ b/test/test_extra_capabilities.py @@ -23,8 +23,9 @@ def test_every_declared_extra_has_a_non_skipping_probe(monkeypatch): def test_unknown_extra_is_an_error(monkeypatch): + probe_module = probes(monkeypatch) with pytest.raises(ValueError, match="extra"): - probes(monkeypatch).run_probe("unregistered") + probe_module.run_probe("unregistered") def test_missing_declared_capability_is_a_failure(monkeypatch): diff --git a/test/test_extras_matrix.py b/test/test_extras_matrix.py index 2a20664..ff97b82 100644 --- a/test/test_extras_matrix.py +++ b/test/test_extras_matrix.py @@ -42,8 +42,9 @@ def test_schedule_matrix_covers_all_supported_versions(metadata): def test_unsafe_extra_names_are_rejected_before_shell_parameters(tmp_path): path = tmp_path / "pyproject.toml" path.write_text('[project.optional-dependencies]\n"bad;echo x" = []\n', encoding="utf-8") + matrix_module = load_matrix_module() with pytest.raises(ValueError, match="extra"): - load_matrix_module().build_matrix(path) + matrix_module.build_matrix(path) def test_cli_writes_checkout_matrix_independently_of_current_directory(tmp_path): diff --git a/test/test_record_contract.py b/test/test_record_contract.py index 41502c8..9c36ebc 100644 --- a/test/test_record_contract.py +++ b/test/test_record_contract.py @@ -51,9 +51,10 @@ def test_recording_without_explicit_context_is_a_no_op(): def test_malformed_legacy_record_reports_field_location(): + run = context() with pytest.raises(ValueError, match="status_code"): from_legacy_record({"Method": "GET", "test_url": "http://localhost/", "status_code": "bad"}, - context(), "passed") + run, "passed") def test_live_payload_capture_is_disabled_by_default(): diff --git a/test/test_service_probe.py b/test/test_service_probe.py index 840fa28..5ec9d72 100644 --- a/test/test_service_probe.py +++ b/test/test_service_probe.py @@ -18,5 +18,6 @@ def test_protocol_failure_cannot_be_reported_as_a_passing_service_probe(monkeypa def test_missing_protocol_measurement_is_a_probe_failure(monkeypatch): + probe_module = load_probe(monkeypatch) with pytest.raises(RuntimeError, match="Expected 1"): - load_probe(monkeypatch).check_events([], 1) + probe_module.check_events([], 1) From 439c5e77a42e9456cf0898fe6586b067873ce96b Mon Sep 17 00:00:00 2001 From: JeffreyChen Date: Sat, 3 Oct 2026 09:36:37 +0800 Subject: [PATCH 13/18] feat: add isolated runners and distributed record delivery --- README.md | 50 +- README/README_zh-CN.md | 16 +- README/README_zh-TW.md | 16 +- architecture.md | 31 +- .../En/doc/architecture/architecture_doc.rst | 8 +- docs/source/En/doc/gui/gui_doc.rst | 25 +- .../parameter_resolver_doc.rst | 15 +- .../En/doc/start_test/start_test_doc.rst | 50 + .../Zh/doc/architecture/architecture_doc.rst | 8 +- docs/source/Zh/doc/gui/gui_doc.rst | 21 +- .../parameter_resolver_doc.rst | 14 +- .../Zh/doc/start_test/start_test_doc.rst | 41 + .../plans/2026-10-03-desktop-gui.md | 19 + ...026-10-03-distributed-canonical-records.md | 58 + .../plans/2026-10-03-native-async.md | 56 + .../plans/2026-10-03-parameter-isolation.md | 49 + docs/updates/2026-10.md | 10 + docs/updates/README.md | 1 + je_load_density/__init__.py | 990 +++++++++--------- je_load_density/__main__.py | 40 +- je_load_density/engine/async_config.py | 249 +++++ je_load_density/engine/async_http.py | 189 ++++ je_load_density/engine/asyncio_engine.py | 393 ++++--- je_load_density/engine/bench_cli.py | 19 +- je_load_density/engine/entrypoints.py | 79 ++ je_load_density/gui/chart_panel.py | 22 +- je_load_density/gui/child_process.py | 46 + je_load_density/gui/dark_style.py | 55 + .../gui/language_wrapper/english.py | 19 + .../gui/language_wrapper/japanese.py | 19 + .../gui/language_wrapper/korean.py | 19 + .../language_wrapper/traditional_chinese.py | 19 + .../gui/load_density_gui_thread.py | 174 ++- je_load_density/gui/main_widget.py | 344 +++--- je_load_density/gui/main_window.py | 26 +- je_load_density/gui/run_protocol.py | 73 ++ je_load_density/gui/run_worker.py | 185 ++++ je_load_density/gui/stats_panel.py | 34 +- .../utils/executor/action_executor.py | 501 ++++----- .../generate_summary_report.py | 6 +- .../utils/parameterization/__init__.py | 6 + .../parameterization/parameter_resolver.py | 197 +++- je_load_density/utils/scenario/cookie_jar.py | 48 +- .../utils/test_record/distributed_context.py | 99 ++ .../create_locust_env/create_locust_env.py | 59 +- .../wrapper/distributed_records.py | 650 ++++++++++++ je_load_density/wrapper/event/request_hook.py | 6 +- .../wrapper/start_wrapper/locust_start.py | 321 ++++++ .../wrapper/start_wrapper/start_test.py | 418 ++------ .../user_template/async_http_user_template.py | 11 + .../user_template/fast_http_user_template.py | 9 +- .../user_template/http_user_template.py | 9 +- .../wrapper/user_template/request_executor.py | 5 +- progress.md | 7 +- test/smoke/gui_probe.py | 112 ++ test/smoke/platform_probes.py | 16 +- test/test_asyncio_engine.py | 38 +- test/test_distributed_health.py | 8 +- test/test_distributed_records.py | 710 +++++++++++++ test/test_expansion2_smoke.py | 47 +- test/test_gui_lifecycle.py | 331 ++++++ test/test_legacy_cli_contract.py | 28 + test/test_native_async_runner.py | 437 ++++++++ test/test_native_imports.py | 73 ++ test/test_parameter_context.py | 379 +++++++ test/test_record_engine_integration.py | 73 +- 66 files changed, 6462 insertions(+), 1624 deletions(-) create mode 100644 docs/superpowers/plans/2026-10-03-desktop-gui.md create mode 100644 docs/superpowers/plans/2026-10-03-distributed-canonical-records.md create mode 100644 docs/superpowers/plans/2026-10-03-native-async.md create mode 100644 docs/superpowers/plans/2026-10-03-parameter-isolation.md create mode 100644 je_load_density/engine/async_config.py create mode 100644 je_load_density/engine/async_http.py create mode 100644 je_load_density/engine/entrypoints.py create mode 100644 je_load_density/gui/child_process.py create mode 100644 je_load_density/gui/dark_style.py create mode 100644 je_load_density/gui/run_protocol.py create mode 100644 je_load_density/gui/run_worker.py create mode 100644 je_load_density/utils/test_record/distributed_context.py create mode 100644 je_load_density/wrapper/distributed_records.py create mode 100644 je_load_density/wrapper/start_wrapper/locust_start.py create mode 100644 test/smoke/gui_probe.py create mode 100644 test/test_distributed_records.py create mode 100644 test/test_gui_lifecycle.py create mode 100644 test/test_native_async_runner.py create mode 100644 test/test_native_imports.py create mode 100644 test/test_parameter_context.py diff --git a/README.md b/README.md index 3fc25c3..a7763a7 100644 --- a/README.md +++ b/README.md @@ -533,6 +533,7 @@ Placeholders are expanded automatically on every task: | Placeholder | Resolves to | |-------------|-------------| | `${var.NAME}` | Value passed to `register_variable(s)` | +| `${session.NAME}` | Value extracted with `scope: "session"` in the current virtual user | | `${env.NAME}` | Environment variable `NAME` | | `${csv.SOURCE.COL}` | Next row from CSV source `SOURCE` (cycles by default) | | `${faker.METHOD}` | `Faker().METHOD()` (lazy import) | @@ -556,6 +557,13 @@ Or from action JSON: Unknown placeholders are left in place so missing data is visible during a dry run. +HTTP, FastHTTP and the Locust HTTPX user keep independent variable/session state for each +virtual user. Extraction writes only to that user's resolver; `scope: "session"` selects +`${session.NAME}`. CSV/DB fields in one resolved task use the same row, with synchronized +row allocation across users. Python callers can select an explicit fork with +`with use_resolver(get_resolver().fork()):`. Other protocol templates retain legacy scope. +Package, native async and executor imports load Locust only when a Locust API is selected. + ## Scenario Modes ```json @@ -648,6 +656,19 @@ All three are loaded lazily and gated by the matching install extra. ## Distributed Master / Worker +Canonical aggregation is opt-in: the master passes +`run_context=DistributedRunContext()` from +`je_load_density.utils.test_record.distributed_context`; each worker sets +`distributed_records=True`. This requires the coordinated ActionCore record API. +The master validates transport/run/worker identity and deduplicates record IDs; +accepted records populate legacy reports once. `env.record_delivery.snapshot()` +exposes queue/delivery diagnostics. Defaults are 100 records / 262,144 bytes per +batch, 65,536 bytes per record, 1,000 records / 4,194,304 queued bytes, 0.1-second +retry and 2-second final drain/acknowledgement budgets. Overflow or incomplete +delivery fails explicitly. Buffers are in memory; delivered history needs export +for durable storage. Ongoing load rebalancing recreates capacity; it does not +replay HTTP side effects, migrate sessions or promise exactly-once execution. + ```python # master start_test( @@ -702,7 +723,7 @@ return keys and existing dashboard snapshot keys remain compatible; `latency_windows` adds bounded chart data. SSE clients can stream while other clients fetch snapshots, and stopping the dashboard closes the stream. -The desktop GUI controls and execution lifecycle are still tracked separately. +The desktop GUI adds engine/load controls and isolated run processes with Start/Stop lifecycle. ## HAR Record / Replay @@ -796,7 +817,14 @@ window.show() sys.exit(app.exec()) ``` -The GUI ships English, Traditional Chinese, Japanese, and Korean translations and a live stats panel that polls `test_record_instance` once a second (RPS, average / p95 latency, failure count). +The GUI ships English, Traditional Chinese, Japanese, and Korean translations. Settings appear +left; run state, Start/Stop, metrics, charts and recent requests appear right. Each run uses an +isolated interpreter for Locust or asyncio. Stop requests cooperative cancellation and escalates +after three seconds. Completed/failed results remain visible; recent requests retain at most 200 sanitized +rows and logs retain 500 blocks. Existing persisted history remains available. Action-file runs +preserve workload settings and report actions; the selected engine applies to `LD_start_test`. +Frames are limited to 128 KiB, retaining fewer request rows when needed. Charts receive +up to 120 windows calculated from the complete child records, independently of the request tail. ## CLI Usage @@ -1107,7 +1135,21 @@ Chrome extension and builds the JetBrains plugin on every change under `editors/ Added in the 2026-05 expansion. Each one is imported lazily and needs only its own extra. -- **Asyncio engine.** `je_load_density.engine.asyncio_engine.run_async_load` drives an HTTP target from asyncio without Locust and writes the same records as Locust users do, with a 4xx/5xx counted as a failure. The `bench` subcommand wraps it: +- **Asyncio engine.** `start_test(..., engine="asyncio")` and `LD_start_test` select native HTTP + execution; Locust remains the default. `await run_async_load(...)` works in an existing event + loop. Native runs support request kwargs, all five HTTP assertion types, extractors, per-user + cookies/session variables, sequence/weighted/conditional scenarios, retry, think time, token + buckets, ramp and stages/spike/soak. HTTP 4xx/5xx fail unless an explicit passing status-code + assertion expects that response. Preflight rejects unsupported protocols, distributed modes + and malformed options before requests. `AsyncRunHandle` exposes `start`, `stop`, `wait` and + `snapshot`; cancellation closes clients and tasks without recording target failures. Each run + returns an isolated `summary` compatible with SLA gates; legacy global report records and + optional canonical recording remain available. `requests` retains its success-count meaning; + `summary.totals.requests` counts all measured attempts. Exporter and full protocol/distributed + parity remain outstanding. The `bench` subcommand wraps native execution: + + Selecting Locust patches its interpreter. Run native I/O in a fresh interpreter + after Locust; CLI `bench` and the desktop supervisor provide this isolation. ```bash python -m je_load_density bench https://api.example.com/health --users 10 --duration 10 @@ -1118,7 +1160,7 @@ Added in the 2026-05 expansion. Each one is imported lazily and needs only its o Cloud launchers validate counts/resources before contacting providers. `cloud.CloudLaunchError` retains prior accepted responses, failed worker indices and available failed response details; provider exceptions remain chained. Fargate rejects partial/malformed submissions. Lambda distinguishes execution success from Event acceptance and DryRun validation, closes payload streams and preserves FunctionError payloads. Cloud Run refreshes credentials per call; configure `parallelism` on the deployed Job, since per-run overrides support `task_count` but reject `parallelism`. ACI waits for provisioning and returns unique `name`, `status="Succeeded"` and `resource_id`. Launchers do not roll back accepted resources or retry launch requests. - **Chaos helpers**: `utils.chaos.toxiproxy` adds and removes latency or bandwidth toxics on a Toxiproxy instance (`install_latency`, `install_bandwidth`, `reset_all`); `utils.chaos.chaos_mesh` builds and applies Chaos Mesh manifests (`build_network_delay`, `apply_manifest`, `delete_manifest`). -- **Stub server**: `utils.stub_server.start_stub_server` / `stop_stub_server` serve canned responses so a scenario can run against a fake backend. It serves from a thread, which works beside Locust's gevent users; for the asyncio engine start it in a separate process, because a server thread in the engine's own process never gets scheduled. +- **Stub server**: `utils.stub_server.start_stub_server` / `stop_stub_server` serve canned responses from a thread. If the test process has selected Locust, run the native asyncio client and server in fresh interpreters to avoid gevent scheduling changes. - **More report formats** next to the seven above: Allure, cost, CycloneDX, Excel, latency histogram, PDF (`pdf` extra), SARIF and a service map, one `generate_*_report.py` module each under `utils/generate_report/`. - **Deployment templates** in `deploy/`: a Helm chart, a Kubernetes operator (`k8s` extra), Terraform, a Grafana dashboard and CI templates. diff --git a/README/README_zh-CN.md b/README/README_zh-CN.md index fa29eb1..7ea4224 100644 --- a/README/README_zh-CN.md +++ b/README/README_zh-CN.md @@ -556,6 +556,8 @@ register_csv_source("users", "users.csv") 未知的占位符会原样保留,因此 dry run 时缺少的数据会显而易见。 +HTTP、FastHTTP 与 Locust HTTPX 用户各自保存独立的变量及 session。提取值只会写入当前用户的解析器；`scope: "session"` 对应 `${session.NAME}`。同一个 task 的 CSV／DB 字段使用同一行数据，跨用户取行则同步分配。Python 可用 `with use_resolver(get_resolver().fork()):` 选择独立状态。其他协议模板保留现有 scope。包、原生 async 与 executor 导入只会在选择 Locust API 时加载 Locust。 + ## 场景模式 ```json @@ -648,6 +650,8 @@ start_opentelemetry_exporter(endpoint="http://otel-collector:4317", ## 分布式 Master / Worker +Canonical 聚合需明确启用：master 从 `je_load_density.utils.test_record.distributed_context` 导入并传入 `run_context=DistributedRunContext()`，每个 worker 设 `distributed_records=True`，需要协调中的 ActionCore record API。Master 验证 transport／run／worker 身份、按 record ID 去重，接受的记录只写入现有报告一次。`env.record_delivery.snapshot()` 显示 queue／传送诊断。默认每批 100 条／262,144 bytes、每条 65,536 bytes、待送 1,000 条／4,194,304 bytes、重试间隔 0.1 秒，最后 drain／acknowledgement 各最多 2 秒。超额或传送不完整会明确失败。Buffer 留在内存，已送历史须导出才能持久保存；持续负载重新分配只重建容量，不重播 HTTP 副作用、迁移 session 或承诺 exactly-once。 + ```python # master start_test( @@ -696,7 +700,7 @@ Qt、浏览器与 `[charts]` PNG 报告共用以请求开始时间分桶的统 报告文件名、返回键与原有 dashboard snapshot 键保持兼容， 新增 `latency_windows` 提供有上限的图表数据。SSE 流不阻挡其他 snapshot 请求，停止 dashboard 时关闭流。 -桌面 GUI 的控制界面与执行生命周期仍另列待办。 +桌面 GUI 提供引擎／负载设置、独立执行进程与 Start／Stop 生命周期。 ## HAR 录制/重放 @@ -790,7 +794,9 @@ window.show() sys.exit(app.exec()) ``` -GUI 内附英文、繁体中文、日文与韩文翻译,以及一个每秒轮询 `test_record_instance` 一次的实时统计面板(RPS、平均 / p95 latency、失败计数)。 +GUI 提供英文、繁体中文、日文与韩文翻译。左侧是设置，右侧是执行状态、Start／Stop、指标、图表与最近请求。每次 Locust／asyncio 执行都使用独立解释器；Stop 先协作式取消，三秒后仍未停止则终止子进程。完成／失败结果会保留，最近请求最多 200 条已清理数据，日志最多 500 段，现有持久化历史仍可查阅。动作文件保留负载参数与报告动作，所选引擎应用于 `LD_start_test`。 + +GUI 消息限制为 128 KiB，必要时减少最近请求行数。图表接收最多 120 个以完整子进程记录计算的时间窗，不受最近请求列表长度影响。 ## CLI 用法 @@ -1099,18 +1105,18 @@ diagnostics。以 `npm install && npm run package` 构建,再安装 于 2026-05 扩充加入。每一个都延迟导入,且只需要它自己的 extra。 -- **Asyncio 引擎。** `je_load_density.engine.asyncio_engine.run_async_load` 不通过 Locust,直接以 asyncio 驱动一个 HTTP 目标,并写出与 Locust 用户相同的记录,4xx/5xx 一律计为失败。`bench` 子命令包装了它: +- **Asyncio 引擎。** `start_test(..., engine="asyncio")` 与 `LD_start_test` 可选原生 HTTP 执行，默认仍是 Locust；现有 event loop 可用 `await run_async_load(...)`。支持请求参数、五种 HTTP 断言、提取、各用户独立 cookie／session、sequence／weighted／conditional、重试、think time、token bucket、ramp 与 stages／spike／soak。HTTP 4xx／5xx 计为失败，除非明确的 status-code 断言通过。请求前拒绝不支持的协议、分布式模式与无效设置。`AsyncRunHandle` 提供 `start`／`stop`／`wait`／`snapshot`；取消会关闭 client 与 task，不计为目标失败。每次返回独立的 `summary` 可供 SLA 评估，现有全局报告记录与可选 canonical 格式仍保留。`requests` 仍表示成功次数，`summary.totals.requests` 是所有测量次数。Exporter 与完整协议／分布式对等仍列待办。`bench` 子命令包装了它: ```bash python -m je_load_density bench https://api.example.com/health --users 10 --duration 10 ``` - 选项:`--method`、`--body`、`--http2`、`--max-in-flight`。 + 选项:`--method`、`--body`、`--http2`、`--max-in-flight`。选择 Locust 后，该进程会应用 gevent patch；随后执行原生 I/O 时请使用新进程。CLI bench 与桌面 GUI 已提供此隔离，嵌入式进程的引擎切换仍列待办。 - **Cloud workers**(`aws`、`gcp`、`azure` 或 `cloud` extras):`cloud.aws_fargate.launch_fargate_workers`、`cloud.aws_lambda.invoke_lambda_workers`(以 `lambda_worker_handler` 作为函数入口)、`cloud.azure_aci.launch_aci_workers` 与 `cloud.gcp_cloud_run.run_cloud_run_job` 为分布式跑法启动远端 worker。 云端 launcher 在联系 provider 前验证数量／资源参数。`cloud.CloudLaunchError` 保留先前接受的响应、失败 worker 索引及可取得的失败响应，并链接 provider 异常。Fargate 拒绝部分失败／格式错误的提交。Lambda 区分执行成功、Event 接受与 DryRun 验证，关闭 payload stream 并保留 FunctionError payload。Cloud Run 每次刷新 credentials；`parallelism` 请设置于部署的 Job，每次执行可覆盖 `task_count`，但拒绝 `parallelism`。ACI 等待 provisioning 并返回唯一 `name`、`status="Succeeded"` 与 `resource_id`。Launcher 不回滚已接受的资源，也不重试启动请求。 - **Chaos 辅助工具**:`utils.chaos.toxiproxy` 在 Toxiproxy 实例上新增与移除 latency 或 bandwidth toxic(`install_latency`、`install_bandwidth`、`reset_all`);`utils.chaos.chaos_mesh` 构建并套用 Chaos Mesh manifest(`build_network_delay`、`apply_manifest`、`delete_manifest`)。 -- **Stub server**:`utils.stub_server.start_stub_server` / `stop_stub_server` 供应罐头响应,让场景能对一个假后端执行。它从一个线程供应,可与 Locust 的 gevent 用户并存;对 asyncio 引擎则要在独立进程启动它,因为引擎自己进程里的服务器线程永远得不到调度。 +- **Stub server**:`utils.stub_server.start_stub_server` / `stop_stub_server` 供应罐头响应，让场景能对假后端执行。它使用线程；若测试进程已选择 Locust，原生 asyncio client 与 server 请使用新进程，避免 gevent 调度影响。 - **更多报告格式**,在上述七种之外:Allure、cost、CycloneDX、Excel、latency histogram、PDF(`pdf` extra)、SARIF 与一张 service map,各自对应 `utils/generate_report/` 下一个 `generate_*_report.py` 模块。 - **部署模板** 位于 `deploy/`:一份 Helm chart、一个 Kubernetes operator(`k8s` extra)、Terraform、一张 Grafana dashboard 与 CI 模板。 diff --git a/README/README_zh-TW.md b/README/README_zh-TW.md index d92fd7a..3fdca1c 100644 --- a/README/README_zh-TW.md +++ b/README/README_zh-TW.md @@ -556,6 +556,8 @@ register_csv_source("users", "users.csv") 未知的占位符會原樣保留,因此 dry run 時缺少的資料會顯而易見。 +HTTP、FastHTTP 與 Locust HTTPX 使用者各自保存獨立的變數及 session。擷取值只會寫入目前使用者的解析器；`scope: "session"` 對應 `${session.NAME}`。同一個 task 的 CSV／DB 欄位使用同一列資料，跨使用者取列則同步分配。Python 可用 `with use_resolver(get_resolver().fork()):` 選擇獨立狀態。其他協定模板保留既有 scope。套件、原生 async 與 executor 匯入只會在選擇 Locust API 時載入 Locust。 + ## 情境模式 ```json @@ -648,6 +650,8 @@ start_opentelemetry_exporter(endpoint="http://otel-collector:4317", ## 分散式 Master / Worker +Canonical 聚合需明確啟用：master 從 `je_load_density.utils.test_record.distributed_context` 匯入並傳入 `run_context=DistributedRunContext()`，每個 worker 設 `distributed_records=True`，需要協調中的 ActionCore record API。Master 驗證 transport／run／worker 身分、依 record ID 去重，接受的紀錄只寫入既有報告一次。`env.record_delivery.snapshot()` 顯示 queue／傳送診斷。預設每批 100 筆／262,144 bytes、每筆 65,536 bytes、待送 1,000 筆／4,194,304 bytes、重試間隔 0.1 秒，最後 drain／acknowledgement 各最多 2 秒。超額或傳送不完整會明確失敗。Buffer 留在記憶體，已送歷史須匯出才能持久保存；持續負載重新分配只重建容量，不重播 HTTP 副作用、搬移 session 或承諾 exactly-once。 + ```python # master start_test( @@ -696,7 +700,7 @@ Qt、瀏覽器與 `[charts]` PNG 報告共用以請求開始時間分桶的統 報告檔名、回傳鍵與原有 dashboard snapshot 鍵維持相容， 新增 `latency_windows` 提供有上限的圖表資料。SSE 串流不阻擋其他 snapshot 請求，停止 dashboard 時關閉串流。 -桌面 GUI 的控制介面與執行生命週期仍另列待辦。 +桌面 GUI 提供引擎／負載設定、獨立執行行程與 Start／Stop 生命週期。 ## HAR 錄製/重放 @@ -790,7 +794,9 @@ window.show() sys.exit(app.exec()) ``` -GUI 內附英文、繁體中文、日文與韓文翻譯,以及一個每秒輪詢 `test_record_instance` 一次的即時統計面板(RPS、平均 / p95 latency、失敗計數)。 +GUI 提供英文、繁體中文、日文與韓文翻譯。左側是設定，右側是執行狀態、Start／Stop、指標、圖表與最近請求。每次 Locust／asyncio 執行都使用獨立直譯器；Stop 先合作式取消，三秒後仍未停止則終止子行程。完成／失敗結果會保留，最近請求最多 200 筆已清理資料，日誌最多 500 段，既有持久化歷史仍可查閱。動作檔保留負載參數與報告動作，所選引擎套用於 `LD_start_test`。 + +GUI 訊息限制為 128 KiB，必要時減少最近請求列數。圖表接收最多 120 個以完整子行程紀錄計算的時間窗，不受最近請求清單長度影響。 ## CLI 用法 @@ -1099,18 +1105,18 @@ diagnostics。以 `npm install && npm run package` 建置,再安裝 於 2026-05 擴充加入。每一個都延遲匯入,且只需要它自己的 extra。 -- **Asyncio 引擎。** `je_load_density.engine.asyncio_engine.run_async_load` 不透過 Locust,直接以 asyncio 驅動一個 HTTP 目標,並寫出與 Locust 使用者相同的紀錄,4xx/5xx 一律計為失敗。`bench` 子指令包裝了它: +- **Asyncio 引擎。** `start_test(..., engine="asyncio")` 與 `LD_start_test` 可選原生 HTTP 執行，預設仍是 Locust；現有 event loop 可用 `await run_async_load(...)`。支援請求參數、五種 HTTP 斷言、擷取、各使用者獨立 cookie／session、sequence／weighted／conditional、重試、think time、token bucket、ramp 與 stages／spike／soak。HTTP 4xx／5xx 計為失敗，除非明確的 status-code 斷言通過。請求前拒絕不支援的協定、分散式模式與無效設定。`AsyncRunHandle` 提供 `start`／`stop`／`wait`／`snapshot`；取消會關閉 client 與 task，不計為目標失敗。每次回傳獨立的 `summary` 可供 SLA 評估，既有全域報告紀錄與選用 canonical 格式仍保留。`requests` 仍表示成功次數，`summary.totals.requests` 是所有量測次數。Exporter 與完整協定／分散式對等仍列待辦。`bench` 子指令包裝了它: ```bash python -m je_load_density bench https://api.example.com/health --users 10 --duration 10 ``` - 選項:`--method`、`--body`、`--http2`、`--max-in-flight`。 + 選項:`--method`、`--body`、`--http2`、`--max-in-flight`。選擇 Locust 後，該行程會套用 gevent patch；接著執行原生 I/O 時請使用新行程。CLI bench 與桌面 GUI 已提供此隔離，嵌入式行程的引擎切換仍列待辦。 - **Cloud workers**(`aws`、`gcp`、`azure` 或 `cloud` extras):`cloud.aws_fargate.launch_fargate_workers`、`cloud.aws_lambda.invoke_lambda_workers`(以 `lambda_worker_handler` 作為函式進入點)、`cloud.azure_aci.launch_aci_workers` 與 `cloud.gcp_cloud_run.run_cloud_run_job` 為分散式跑法啟動遠端 worker。 雲端 launcher 在聯絡 provider 前驗證數量／資源參數。`cloud.CloudLaunchError` 保留先前接受的回應、失敗 worker 索引及可取得的失敗回應，並串接 provider 例外。Fargate 拒絕部分失敗／格式錯誤的提交。Lambda 區分執行成功、Event 接受與 DryRun 驗證，關閉 payload stream 並保留 FunctionError payload。Cloud Run 每次刷新 credentials；`parallelism` 請設定於部署的 Job，每次執行可覆寫 `task_count`，但拒絕 `parallelism`。ACI 等待 provisioning 並回傳唯一 `name`、`status="Succeeded"` 與 `resource_id`。Launcher 不回滾已接受的資源，也不重試啟動請求。 - **Chaos 輔助工具**:`utils.chaos.toxiproxy` 在 Toxiproxy 執行個體上新增與移除 latency 或 bandwidth toxic(`install_latency`、`install_bandwidth`、`reset_all`);`utils.chaos.chaos_mesh` 建構並套用 Chaos Mesh manifest(`build_network_delay`、`apply_manifest`、`delete_manifest`)。 -- **Stub server**:`utils.stub_server.start_stub_server` / `stop_stub_server` 供應罐頭回應,讓情境能對一個假後端執行。它從一個執行緒供應,可與 Locust 的 gevent 使用者並存;對 asyncio 引擎則要在獨立行程啟動它,因為引擎自己行程裡的伺服器執行緒永遠得不到排程。 +- **Stub server**:`utils.stub_server.start_stub_server` / `stop_stub_server` 供應罐頭回應，讓情境能對假後端執行。它使用執行緒；若測試行程已選擇 Locust，原生 asyncio client 與 server 請使用新行程，避免 gevent 排程影響。 - **更多報告格式**,在上述七種之外:Allure、cost、CycloneDX、Excel、latency histogram、PDF(`pdf` extra)、SARIF 與一張 service map,各自對應 `utils/generate_report/` 下一個 `generate_*_report.py` 模組。 - **部署範本** 位於 `deploy/`:一份 Helm chart、一個 Kubernetes operator(`k8s` extra)、Terraform、一張 Grafana dashboard 與 CI 範本。 diff --git a/architecture.md b/architecture.md index ae2a81c..d1d7400 100644 --- a/architecture.md +++ b/architecture.md @@ -1,7 +1,7 @@ # LoadDensity Architecture > Short overview for people and agents. -> Last verified: 2026-09-22 against `7cf7901` on `dev`. +> Last verified: 2026-10-03 against the working tree on `feat/testing-platform`. ## 1. Purpose @@ -214,6 +214,35 @@ the published dependency floor. The old action-executor record contract is separ Cloud launchers preflight counts/resources and distinguish accepted submissions from successful execution/provisioning. Public cloud.CloudLaunchError retains accepted worker responses and failure indices. Cloud Run per-run parallelism is rejected (configure the deployed Job); ACI waits its poller and returns unique names/resource IDs. No adapter retries or rolls back launches. +Public package exports and executor command functions resolve their fixed module bindings +on first use. Importing the package, CLI parser or native HTTP engine does not load Locust +or patch socket/TLS/threading. Selecting a Locust environment registers its request hook. +`get_resolver`, `use_resolver` and `register_session_variable` are public APIs. HTTP user +templates fork variable/session state once per user; ContextVar scopes restore selection, +while locked CSV/DB providers allocate one coherent row per recursive task resolution. +CLI action failures are counted through a scoped reporter around public execute_action; +report callbacks and settings restoration do not require the unreleased core collector API. + +The public start_test signature remains compatible and accepts engine through kwargs: +locust defaults to the existing environment path, asyncio dispatches local native HTTP. +AsyncRunHandle owns clients/tasks, user resolvers, run summary and cooperative callbacks; +legacy record/report and canonical opt-in paths remain available. Unsupported protocols, +distributed options and exporter integrations are retained as outstanding capabilities. +The desktop Qt supervisor communicates with a fresh engine interpreter through bounded +JSON frames and a unique cancellation file; no worker mutates Qt objects. Existing PyBreeze +LoadDensityWidget controls and tab attributes remain available. +The 128 KiB frame budget retains recent complete request rows; charts consume at most +120 child-computed windows covering full measurements rather than the bounded row tail. + +Locust master opt-in DistributedRunContext composes public strict ActionCore worker contexts. +Worker distributed_records delivery binds producer epochs/generations to master run identity, +uses bounded sequenced batches with ACK retry and final drain, and preserves record IDs across +reconnection. Master whole-batch validation and global deduplication feed legacy reports once. +env.record_delivery exposes pending/incomplete diagnostics; failure cleanup preserves caller +exceptions. No durable queue, finite shard replay, session migration or HTTP exactly-once promise +is part of this native ongoing-load contract. Legacy start-wrapper imports now defer the Locust +implementation to locust_start.py while preserving the original registry/import attributes. + ## 7. Design constraints - SOLID, composition over inheritance, patterns only where they reduce complexity (CLAUDE.md diff --git a/docs/source/En/doc/architecture/architecture_doc.rst b/docs/source/En/doc/architecture/architecture_doc.rst index e8eb71e..2b77f01 100644 --- a/docs/source/En/doc/architecture/architecture_doc.rst +++ b/docs/source/En/doc/architecture/architecture_doc.rst @@ -237,12 +237,14 @@ Action lifecycle, step by step #. ``Executor.execute_action`` dispatches each step against ``event_dict`` (``LD_*`` commands plus safe builtins). #. When the step is ``LD_start_test``, the dispatcher selects a user - template (one of 12), seeds the parameter resolver from any + engine (Locust by default, native asyncio HTTP by explicit selection), + selects a Locust user template when applicable, seeds per-user resolver forks from ``variables`` / ``csv_sources`` / ``db_sources``, optionally constructs a ``LoadTestShape`` from ``load_shape`` / ``shape_config``, then calls ``prepare_env``. -#. ``prepare_env`` builds a Locust ``Environment`` in local, master, - or worker mode and starts the run. +#. Locust ``prepare_env`` builds an ``Environment`` in local, master or + worker mode. Native ``AsyncRunHandle`` owns local HTTP users, clients, tasks + and cooperative cancellation without importing Locust. #. Each user runs ``run_scenario`` (or the protocol equivalent) per tick. Before each ``execute_task``: throttle bucket → network conditioner → ``${...}`` expansion → retry policy → request. diff --git a/docs/source/En/doc/gui/gui_doc.rst b/docs/source/En/doc/gui/gui_doc.rst index 4708df3..4eff02f 100644 --- a/docs/source/En/doc/gui/gui_doc.rst +++ b/docs/source/En/doc/gui/gui_doc.rst @@ -5,9 +5,17 @@ Overview -------- LoadDensity ships an optional PySide6 graphical front-end. It carries -the form controls for kicking off a quick HTTP test, a log panel that -mirrors the framework log, and a live stats panel that polls -``test_record_instance`` once a second. +settings on the left and state, Start/Stop, metrics, charts and recent requests +on the right. Choose a target or action file, Locust/asyncio engine and load +controls. Each run uses a fresh interpreter; the Qt parent never imports Locust. +Action files retain their workload and report actions. + +Stop requests cooperative cancellation and escalates after three seconds. +Closing an active window stops the child and joins its QThread. Results remain +visible after completion/failure. Recent requests retain at most 200 sanitized rows, +logs retain 500 blocks and existing persisted history remains available. +The 128 KiB frame limit may retain fewer request rows. Charts receive up to 120 +windows computed from complete child records, independently of the request tail. Install ------- @@ -38,11 +46,10 @@ Launch Layout ------ -* **Test parameter form** — URL, test duration, user count, spawn rate, - HTTP method. -* **Start button** — Launches the load test in a background ``QThread``. +* **Test parameter form** — Target/action file, engine, duration, users, spawn rate and HTTP method. +* **Start/Stop** — Launch or cancel a child supervised by a background ``QThread``. * **Live stats panel** — Total requests, current rate, average and p95 - latency, failure count. Refreshes every 1 s. + latency, failure count, from child snapshots. * **Log panel** — Real-time framework log feed. * **Material Design theme** — ``dark_amber.xml`` from ``qt-material``. @@ -74,9 +81,9 @@ Architecture * - ``LoadDensityWidget`` - Form + start button + stats panel + log panel. * - ``StatsPanel`` - - QTimer-driven panel reading ``test_record_instance``. + - Receives cumulative run snapshots through GUI-thread signals. * - ``LoadDensityGUIThread`` - - Background ``QThread`` that runs the test without blocking the UI. + - Background ``QThread`` supervising an isolated interpreter and bounded JSON frames. * - ``InterceptAllFilter`` - Captures log records into a thread-safe queue. * - ``log_message_queue`` diff --git a/docs/source/En/doc/parameter_resolver/parameter_resolver_doc.rst b/docs/source/En/doc/parameter_resolver/parameter_resolver_doc.rst index 5a87389..6d5e11a 100644 --- a/docs/source/En/doc/parameter_resolver/parameter_resolver_doc.rst +++ b/docs/source/En/doc/parameter_resolver/parameter_resolver_doc.rst @@ -54,8 +54,19 @@ Registering data {"name": "products", "file_path": "products.csv", "cycle": False}, ]) -CSV files must have a header row. Each call to ``${csv.name.col}`` -returns the value at column ``col`` from the next row. +CSV files must have a header row. One recursive ``resolve(task)`` uses one row +per referenced CSV/DB source, so credentials in different fields stay paired. +Separate calls advance the source; forked users share synchronized allocation. + +Virtual-user sessions +--------------------- + +HTTP, FastHTTP and Locust HTTPX users fork variable/session state once per user. +Extracted values persist in that user's journey. ``scope: "session"`` writes to +the separate ``${session.NAME}`` namespace; omitted scope uses ``${var.NAME}``. +Other protocol templates retain the legacy resolver scope. Python callers can +select explicit state with ``with use_resolver(get_resolver().fork()):``; +nested scopes restore the previous resolver even after an exception. Action-JSON usage ----------------- diff --git a/docs/source/En/doc/start_test/start_test_doc.rst b/docs/source/En/doc/start_test/start_test_doc.rst index fd7a5b9..514bcd7 100644 --- a/docs/source/En/doc/start_test/start_test_doc.rst +++ b/docs/source/En/doc/start_test/start_test_doc.rst @@ -110,3 +110,53 @@ Worker (run on each node, on the same network as master):: The master waits for ``expected_workers`` workers to register before ramping up. Workers join the master and run the requested user count proportional to the cluster size. + +Native asyncio HTTP +------------------- + +Pass ``engine="asyncio"`` to the public ``start_test`` or ``LD_start_test``. +The default stays ``locust``. In an active event loop, use +``await run_async_load(tasks, users, duration_seconds, ...)`` instead. +Package/native imports do not load Locust or patch process I/O. +After selecting Locust, use a fresh interpreter for native I/O; CLI bench and +the desktop supervisor provide this isolation. + +Native HTTP supports request parameters/body/headers/cookies/auth/redirects, +all five HTTP assertions and three extractor kinds, independent user sessions, +sequence/weighted/conditional scenarios, retry, asynchronous think time/token +buckets, ramp and stages/spike/soak shapes. Client TLS/proxy settings are reused +per user. Certificates never modify another user's trust context. If supplying +an SSLContext, configure its certificate directly and omit cert/client_cert. +Malformed or unsupported options are rejected before requests; protocols other +than HTTP and master/worker modes remain unsupported. Metrics exporter parity +is still outstanding. + +``AsyncRunHandle`` exposes awaitable ``start``/``wait`` and ``stop``/``snapshot``. +``stop_requested`` and ``on_environment`` callbacks work with both engines. +Cancellation closes owned tasks and clients, while unexpected worker/cleanup +errors propagate. Each result includes an isolated ``summary`` suitable for +SLA evaluation. Legacy report lists and optional canonical RunContext recording +remain compatible. ``requests`` counts successes; ``summary.totals.requests`` +counts all measured attempts. HTTP 4xx/5xx fail unless a passing status-code +assertion explicitly expects that status. Native durations/counts/rates must be +positive finite numbers; counts must be integral. + +Canonical distributed results +----------------------------- + +The opt-in master passes ``run_context=DistributedRunContext()`` imported from +``je_load_density.utils.test_record.distributed_context``. Every worker enables +``distributed_records=True``. This requires the coordinated ActionCore record +API; legacy-only environments retain their published dependency floor. +Master ingestion preserves worker identity, validates the whole batch before +mutation and globally deduplicates record IDs. Newly accepted records populate +legacy reports once. Inspect ``env.record_delivery.snapshot()`` for diagnostics. + +Default limits are 100 records/262144 bytes per batch, 65536 bytes per record, +1000 records/4194304 queued bytes, 0.1-second retries and 2-second drain/terminal +acknowledgement budgets. Configure with ``record_batch_size``, +``record_batch_bytes``, ``record_max_bytes``, ``record_max_pending``, +``record_pending_bytes``, ``record_flush_interval`` and ``record_drain_timeout``. +Queue overflow, conflicting retries and incomplete final delivery fail explicitly. +Storage is in memory until exported. Native ongoing-user rebalancing does not +replay requests, migrate sessions or supply finite shards/exactly-once execution. diff --git a/docs/source/Zh/doc/architecture/architecture_doc.rst b/docs/source/Zh/doc/architecture/architecture_doc.rst index afe27da..1184d77 100644 --- a/docs/source/Zh/doc/architecture/architecture_doc.rst +++ b/docs/source/Zh/doc/architecture/architecture_doc.rst @@ -234,12 +234,14 @@ User 派發 ``execute_action(...)`` 送進一份動作 JSON。 #. ``Executor.execute_action`` 依 ``event_dict``(``LD_*`` + 安全 builtins)派發每一步驟。 -#. 當步驟為 ``LD_start_test``,派發器選擇 user template(12 種其一), +#. 當步驟為 ``LD_start_test``，派發器選擇引擎（預設 Locust， + 明確選擇可用原生 asyncio HTTP），必要時再選擇 Locust user template， 依 ``variables`` / ``csv_sources`` / ``db_sources`` 填入參數解析器, 並依 ``load_shape`` / ``shape_config`` 構造可選的 ``LoadTestShape``, 再呼叫 ``prepare_env``。 -#. ``prepare_env`` 以 local、master 或 worker 模式建立 Locust - ``Environment`` 並啟動執行。 +#. Locust ``prepare_env`` 以 local、master 或 worker 模式建立環境。 + 原生 ``AsyncRunHandle`` 管理 local HTTP 使用者、client、task 及 + 合作式取消，不載入 Locust。 #. 每個 user 每 tick 執行 ``run_scenario``。對每個 ``execute_task``: throttle bucket → network conditioner → ``${...}`` 展開 → retry policy → 請求。 diff --git a/docs/source/Zh/doc/gui/gui_doc.rst b/docs/source/Zh/doc/gui/gui_doc.rst index 32033ce..1851113 100644 --- a/docs/source/Zh/doc/gui/gui_doc.rst +++ b/docs/source/Zh/doc/gui/gui_doc.rst @@ -4,7 +4,16 @@ GUI 圖形介面 概觀 ---- -LoadDensity 內含選用的 PySide6 圖形前端。提供啟動快速 HTTP 測試的表單控制元件、鏡像框架日誌的 log panel，以及每秒輪詢 ``test_record_instance`` 的即時統計面板。 +LoadDensity 提供選用的 PySide6 圖形前端。左側設定目標／動作檔、 +Locust／asyncio 引擎與負載，右側顯示執行狀態、Start／Stop、指標、 +圖表與最近請求。每次執行使用獨立直譯器，Qt 主行程不載入 Locust。 +動作檔保留負載參數與報告動作。 + +Stop 先合作式取消，三秒後仍未停止則終止子行程；關閉執行中的視窗 +會停止子行程並等待 QThread。完成／失敗結果保留，最近請求最多 +200 筆已清理資料，日誌最多 500 段，既有持久化歷史仍可查閱。 +訊息限制為 128 KiB，必要時減少請求列數。圖表接收最多 120 個以完整 +子行程紀錄計算的時間窗，不受最近請求清單長度影響。 安裝 ---- @@ -35,9 +44,9 @@ LoadDensity 內含選用的 PySide6 圖形前端。提供啟動快速 HTTP 測 版面 ---- -* **測試參數表單** — URL、測試時間、user 數、spawn rate、HTTP method。 -* **開始按鈕** — 在背景 ``QThread`` 啟動壓測。 -* **即時統計面板** — 總請求、目前速率、平均與 p95 延遲、失敗數。每 1 秒重新整理。 +* **測試參數表單** — 目標／動作檔、引擎、測試時間、user 數、spawn rate、HTTP method。 +* **Start／Stop** — 由背景 ``QThread`` 管理子行程的啟動與取消。 +* **即時統計面板** — 從子行程快照顯示總請求、速率、平均／p95 延遲及失敗數。 * **Log panel** — 即時框架日誌。 * **Material Design 主題** — ``qt-material`` 的 ``dark_amber.xml``。 @@ -67,9 +76,9 @@ GUI 內含英文、繁體中文、日文、韓文翻譯。透過 ``LanguageWrapp * - ``LoadDensityWidget`` - 表單 + 開始按鈕 + 統計面板 + log panel。 * - ``StatsPanel`` - - 由 QTimer 驅動、讀取 ``test_record_instance`` 的面板。 + - 透過 GUI thread signal 接收累積執行快照。 * - ``LoadDensityGUIThread`` - - 在背景跑測試的 ``QThread``，避免阻擋 UI。 + - 管理獨立直譯器及有界 JSON 訊息的背景 ``QThread``。 * - ``InterceptAllFilter`` - 將 log records 攔截至 thread-safe queue。 * - ``log_message_queue`` diff --git a/docs/source/Zh/doc/parameter_resolver/parameter_resolver_doc.rst b/docs/source/Zh/doc/parameter_resolver/parameter_resolver_doc.rst index c4454bb..6f2f1cd 100644 --- a/docs/source/Zh/doc/parameter_resolver/parameter_resolver_doc.rst +++ b/docs/source/Zh/doc/parameter_resolver/parameter_resolver_doc.rst @@ -32,6 +32,17 @@ 未知占位符會原樣保留，便於 dry run 時偵測缺值。 +使用者隔離 +---------- + +HTTP、FastHTTP 與 Locust HTTPX 使用者各自 fork 變數及 session 狀態。 +擷取規則的 ``scope: "session"`` 對應獨立的 ``${session.NAME}``； +省略 scope 則寫入目前使用者的 ``${var.NAME}``。 +其他協定模板保留既有 scope。 +Python 可用 ``with use_resolver(get_resolver().fork()):`` 選擇解析器； +巢狀 scope 在例外後也會還原。每次遞迴 ``resolve(task)`` 對每個 +CSV／DB 來源只取一列，欄位保持配對，使用者間同步分配資料列。 + 註冊資料 -------- @@ -50,7 +61,8 @@ {"name": "products", "file_path": "products.csv", "cycle": False}, ]) -CSV 必須有 header；每次呼叫 ``${csv.name.col}`` 取下一行對應欄位。 +CSV 必須有 header；每次 ``resolve(task)`` 對每個來源取一列，該 task +所有欄位共用這列資料，下次 resolve 才會前進。 動作 JSON 用法 -------------- diff --git a/docs/source/Zh/doc/start_test/start_test_doc.rst b/docs/source/Zh/doc/start_test/start_test_doc.rst index 63f16d6..4defae4 100644 --- a/docs/source/Zh/doc/start_test/start_test_doc.rst +++ b/docs/source/Zh/doc/start_test/start_test_doc.rst @@ -104,3 +104,44 @@ Worker（在每個壓測節點執行，與 master 同網段）:: ) Master 在開始 ramp 前會等待 ``expected_workers`` 個 worker 註冊完成。Workers 加入 master 後，會依群集規模分擔 user count。 + +原生 asyncio HTTP +----------------- + +公開 ``start_test`` 與 ``LD_start_test`` 可傳入 ``engine="asyncio"``； +預設仍為 Locust。現有 event loop 請用 ``await run_async_load(...)``。 +套件及原生匯入不載入 Locust，也不修改 socket／TLS／threading。 +Locust 會修改所處直譯器；之後執行原生 I/O 請用獨立行程，CLI bench +及桌面 supervisor 均提供此隔離。 +支援請求參數、body、header、cookie、auth、redirect、五種 HTTP 斷言、 +三種擷取、獨立使用者 session、sequence／weighted／conditional、重試、 +非同步 think time／token bucket、ramp 及 stages／spike／soak。 +每位使用者重用 TLS／proxy client，憑證不修改其他使用者的 trust context。 +自訂 SSLContext 請直接配置憑證並省略 cert／client_cert。 +無效或未支援設定在請求前拒絕；非 HTTP 協定、master／worker 與 +metrics exporter 對等仍待補齊。數量、速率、期間須為正數且有限，數量須為整數。 + +``AsyncRunHandle`` 提供 awaitable ``start``／``wait`` 及 ``stop``／``snapshot``。 +兩個引擎均支援 ``stop_requested`` 與 ``on_environment`` callback。 +取消會關閉 task／client；worker／清理例外會向上傳遞。 +每次回傳獨立 ``summary`` 可交給 SLA 評估，既有報告清單與選用 canonical +RunContext 仍相容。``requests`` 是成功次數，``summary.totals.requests`` +是所有量測次數。HTTP 4xx／5xx 計為失敗，除非 status-code 斷言明確通過。 + +Canonical 分散式結果 +-------------------- + +Master 從 ``je_load_density.utils.test_record.distributed_context`` 匯入 +``DistributedRunContext`` 並傳入 ``run_context=DistributedRunContext()``， +每個 worker 設 ``distributed_records=True``；需協調中的 ActionCore +record API，既有 legacy 環境仍相容於已發布的相依下限。 +Master 保留 worker 身分，整批驗證後才寫入，依 record ID 全域去重。 +新接受的紀錄只寫入既有報告一次，診斷見 ``env.record_delivery.snapshot()``。 + +預設每批 100 筆／262144 bytes、每筆 65536 bytes、待送 1000 筆／4194304 +bytes、重試間隔 0.1 秒，drain／terminal acknowledgement 各最多 2 秒。 +可用 ``record_batch_size``、``record_batch_bytes``、``record_max_bytes``、 +``record_max_pending``、``record_pending_bytes``、``record_flush_interval``、 +``record_drain_timeout`` 調整。超額、重試衝突及最後傳送不完整均明確失敗。 +資料留在記憶體直到匯出；持續使用者重新分配不重播請求、不搬移 session， +也不提供 finite shards／exactly-once 執行。 diff --git a/docs/superpowers/plans/2026-10-03-desktop-gui.md b/docs/superpowers/plans/2026-10-03-desktop-gui.md new file mode 100644 index 0000000..81e3e45 --- /dev/null +++ b/docs/superpowers/plans/2026-10-03-desktop-gui.md @@ -0,0 +1,19 @@ +# Desktop GUI lifecycle execution plan + +Approved roadmap: `docs/superpowers/specs/2026-10-02-testing-platform-roadmap.md`, UI section, items 18/24. + +- Preserve public controls and the Log / Live Chart / Run History tab contract while arranging settings left and run inspection right. Translate new settings, states, request results and controls in all four existing languages. +- Replace the Qt thread's direct engine call with a fresh interpreter for every run. The parent imports no Locust; the child selects the scheduler before starting its metrics pump. +- Pass a unique control file through `stop_requested`, collect environment handles through `on_environment`, signal all background updates to Qt, reject duplicate starts and bound close/join cleanup. Escalate child termination after a three second cooperative grace period. +- Transport cumulative metrics and at most 200 sanitized request rows. Keep at most 500 log blocks and freeze completed charts. Preserve workload arguments and report actions in action files, while recording all action failures as a failed terminal state. +- Verify initial red lifecycle contracts, focused regression tests, existing tabs/bands, and a required offscreen installed-wheel probe against an independent local HTTP server. Exercise completion/cancellation for Locust and asyncio, then close an active run and confirm no child or QThread survives. + +Official API references: [Qt QThread wait and signal threading](https://doc.qt.io/qtforpython-6/PySide6/QtCore/QThread.html), [Qt QProcess lifecycle and native bounded waits](https://doc.qt.io/qtforpython-6/PySide6/QtCore/QProcess.html). The supervisor and smoke HTTP server use QProcess rather than monkey-patched Python subprocess/thread/queue I/O. + +Final bounded-frame regression: 200 request rows with long CJK names plus 120 complete window aggregates serialize to 618,967 bytes without limiting and exceed the unchanged 131,072-byte reader cap. The serializer preserves the full cumulative summary and all bounded aggregates, then retains the latest complete request rows under that byte limit. The regression frame is 129,733 bytes, retains the latest 38 rows through request 199 and all 120 windows, and still reports 200 total requests; the source snapshot remains unmodified. + +Chart accuracy regression: 250 requests at epoch 100.5 with latency 10 ms plus 250 at epoch 101.5 with latency 100 ms retain only 200 rows in the request table, but both rendered child windows retain their true count of 250 and p95 values of 10/100 ms. The chart consumes aggregates computed from complete child records, retains up to 120 windows, and preserves default global-record behavior for callers that do not supply snapshots. + +Final GUI lifecycle/tabs/bands verification: 25 passed; bounded Ruff checks passed. The final source Locust-prepatched HTTP completion/cancel/close probe also passed. Earlier installed-wheel checks passed before these final serialization/window-aggregate changes; the integrating agent rebuilds the final wheel. + +Ownership: `gui/*`, including the approved direct-window snapshot adapter in `chart_panel.py`; `test/test_gui_lifecycle.py`; `test/smoke/gui_probe.py`; and only `platform_probes.gui`. Existing band rendering and default global-record behavior remain covered by their original tests. Shared docs and commits belong to the integrating agent. diff --git a/docs/superpowers/plans/2026-10-03-distributed-canonical-records.md b/docs/superpowers/plans/2026-10-03-distributed-canonical-records.md new file mode 100644 index 0000000..1ba8f28 --- /dev/null +++ b/docs/superpowers/plans/2026-10-03-distributed-canonical-records.md @@ -0,0 +1,58 @@ +# Distributed Canonical Records Implementation Plan + +> Implement the approved aggregate composition and native message protocol using existing lifecycle boundaries. + +**Goal:** Deliver canonical worker request results to an explicitly selected master run without changing default legacy execution. + +**Architecture:** `DistributedRunContext` composes strict ActionCore contexts per source worker and validates full batches before committing. A native Locust hello/config/ready exchange binds producer epochs to master run identity. Bounded worker queues send a sequenced batch until acknowledged, retaining identical request record identifiers across retries. Master acceptance validates transport identity and projects newly accepted records to legacy reports. + +**Constraints:** No ActionCore internals or identity rewriting. Bound record bytes, batch bytes/count, and pending bytes/count. Stale generations, foreign identities, malformed records, conflicting retries and queue overflow are explicit failures. Final drains are bounded; missing acknowledgements remain observable. Ongoing virtual-user capacity redistribution is independent of record delivery. There is no finite work queue, request exactly-once guarantee, or session migration. + +## Tasks + +- [x] Red/green aggregate identity, global deduplication, JSON conflict and atomic batch tests. +- [x] Red/green bounded producer batching, retries, acknowledgement, generation and failure tests. +- [x] Wire opt-in contexts and sinks into isolated environments, readiness and cleanup. +- [x] Exercise real loopback Locust messages, final drain, cancellation, loss/reconnection and legacy report counts. +- [x] Replace the ramp test's original-dispatcher completion assumption with native final capacity completion. +- [x] Run focused and lint checks; report limitations and exact files to the parent for full-suite integration. + +## API + +Master selects `run_context=DistributedRunContext(...)`; every worker opts in with `distributed_records=True`. Import the aggregate from `je_load_density.utils.test_record.distributed_context`. Worker limits are provided through existing kwargs. An aggregate exposes `run_id`, `snapshot()`, `to_json()`, and transactional `ingest_batch(worker_id, records)`. Delivery diagnostics are available on `env.record_delivery.snapshot()`. Strict ordinary ActionCore contexts and default legacy environments retain their current behavior. A canonical master counts only workers that have completed both native and canonical readiness; default fail startup prevents load starting with an unbound producer. + +| Existing kwargs | Default | Meaning | +|---|---:|---| +| `record_batch_size` | 100 | Maximum records per batch | +| `record_batch_bytes` | 262144 | Maximum UTF-8 JSON bytes for the whole batch envelope | +| `record_max_bytes` | 65536 | Maximum UTF-8 JSON bytes per canonical record | +| `record_max_pending` | 1000 | Maximum queued records, including the in-flight batch | +| `record_pending_bytes` | 4194304 | Maximum queued serialized record bytes | +| `record_flush_interval` | 0.1 | Flush and identical retry interval in seconds | +| `record_drain_timeout` | 2 | Bounded drain and terminal acknowledgement budgets in seconds | + +Master bounds are negotiated with each worker using the smaller configured limits. Positive counts and finite positive times are required; an individual record must fit inside the batch and pending byte limits. Queue overflow fails the run without discarding old entries or growing an unbounded retry queue. Queued measurements captured before the handshake are bound once to the authenticated master identity. Each producer assigns UUID record IDs once; reconnects and identical retries preserve them. Record ID conflicts anywhere in a run reject the entire batch. + +The master projects only newly accepted canonical records into the existing success/error report lists. Global report percentiles therefore use delivered individual measurements, without averaging per-worker percentiles. Payload capture remains disabled by the existing canonical adapter. SQLite canonical export accepts the aggregate's public `run_id` and detached `snapshot()`. + +## Shutdown and limits + +Custom handlers use the existing native message channel; no second heartbeat or user dispatcher is added. A final flush stops worker users in an owned auxiliary greenlet, while the native RPC reader remains able to process acknowledgements. Concurrent cleanup waits for the active drain. Worker close also waits for a terminal master acknowledgement before transport closure; pending loss and missing workers fail explicitly. Closed producers ignore late batch acknowledgements, preserving final loss diagnostics. + +Terminal control messages enforce exact field sets, a maximum 128-byte UTF-8 nonce, and the configured envelope byte bound. Completion acknowledgements include only protocol fields; incoming data is never reflected wholesale. The sender also refuses oversized or non-serializable outgoing messages. Per-worker session history retains at most 128 retired producer epochs. Retired bootstrap hellos are ignored without changing the active producer or failing its valid traffic. When retirement history reaches capacity, a further replacement fails explicitly without eviction or session mutation, preventing resurrection of old epochs. + +This is bounded in-memory transport, not durable recovery: process termination can lose pending records, and the aggregate retains delivered run history in memory until exported. A master restart does not restore its generation/index state. A worker with pending records rejects a different run identity. All canonical workers must opt in; an explicitly degraded mixed startup can still end with incomplete delivery diagnostics. Locust's native load shapes, virtual-user redistribution and stateful session limitations remain independent of record delivery. There is no finite scenario shard queue or lease/retry API in this implementation, and no HTTP request replay, exactly-once execution or session migration guarantee. + +## Verification evidence + +The initial aggregate tests failed because the module was absent; producer/master tests then exposed missing acceptance and changed in-flight payloads on repeated configuration. Real native cancellation reproduced early RPC closure while another greenlet was draining, and local worker stop reproduced loss of its final completion message. Regressions cover both fixes, retry identity, malformed/foreign batches, stale generations/configuration, byte/count boundaries, reconnects, two-worker provenance, final flush and cleanup. A stalled-send regression verifies that transport retries cannot defeat the drain deadline, and an aggregate SQLite roundtrip preserves worker identity and record deduplication. Cleanup records delivery failures without masking an active caller/callback exception. The 56 aggregation tests and existing health/record regressions are checked with the coordinated ActionCore checkout; Ruff E/F/W/C90/I runs across all six owned Python paths. Controlled health tests bind random ports to permit parallel execution. Parent integration owns the full suite and shared documentation. + +Pre-review focused verification: **97 passed** (49 aggregation, 31 health, 17 canonical/engine/legacy record checks) in 24.39 seconds. Ruff E/F/W/C90/I and owned tracked-file whitespace checks passed. Earlier integrated verification also included native import regressions. Existing Locust late monkey-patching and legacy `TestRecord` collection warnings remain. + +Final review reproduced six failures: oversized/extra-field DONE messages finalized sessions and reflected payloads, malformed FLUSH/FINISHED controls were accepted, retired bootstrap epochs resurrected, and retirement history had no capacity limit. Those seven new cases (including an already passing normal DONE acknowledgement) now pass. Final follow-up verification: **87 passed** (56 aggregation plus 31 health) in 28.15 seconds; Ruff E/F/W/C90/I and owned-file whitespace checks passed. The canonical suite now calls `pytest.importorskip("je_action_core.request_context")` before importing the LoadDensity adapter, preserving collection compatibility when published ActionCore lacks the optional canonical API. Parent integration verifies the old-core wheel and full suite. + +## References + +- https://docs.locust.io/en/stable/running-distributed.html#communicating-across-nodes +- https://docs.locust.io/en/stable/api.html#event-hooks +- Installed Locust 2.43.4 public runner message methods and event hooks; current official stable documentation was verified at 2.46.6. diff --git a/docs/superpowers/plans/2026-10-03-native-async.md b/docs/superpowers/plans/2026-10-03-native-async.md new file mode 100644 index 0000000..e85feff --- /dev/null +++ b/docs/superpowers/plans/2026-10-03-native-async.md @@ -0,0 +1,56 @@ +# Native async engine implementation plan + +Execute the previously approved testing-platform roadmap in the existing isolated +feature worktree. Existing public functions, action shape, report filenames and +summary keys remain compatible. + +## Dependencies and boundaries + +- Per-user resolver forks/context scopes are a coordinated independent task. +- HTTP execution uses genuine httpx async I/O. HTTP/2 is opt-in and requires h2. +- TLS verify/cert/proxy belong to reusable clients, not individual httpx.request + keyword arguments. Keep a user-owned client pool keyed by TLS/proxy settings; + share its cookie jar only within that user. Close every client on exit. +- Locust imports and hook registration occur only when selecting Locust APIs. +- Non-HTTP protocol and unsupported native distributed options must fail preflight + until a genuine async implementation and worker contract exists. Completing + HTTP support does not close the full protocol/distributed parity item. + +## Steps + +1. Prove fresh-process package/native imports preserve socket/threading/TLS and do + not load Locust/gevent. Replace eager public exports and command imports with + lazy bindings while preserving all existing names and command registry policy. +2. Add start_test engine dispatch for Python/action/CLI. Retain Locust defaults and + the awaitable run_async_load positional arguments. Validate capabilities, + numbers, scenarios/assertion/extractor kinds and transport options before I/O. +3. Implement named HTTP kwargs/assertions/extractors, isolated user cookie/variable + sessions and coherent data sources. Record every measured attempt, including + assertion/transport/HTTP failures, in legacy and optional canonical formats. +4. Implement sequence/weighted/conditional scenarios, retries, think time, + asynchronous token buckets, ramp and stages/spike/soak. Separate per-run counts + from the compatible legacy record sink; expose start/stop/wait/snapshot through + a RunHandle and callbacks. Cancellation closes clients/tasks and does not become + a target failure. +5. Wire per-run summary/SLA/report persistence and engine-neutral metric events. + Verify real HTTP request shapes, cookie/token isolation, bounded concurrency, + ramp/shape changes, cancellation during I/O, repeated runs and failed preflight. +6. Update three README languages, Sphinx, architecture section6, outstanding work + and update logs; run full suite and installed-wheel smoke, review then commit. + +## HTTP batch boundary + +Steps 1-4 and the per-run summary/canonical-record portion of step5 are implemented. +HTTPX transport and lifecycle regressions verify actual async execution, session/cookie +separation, expected-status assertions, retry classification, malformed preflight, +shape concurrency, worker exceptions and cancellation cleanup. Legacy report actions +remain available. Native metric exporters, all non-HTTP protocols and master/worker +execution stay in progress item16; this batch does not close full parity. +Selecting Locust patches its process scheduler. Use a fresh interpreter for native I/O +after a Locust run; CLI bench and the desktop supervisor provide that isolation. + +## Primary references + +- https://www.python-httpx.org/api/ +- https://www.python-httpx.org/async/ +- Existing Locust shape tick helpers define shared stage/spike/soak semantics. diff --git a/docs/superpowers/plans/2026-10-03-parameter-isolation.md b/docs/superpowers/plans/2026-10-03-parameter-isolation.md new file mode 100644 index 0000000..81f73e4 --- /dev/null +++ b/docs/superpowers/plans/2026-10-03-parameter-isolation.md @@ -0,0 +1,49 @@ +# Per-user parameter isolation + +Approved predecessor to asyncio HTTP parity in the testing-platform roadmap. +Execute inline in the existing worktree; the coordinating worker owns shared +documentation, commits, facade changes and final review. + +## Contract + +`ParameterResolver.fork()` deep-copies variable and session state, while CSV/DB +providers remain shared and synchronize row advancement. `use_resolver(instance)` +selects a resolver for an explicit thread/task/greenlet scope and restores the +previous selection on exit. `get_resolver()` exposes the selected/default instance. +Existing resolver facade and registration functions retain legacy defaults. +`${session.NAME}` and session-scoped extraction use a separate local namespace. +One recursive `resolve(value)` consumes at most one row per referenced source. + +## Tasks + +- [x] Add failing tests for forks, scopes, concurrent extraction and coherent CSV/DB rows. +- [x] Implement scoped facade, detached variable state and locked shared row providers. +- [x] Wire persistent resolver forks into HttpUser/FastHttpUser/httpx-backed Locust users. +- [x] Verify legacy source registration, scenarios, templates and the full suite. + +## Review focus + +Do not advance CSV rows for each field, share extracted tokens between users, +leak context selection after errors, or import Locust from parameterization. +Asyncio callers explicitly enter one fork per user; child tasks inheriting a scope +share that selected resolver until they explicitly choose their own fork. + +## Execution evidence + +The initial eight parameter tests failed on missing fork/scope APIs. Six additional +HTTP integration regressions reproduced token contamination, missing httpx +extraction and missing DB source registration; all pass after user scope wiring. +Two database cleanup regressions verified that connection closure alone retained +the pool; the cached source now disposes its engine on success and query failure. + +Cookie storage was created before Locust patched threading under the new lazy +facade. A subprocess reproduced sharing a parent cookie jar across patched users, +and async tasks inherited their parent's jar. Context-local storage now detaches +inherited task owners and preserves the legacy explicit user-ID registry. + +Focused verification: 618 parameter, source, scenario and user-template tests +passed, with two optional SQLAlchemy skips. The standalone isolation suite now +has 22 passing tests, including real native-thread concurrency and subprocess +checks proving parameter-only imports load neither Locust nor gevent. The same +22 isolation tests also pass with published ActionCore 0.0.2. Final integrated +verification and completion evidence are recorded in the 2026-10 update log. diff --git a/docs/updates/2026-10.md b/docs/updates/2026-10.md index 8c6bf96..ceafac2 100644 --- a/docs/updates/2026-10.md +++ b/docs/updates/2026-10.md @@ -257,3 +257,13 @@ Added a stdlib smoke harness that exercises local HTTP requests in isolated proc - Refactored hook metadata construction while preserving legacy success-only fields, falsey HTTP-error responses, key order and canonical binding. Differential characterization108 combinations matched old/new records. - Verification:205 focused integration/contract/doc tests passed;560 report/protocol/matrix tests passed with one existing optional skip;139 cloud mandatory and12 isolated official SDK tests passed. Bounded Ruff E/F/W/C90/I and diffcheck pass. Remote quality analysis reruns after commit. - Remaining Codacy security audits concern fixed argument-vector subprocesses, parameterized test imports and locally generated XML in a stdlib-only harness; no shell is used and no external XML is accepted. No dependency/scan-rule bypass was introduced to suppress these audits. + +## U-20261003-02 · 2026-10-03 · Add isolated runners and canonical worker delivery · #done #async #gui #distributed #parameters #15 #24 #29 + +- **What**: Closed progress.md #15 (per-user parameters/session extraction), #24 (installed-wheel GUI lifecycle smoke) and #29 (canonical native worker aggregation). Implemented native HTTP lifecycle/scenarios and the desktop portion of #16/#18; those items retain only their outstanding capabilities. +- **Behavior**: Fixed lazy package/command/legacy-wrapper dispatch keeps native imports unpatched and preserves Locust hooks/defaults. AsyncRunHandle owns tasks, clients, user state and independent summaries; HTTP assertions/extractors, retries, pacing, shapes and cancellation preserve compatible request/record APIs. Static TLS/proxy failures are checked before target traffic, including alongside dynamic options. CLI failure tracking uses the published core reporter API rather than an unreleased collector. +- **GUI**: Fresh engine interpreters supervised by QProcess support Start/Stop, retained results, action files and all four translations. Frames stay within 128 KiB, recent rows are at most 200 and logs 500 blocks. Up to 120 windows are calculated from full child measurements; truncating rows does not change chart counts/percentiles. URL display redacts credentials/query/fragment, cleanup still emits terminal failure and session package commands preserve the existing gate/registry. +- **Distributed**: Explicit DistributedRunContext composes strict worker contexts. Native bounded batches, identical ACK retry, global record-ID conflict checks and terminal drain retain provenance and feed legacy reports once. Retired epochs cannot resurrect; each worker retains 128 retired epochs without eviction, failing explicitly at capacity. Control envelopes/fields and 128-byte UTF-8 nonces are bounded. Delivery failures preserve primary callback exceptions. +- **Verification**: Final full suite 1590 passed/14 optional skips/four existing warnings; changed Python Ruff E/F/W/C 90/I and whitespace checks passed. Six rebuilt installed-wheel smoke tests passed against published ActionCore 0.0.2. The final installed-wheel GUI probe with a Locust-prepatched parent completed/cancelled both engines and closed an active window without leaked child/QThread. Published-core focused native/parameter checks 61 passed; GUI 19 passed with canonical tests skipped; legacy CLI plus checkout smoke 16 passed. Sphinx dummy parsing succeeded with 576 autosummary/mock/reference/static warnings; no claim of a warning-free documentation build. +- **Evidence**: test/test_native_async_runner.py, test/test_native_imports.py, test/test_parameter_context.py, test/test_distributed_records.py, test/test_gui_lifecycle.py, test/test_legacy_cli_contract.py and test/smoke/gui_probe.py; four 2026-10-03 plans. Independent review findings were reproduced and corrected. Actual desktop 1200 x 850 rendering was inspected. Three README languages, En/Zh Sphinx and architecture section 6 describe the implemented boundary. +- **Open items**: See progress.md #16/#18/#28/#30. Native exporters/non-HTTP/master-worker parity, embedded scheduler switching, shared-record release activation and a unified functional/load/synthetic script remain open. Native I/O after selecting Locust uses a fresh interpreter. Delivery is bounded in-memory transport without finite-work leases, request replay or session migration. Docker is unavailable locally; updated remote CI follows the feature-branch push. diff --git a/docs/updates/README.md b/docs/updates/README.md index e611a8d..5e1a46d 100644 --- a/docs/updates/README.md +++ b/docs/updates/README.md @@ -58,6 +58,7 @@ In the same commit: delete the item from `progress.md`, add a `#done` entry here | ID | Date | Title | Tags | Batch | |---|---|---|---|---| +| U-20261003-02 | 2026-10-03 | Add isolated runners and canonical worker delivery | #done #async #gui #distributed #parameters #15 #24 #29 | [2026-10](2026-10.md) | | U-20261003-01 | 2026-10-03 | Verify extras matrix and resolve quality findings | #done #ci #quality #25 | [2026-10](2026-10.md) | | U-20261002-11 | 2026-10-02 | Share percentile bands and redesign live dashboard | #done #charts #ui #26 | [2026-10](2026-10.md) | | U-20261002-10 | 2026-10-02 | Monitor worker health and rebalance native load | #done #distributed | [2026-10](2026-10.md) | diff --git a/je_load_density/__init__.py b/je_load_density/__init__.py index c4e7779..df0d87f 100644 --- a/je_load_density/__init__.py +++ b/je_load_density/__init__.py @@ -1,541 +1,527 @@ -# hook (side-effect import: registers Locust request hooks) -from je_load_density.wrapper.event.request_hook import request_hook # noqa: F401 - -# Executor + action plumbing -from je_load_density.utils.executor.action_executor import ( - add_command_to_executor, - execute_action, - execute_files, - executor, -) -from je_load_density.utils.file_process.get_dir_file_list import get_dir_files_as_list - -# Reports -from je_load_density.utils.generate_report.generate_allure_report import ( - generate_allure_report, -) -from je_load_density.utils.generate_report.generate_chart_report import ( - generate_chart_report, -) -from je_load_density.utils.generate_report.generate_histogram_report import ( - generate_histogram_report, -) -from je_load_density.utils.generate_report.generate_pdf_report import ( - generate_pdf_report, -) -from je_load_density.utils.generate_report.generate_csv_report import generate_csv_report -from je_load_density.utils.generate_report.generate_html_report import ( - generate_html, - generate_html_report, -) -from je_load_density.utils.generate_report.generate_json_report import ( - generate_json, - generate_json_report, -) -from je_load_density.utils.generate_report.generate_junit_report import generate_junit_report -from je_load_density.utils.generate_report.generate_summary_report import ( - build_summary, - generate_summary_report, -) -from je_load_density.utils.generate_report.generate_xml_report import ( - generate_xml, - generate_xml_report, -) - -# JSON / YAML / TOML IO -from je_load_density.utils.json.json_file.json_file import read_action_json -from je_load_density.utils.json.json_file.toml_file import ( - read_action_toml, - write_action_toml, -) -from je_load_density.utils.json.json_file.yaml_file import ( - read_action_yaml, - write_action_yaml, -) - -# Metrics -from je_load_density.utils.metrics import ( - start_influxdb_sink, - start_opentelemetry_exporter, - start_prometheus_exporter, - stop_influxdb_sink, - stop_opentelemetry_exporter, - stop_prometheus_exporter, -) -from je_load_density.utils.metrics.datadog_apm_exporter import ( - start_datadog_apm_exporter, - stop_datadog_apm_exporter, -) -from je_load_density.utils.metrics.opentelemetry_tracing_exporter import ( - start_opentelemetry_tracing_exporter, - stop_opentelemetry_tracing_exporter, -) - -# Parameterisation -from je_load_density.utils.parameterization import ( - parameter_resolver, - register_csv_source, - register_csv_sources, - register_db_source, - register_db_sources, - register_variable, - register_variables, - resolve, -) - -# Recording / replay -from je_load_density.utils.recording.curl_importer import curl_to_task -from je_load_density.utils.recording.har_importer import ( - har_to_action_json, - har_to_tasks, - load_har, -) -from je_load_density.utils.recording.openapi_importer import ( - load_openapi, - openapi_to_action_json, - openapi_to_tasks, -) -from je_load_density.utils.recording.postman_importer import ( - load_postman_collection, - postman_to_action_json, - postman_to_tasks, -) - -# Project scaffolding -from je_load_density.utils.project.create_project_structure import create_project_dir - -# Control socket -from je_load_density.utils.socket_server.load_density_socket_server import ( - start_load_density_socket_server, -) - -# Test records -from je_load_density.utils.test_record.sqlite_persistence import ( - fetch_run_records, - list_runs, - persist_records, -) -from je_load_density.utils.test_record.test_record_class import test_record_instance - -# Locust environment + start -from je_load_density.wrapper.create_locust_env.create_locust_env import ( - create_env, - prepare_env, -) -from je_load_density.wrapper.proxy.proxy_user import locust_wrapper_proxy -from je_load_density.wrapper.start_wrapper.start_test import start_test - -# Locust re-exports -from locust import SequentialTaskSet, TaskSet, task - -# Callback executor -from je_load_density.utils.callback.callback_function_executor import callback_executor - -# DX / Quality -from je_load_density.utils.action_generator.generate import ( - generate_from_curls, - generate_from_openapi, - merge_actions, -) -from je_load_density.utils.ci_annotations.github_actions import ( - emit_github_annotations, - format_github_annotation, -) -from je_load_density.utils.linter.action_formatter import ( - format_action_document, - format_action_file, - format_action_string, -) -from je_load_density.utils.linter.action_linter import lint_action, lint_action_file -from je_load_density.utils.schema.action_schema import ( - action_json_schema, - export_schema, -) - -# SLA / regression -from je_load_density.utils.regression.diff import diff_runs, summarise_records -from je_load_density.utils.regression.error_clustering import cluster_errors -from je_load_density.utils.regression.multi_run_trend import trend_runs -from je_load_density.utils.sla.sla_gates import assert_sla, evaluate_sla - -# Load shapes -from je_load_density.utils.load_shapes.shapes import ( - SoakShape, - SpikeShape, - StagesShape, - build_load_shape, -) - -# GraphQL helper -from je_load_density.utils.graphql.graphql_task import ( - extract_field, - graphql_to_http_task, -) - -# Throttle -from je_load_density.utils.throttle.rps_throttle import ( - RpsThrottle, - get_throttle, - reset_throttles, -) - -# Reliability -from je_load_density.utils.reliability.adaptive_retry import ( - AdaptiveRetryPolicy, - classify_error, - run_with_retry, -) -from je_load_density.utils.reliability.failure_budget import ( - CircuitOpenError, - FailureBudget, - install_failure_budget, - uninstall_failure_budget, -) -from je_load_density.utils.reliability.network_conditioner import ( - NetworkConditioner, - install_network_conditioner, - uninstall_network_conditioner, -) -from je_load_density.utils.reliability.process_supervisor import ( - ProcessSupervisor, - with_watchdog, -) - -# Dashboard -from je_load_density.utils.dashboard.live_dashboard import ( - snapshot_metrics, - start_dashboard, - stop_dashboard, -) - -# Notifier -from je_load_density.utils.notifier.gitlab import ( - build_gitlab_mr_note, - post_gitlab_mr_summary, -) -from je_load_density.utils.notifier.opsgenie import ( - build_opsgenie_alert, - post_opsgenie_alert, -) -from je_load_density.utils.notifier.pagerduty import ( - build_pagerduty_event, - post_pagerduty_event, -) -from je_load_density.utils.notifier.slack import ( - build_slack_summary, - post_slack_summary, -) -from je_load_density.utils.notifier.teams import ( - build_teams_summary, - post_teams_summary, -) - -# StatsD -from je_load_density.utils.metrics.statsd_sink import ( - start_statsd_sink, - stop_statsd_sink, -) - -# Importers (k6 + JMeter) -from je_load_density.utils.recording.jmeter_importer import ( - jmeter_to_action_json, - jmeter_to_tasks, - load_jmeter_jmx, -) -from je_load_density.utils.recording.k6_importer import ( - k6_script_to_action_json, - k6_script_to_tasks, - load_k6_script, -) - -# Scenario / Security / Stub / Chaos -from je_load_density.utils.chaos.chaos_mesh import ( - apply_manifest as chaos_apply_manifest, - build_network_delay as chaos_build_network_delay, - delete_manifest as chaos_delete_manifest, -) -from je_load_density.utils.chaos.toxiproxy import ( - add_toxic as toxiproxy_add_toxic, - create_proxy as toxiproxy_create_proxy, - install_bandwidth as toxiproxy_install_bandwidth, - install_latency as toxiproxy_install_latency, - list_proxies as toxiproxy_list_proxies, - remove_proxies as toxiproxy_remove_proxies, - remove_toxic as toxiproxy_remove_toxic, - reset_all as toxiproxy_reset_all, -) -from je_load_density.utils.scenario.cookie_jar import ( - jar_for_user, - reset_all_jars, - reset_user_jar, -) -from je_load_density.utils.scenario.fsm import FsmRunner -from je_load_density.utils.security.fuzz import ( - expand_task_fuzz, - fuzz_query_string, - mutate_json, - mutate_string, -) -from je_load_density.utils.security.owasp_checks import ( - check_broken_object_level_auth, - check_excessive_data_exposure, - check_security_headers, - check_sensitive_token_leak, - run_owasp_checks, -) -from je_load_density.utils.stub_server.stub_server import ( - start_stub_server, - stop_stub_server, -) - -# Native asyncio engine -from je_load_density.engine.asyncio_engine import run_async_load - -# AI features -from je_load_density.utils.ai.auto_baseline import calibrate_sla -from je_load_density.utils.ai.auto_tune import AutoTuner -from je_load_density.utils.ai.root_cause import ( - build_root_cause_prompt, - render_prompt_text, -) -from je_load_density.utils.ai.smart_shape import find_breaking_point - -# Cloud worker adapters -from je_load_density.cloud.aws_fargate import launch_fargate_workers -from je_load_density.cloud.aws_lambda import ( - invoke_lambda_workers, - lambda_worker_handler, -) -from je_load_density.cloud.azure_aci import launch_aci_workers -from je_load_density.cloud.gcp_cloud_run import run_cloud_run_job - -# CI / canary -from je_load_density.utils.ci_annotations.canary_analysis import canary_verdict - -# Data / state utilities -from je_load_density.utils.data.db_fixtures import apply_fixture, run_teardown -from je_load_density.utils.data.factory import build_user, build_user_pool -from je_load_density.utils.data.pii_anonymizer import ( - find_pii, - scrub as pii_scrub, - scrub_string as pii_scrub_string, -) - -# DX / tooling -from je_load_density.utils.dx.i18n import ( - available_locales, - get_current_locale, - t as translate, -) -from je_load_density.utils.dx.leak_detector import ( - detect_growing_allocations, - start_leak_detector, - stop_leak_detector, -) -from je_load_density.utils.dx.profiler import memory_snapshot, profile_call -from je_load_density.utils.dx.repl import start_repl - -# Governance -from je_load_density.utils.governance.audit_log import ( - append_audit_entry, - read_audit_log, -) -from je_load_density.utils.governance.run_tagging import ( - list_tags, - search_runs_by_tag, - tag_run, -) -from je_load_density.utils.governance.share_link import ( - issue_share_link, - verify_share_link, -) -from je_load_density.utils.governance.test_catalog import index_catalog, search_catalog - -# Reports (additional) -from je_load_density.utils.generate_report.generate_cost_report import ( - estimate_run_cost, - generate_cost_report, -) -from je_load_density.utils.generate_report.generate_cyclonedx_report import ( - generate_cyclonedx_report, -) -from je_load_density.utils.generate_report.generate_excel_report import ( - generate_excel_report, -) -from je_load_density.utils.generate_report.generate_sarif_report import ( - generate_sarif_report, -) -from je_load_density.utils.generate_report.generate_service_map import ( - build_service_map, - generate_service_map, -) - -# Recording (additional) -from je_load_density.utils.recording.cdp_capture import ( - capture_cdp_session, - capture_cdp_to_har, - discover_targets as cdp_discover_targets, -) - -# Security: extended attack toolkits -from je_load_density.utils.security.graphql_checks import ( - build_alias_batching_attack, - build_depth_attack, - build_introspection_payload, - graphql_attack_pack, -) -from je_load_density.utils.security.jwt_attacks import ( - craft_alg_confusion_token, - craft_alg_none_token, - craft_attack_pack as craft_jwt_attack_pack, - craft_expired_token, - craft_kid_traversal_token, -) -from je_load_density.utils.security.rate_limit_probe import probe_rate_limit -from je_load_density.utils.security.smuggling_checks import ( - build_cl_te, - build_te_cl, - build_te_te, - smuggling_attack_pack, -) -from je_load_density.utils.security.ssrf_checks import ( - build_ssrf_targets, - find_metadata_leak, - render_ssrf_tasks, -) - -# Tracing exporters - -# Auth -from je_load_density.utils.auth.aws_sigv4 import sign_aws_request -from je_load_density.utils.auth.jwt_signer import decode_jwt, sign_jwt -from je_load_density.utils.auth.oauth2 import ( - OAuth2Client, - fetch_client_credentials_token, - fetch_password_token, - refresh_token, -) +"""Public APIs are loaded on demand; native asyncio imports do not patch process I/O.""" + +import importlib + +_EXPORTS = { + "AsyncRunHandle": ("je_load_density.engine.asyncio_engine", "AsyncRunHandle"), + "AdaptiveRetryPolicy": ("je_load_density.utils.reliability.adaptive_retry", "AdaptiveRetryPolicy"), + "AutoTuner": ("je_load_density.utils.ai.auto_tune", "AutoTuner"), + "CircuitOpenError": ("je_load_density.utils.reliability.failure_budget", "CircuitOpenError"), + "FailureBudget": ("je_load_density.utils.reliability.failure_budget", "FailureBudget"), + "FsmRunner": ("je_load_density.utils.scenario.fsm", "FsmRunner"), + "NetworkConditioner": ("je_load_density.utils.reliability.network_conditioner", "NetworkConditioner"), + "OAuth2Client": ("je_load_density.utils.auth.oauth2", "OAuth2Client"), + "ProcessSupervisor": ("je_load_density.utils.reliability.process_supervisor", "ProcessSupervisor"), + "RpsThrottle": ("je_load_density.utils.throttle.rps_throttle", "RpsThrottle"), + "SequentialTaskSet": ("locust", "SequentialTaskSet"), + "SoakShape": ("je_load_density.utils.load_shapes.shapes", "SoakShape"), + "SpikeShape": ("je_load_density.utils.load_shapes.shapes", "SpikeShape"), + "StagesShape": ("je_load_density.utils.load_shapes.shapes", "StagesShape"), + "TaskSet": ("locust", "TaskSet"), + "action_json_schema": ("je_load_density.utils.schema.action_schema", "action_json_schema"), + "add_command_to_executor": ("je_load_density.utils.executor.action_executor", "add_command_to_executor"), + "append_audit_entry": ("je_load_density.utils.governance.audit_log", "append_audit_entry"), + "apply_fixture": ("je_load_density.utils.data.db_fixtures", "apply_fixture"), + "assert_sla": ("je_load_density.utils.sla.sla_gates", "assert_sla"), + "available_locales": ("je_load_density.utils.dx.i18n", "available_locales"), + "build_alias_batching_attack": ("je_load_density.utils.security.graphql_checks", "build_alias_batching_attack"), + "build_cl_te": ("je_load_density.utils.security.smuggling_checks", "build_cl_te"), + "build_depth_attack": ("je_load_density.utils.security.graphql_checks", "build_depth_attack"), + "build_gitlab_mr_note": ("je_load_density.utils.notifier.gitlab", "build_gitlab_mr_note"), + "build_introspection_payload": ("je_load_density.utils.security.graphql_checks", "build_introspection_payload"), + "build_load_shape": ("je_load_density.utils.load_shapes.shapes", "build_load_shape"), + "build_opsgenie_alert": ("je_load_density.utils.notifier.opsgenie", "build_opsgenie_alert"), + "build_pagerduty_event": ("je_load_density.utils.notifier.pagerduty", "build_pagerduty_event"), + "build_root_cause_prompt": ("je_load_density.utils.ai.root_cause", "build_root_cause_prompt"), + "build_service_map": ("je_load_density.utils.generate_report.generate_service_map", "build_service_map"), + "build_slack_summary": ("je_load_density.utils.notifier.slack", "build_slack_summary"), + "build_ssrf_targets": ("je_load_density.utils.security.ssrf_checks", "build_ssrf_targets"), + "build_summary": ("je_load_density.utils.generate_report.generate_summary_report", "build_summary"), + "build_te_cl": ("je_load_density.utils.security.smuggling_checks", "build_te_cl"), + "build_te_te": ("je_load_density.utils.security.smuggling_checks", "build_te_te"), + "build_teams_summary": ("je_load_density.utils.notifier.teams", "build_teams_summary"), + "build_user": ("je_load_density.utils.data.factory", "build_user"), + "build_user_pool": ("je_load_density.utils.data.factory", "build_user_pool"), + "calibrate_sla": ("je_load_density.utils.ai.auto_baseline", "calibrate_sla"), + "callback_executor": ("je_load_density.utils.callback.callback_function_executor", "callback_executor"), + "canary_verdict": ("je_load_density.utils.ci_annotations.canary_analysis", "canary_verdict"), + "capture_cdp_session": ("je_load_density.utils.recording.cdp_capture", "capture_cdp_session"), + "capture_cdp_to_har": ("je_load_density.utils.recording.cdp_capture", "capture_cdp_to_har"), + "cdp_discover_targets": ("je_load_density.utils.recording.cdp_capture", "discover_targets"), + "chaos_apply_manifest": ("je_load_density.utils.chaos.chaos_mesh", "apply_manifest"), + "chaos_build_network_delay": ("je_load_density.utils.chaos.chaos_mesh", "build_network_delay"), + "chaos_delete_manifest": ("je_load_density.utils.chaos.chaos_mesh", "delete_manifest"), + "check_broken_object_level_auth": ("je_load_density.utils.security.owasp_checks", "check_broken_object_level_auth"), + "check_excessive_data_exposure": ("je_load_density.utils.security.owasp_checks", "check_excessive_data_exposure"), + "check_security_headers": ("je_load_density.utils.security.owasp_checks", "check_security_headers"), + "check_sensitive_token_leak": ("je_load_density.utils.security.owasp_checks", "check_sensitive_token_leak"), + "classify_error": ("je_load_density.utils.reliability.adaptive_retry", "classify_error"), + "cluster_errors": ("je_load_density.utils.regression.error_clustering", "cluster_errors"), + "craft_alg_confusion_token": ("je_load_density.utils.security.jwt_attacks", "craft_alg_confusion_token"), + "craft_alg_none_token": ("je_load_density.utils.security.jwt_attacks", "craft_alg_none_token"), + "craft_expired_token": ("je_load_density.utils.security.jwt_attacks", "craft_expired_token"), + "craft_jwt_attack_pack": ("je_load_density.utils.security.jwt_attacks", "craft_attack_pack"), + "craft_kid_traversal_token": ("je_load_density.utils.security.jwt_attacks", "craft_kid_traversal_token"), + "create_env": ("je_load_density.wrapper.create_locust_env.create_locust_env", "create_env"), + "create_project_dir": ("je_load_density.utils.project.create_project_structure", "create_project_dir"), + "curl_to_task": ("je_load_density.utils.recording.curl_importer", "curl_to_task"), + "decode_jwt": ("je_load_density.utils.auth.jwt_signer", "decode_jwt"), + "detect_growing_allocations": ("je_load_density.utils.dx.leak_detector", "detect_growing_allocations"), + "diff_runs": ("je_load_density.utils.regression.diff", "diff_runs"), + "emit_github_annotations": ("je_load_density.utils.ci_annotations.github_actions", "emit_github_annotations"), + "estimate_run_cost": ("je_load_density.utils.generate_report.generate_cost_report", "estimate_run_cost"), + "evaluate_sla": ("je_load_density.utils.sla.sla_gates", "evaluate_sla"), + "execute_action": ("je_load_density.utils.executor.action_executor", "execute_action"), + "execute_files": ("je_load_density.utils.executor.action_executor", "execute_files"), + "executor": ("je_load_density.utils.executor.action_executor", "executor"), + "expand_task_fuzz": ("je_load_density.utils.security.fuzz", "expand_task_fuzz"), + "export_schema": ("je_load_density.utils.schema.action_schema", "export_schema"), + "extract_field": ("je_load_density.utils.graphql.graphql_task", "extract_field"), + "fetch_client_credentials_token": ("je_load_density.utils.auth.oauth2", "fetch_client_credentials_token"), + "fetch_password_token": ("je_load_density.utils.auth.oauth2", "fetch_password_token"), + "fetch_run_records": ("je_load_density.utils.test_record.sqlite_persistence", "fetch_run_records"), + "find_breaking_point": ("je_load_density.utils.ai.smart_shape", "find_breaking_point"), + "find_metadata_leak": ("je_load_density.utils.security.ssrf_checks", "find_metadata_leak"), + "find_pii": ("je_load_density.utils.data.pii_anonymizer", "find_pii"), + "format_action_document": ("je_load_density.utils.linter.action_formatter", "format_action_document"), + "format_action_file": ("je_load_density.utils.linter.action_formatter", "format_action_file"), + "format_action_string": ("je_load_density.utils.linter.action_formatter", "format_action_string"), + "format_github_annotation": ("je_load_density.utils.ci_annotations.github_actions", "format_github_annotation"), + "fuzz_query_string": ("je_load_density.utils.security.fuzz", "fuzz_query_string"), + "generate_allure_report": ( + "je_load_density.utils.generate_report.generate_allure_report", + "generate_allure_report", + ), + "generate_chart_report": ("je_load_density.utils.generate_report.generate_chart_report", "generate_chart_report"), + "generate_cost_report": ("je_load_density.utils.generate_report.generate_cost_report", "generate_cost_report"), + "generate_csv_report": ("je_load_density.utils.generate_report.generate_csv_report", "generate_csv_report"), + "generate_cyclonedx_report": ( + "je_load_density.utils.generate_report.generate_cyclonedx_report", + "generate_cyclonedx_report", + ), + "generate_excel_report": ("je_load_density.utils.generate_report.generate_excel_report", "generate_excel_report"), + "generate_from_curls": ("je_load_density.utils.action_generator.generate", "generate_from_curls"), + "generate_from_openapi": ("je_load_density.utils.action_generator.generate", "generate_from_openapi"), + "generate_histogram_report": ( + "je_load_density.utils.generate_report.generate_histogram_report", + "generate_histogram_report", + ), + "generate_html": ("je_load_density.utils.generate_report.generate_html_report", "generate_html"), + "generate_html_report": ("je_load_density.utils.generate_report.generate_html_report", "generate_html_report"), + "generate_json": ("je_load_density.utils.generate_report.generate_json_report", "generate_json"), + "generate_json_report": ("je_load_density.utils.generate_report.generate_json_report", "generate_json_report"), + "generate_junit_report": ("je_load_density.utils.generate_report.generate_junit_report", "generate_junit_report"), + "generate_pdf_report": ("je_load_density.utils.generate_report.generate_pdf_report", "generate_pdf_report"), + "generate_sarif_report": ("je_load_density.utils.generate_report.generate_sarif_report", "generate_sarif_report"), + "generate_service_map": ("je_load_density.utils.generate_report.generate_service_map", "generate_service_map"), + "generate_summary_report": ( + "je_load_density.utils.generate_report.generate_summary_report", + "generate_summary_report", + ), + "generate_xml": ("je_load_density.utils.generate_report.generate_xml_report", "generate_xml"), + "generate_xml_report": ("je_load_density.utils.generate_report.generate_xml_report", "generate_xml_report"), + "get_current_locale": ("je_load_density.utils.dx.i18n", "get_current_locale"), + "get_dir_files_as_list": ("je_load_density.utils.file_process.get_dir_file_list", "get_dir_files_as_list"), + "get_throttle": ("je_load_density.utils.throttle.rps_throttle", "get_throttle"), + "graphql_attack_pack": ("je_load_density.utils.security.graphql_checks", "graphql_attack_pack"), + "graphql_to_http_task": ("je_load_density.utils.graphql.graphql_task", "graphql_to_http_task"), + "har_to_action_json": ("je_load_density.utils.recording.har_importer", "har_to_action_json"), + "har_to_tasks": ("je_load_density.utils.recording.har_importer", "har_to_tasks"), + "index_catalog": ("je_load_density.utils.governance.test_catalog", "index_catalog"), + "install_failure_budget": ("je_load_density.utils.reliability.failure_budget", "install_failure_budget"), + "install_network_conditioner": ( + "je_load_density.utils.reliability.network_conditioner", + "install_network_conditioner", + ), + "invoke_lambda_workers": ("je_load_density.cloud.aws_lambda", "invoke_lambda_workers"), + "issue_share_link": ("je_load_density.utils.governance.share_link", "issue_share_link"), + "jar_for_user": ("je_load_density.utils.scenario.cookie_jar", "jar_for_user"), + "jmeter_to_action_json": ("je_load_density.utils.recording.jmeter_importer", "jmeter_to_action_json"), + "jmeter_to_tasks": ("je_load_density.utils.recording.jmeter_importer", "jmeter_to_tasks"), + "k6_script_to_action_json": ("je_load_density.utils.recording.k6_importer", "k6_script_to_action_json"), + "k6_script_to_tasks": ("je_load_density.utils.recording.k6_importer", "k6_script_to_tasks"), + "lambda_worker_handler": ("je_load_density.cloud.aws_lambda", "lambda_worker_handler"), + "launch_aci_workers": ("je_load_density.cloud.azure_aci", "launch_aci_workers"), + "launch_fargate_workers": ("je_load_density.cloud.aws_fargate", "launch_fargate_workers"), + "lint_action": ("je_load_density.utils.linter.action_linter", "lint_action"), + "lint_action_file": ("je_load_density.utils.linter.action_linter", "lint_action_file"), + "list_runs": ("je_load_density.utils.test_record.sqlite_persistence", "list_runs"), + "list_tags": ("je_load_density.utils.governance.run_tagging", "list_tags"), + "load_har": ("je_load_density.utils.recording.har_importer", "load_har"), + "load_jmeter_jmx": ("je_load_density.utils.recording.jmeter_importer", "load_jmeter_jmx"), + "load_k6_script": ("je_load_density.utils.recording.k6_importer", "load_k6_script"), + "load_openapi": ("je_load_density.utils.recording.openapi_importer", "load_openapi"), + "load_postman_collection": ("je_load_density.utils.recording.postman_importer", "load_postman_collection"), + "locust_wrapper_proxy": ("je_load_density.wrapper.proxy.proxy_user", "locust_wrapper_proxy"), + "memory_snapshot": ("je_load_density.utils.dx.profiler", "memory_snapshot"), + "merge_actions": ("je_load_density.utils.action_generator.generate", "merge_actions"), + "mutate_json": ("je_load_density.utils.security.fuzz", "mutate_json"), + "mutate_string": ("je_load_density.utils.security.fuzz", "mutate_string"), + "openapi_to_action_json": ("je_load_density.utils.recording.openapi_importer", "openapi_to_action_json"), + "openapi_to_tasks": ("je_load_density.utils.recording.openapi_importer", "openapi_to_tasks"), + "parameter_resolver": ("je_load_density.utils.parameterization", "parameter_resolver"), + "persist_records": ("je_load_density.utils.test_record.sqlite_persistence", "persist_records"), + "pii_scrub": ("je_load_density.utils.data.pii_anonymizer", "scrub"), + "pii_scrub_string": ("je_load_density.utils.data.pii_anonymizer", "scrub_string"), + "post_gitlab_mr_summary": ("je_load_density.utils.notifier.gitlab", "post_gitlab_mr_summary"), + "post_opsgenie_alert": ("je_load_density.utils.notifier.opsgenie", "post_opsgenie_alert"), + "post_pagerduty_event": ("je_load_density.utils.notifier.pagerduty", "post_pagerduty_event"), + "post_slack_summary": ("je_load_density.utils.notifier.slack", "post_slack_summary"), + "post_teams_summary": ("je_load_density.utils.notifier.teams", "post_teams_summary"), + "postman_to_action_json": ("je_load_density.utils.recording.postman_importer", "postman_to_action_json"), + "postman_to_tasks": ("je_load_density.utils.recording.postman_importer", "postman_to_tasks"), + "prepare_env": ("je_load_density.wrapper.create_locust_env.create_locust_env", "prepare_env"), + "probe_rate_limit": ("je_load_density.utils.security.rate_limit_probe", "probe_rate_limit"), + "profile_call": ("je_load_density.utils.dx.profiler", "profile_call"), + "read_action_json": ("je_load_density.utils.json.json_file.json_file", "read_action_json"), + "read_action_toml": ("je_load_density.utils.json.json_file.toml_file", "read_action_toml"), + "read_action_yaml": ("je_load_density.utils.json.json_file.yaml_file", "read_action_yaml"), + "read_audit_log": ("je_load_density.utils.governance.audit_log", "read_audit_log"), + "refresh_token": ("je_load_density.utils.auth.oauth2", "refresh_token"), + "register_csv_source": ("je_load_density.utils.parameterization", "register_csv_source"), + "register_csv_sources": ("je_load_density.utils.parameterization", "register_csv_sources"), + "register_db_source": ("je_load_density.utils.parameterization", "register_db_source"), + "register_db_sources": ("je_load_density.utils.parameterization", "register_db_sources"), + "register_variable": ("je_load_density.utils.parameterization", "register_variable"), + "get_resolver": ("je_load_density.utils.parameterization", "get_resolver"), + "use_resolver": ("je_load_density.utils.parameterization", "use_resolver"), + "register_session_variable": ("je_load_density.utils.parameterization", "register_session_variable"), + "register_variables": ("je_load_density.utils.parameterization", "register_variables"), + "render_prompt_text": ("je_load_density.utils.ai.root_cause", "render_prompt_text"), + "render_ssrf_tasks": ("je_load_density.utils.security.ssrf_checks", "render_ssrf_tasks"), + "request_hook": ("je_load_density.wrapper.event.request_hook", "request_hook"), + "reset_all_jars": ("je_load_density.utils.scenario.cookie_jar", "reset_all_jars"), + "reset_throttles": ("je_load_density.utils.throttle.rps_throttle", "reset_throttles"), + "reset_user_jar": ("je_load_density.utils.scenario.cookie_jar", "reset_user_jar"), + "resolve": ("je_load_density.utils.parameterization", "resolve"), + "run_async_load": ("je_load_density.engine.asyncio_engine", "run_async_load"), + "run_cloud_run_job": ("je_load_density.cloud.gcp_cloud_run", "run_cloud_run_job"), + "run_owasp_checks": ("je_load_density.utils.security.owasp_checks", "run_owasp_checks"), + "run_teardown": ("je_load_density.utils.data.db_fixtures", "run_teardown"), + "run_with_retry": ("je_load_density.utils.reliability.adaptive_retry", "run_with_retry"), + "search_catalog": ("je_load_density.utils.governance.test_catalog", "search_catalog"), + "search_runs_by_tag": ("je_load_density.utils.governance.run_tagging", "search_runs_by_tag"), + "sign_aws_request": ("je_load_density.utils.auth.aws_sigv4", "sign_aws_request"), + "sign_jwt": ("je_load_density.utils.auth.jwt_signer", "sign_jwt"), + "smuggling_attack_pack": ("je_load_density.utils.security.smuggling_checks", "smuggling_attack_pack"), + "snapshot_metrics": ("je_load_density.utils.dashboard.live_dashboard", "snapshot_metrics"), + "start_dashboard": ("je_load_density.utils.dashboard.live_dashboard", "start_dashboard"), + "start_datadog_apm_exporter": ("je_load_density.utils.metrics.datadog_apm_exporter", "start_datadog_apm_exporter"), + "start_influxdb_sink": ("je_load_density.utils.metrics", "start_influxdb_sink"), + "start_leak_detector": ("je_load_density.utils.dx.leak_detector", "start_leak_detector"), + "start_load_density_socket_server": ( + "je_load_density.utils.socket_server.load_density_socket_server", + "start_load_density_socket_server", + ), + "start_opentelemetry_exporter": ("je_load_density.utils.metrics", "start_opentelemetry_exporter"), + "start_opentelemetry_tracing_exporter": ( + "je_load_density.utils.metrics.opentelemetry_tracing_exporter", + "start_opentelemetry_tracing_exporter", + ), + "start_prometheus_exporter": ("je_load_density.utils.metrics", "start_prometheus_exporter"), + "start_repl": ("je_load_density.utils.dx.repl", "start_repl"), + "start_statsd_sink": ("je_load_density.utils.metrics.statsd_sink", "start_statsd_sink"), + "start_stub_server": ("je_load_density.utils.stub_server.stub_server", "start_stub_server"), + "start_test": ("je_load_density.engine.entrypoints", "start_test"), + "stop_dashboard": ("je_load_density.utils.dashboard.live_dashboard", "stop_dashboard"), + "stop_datadog_apm_exporter": ("je_load_density.utils.metrics.datadog_apm_exporter", "stop_datadog_apm_exporter"), + "stop_influxdb_sink": ("je_load_density.utils.metrics", "stop_influxdb_sink"), + "stop_leak_detector": ("je_load_density.utils.dx.leak_detector", "stop_leak_detector"), + "stop_opentelemetry_exporter": ("je_load_density.utils.metrics", "stop_opentelemetry_exporter"), + "stop_opentelemetry_tracing_exporter": ( + "je_load_density.utils.metrics.opentelemetry_tracing_exporter", + "stop_opentelemetry_tracing_exporter", + ), + "stop_prometheus_exporter": ("je_load_density.utils.metrics", "stop_prometheus_exporter"), + "stop_statsd_sink": ("je_load_density.utils.metrics.statsd_sink", "stop_statsd_sink"), + "stop_stub_server": ("je_load_density.utils.stub_server.stub_server", "stop_stub_server"), + "summarise_records": ("je_load_density.utils.regression.diff", "summarise_records"), + "tag_run": ("je_load_density.utils.governance.run_tagging", "tag_run"), + "task": ("locust", "task"), + "test_record_instance": ("je_load_density.utils.test_record.test_record_class", "test_record_instance"), + "toxiproxy_add_toxic": ("je_load_density.utils.chaos.toxiproxy", "add_toxic"), + "toxiproxy_create_proxy": ("je_load_density.utils.chaos.toxiproxy", "create_proxy"), + "toxiproxy_install_bandwidth": ("je_load_density.utils.chaos.toxiproxy", "install_bandwidth"), + "toxiproxy_install_latency": ("je_load_density.utils.chaos.toxiproxy", "install_latency"), + "toxiproxy_list_proxies": ("je_load_density.utils.chaos.toxiproxy", "list_proxies"), + "toxiproxy_remove_proxies": ("je_load_density.utils.chaos.toxiproxy", "remove_proxies"), + "toxiproxy_remove_toxic": ("je_load_density.utils.chaos.toxiproxy", "remove_toxic"), + "toxiproxy_reset_all": ("je_load_density.utils.chaos.toxiproxy", "reset_all"), + "translate": ("je_load_density.utils.dx.i18n", "t"), + "trend_runs": ("je_load_density.utils.regression.multi_run_trend", "trend_runs"), + "uninstall_failure_budget": ("je_load_density.utils.reliability.failure_budget", "uninstall_failure_budget"), + "uninstall_network_conditioner": ( + "je_load_density.utils.reliability.network_conditioner", + "uninstall_network_conditioner", + ), + "verify_share_link": ("je_load_density.utils.governance.share_link", "verify_share_link"), + "with_watchdog": ("je_load_density.utils.reliability.process_supervisor", "with_watchdog"), + "write_action_toml": ("je_load_density.utils.json.json_file.toml_file", "write_action_toml"), + "write_action_yaml": ("je_load_density.utils.json.json_file.yaml_file", "write_action_yaml"), +} __all__ = [ - "create_env", "start_test", + "AsyncRunHandle", + "create_env", + "start_test", "locust_wrapper_proxy", "prepare_env", "test_record_instance", - "execute_action", "execute_files", "executor", "add_command_to_executor", + "execute_action", + "execute_files", + "executor", + "add_command_to_executor", "get_dir_files_as_list", - "generate_html", "generate_html_report", - "generate_json", "generate_json_report", - "generate_xml", "generate_xml_report", - "generate_csv_report", "generate_junit_report", "generate_summary_report", + "generate_html", + "generate_html_report", + "generate_json", + "generate_json_report", + "generate_xml", + "generate_xml_report", + "generate_csv_report", + "generate_junit_report", + "generate_summary_report", "generate_chart_report", "build_summary", "read_action_json", "start_load_density_socket_server", - "SequentialTaskSet", "task", "TaskSet", - "callback_executor", "create_project_dir", - "parameter_resolver", "resolve", - "register_variable", "register_variables", - "register_csv_source", "register_csv_sources", - "register_db_source", "register_db_sources", - "har_to_action_json", "har_to_tasks", "load_har", - "postman_to_action_json", "postman_to_tasks", "load_postman_collection", - "openapi_to_action_json", "openapi_to_tasks", "load_openapi", + "SequentialTaskSet", + "task", + "TaskSet", + "callback_executor", + "create_project_dir", + "parameter_resolver", + "resolve", + "register_variable", + "get_resolver", + "use_resolver", + "register_session_variable", + "register_variables", + "register_csv_source", + "register_csv_sources", + "register_db_source", + "register_db_sources", + "har_to_action_json", + "har_to_tasks", + "load_har", + "postman_to_action_json", + "postman_to_tasks", + "load_postman_collection", + "openapi_to_action_json", + "openapi_to_tasks", + "load_openapi", "curl_to_task", - "persist_records", "list_runs", "fetch_run_records", - "start_prometheus_exporter", "stop_prometheus_exporter", - "start_influxdb_sink", "stop_influxdb_sink", - "start_opentelemetry_exporter", "stop_opentelemetry_exporter", - "lint_action", "lint_action_file", - "action_json_schema", "export_schema", - "emit_github_annotations", "format_github_annotation", - "evaluate_sla", "assert_sla", - "diff_runs", "summarise_records", - "SoakShape", "SpikeShape", "StagesShape", "build_load_shape", - "graphql_to_http_task", "extract_field", - "RpsThrottle", "get_throttle", "reset_throttles", - "AdaptiveRetryPolicy", "classify_error", "run_with_retry", - "FailureBudget", "CircuitOpenError", - "install_failure_budget", "uninstall_failure_budget", + "persist_records", + "list_runs", + "fetch_run_records", + "start_prometheus_exporter", + "stop_prometheus_exporter", + "start_influxdb_sink", + "stop_influxdb_sink", + "start_opentelemetry_exporter", + "stop_opentelemetry_exporter", + "lint_action", + "lint_action_file", + "action_json_schema", + "export_schema", + "emit_github_annotations", + "format_github_annotation", + "evaluate_sla", + "assert_sla", + "diff_runs", + "summarise_records", + "SoakShape", + "SpikeShape", + "StagesShape", + "build_load_shape", + "graphql_to_http_task", + "extract_field", + "RpsThrottle", + "get_throttle", + "reset_throttles", + "AdaptiveRetryPolicy", + "classify_error", + "run_with_retry", + "FailureBudget", + "CircuitOpenError", + "install_failure_budget", + "uninstall_failure_budget", "NetworkConditioner", - "install_network_conditioner", "uninstall_network_conditioner", - "ProcessSupervisor", "with_watchdog", - "start_dashboard", "stop_dashboard", "snapshot_metrics", - "post_slack_summary", "build_slack_summary", - "post_teams_summary", "build_teams_summary", - "start_statsd_sink", "stop_statsd_sink", - "jmeter_to_action_json", "jmeter_to_tasks", "load_jmeter_jmx", - "k6_script_to_action_json", "k6_script_to_tasks", "load_k6_script", + "install_network_conditioner", + "uninstall_network_conditioner", + "ProcessSupervisor", + "with_watchdog", + "start_dashboard", + "stop_dashboard", + "snapshot_metrics", + "post_slack_summary", + "build_slack_summary", + "post_teams_summary", + "build_teams_summary", + "start_statsd_sink", + "stop_statsd_sink", + "jmeter_to_action_json", + "jmeter_to_tasks", + "load_jmeter_jmx", + "k6_script_to_action_json", + "k6_script_to_tasks", + "load_k6_script", "OAuth2Client", - "fetch_client_credentials_token", "fetch_password_token", "refresh_token", - "sign_jwt", "decode_jwt", "sign_aws_request", + "fetch_client_credentials_token", + "fetch_password_token", + "refresh_token", + "sign_jwt", + "decode_jwt", + "sign_aws_request", # New report formats - "generate_histogram_report", "generate_pdf_report", "generate_allure_report", + "generate_histogram_report", + "generate_pdf_report", + "generate_allure_report", # YAML / TOML loaders - "read_action_yaml", "write_action_yaml", - "read_action_toml", "write_action_toml", + "read_action_yaml", + "write_action_yaml", + "read_action_toml", + "write_action_toml", # Tracing & APM exporters - "start_opentelemetry_tracing_exporter", "stop_opentelemetry_tracing_exporter", - "start_datadog_apm_exporter", "stop_datadog_apm_exporter", + "start_opentelemetry_tracing_exporter", + "stop_opentelemetry_tracing_exporter", + "start_datadog_apm_exporter", + "stop_datadog_apm_exporter", # Notifier - "post_pagerduty_event", "build_pagerduty_event", - "post_opsgenie_alert", "build_opsgenie_alert", - "post_gitlab_mr_summary", "build_gitlab_mr_note", + "post_pagerduty_event", + "build_pagerduty_event", + "post_opsgenie_alert", + "build_opsgenie_alert", + "post_gitlab_mr_summary", + "build_gitlab_mr_note", # Trend & error clustering - "trend_runs", "cluster_errors", + "trend_runs", + "cluster_errors", # DX - "format_action_document", "format_action_file", "format_action_string", - "generate_from_openapi", "generate_from_curls", "merge_actions", + "format_action_document", + "format_action_file", + "format_action_string", + "generate_from_openapi", + "generate_from_curls", + "merge_actions", # Scenario - "FsmRunner", "jar_for_user", "reset_user_jar", "reset_all_jars", + "FsmRunner", + "jar_for_user", + "reset_user_jar", + "reset_all_jars", # Stub server - "start_stub_server", "stop_stub_server", + "start_stub_server", + "stop_stub_server", # Security - "mutate_string", "mutate_json", "fuzz_query_string", "expand_task_fuzz", + "mutate_string", + "mutate_json", + "fuzz_query_string", + "expand_task_fuzz", "run_owasp_checks", - "check_broken_object_level_auth", "check_excessive_data_exposure", - "check_security_headers", "check_sensitive_token_leak", + "check_broken_object_level_auth", + "check_excessive_data_exposure", + "check_security_headers", + "check_sensitive_token_leak", # Chaos - "toxiproxy_create_proxy", "toxiproxy_list_proxies", - "toxiproxy_add_toxic", "toxiproxy_remove_toxic", - "toxiproxy_install_latency", "toxiproxy_install_bandwidth", - "toxiproxy_reset_all", "toxiproxy_remove_proxies", - "chaos_apply_manifest", "chaos_delete_manifest", "chaos_build_network_delay", + "toxiproxy_create_proxy", + "toxiproxy_list_proxies", + "toxiproxy_add_toxic", + "toxiproxy_remove_toxic", + "toxiproxy_install_latency", + "toxiproxy_install_bandwidth", + "toxiproxy_reset_all", + "toxiproxy_remove_proxies", + "chaos_apply_manifest", + "chaos_delete_manifest", + "chaos_build_network_delay", # Asyncio engine "run_async_load", # AI - "AutoTuner", "build_root_cause_prompt", "render_prompt_text", - "find_breaking_point", "calibrate_sla", + "AutoTuner", + "build_root_cause_prompt", + "render_prompt_text", + "find_breaking_point", + "calibrate_sla", # Cloud workers - "invoke_lambda_workers", "lambda_worker_handler", - "launch_fargate_workers", "launch_aci_workers", "run_cloud_run_job", + "invoke_lambda_workers", + "lambda_worker_handler", + "launch_fargate_workers", + "launch_aci_workers", + "run_cloud_run_job", # CI / canary "canary_verdict", # Data / state - "apply_fixture", "run_teardown", - "build_user", "build_user_pool", - "pii_scrub", "pii_scrub_string", "find_pii", + "apply_fixture", + "run_teardown", + "build_user", + "build_user_pool", + "pii_scrub", + "pii_scrub_string", + "find_pii", # DX - "start_repl", "profile_call", "memory_snapshot", - "start_leak_detector", "stop_leak_detector", "detect_growing_allocations", - "translate", "available_locales", "get_current_locale", + "start_repl", + "profile_call", + "memory_snapshot", + "start_leak_detector", + "stop_leak_detector", + "detect_growing_allocations", + "translate", + "available_locales", + "get_current_locale", # Governance - "append_audit_entry", "read_audit_log", - "tag_run", "list_tags", "search_runs_by_tag", - "issue_share_link", "verify_share_link", - "index_catalog", "search_catalog", + "append_audit_entry", + "read_audit_log", + "tag_run", + "list_tags", + "search_runs_by_tag", + "issue_share_link", + "verify_share_link", + "index_catalog", + "search_catalog", # Reports (additional) - "generate_excel_report", "generate_sarif_report", "generate_cyclonedx_report", - "generate_service_map", "build_service_map", - "generate_cost_report", "estimate_run_cost", + "generate_excel_report", + "generate_sarif_report", + "generate_cyclonedx_report", + "generate_service_map", + "build_service_map", + "generate_cost_report", + "estimate_run_cost", # Recording (additional) - "cdp_discover_targets", "capture_cdp_session", "capture_cdp_to_har", + "cdp_discover_targets", + "capture_cdp_session", + "capture_cdp_to_har", # Security: extended - "craft_alg_none_token", "craft_alg_confusion_token", - "craft_expired_token", "craft_kid_traversal_token", + "craft_alg_none_token", + "craft_alg_confusion_token", + "craft_expired_token", + "craft_kid_traversal_token", "craft_jwt_attack_pack", - "build_introspection_payload", "build_depth_attack", - "build_alias_batching_attack", "graphql_attack_pack", - "build_ssrf_targets", "render_ssrf_tasks", "find_metadata_leak", - "build_cl_te", "build_te_cl", "build_te_te", "smuggling_attack_pack", + "build_introspection_payload", + "build_depth_attack", + "build_alias_batching_attack", + "graphql_attack_pack", + "build_ssrf_targets", + "render_ssrf_tasks", + "find_metadata_leak", + "build_cl_te", + "build_te_cl", + "build_te_te", + "smuggling_attack_pack", "probe_rate_limit", ] + + +def __getattr__(name: str): + target = _EXPORTS.get(name) + if target is None: + raise AttributeError(f"module {__name__!r} has no attribute {name!r}") + module, attribute = target + value = getattr(importlib.import_module(module), attribute) + if module == "locust": + importlib.import_module("je_load_density.wrapper.event.request_hook") + globals()[name] = value + return value + + +def __dir__() -> list[str]: + return sorted(set(globals()) | set(_EXPORTS)) diff --git a/je_load_density/__main__.py b/je_load_density/__main__.py index 0d2f022..b6fca28 100644 --- a/je_load_density/__main__.py +++ b/je_load_density/__main__.py @@ -1,25 +1,51 @@ import argparse import json import sys +from dataclasses import dataclass, replace from typing import List, Optional +from je_action_core.reporting import ExecutionReporter + from je_load_density.utils.exception.exception_tags import argparse_get_wrong_data from je_load_density.utils.exception.exceptions import LoadDensityTestExecuteException from je_load_density.utils.executor.action_executor import executor from je_load_density.utils.file_process.get_dir_file_list import get_dir_files_as_list from je_load_density.utils.json.json_file.json_file import read_action_json from je_load_density.utils.project.create_project_structure import create_project_dir -from je_load_density.utils.socket_server.load_density_socket_server import ( - start_load_density_socket_server, -) + + +def start_load_density_socket_server(*args, **kwargs): + """Select the control server's scheduler only when the serve command is used.""" + from je_load_density.utils.socket_server.load_density_socket_server import ( + start_load_density_socket_server as start_server, + ) + return start_server(*args, **kwargs) + + +@dataclass +class _CliReporter: + delegate: ExecutionReporter + failed: int = 0 + + def __getattr__(self, name: str): + return getattr(self.delegate, name) + + def on_failure(self, action, error: Exception) -> None: + self.failed += 1 + self.delegate.on_failure(action, error) def _execute_cli_actions(actions: object) -> None: """Keep batch execution/reporting, then fail the CLI if an action raised.""" - records, failed = executor.collect_action_results(actions) - executor.settings.reporter.on_records(records) - if failed: - raise LoadDensityTestExecuteException(f"{len(failed)} action(s) failed") + settings = executor.settings + reporter = _CliReporter(settings.reporter) + executor.settings = replace(settings, reporter=reporter) + try: + executor.execute_action(actions) + finally: + executor.settings = settings + if reporter.failed: + raise LoadDensityTestExecuteException(f"{reporter.failed} action(s) failed") def _execute_cli_files(directory: str) -> None: diff --git a/je_load_density/engine/async_config.py b/je_load_density/engine/async_config.py new file mode 100644 index 0000000..ab657c5 --- /dev/null +++ b/je_load_density/engine/async_config.py @@ -0,0 +1,249 @@ +"""Capability validation for the native HTTP engine, performed before I/O.""" + +import math +import ssl +from collections.abc import Mapping +from typing import Any + +from je_load_density.utils.load_shapes.shapes import _soak_tick, _spike_tick, _stages_tick +from je_load_density.wrapper.user_template.scenario_runner import _coerce_tasks_payload + +_TASK_FIELDS = frozenset( + { + "method", + "request_url", + "url", + "name", + "params", + "headers", + "cookies", + "json", + "data", + "timeout", + "allow_redirects", + "verify", + "files", + "cert", + "client_cert", + "auth", + "proxy", + "assertions", + "extract", + "weight", + "run_if", + "skip_if", + "retry", + "think_time", + "throttle", + "step_id", + "scenario_id", + } +) +_ASSERTIONS = frozenset({"status_code", "contains", "not_contains", "json_path", "header"}) +_EXTRACTORS = frozenset({"json_path", "header", "status_code"}) +_METHODS = frozenset({"get", "post", "put", "patch", "delete", "head", "options", "trace", "connect"}) +_RETRY_FIELDS = { + "transient": 5, + "flaky": 2, + "permanent": 0, + "base_delay": 0.1, + "max_delay": 5, + "backoff_factor": 2, + "jitter": 0.25, +} + + +def _condition(expression: Any) -> None: + if expression is None: + return + if isinstance(expression, (str, bool, int)): + return + if not isinstance(expression, dict) or len(expression) != 1: + raise ValueError("Conditions require one supported operator") + operator, arguments = next(iter(expression.items())) + if operator == "truthy": + return + if operator not in {"equals", "not_equals", "in"}: + raise ValueError("Unsupported condition operator") + if not isinstance(arguments, list) or len(arguments) != 2: + raise ValueError("Condition operator requires two arguments") + + +def _request_types(task: dict) -> None: + accepted = { + "headers": (Mapping, list, tuple), + "params": (Mapping, list, tuple, str, bytes), + "cookies": (Mapping,), + "data": (Mapping, str, bytes), + "files": (Mapping, list, tuple), + "allow_redirects": (bool,), + "verify": (bool, str, ssl.SSLContext), + "proxy": (str,), + "cert": (str, list, tuple), + "client_cert": (str, list, tuple), + "name": (str,), + } + for key, types in accepted.items(): + value = task.get(key) + if value is not None and not isinstance(value, types): + raise ValueError(f"Invalid native request field {key}") + for key in ("run_if", "skip_if"): + if key in task: + _condition(task[key]) + if isinstance(task.get("verify"), ssl.SSLContext) and ("cert" in task or "client_cert" in task): + raise ValueError("Configure the certificate on your SSLContext and omit cert/client_cert") + + +def number(value: Any, name: str, *, integer: bool = False, minimum: float = 0) -> float: + if isinstance(value, bool) or not isinstance(value, (int, float)): + raise ValueError(f"{name} must be a finite number") + if not math.isfinite(value) or value < minimum: + raise ValueError(f"{name} must be finite and >= {minimum}") + if integer and int(value) != value: + raise ValueError(f"{name} must be an integer") + return value + + +def _validate_rules(task: dict) -> None: + for rule in task.get("assertions") or []: + if not isinstance(rule, dict) or rule.get("type") not in _ASSERTIONS: + raise ValueError("Unsupported native assertion") + _rule_fields(rule, {"type", "value", "path", "name"}) + if "value" not in rule: + raise ValueError("Assertions require value") + if rule["type"] == "status_code": + number(rule.get("value"), "assertion status_code", integer=True, minimum=100) + for rule in task.get("extract") or []: + if not isinstance(rule, dict) or rule.get("from", "json_path") not in _EXTRACTORS: + raise ValueError("Unsupported native extractor") + _rule_fields(rule, {"from", "var", "scope", "path", "name"}) + if not rule.get("var") or rule.get("scope", "var") not in {"var", "session"}: + raise ValueError("Extractors require var and scope var/session") + + +def _rule_fields(rule: dict, fields: set) -> None: + if set(rule) - fields: + raise ValueError("Unsupported assertion/extractor configuration") + for key in ("path", "name", "var"): + if key in rule and not isinstance(rule[key], str): + raise ValueError(f"Rule {key} must be a string") + + +def _validate_timing(task: dict) -> None: + retry = task.get("retry") + if retry is not None: + if not isinstance(retry, dict) or set(retry) - _RETRY_FIELDS.keys(): + raise ValueError("Unsupported retry configuration") + for key, value in retry.items(): + number(value, f"retry.{key}", integer=key in {"transient", "flaky", "permanent"}) + think = task.get("think_time", 0) + if isinstance(think, dict): + if set(think) - {"min", "max"}: + raise ValueError("Unsupported think_time configuration") + number(think.get("min", 0), "think_time.min") + number(think.get("max", think.get("min", 0)), "think_time.max") + else: + number(think, "think_time") + throttle = task.get("throttle") + if throttle is not None: + if not isinstance(throttle, dict) or set(throttle) - {"key", "rps", "burst"}: + raise ValueError("Unsupported throttle configuration") + number(throttle.get("rps"), "throttle.rps", minimum=0.000001) + if "burst" in throttle: + number(throttle["burst"], "throttle.burst", integer=True, minimum=1) + + +def validate_tasks(raw_tasks: Any) -> dict: + if not isinstance(raw_tasks, (list, dict)): + raise ValueError("tasks must be a list or scenario mapping") + if isinstance(raw_tasks, dict) and "tasks" in raw_tasks: + if set(raw_tasks) - {"mode", "tasks"}: + raise ValueError("Unsupported scenario configuration") + _validate_task_items(raw_tasks["tasks"]) + else: + _validate_task_items(raw_tasks) + payload = _coerce_tasks_payload(raw_tasks) + if payload["mode"] not in {"sequence", "weighted", "conditional"}: + raise ValueError("Native scenarios support sequence, weighted and conditional") + if not payload["tasks"]: + raise ValueError("At least one HTTP task is required") + for task in payload["tasks"]: + _validate_task(task) + return payload + + +def _validate_task_items(raw: Any) -> None: + if isinstance(raw, list) and raw and all(isinstance(item, dict) for item in raw): + return + if isinstance(raw, dict) and raw and all(isinstance(item, dict) for item in raw.values()): + return + raise ValueError("Every task must be a mapping") + + +def _validate_task(task: dict) -> None: + unknown = set(task) - _TASK_FIELDS + if unknown: + raise ValueError(f"Unsupported native task fields: {sorted(unknown)}") + if str(task.get("method", "get")).lower() not in _METHODS: + raise ValueError("Unsupported HTTP method") + url = task.get("request_url") or task.get("url") + if not isinstance(url, str) or not url: + raise ValueError("HTTP task requires request_url") + if "${" not in url and not url.startswith(("http://", "https://", "/")): + raise ValueError("Only HTTP(S) URLs are supported") + if "timeout" in task and "${" not in str(task["timeout"]): + number(task["timeout"], "timeout", minimum=0.000001) + auth = task.get("auth") + if auth is not None and (not isinstance(auth, dict) or auth.get("type") not in {"basic", "bearer"}): + raise ValueError("Native auth supports basic and bearer") + if auth is not None: + names = {"type", "username", "password"} if auth["type"] == "basic" else {"type", "token"} + if set(auth) - names: + raise ValueError("Unsupported auth configuration") + _request_types(task) + number(task.get("weight", 1), "weight", integer=True) + _validate_rules(task) + _validate_timing(task) + + +def validate_shape(name: str | None, config: dict | None) -> None: + if name is None: + return + config = config or {} + if not isinstance(config, dict): + raise ValueError("Shape configuration must be a mapping") + if name not in {"stages", "spike", "soak"}: + raise ValueError("Unsupported native load shape") + fields = { + "stages": {"stages"}, + "spike": {"baseline_users", "spike_users", "spawn_rate", "pre_seconds", "spike_seconds", "post_seconds"}, + "soak": {"users", "spawn_rate", "ramp_seconds", "hold_seconds", "cooldown_seconds"}, + } + if set(config) - fields[name]: + raise ValueError("Unsupported shape configuration") + entries = config.get("stages", []) if name == "stages" else [config] + if not entries: + raise ValueError("Stages shape requires stages") + for stage in entries: + _validate_stage(stage, name) + + +def _validate_stage(stage: Any, name: str) -> None: + if not isinstance(stage, dict): + raise ValueError("Shape stages must be mappings") + if name == "stages" and set(stage) - {"duration", "users", "spawn_rate"}: + raise ValueError("Unsupported stage configuration") + for key, value in stage.items(): + number(value, f"shape.{key}", integer=key.endswith("users") or key == "spawn_rate") + if stage.get("spawn_rate", 1) <= 0: + raise ValueError("Shape spawn_rate must be positive") + + +def shape_target(name: str | None, config: dict, elapsed: float, users: int, spawn_rate: float): + if name == "stages": + return _stages_tick(config["stages"], elapsed) + if name == "spike": + return _spike_tick(config, elapsed) + if name == "soak": + return _soak_tick(config, elapsed) + return users, spawn_rate diff --git a/je_load_density/engine/async_http.py b/je_load_density/engine/async_http.py new file mode 100644 index 0000000..aa7e1f9 --- /dev/null +++ b/je_load_density/engine/async_http.py @@ -0,0 +1,189 @@ +"""User-owned native HTTP clients, measured attempts and asynchronous pacing.""" + +import asyncio +import time +from contextlib import AsyncExitStack +from dataclasses import dataclass +from urllib.parse import urljoin + +from je_load_density.utils.parameterization import parameter_resolver +from je_load_density.utils.reliability.adaptive_retry import AdaptiveRetryPolicy, classify_error +from je_load_density.utils.test_record.contract import record_request +from je_load_density.utils.test_record.test_record_class import test_record_instance +from je_load_density.wrapper.user_template.request_executor import ( + _apply_extractors, + _build_kwargs, + _check_assertions, +) +from je_load_density.wrapper.user_template.scenario_runner import _think_time_seconds + + +@dataclass +class Attempt: + entry: dict + error: Exception | None + + +class AsyncThrottle: + """Run-local token bucket; waiting never blocks the asyncio scheduler.""" + + def __init__(self, rps: float, burst: int) -> None: + self.rps = rps + self.burst = burst + self.tokens = float(burst) + self.updated = time.monotonic() + self.lock = asyncio.Lock() + + async def acquire(self) -> None: + while True: + async with self.lock: + now = time.monotonic() + self.tokens = min(self.burst, self.tokens + (now - self.updated) * self.rps) + self.updated = now + if self.tokens >= 1: + self.tokens -= 1 + return + delay = (1 - self.tokens) / self.rps + await asyncio.sleep(delay) + + +class UserClientPool: + """Reusable transport configurations belong to one virtual user's cookie jar.""" + + def __init__(self, httpx, http2: bool, verify_context=None) -> None: + self.httpx = httpx + self.http2 = http2 + self.verify_context = verify_context + self.clients = {} + self.cookies = httpx.Cookies() + self.stack = AsyncExitStack() + + async def __aenter__(self): + await self.stack.__aenter__() + return self + + async def __aexit__(self, *args): + return await self.stack.__aexit__(*args) + + async def client(self, task: dict): + options = {"verify": task.get("verify", True), "http2": self.http2} + cert = task.get("cert", task.get("client_cert")) + if options["verify"] is True and self.verify_context is not None and cert is None: + options["verify"] = self.verify_context + if cert is not None: + options["cert"] = cert + if task.get("proxy") is not None: + options["proxy"] = task["proxy"] + key = tuple((name, repr(value)) for name, value in options.items()) + if key not in self.clients: + self.clients[key] = await self.stack.enter_async_context(self.httpx.AsyncClient(**options)) + client = self.clients[key] + client.cookies.update(self.cookies) + return client + + async def send(self, task: dict, base_url: str): + client = await self.client(task) + kwargs = _build_kwargs(task) + for key in ("name", "verify", "cert"): + kwargs.pop(key, None) + kwargs["follow_redirects"] = kwargs.pop("allow_redirects", True) + kwargs.setdefault("timeout", 10.0) + url = task.get("request_url") or task.get("url") + url = urljoin(base_url, url) + response = await client.request(str(task.get("method", "get")).upper(), url, **kwargs) + # Share the actual CookieJar, including expiry/deletion, between this user's clients. + self.cookies = client.cookies + for other in self.clients.values(): + other.cookies.jar = self.cookies.jar + return response + + +def _response_result(response, task: dict) -> tuple[Exception | None, list]: + results = [] + error = None + for assertion in task.get("assertions") or []: + passed, reason = _check_assertions(response, [assertion]) + results.append({"type": assertion["type"], "passed": passed}) + if not passed and error is None: + error = AssertionError(reason) + expected_status = any(rule["type"] == "status_code" for rule in task.get("assertions") or []) + if error is None and response.status_code >= 400 and not expected_status: + error = RuntimeError(f"HTTP {response.status_code}") + if error is None: + _apply_extractors(response, task.get("extract") or []) + return error, results + + +async def measure(pool: UserClientPool, task: dict, base_url: str) -> Attempt: + started, clock = time.time(), time.monotonic() + url = urljoin(base_url, task.get("request_url") or task.get("url")) + entry = { + "Method": str(task.get("method", "get")).upper(), + "test_url": url, + "name": task.get("name") or url, + "start_time": started, + "status_code": "0", + "response_length": 0, + } + try: + response = await pool.send(task, base_url) + except pool.httpx.HTTPError as error: + entry["error_kind"] = "timeout" if isinstance(error, pool.httpx.TimeoutException) else "connection" + outcome_error = error + else: + entry.update(status_code=str(response.status_code), response_length=len(response.content)) + outcome_error, assertions = _response_result(response, task) + entry["assertions"] = assertions + if outcome_error is not None: + entry["error_kind"] = "assertion" if isinstance(outcome_error, AssertionError) else "http_status" + entry["response_time_ms"] = (time.monotonic() - clock) * 1000 + entry["error"] = str(outcome_error) if outcome_error is not None else None + for field in ("step_id", "scenario_id"): + if task.get(field) is not None: + entry[field] = task[field] + return Attempt(entry, outcome_error) + + +def record_attempt(attempt: Attempt, run) -> None: + failed = attempt.error is not None + legacy = test_record_instance.error_record_list if failed else test_record_instance.test_record_list + legacy.append(attempt.entry) + record_request(attempt.entry, "failed" if failed else "passed") + (run.failures if failed else run.successes).append(attempt.entry) + + +def retry_policy(config: dict) -> AdaptiveRetryPolicy: + names = {"transient": "transient_budget", "flaky": "flaky_budget", "permanent": "permanent_budget"} + return AdaptiveRetryPolicy( + classifier=_classify_native_error, **{names.get(key, key): value for key, value in config.items()} + ) + + +def _classify_native_error(error: BaseException) -> str: + import httpx + + if isinstance(error, (httpx.NetworkError, httpx.TimeoutException, httpx.RemoteProtocolError)): + return "transient" + return classify_error(error) + + +async def execute_step(pool: UserClientPool, raw_task: dict, run) -> None: + # Resolve once, keeping CSV/DB rows and idempotency values stable across retries. + task = parameter_resolver.resolve(raw_task) + throttle = task.get("throttle") + policy = retry_policy(task["retry"]) if "retry" in task else None + attempt_number = 0 + while True: + if throttle: + await run.throttle(throttle).acquire() + async with run.semaphore: + attempt = await measure(pool, task, run.base_url) + record_attempt(attempt, run) + attempt_number += 1 + if attempt.error is None or policy is None: + break + decision = policy.decide(attempt.error, attempt_number) + if not decision.will_retry: + break + await asyncio.sleep(decision.delay_seconds) + await asyncio.sleep(_think_time_seconds(task.get("think_time"))) diff --git a/je_load_density/engine/asyncio_engine.py b/je_load_density/engine/asyncio_engine.py index 6cc429e..4d5121d 100644 --- a/je_load_density/engine/asyncio_engine.py +++ b/je_load_density/engine/asyncio_engine.py @@ -1,119 +1,277 @@ -""" -Native asyncio load engine. - -Independent of Locust / gevent — uses stdlib ``asyncio`` and ``httpx`` -(lazy) to drive HTTP load. Writes results into the same -``test_record_instance`` so reports, SLA gates, and persistence behave -identically. - -Usage:: - - import asyncio - from je_load_density.engine.asyncio_engine import run_async_load - - asyncio.run(run_async_load( - tasks=[{"method": "get", "request_url": "https://x"}], - users=50, duration_seconds=30, - )) -""" +"""Native asyncio HTTP runs with owned clients, tasks, records and cancellation.""" import asyncio +import ssl import time -from contextlib import nullcontext +from contextlib import AsyncExitStack, nullcontext from typing import Any, Dict, List, Optional -from je_load_density.utils.logging.loggin_instance import load_density_logger -from je_load_density.utils.parameterization import parameter_resolver -from je_load_density.utils.test_record.contract import record_request -from je_load_density.utils.test_record.test_record_class import test_record_instance +from je_load_density.engine.async_config import number, shape_target, validate_shape, validate_tasks +from je_load_density.engine.async_http import AsyncThrottle, UserClientPool, execute_step +from je_load_density.utils.generate_report.generate_summary_report import build_summary +from je_load_density.utils.parameterization import ( + get_resolver, + register_csv_sources, + register_db_sources, + register_variables, + use_resolver, +) +from je_load_density.wrapper.user_template.scenario_runner import _condition_passes, _pick_weighted + +_OPTIONS = frozenset( + { + "spawn_rate", + "load_shape", + "shape_config", + "runner_mode", + "base_url", + "stop_requested", + "on_environment", + "variables", + "csv_sources", + "db_sources", + } +) def _import_httpx(): try: import httpx except ImportError as error: - raise RuntimeError( - "httpx is required for the asyncio engine; " - "install with: pip install 'httpx[http2]'" - ) from error + raise RuntimeError("httpx is required for the asyncio engine") from error return httpx -def _record(method: str, url: str, status: int, elapsed_ms: float, length: int, started: float) -> bool: - """Record one response. A 4xx/5xx is a failure, as Locust's HTTP users count it, so SLA gates - and failure-rate reports mean the same thing whichever engine produced the records.""" - entry = { - "Method": method.upper(), - "test_url": url, - "name": url, - "status_code": str(status), - "response_time_ms": elapsed_ms, - "response_length": length, - "start_time": started, - } - if status >= 400: - entry["error"] = f"HTTP {status}" - test_record_instance.error_record_list.append(entry) - record_request({**entry, "error_kind": "http_status"}, "failed") - return False - entry["error"] = None - test_record_instance.test_record_list.append(entry) - record_request(entry, "passed") - return True - - -def _record_error(method: str, url: str, error: Exception, started: float, elapsed_ms: float) -> bool: - entry = { - "Method": method.upper(), - "test_url": url, - "name": url, - "status_code": "0", - "response_time_ms": elapsed_ms, - "response_length": 0, - "error": repr(error), - "start_time": started, - } - test_record_instance.error_record_list.append(entry) - kind = "timeout" if "timeout" in type(error).__name__.lower() else "connection" - record_request({**entry, "error_kind": kind}, "failed") - return False - - -async def _send_one(client, raw_task: Dict[str, Any]) -> bool: - task = parameter_resolver.resolve(raw_task) - method = str(task.get("method", "get")).lower() - url = task["request_url"] - headers = task.get("headers") or None - body = task.get("json") - started = time.time() - start = time.monotonic() - try: - response = await client.request( - method.upper(), url, headers=headers, json=body, - timeout=float(task.get("timeout", 10.0)), +class AsyncRunHandle: + """One run's awaitable lifecycle; stop cooperatively cancels owned I/O.""" + + def __init__(self, tasks, users=10, duration_seconds=10.0, **options) -> None: + self.payload = validate_tasks(tasks) + self.users = int(number(users, "users", integer=True, minimum=1)) + self.duration = number(duration_seconds, "duration_seconds", minimum=0.000001) + self.http2 = options.pop("http2", False) + if not isinstance(self.http2, bool): + raise ValueError("http2 must be boolean") + self.run_context = options.pop("run_context", None) + limit = options.pop("max_in_flight", None) + self._configure(options) + self.limit = ( + self._peak_users() if limit is None else int(number(limit, "max_in_flight", integer=True, minimum=1)) ) - elapsed_ms = (time.monotonic() - start) * 1000 - return _record(method, url, response.status_code, elapsed_ms, len(response.content), started) - except Exception as error: # noqa: BLE001 - return _record_error(method, url, error, started, (time.monotonic() - start) * 1000) + self.successes, self.failures = [], [] + self.workers = [] + self.state = "created" + self.started = None + self.finished = None + self._task = None + self._loop = None + self._stop = asyncio.Event() + self._throttles = {} + self.resolver = get_resolver().fork() + self._warm_pool = None + def _configure(self, options: dict) -> None: + unknown = set(options) - _OPTIONS + if unknown: + raise ValueError(f"Unsupported native run options: {sorted(unknown)}") + if options.get("runner_mode", "local") != "local": + raise ValueError("Native asyncio currently supports local HTTP runs only") + self.options = options + self.spawn_rate = number(options.get("spawn_rate", self.users), "spawn_rate", minimum=0.000001) + self.shape = options.get("load_shape") + self.shape_config = options.get("shape_config") or {} + validate_shape(self.shape, self.shape_config) + self.base_url = options.get("base_url", "") + if not isinstance(self.base_url, str): + raise ValueError("base_url must be a string") + if not self.base_url and any( + (task.get("request_url") or task.get("url")).startswith("/") for task in self.payload["tasks"] + ): + raise ValueError("Relative HTTP tasks require base_url") + for name in ("stop_requested", "on_environment"): + if options.get(name) is not None and not callable(options[name]): + raise ValueError(f"{name} must be callable") -async def _worker( - client, - tasks: List[Dict[str, Any]], - deadline: float, - semaphore: asyncio.Semaphore, -) -> tuple[int, int]: - successes = failures = 0 - while time.monotonic() < deadline: - for task in tasks: - if time.monotonic() >= deadline: - return successes, failures - async with semaphore: - passed = await _send_one(client, task) - successes += int(passed) - failures += int(not passed) - return successes, failures + def _peak_users(self) -> int: + if self.shape == "stages": + return max(self.users, *(int(stage.get("users", 0)) for stage in self.shape_config["stages"])) + return max( + self.users, + int(self.shape_config.get("users", 0)), + int(self.shape_config.get("baseline_users", 0)), + int(self.shape_config.get("spike_users", 0)), + ) + + async def start(self): + if self._task is not None: + raise RuntimeError("A run handle can only be started once") + self._loop = asyncio.get_running_loop() + self.semaphore = asyncio.Semaphore(self.limit) + self._task = asyncio.create_task(self._execute()) + return self + + def stop(self) -> None: + if self._loop is None: + self._stop.set() + else: + self._loop.call_soon_threadsafe(self._stop.set) + + async def wait(self) -> dict: + if self._task is None: + raise RuntimeError("Start the run before waiting") + return await self._task + + def snapshot(self) -> dict: + elapsed = 0 if self.started is None else (self.finished or time.monotonic()) - self.started + total = len(self.successes) + len(self.failures) + return { + "engine": "asyncio", + "state": self.state, + "users": sum(not task.done() for task in self.workers), + "target": self.base_url, + "elapsed_seconds": elapsed, + "rps": total / max(elapsed, 0.000001), + "requests": len(self.successes), + "failures": len(self.failures), + "summary": build_summary(self.successes, self.failures), + } + + def throttle(self, config: dict) -> AsyncThrottle: + key = str(config.get("key", "default")) + settings = (float(config["rps"]), int(config.get("burst", 1))) + if key not in self._throttles: + self._throttles[key] = (settings, AsyncThrottle(*settings)) + previous, throttle = self._throttles[key] + if settings != previous: + raise ValueError(f"Throttle {key!r} has conflicting settings") + return throttle + + def _prepare_sources(self) -> None: + register_variables(self.options.get("variables") or {}) + register_csv_sources(self.options.get("csv_sources") or []) + register_db_sources(self.options.get("db_sources") or []) + + async def _prepare_clients(self) -> None: + """Reject static TLS/proxy setup failures before starting target traffic.""" + await self._warm_pool.client({}) + for task in self.payload["tasks"]: + # Validate independent static options even alongside user-dependent values. + transport = {key: task[key] for key in ("verify", "cert", "client_cert", "proxy") + if key in task and "${" not in repr(task[key])} + await self._warm_pool.client(transport) + + async def _execute(self) -> dict: + primary_failure = False + resources = AsyncExitStack() + scope = nullcontext() + if self.run_context is not None: + from je_load_density.utils.test_record.run_context import use_run_context + + scope = use_run_context(self.run_context) + try: + with scope, use_resolver(self.resolver): + self._prepare_sources() + httpx = _import_httpx() + # Transport initialization precedes the load clock; users share TLS trust, + # but keep separate clients, sockets and cookies. + self.verify_context = ssl.create_default_context() + self._warm_pool = await resources.enter_async_context( + UserClientPool(httpx, self.http2, self.verify_context) + ) + await self._prepare_clients() + self.started = time.monotonic() + self.state = "running" + callback = self.options.get("on_environment") + if callback is not None: + callback(self) + await self._schedule(httpx) + self.state = "cancelled" if self._stop.is_set() else "completed" + except asyncio.CancelledError: + primary_failure = True + self.state = "cancelled" + raise + except Exception: + primary_failure = True + self.state = "failed" + raise + finally: + results = await self._cancel_workers() + try: + if not primary_failure: + self._raise_worker_errors(results) + finally: + await resources.aclose() + self.finished = time.monotonic() + return self.snapshot() + + async def _cancel_workers(self) -> list: + for task in self.workers: + if not task.done(): + task.cancel() + return await asyncio.gather(*self.workers, return_exceptions=True) + + def _raise_worker_errors(self, results: list) -> None: + for result in results: + if isinstance(result, Exception): + self.state = "failed" + raise result + + def _check_workers(self) -> None: + for task in self.workers: + if task.done() and not task.cancelled(): + task.result() + self.workers = [task for task in self.workers if not task.done()] + + async def _schedule(self, httpx) -> None: + credits, previous = 1.0, self.started + while not self._stop.is_set(): + self._check_workers() + now = time.monotonic() + elapsed = now - self.started + if elapsed >= self.duration: + return + callback = self.options.get("stop_requested") + if callback is not None and callback(): + self._stop.set() + return + target = shape_target(self.shape, self.shape_config, elapsed, self.users, self.spawn_rate) + if target is None: + return + count, rate = target + credits = min(max(count, 1), credits + (now - previous) * rate) + previous = now + await self._resize(httpx, count, int(credits)) + credits -= self._spawned + await asyncio.sleep(min(0.02, max(0, self.duration - elapsed))) + + async def _resize(self, httpx, count: int, available: int) -> None: + excess = self.workers[count:] + for task in excess: + task.cancel() + if excess: + results = await asyncio.gather(*excess, return_exceptions=True) + self._raise_worker_errors(results) + self.workers = self.workers[:count] + self._spawned = min(max(count - len(self.workers), 0), available) + for _ in range(self._spawned): + self.workers.append(asyncio.create_task(self._user(httpx, self.resolver.fork()))) + + async def _user(self, httpx, resolver) -> None: + pool = self._warm_pool + self._warm_pool = None + if pool is None: + pool = UserClientPool(httpx, self.http2, self.verify_context) + with use_resolver(resolver): + async with pool: + while not self._stop.is_set(): + tasks = self.payload["tasks"] + if self.payload["mode"] == "weighted": + chosen = _pick_weighted(tasks) + tasks = [] if chosen is None else [chosen] + for task in tasks: + if _condition_passes(task): + await execute_step(pool, task, self) + await asyncio.sleep(0.001) async def run_async_load( @@ -123,34 +281,11 @@ async def run_async_load( http2: bool = False, max_in_flight: Optional[int] = None, run_context=None, + **options, ) -> Dict[str, Any]: - """Run a pure-asyncio HTTP load test. Returns a summary dict.""" - httpx = _import_httpx() - semaphore = asyncio.Semaphore(max_in_flight or max(users, 1)) - scope = nullcontext() - if run_context is not None: - from je_load_density.utils.test_record.run_context import use_run_context - scope = use_run_context(run_context) - with scope: - results = await _run_workers(httpx, tasks, users, duration_seconds, http2, semaphore) - summary = {"requests": sum(result[0] for result in results), "failures": sum(result[1] for result in results)} - load_density_logger.info(f"asyncio engine done — {summary['requests']} ok, {summary['failures']} err") - return summary - - -async def _run_workers(httpx, tasks, users, duration_seconds, http2, semaphore): - async with httpx.AsyncClient(http2=http2) as client: - # Start the clock once the client exists, so duration_seconds is time under load and a - # slow client setup cannot eat a short run's whole budget. - deadline = time.monotonic() + duration_seconds - workers = [ - asyncio.create_task(_worker(client, tasks, deadline, semaphore)) - for _ in range(users) - ] - try: - return await asyncio.gather(*workers) - finally: - for worker in workers: - if not worker.done(): - worker.cancel() - await asyncio.gather(*workers, return_exceptions=True) + """Await HTTP load; legacy requests counts successes, summary counts attempts.""" + handle = AsyncRunHandle( + tasks, users, duration_seconds, http2=http2, max_in_flight=max_in_flight, run_context=run_context, **options + ) + await handle.start() + return await handle.wait() diff --git a/je_load_density/engine/bench_cli.py b/je_load_density/engine/bench_cli.py index 547d0af..887cfe2 100644 --- a/je_load_density/engine/bench_cli.py +++ b/je_load_density/engine/bench_cli.py @@ -12,7 +12,6 @@ from typing import List, Optional from je_load_density.engine.asyncio_engine import run_async_load -from je_load_density.utils.generate_report.generate_summary_report import build_summary def _build_tasks(url: str, method: str, body: Optional[str]) -> List[dict]: @@ -39,14 +38,16 @@ def parse_args(argv: List[str]) -> argparse.Namespace: def main(argv: Optional[List[str]] = None) -> int: args = parse_args(argv if argv is not None else sys.argv[1:]) - asyncio.run(run_async_load( - tasks=_build_tasks(args.url, args.method, args.body), - users=args.users, - duration_seconds=args.duration, - http2=args.http2, - max_in_flight=args.max_in_flight, - )) - summary = build_summary() + result = asyncio.run( + run_async_load( + tasks=_build_tasks(args.url, args.method, args.body), + users=args.users, + duration_seconds=args.duration, + http2=args.http2, + max_in_flight=args.max_in_flight, + ) + ) + summary = result["summary"] print(json.dumps(summary, indent=2)) return 0 diff --git a/je_load_density/engine/entrypoints.py b/je_load_density/engine/entrypoints.py new file mode 100644 index 0000000..8f62e7d --- /dev/null +++ b/je_load_density/engine/entrypoints.py @@ -0,0 +1,79 @@ +"""Scheduler selection without importing or patching the unselected engine.""" + +import asyncio +from typing import Any, Dict, Optional + + +def start_test( + user_detail_dict: Dict[str, Any], + user_count: int = 50, + spawn_rate: int = 10, + test_time: Optional[int] = 60, + web_ui_dict: Optional[Dict[str, Any]] = None, + runner_mode: str = "local", + load_shape: Optional[str] = None, + shape_config: Optional[Dict[str, Any]] = None, + **kwargs, +) -> Dict[str, Any]: + """Select Locust (default) or local native asyncio HTTP with the legacy signature.""" + engine = kwargs.pop("engine", "locust") + arguments = ( + user_detail_dict, + user_count, + spawn_rate, + test_time, + web_ui_dict, + runner_mode, + load_shape, + shape_config, + ) + if engine == "locust": + from je_load_density.wrapper.start_wrapper.locust_start import start_test as start_locust + + return start_locust(*arguments, **kwargs) + if engine != "asyncio": + raise ValueError(f"Unsupported engine: {engine!r}") + return _start_native(arguments, kwargs) + + +def _start_native(arguments: tuple, options: dict) -> dict: + detail, users, rate, duration, web_ui, mode, shape, shape_config = arguments + if detail.get("user", "fast_http_user") not in {"http_user", "fast_http_user", "async_http_user"}: + raise ValueError("Native asyncio currently supports HTTP users only") + if set(detail) - {"user", "host", "base_url"}: + raise ValueError("Unsupported native user configuration") + if web_ui: + raise ValueError("Native asyncio does not provide the Locust web UI") + try: + asyncio.get_running_loop() + except RuntimeError: + pass + else: + raise RuntimeError("Use await run_async_load() inside an active asyncio event loop") + from je_load_density.engine.asyncio_engine import run_async_load + + tasks = options.pop("tasks", None) + options.setdefault("base_url", detail.get("host", detail.get("base_url", ""))) + native = asyncio.run( + run_async_load( + tasks, + users, + duration, + spawn_rate=rate, + runner_mode=mode, + load_shape=shape, + shape_config=shape_config, + **options, + ) + ) + return { + "user_detail": detail, + "user_count": users, + "spawn_rate": rate, + "test_time": duration, + "web_ui": web_ui, + "runner_mode": mode, + "engine": "asyncio", + "state": native["state"], + "summary": native["summary"], + } diff --git a/je_load_density/gui/chart_panel.py b/je_load_density/gui/chart_panel.py index a99910e..0405dda 100644 --- a/je_load_density/gui/chart_panel.py +++ b/je_load_density/gui/chart_panel.py @@ -73,12 +73,30 @@ def __init__(self, parent=None) -> None: self._timer.start() def reset_history(self, start_time: float | None = None) -> None: + self._snapshot_windows = None self._start_epoch = math.floor(time.time()) if start_time is None else start_time self._rps_series.clear() self._latency_chart.removeAllSeries() self._latency_lines.clear() self._band_series.clear() + def set_snapshot(self, snapshot: dict) -> None: + """Render complete child window aggregates independently of the bounded request table.""" + windows = snapshot.get("windows") + self._snapshot_windows = list(windows[-_HISTORY_SECONDS:]) if windows is not None else None + self.refresh() + + def _current_windows(self): + if self._snapshot_windows is not None: + end = self._start_epoch + if self._snapshot_windows: + last = self._snapshot_windows[-1] + end = last["start_time"] + last["duration_seconds"] + return self._snapshot_windows, max(end, self._start_epoch) + now = max(time.time(), self._start_epoch) + records = chain(test_record_instance.test_record_list, test_record_instance.error_record_list) + return latency_windows(records, start=self._start_epoch, end=now, max_buckets=_HISTORY_SECONDS), now + def _add_latency(self, series) -> None: self._latency_chart.addSeries(series) series.attachAxis(self._latency_axis_x) @@ -120,9 +138,7 @@ def _render_latency(self, windows) -> None: marker.setVisible(False) def refresh(self) -> None: - now = max(time.time(), self._start_epoch) - records = chain(test_record_instance.test_record_list, test_record_instance.error_record_list) - windows = latency_windows(records, start=self._start_epoch, end=now, max_buckets=_HISTORY_SECONDS) + windows, now = self._current_windows() self._rps_series.replace([QPointF(item["start_time"] - self._start_epoch, item["rps"]) for item in windows]) self._render_latency(windows) left = windows[0]["start_time"] - self._start_epoch if windows else 0 diff --git a/je_load_density/gui/child_process.py b/je_load_density/gui/child_process.py new file mode 100644 index 0000000..aa6be53 --- /dev/null +++ b/je_load_density/gui/child_process.py @@ -0,0 +1,46 @@ +"""Native Qt process I/O, independent of gevent patched Python threads and subprocesses.""" + +from PySide6.QtCore import QProcess + + +class ChildProcess: + """Own a process in its QThread and buffer only one bounded protocol line.""" + + def __init__(self, argv: list[str]) -> None: + self._process = QProcess() + self._process.setProcessChannelMode(QProcess.MergedChannels) + self._process.setProgram(argv[0]) + self._process.setArguments(argv[1:]) + self._buffer = b"" + self._process.start() + if not self._process.waitForStarted(3000): + raise OSError("Unable to start worker interpreter") + + def poll(self) -> int | None: + """Read the native process state without Python scheduler involvement.""" + if self._process.state() != QProcess.NotRunning: + return None + return self._process.exitCode() if self._process.exitStatus() == QProcess.NormalExit else 1 + + @property + def returncode(self) -> int | None: + """Preserve the supervisor's process inspection interface.""" + return self.poll() + + def kill(self) -> None: + """Escalate from the owning QThread only after cooperative cancellation expires.""" + self._process.kill() + + def wait(self, timeout: float) -> None: + """Wait with a native Qt millisecond timeout and report unexpected cleanup failure.""" + if self.poll() is None and not self._process.waitForFinished(int(timeout * 1000)): + raise OSError("Worker cleanup timeout") + + def read_lines(self) -> list[str]: + """Read available complete frames; native waits never need a Python reader thread.""" + if self.poll() is None: + self._process.waitForReadyRead(50) + data = self._buffer + bytes(self._process.readAllStandardOutput()) + lines = data.split(b"\n") + self._buffer = lines.pop()[-131072:] + return [line.decode("utf-8", errors="replace") for line in lines if len(line) <= 131072] diff --git a/je_load_density/gui/dark_style.py b/je_load_density/gui/dark_style.py new file mode 100644 index 0000000..1f0ddd9 --- /dev/null +++ b/je_load_density/gui/dark_style.py @@ -0,0 +1,55 @@ +"""Desktop contrast independent of platform styles and optional theme packages.""" + +import os +from functools import lru_cache +from pathlib import Path + +from PySide6.QtGui import QFont, QFontDatabase +from PySide6.QtWidgets import QWidget + +DARK_STYLE = """ +QWidget { background-color: #171c25; color: #eef2f7; font-size: 13px; } +QGroupBox { border: 1px solid #394354; border-radius: 8px; margin-top: 13px; padding: 10px; } +QGroupBox::title { subcontrol-origin: margin; left: 12px; padding: 0 5px; color: #ffcf70; } +QLabel { background: transparent; color: #eef2f7; } +QLineEdit, QComboBox, QTextEdit, QTableWidget { + background-color: #232c3b; color: #eef2f7; border: 1px solid #46536a; + border-radius: 4px; padding: 5px; selection-background-color: #466486; +} +QLineEdit:focus, QComboBox:focus { border-color: #ffcf70; } +QComboBox QAbstractItemView { background: #232c3b; color: #eef2f7; selection-background-color: #466486; } +QPushButton { background: #34445c; color: #ffffff; border: 1px solid #526781; border-radius: 5px; padding: 7px 16px; } +QPushButton:hover { background: #425a7b; } +QPushButton:pressed { background: #26364e; } +QPushButton:disabled, QWidget:disabled { color: #93a0b5; } +QHeaderView::section { background: #303c50; color: #eef2f7; border: 0; padding: 7px; } +QTableWidget { gridline-color: #3d4960; } +QTabWidget::pane { border: 1px solid #394354; } +QTabBar::tab { background: #273246; color: #cdd7e5; padding: 9px 16px; border: 1px solid #394354; } +QTabBar::tab:selected { background: #394b67; color: #ffcf70; } +QSplitter::handle { background: #394354; } +QScrollBar { background: #1b2230; } +QScrollBar::handle { background: #526781; border-radius: 4px; min-height: 20px; min-width: 20px; } +""" + + +@lru_cache(maxsize=1) +def _desktop_font() -> str: + # Offscreen Qt may not scan the operating system font directories itself. + candidates = (Path(os.environ.get("WINDIR", "C:/Windows")) / "Fonts/segoeui.ttf", + Path("/usr/share/fonts/truetype/dejavu/DejaVuSans.ttf"), + Path("/System/Library/Fonts/SFNS.ttf")) + for path in candidates: + if path.is_file(): + identifier = QFontDatabase.addApplicationFont(str(path)) + families = QFontDatabase.applicationFontFamilies(identifier) + if families: + return families[0] + return "sans-serif" + + +def apply_dark_style(widget: QWidget) -> None: + """Apply readable controls and register an OS font when using offscreen Qt.""" + family = _desktop_font() + widget.setFont(QFont(family, 10)) + widget.setStyleSheet(DARK_STYLE + f'QWidget {{ font-family: "{family}"; }}') diff --git a/je_load_density/gui/language_wrapper/english.py b/je_load_density/gui/language_wrapper/english.py index 7e599da..eabce80 100644 --- a/je_load_density/gui/language_wrapper/english.py +++ b/je_load_density/gui/language_wrapper/english.py @@ -1,4 +1,23 @@ english_word_dict = { + # Desktop lifecycle and settings + "settings": "Test settings", + "browse": "Browse…", + "action_file": "Action file", + "engine": "Engine", + "stop_button": "Stop", + "idle": "Idle", + "running": "Running", + "stopping": "Stopping", + "completed": "Completed", + "failed": "Failed", + "cancelled": "Cancelled", + "success": "Success", + "results": "Recent requests", + "request": "Request", + "latency": "Latency (ms)", + "result": "Result", + "failure_rate": "Failure rate", + "invalid_input": "Enter an HTTP target and positive load settings.", # Main "application_name": "LoadDensity", # Widget diff --git a/je_load_density/gui/language_wrapper/japanese.py b/je_load_density/gui/language_wrapper/japanese.py index 306c392..9438e32 100644 --- a/je_load_density/gui/language_wrapper/japanese.py +++ b/je_load_density/gui/language_wrapper/japanese.py @@ -1,4 +1,23 @@ japanese_word_dict = { + # Desktop lifecycle and settings + "settings": "テスト設定", + "browse": "参照…", + "action_file": "アクションファイル", + "engine": "エンジン", + "stop_button": "停止", + "idle": "待機", + "running": "実行中", + "stopping": "停止中", + "completed": "完了", + "failed": "失敗", + "cancelled": "キャンセル済み", + "success": "成功", + "results": "最近のリクエスト", + "request": "リクエスト", + "latency": "遅延 (ms)", + "result": "結果", + "failure_rate": "失敗率", + "invalid_input": "HTTP URL と正の負荷設定を入力してください。", # Main "application_name": "LoadDensity", # Widget diff --git a/je_load_density/gui/language_wrapper/korean.py b/je_load_density/gui/language_wrapper/korean.py index 011d5ba..1bba026 100644 --- a/je_load_density/gui/language_wrapper/korean.py +++ b/je_load_density/gui/language_wrapper/korean.py @@ -1,4 +1,23 @@ korean_word_dict = { + # Desktop lifecycle and settings + "settings": "테스트 설정", + "browse": "찾아보기…", + "action_file": "작업 파일", + "engine": "엔진", + "stop_button": "중지", + "idle": "대기", + "running": "실행 중", + "stopping": "중지 중", + "completed": "완료", + "failed": "실패", + "cancelled": "취소됨", + "success": "성공", + "results": "최근 요청", + "request": "요청", + "latency": "지연 (ms)", + "result": "결과", + "failure_rate": "실패율", + "invalid_input": "HTTP 대상과 양수 부하 설정을 입력하세요.", # Main "application_name": "LoadDensity", # Widget diff --git a/je_load_density/gui/language_wrapper/traditional_chinese.py b/je_load_density/gui/language_wrapper/traditional_chinese.py index 344e013..ede5b95 100644 --- a/je_load_density/gui/language_wrapper/traditional_chinese.py +++ b/je_load_density/gui/language_wrapper/traditional_chinese.py @@ -1,4 +1,23 @@ traditional_chinese_word_dict = { + # Desktop lifecycle and settings + "settings": "測試設定", + "browse": "瀏覽…", + "action_file": "動作檔案", + "engine": "引擎", + "stop_button": "停止", + "idle": "待命", + "running": "執行中", + "stopping": "停止中", + "completed": "已完成", + "failed": "失敗", + "cancelled": "已取消", + "success": "成功", + "results": "近期請求", + "request": "請求", + "latency": "延遲 (ms)", + "result": "結果", + "failure_rate": "失敗率", + "invalid_input": "請輸入 HTTP 目標及正整數負載設定。", # Main "application_name": "LoadDensity", # Widget diff --git a/je_load_density/gui/load_density_gui_thread.py b/je_load_density/gui/load_density_gui_thread.py index dd858cc..ef8b40e 100644 --- a/je_load_density/gui/load_density_gui_thread.py +++ b/je_load_density/gui/load_density_gui_thread.py @@ -1,59 +1,131 @@ -from PySide6.QtCore import QThread -from je_load_density.wrapper.start_wrapper.start_test import start_test +"""Qt worker that supervises an isolated engine process without importing Locust.""" + +import json +import sys +import tempfile +import time +from pathlib import Path + +from PySide6.QtCore import QThread, Signal + +from je_load_density.gui.child_process import ChildProcess -# 定義常數，避免硬編碼字串 -# Define constant to avoid hard-coded string DEFAULT_USER_TYPE = "fast_http_user" +STOP_GRACE_SECONDS = 3.0 class LoadDensityGUIThread(QThread): - """ - GUI 測試執行緒 - GUI Test Thread - - 用於在背景執行負載測試，避免阻塞主介面。 - Used to run load tests in the background without blocking the main GUI. - """ - - def __init__(self, - request_url: str = None, - test_duration: int = None, - user_count: int = None, - spawn_rate: int = None, - http_method: str = None): - """ - 初始化執行緒參數 - Initialize thread parameters - - :param request_url: 測試目標 URL (Target request URL) - :param test_duration: 測試持續時間 (Test duration in seconds) - :param user_count: 使用者數量 (Number of simulated users) - :param spawn_rate: 使用者生成速率 (User spawn rate) - :param http_method: HTTP 方法 (HTTP method, e.g., "GET", "POST") - """ + """Deliver bounded child messages through Qt signals; stop and join on shutdown.""" + + state_changed = Signal(str) + snapshot_ready = Signal(dict) + log_message = Signal(str) + completed = Signal(dict) + + def __init__(self, request_url: str | None = None, test_duration: int | None = None, + user_count: int | None = None, spawn_rate: int | None = None, + http_method: str | None = None, **kwargs: object) -> None: super().__init__() - self.request_url = request_url - self.test_duration = test_duration - self.user_count = user_count - self.spawn_rate = spawn_rate + self.request_url, self.test_duration = request_url, test_duration + self.user_count, self.spawn_rate = user_count, spawn_rate self.http_method = http_method + self.engine = kwargs.get("engine", "locust") + self.action_file = kwargs.get("action_file", "") + self._stop = False + self._process = None + self._control = None + self._last_snapshot = {"summary": {}, "records": []} + self._terminal = None + + def request_stop(self) -> None: + """Ask the child to stop cooperatively through a run specific control file.""" + self._stop = True + if self._control is not None: + try: + self._control.touch(exist_ok=True) + except FileNotFoundError: + return # The already finished run removed its private control directory. + + def shutdown(self, timeout_ms: int = 6000) -> bool: + """Cancel, escalate after the grace period and join the QThread within a bound.""" + self.request_stop() + if self.wait(timeout_ms): + return True + self.requestInterruption() + return self.wait(2000) + + def _terminate(self) -> None: + if self._process is not None and self._process.poll() is None: + self._process.kill() + + def _config(self) -> dict: + return {"url": self.request_url, "duration": self.test_duration, "users": self.user_count, + "spawn_rate": self.spawn_rate, "method": self.http_method, + "engine": self.engine, "action_file": self.action_file} + + def run(self) -> None: + """Run a fresh interpreter and report every exit as a terminal lifecycle event.""" + try: + with tempfile.TemporaryDirectory(prefix="loaddensity-gui-") as directory: + config = Path(directory) / "run.json" + config.write_text(json.dumps(self._config()), encoding="utf-8") + config.chmod(0o600) + self._control = Path(directory) / "stop" + if self._stop: + self.request_stop() + self._launch(config) + self._consume() + except (OSError, ValueError): + self.log_message.emit("Unable to run the selected test. Check the configuration and dependencies.") + finally: + self._cleanup() + self._control = None + result = self._terminal or {"state": "cancelled" if self._stop else "failed"} + result.setdefault("summary", self._last_snapshot["summary"]) + self.completed.emit(result) + + def _cleanup(self) -> None: + try: + self._terminate() + if self._process is not None: + self._process.wait(timeout=2) + except OSError: + self._terminal = {"state": "failed"} + self.log_message.emit("Unable to finish worker cleanup. The run has failed.") + + def _launch(self, config: Path) -> None: + argv = [sys.executable, "-m", "je_load_density.gui.run_worker", "--config", str(config), + "--control", str(self._control)] + self._process = ChildProcess(argv) + + def _consume(self) -> None: + deadline = None + while self._process.poll() is None: + if self._stop and deadline is None: + deadline = time.monotonic() + STOP_GRACE_SECONDS + if self.isInterruptionRequested() or (deadline is not None and time.monotonic() >= deadline): + self._terminate() + for line in self._process.read_lines(): + self._message(line) + for line in self._process.read_lines(): + self._message(line) + if self._process.returncode != 0: + self._terminal = None - def run(self): - """ - 執行負載測試 - Run the load test - """ - if not self.request_url or not self.http_method: - # 基本檢查，避免傳入 None - # Basic validation to avoid None values - raise ValueError("Request URL and HTTP method must be provided.") - - start_test( - {"user": DEFAULT_USER_TYPE}, # 使用者類型 (User type) - self.user_count, - self.spawn_rate, - self.test_duration, - tasks={ - self.http_method: {"request_url": self.request_url} - } - ) \ No newline at end of file + def _message(self, line: str) -> None: + try: + message = json.loads(line) + except json.JSONDecodeError: + return # Third party diagnostics may contain request credentials. + if not isinstance(message, dict): + return + kind = message.get("type") + if kind == "snapshot": + self._last_snapshot = message["data"] + self.snapshot_ready.emit(message["data"]) + elif kind == "state": + self.state_changed.emit(message["state"]) + elif kind == "log": + self.log_message.emit(message["text"][:1200]) + elif kind == "terminal": + self._terminal = message["data"] diff --git a/je_load_density/gui/main_widget.py b/je_load_density/gui/main_widget.py index 45441e9..c39545d 100644 --- a/je_load_density/gui/main_widget.py +++ b/je_load_density/gui/main_widget.py @@ -1,146 +1,252 @@ -import logging -from typing import Optional +"""Desktop settings and run inspection; all view mutations happen in Qt slots.""" -from PySide6.QtCore import QTimer -from PySide6.QtGui import QIntValidator +from urllib.parse import urlsplit + +from PySide6.QtCore import QTimer, Slot +from PySide6.QtGui import QIntValidator, QTextCursor from PySide6.QtWidgets import ( - QWidget, QFormLayout, QLineEdit, QComboBox, - QPushButton, QTextEdit, QVBoxLayout, QMessageBox, QTabWidget + QComboBox, + QFileDialog, + QFormLayout, + QGroupBox, + QHBoxLayout, + QHeaderView, + QLabel, + QLineEdit, + QPushButton, + QSplitter, + QTableWidget, + QTableWidgetItem, + QTabWidget, + QTextEdit, + QVBoxLayout, + QWidget, ) from je_load_density.gui.chart_panel import LiveChartPanel -from je_load_density.gui.load_density_gui_thread import LoadDensityGUIThread +from je_load_density.gui.dark_style import apply_dark_style from je_load_density.gui.language_wrapper.multi_language_wrapper import language_wrapper -from je_load_density.gui.log_to_ui_filter import InterceptAllFilter, log_message_queue +from je_load_density.gui.load_density_gui_thread import LoadDensityGUIThread +from je_load_density.gui.log_to_ui_filter import log_message_queue from je_load_density.gui.run_history_panel import RunHistoryPanel from je_load_density.gui.stats_panel import StatsPanel +from je_load_density.utils.test_record.test_record_class import test_record_instance class LoadDensityWidget(QWidget): - """ - 負載測試 GUI 控制元件 - Load Test GUI Widget + """Keep settings left and state, metrics, charts and inspectable results right.""" - 提供使用者輸入測試參數並啟動負載測試， - 並將日誌訊息即時顯示在 GUI 中。 - Provides input fields for test parameters, starts load tests, - and displays log messages in real-time. - """ - - def __init__(self, parent: Optional[QWidget] = None): + def __init__(self, parent: QWidget | None = None) -> None: super().__init__(parent) - - # === 表單區域 (Form Section) === - form_layout = QFormLayout() - - # URL 輸入框 (Target URL input) - self.url_input = QLineEdit() - - # 測試時間 (Test duration, must be int) - self.test_time_input = QLineEdit() - self.test_time_input.setValidator(QIntValidator()) - - # 使用者數量 (User count) - self.user_count_input = QLineEdit() - self.user_count_input.setValidator(QIntValidator()) - - # 生成速率 (Spawn rate) - self.spawn_rate_input = QLineEdit() - self.spawn_rate_input.setValidator(QIntValidator()) - - # HTTP 方法選擇 (HTTP method selection) + apply_dark_style(self) + self.words = language_wrapper.language_word_dict + self.run_state = "idle" + self.run_load_density_thread = None + self.last_summary = {} + self._settings = self._build_settings() + inspection = self._build_inspection() + splitter = QSplitter() + splitter.addWidget(self._settings) + splitter.addWidget(inspection) + splitter.setStretchFactor(1, 1) + layout = QHBoxLayout(self) + layout.addWidget(splitter) + self.pull_log_timer = QTimer(self) + self.pull_log_timer.setInterval(100) + self.pull_log_timer.timeout.connect(self.add_text_to_log) + self._set_state("idle") + + def _build_settings(self) -> QWidget: + settings = QGroupBox(self.words["settings"]) + form = QFormLayout(settings) + self.action_file_input = QLineEdit() + browse = QPushButton(self.words["browse"]) + browse.clicked.connect(self._browse_action) + action_row = QHBoxLayout() + action_row.addWidget(self.action_file_input) + action_row.addWidget(browse) + form.addRow(self.words["action_file"], action_row) + self.url_input = QLineEdit("http://localhost:8080/") + self.engine_combobox = QComboBox() + self.engine_combobox.addItems(["Locust", "asyncio"]) + self.test_time_input = QLineEdit("10") + self.user_count_input = QLineEdit("1") + self.spawn_rate_input = QLineEdit("1") + for field in (self.test_time_input, self.user_count_input, self.spawn_rate_input): + field.setValidator(QIntValidator(1, 2147483647, field)) self.method_combobox = QComboBox() - self.method_combobox.addItems([ - language_wrapper.language_word_dict.get("get"), - language_wrapper.language_word_dict.get("post"), - language_wrapper.language_word_dict.get("put"), - language_wrapper.language_word_dict.get("patch"), - language_wrapper.language_word_dict.get("delete"), - language_wrapper.language_word_dict.get("head"), - language_wrapper.language_word_dict.get("options"), - ]) - - # 將元件加入表單 (Add widgets to form layout) - form_layout.addRow(language_wrapper.language_word_dict.get("url"), self.url_input) - form_layout.addRow(language_wrapper.language_word_dict.get("test_time"), self.test_time_input) - form_layout.addRow(language_wrapper.language_word_dict.get("user_count"), self.user_count_input) - form_layout.addRow(language_wrapper.language_word_dict.get("spawn_rate"), self.spawn_rate_input) - form_layout.addRow(language_wrapper.language_word_dict.get("test_method"), self.method_combobox) - - # === 啟動按鈕 (Start button) === - self.start_button = QPushButton(language_wrapper.language_word_dict.get("start_button")) + self.method_combobox.addItems([self.words[key] for key in + ("get", "post", "put", "patch", "delete", "head", "options")]) + for key, field in (("url", self.url_input), ("engine", self.engine_combobox), + ("user_count", self.user_count_input), ("spawn_rate", self.spawn_rate_input), + ("test_time", self.test_time_input), ("test_method", self.method_combobox)): + form.addRow(self.words[key], field) + settings.setMinimumWidth(270) + return settings + + def _build_inspection(self) -> QWidget: + inspection = QWidget() + layout = QVBoxLayout(inspection) + layout.setContentsMargins(0, 0, 0, 0) + controls = QHBoxLayout() + self.state_label = QLabel() + self.start_button = QPushButton(self.words["start_button"]) + self.stop_button = QPushButton(self.words["stop_button"]) self.start_button.clicked.connect(self.run_load_density) - - # === 日誌面板 (Log panel) === + self.stop_button.clicked.connect(self.stop_load_density) + for widget in (self.state_label, self.start_button, self.stop_button): + controls.addWidget(widget) + layout.addLayout(controls) + self.stats_panel = StatsPanel() + self.stats_panel.set_snapshot({}) + layout.addWidget(self.stats_panel) self.log_panel = QTextEdit() self.log_panel.setReadOnly(True) - - # === 即時統計面板 (Live stats panel) === - self.stats_panel = StatsPanel() - - # === 即時圖表與執行紀錄 (Live chart and run history) === + self.log_panel.document().setMaximumBlockCount(500) self.chart_panel = LiveChartPanel() + self.chart_panel._chart_view.setMinimumHeight(260) + self.chart_panel._rps_view.setMinimumHeight(220) + self.chart_panel.layout().setStretch(0, 1) + self.chart_panel.layout().setStretch(1, 1) self.history_panel = RunHistoryPanel() - - # 日誌、圖表、紀錄共用一組分頁 (Log, chart and history share one tab bar) self.tabs = QTabWidget() - self.tabs.addTab(self.log_panel, language_wrapper.language_word_dict.get("tab_log")) - self.tabs.addTab(self.chart_panel, language_wrapper.language_word_dict.get("tab_chart")) - self.tabs.addTab(self.history_panel, language_wrapper.language_word_dict.get("tab_history")) - - # === 主版面配置 (Main layout) === - main_layout = QVBoxLayout() - main_layout.addLayout(form_layout) - main_layout.addWidget(self.start_button) - main_layout.addWidget(self.stats_panel) - main_layout.addWidget(self.tabs) - - # === 執行緒與計時器 (Thread & Timer) === - self.run_load_density_thread: Optional[LoadDensityGUIThread] = None - self.pull_log_timer = QTimer() - self.pull_log_timer.setInterval(50) # 稍微放大間隔，避免 UI 卡頓 - self.pull_log_timer.timeout.connect(self.add_text_to_log) - - self.setLayout(main_layout) - + for widget, key in ((self.log_panel, "tab_log"), (self.chart_panel, "tab_chart"), + (self.history_panel, "tab_history")): + self.tabs.addTab(widget, self.words[key]) + self.tabs.setCurrentIndex(1) + layout.addWidget(self.tabs, 2) + layout.addWidget(QLabel(self.words["results"])) + self.results_table = QTableWidget(0, 3) + self.results_table.setHorizontalHeaderLabels([self.words["request"], self.words["latency"], + self.words["result"]]) + self.results_table.setEditTriggers(QTableWidget.NoEditTriggers) + self.results_table.setMinimumHeight(110) + self.results_table.setMaximumHeight(220) + header = self.results_table.horizontalHeader() + header.setSectionResizeMode(0, QHeaderView.Stretch) + header.setSectionResizeMode(1, QHeaderView.ResizeToContents) + header.setSectionResizeMode(2, QHeaderView.ResizeToContents) + layout.addWidget(self.results_table, 1) + return inspection + + def _browse_action(self) -> None: + path, _ = QFileDialog.getOpenFileName(self, self.words["action_file"], "", "Actions (*.json *.yaml *.yml)") + if path: + self.action_file_input.setText(path) + + def _configuration(self) -> dict: + users, spawn, duration = (int(field.text()) for field in + (self.user_count_input, self.spawn_rate_input, self.test_time_input)) + if min(users, spawn, duration) <= 0: + raise ValueError("Positive load settings required") + action_file = self.action_file_input.text().strip() + target = self.url_input.text().strip() + if not action_file and (urlsplit(target).scheme not in {"http", "https"} or not urlsplit(target).hostname): + raise ValueError("HTTP target required") + return {"request_url": target, "user_count": users, "spawn_rate": spawn, "test_duration": duration, + "http_method": self.method_combobox.currentText().lower(), + "engine": self.engine_combobox.currentText().lower(), "action_file": action_file} + + @Slot() def run_load_density(self) -> None: - """ - 啟動負載測試 - Start the load test - """ + """Reject duplicate starts and retain prior results until new settings validate.""" + if self.run_state in {"running", "stopping"}: + return + if self.run_load_density_thread is not None and self.run_load_density_thread.isRunning(): + return try: - test_time = int(self.test_time_input.text()) - user_count = int(self.user_count_input.text()) - spawn_rate = int(self.spawn_rate_input.text()) + config = self._configuration() except ValueError: - QMessageBox.warning(self, "Invalid Input", "請輸入有效的數字\nPlease enter valid numbers") + self._append_log(self.words["invalid_input"]) return - - self.run_load_density_thread = LoadDensityGUIThread() - self.run_load_density_thread.request_url = self.url_input.text() - self.run_load_density_thread.test_duration = test_time - self.run_load_density_thread.user_count = user_count - self.run_load_density_thread.spawn_rate = spawn_rate - self.run_load_density_thread.http_method = self.method_combobox.currentText().lower() - - # 設定日誌攔截器 (Attach log filter) - root_logger = logging.getLogger("root") - log_handler_list = [handler for handler in root_logger.handlers if handler.name == "log_reader"] - if log_handler_list: - log_handler = log_handler_list[0] - # 避免重複新增 Filter (Prevent duplicate filters) - if not any(isinstance(f, InterceptAllFilter) for f in log_handler.filters): - log_handler.addFilter(InterceptAllFilter()) - - # 啟動執行緒與計時器 (Start thread & timer) - self.run_load_density_thread.start() - self.pull_log_timer.start() + worker = LoadDensityGUIThread(**config) + self.run_load_density_thread = worker + worker.state_changed.connect(self._worker_state) + worker.snapshot_ready.connect(self._snapshot) + worker.log_message.connect(self._append_log) + worker.completed.connect(self._completed) self.log_panel.clear() + self.last_summary = {} + self.stats_panel.set_snapshot({}) + self.results_table.setRowCount(0) + test_record_instance.clear_records() + self.chart_panel.reset_history() + self.chart_panel._timer.start() + self._set_state("running") + self._append_log(self.words["running"]) + worker.start() + + @Slot() + def stop_load_density(self) -> None: + """Show stopping immediately and ask the isolated engine to cancel cooperatively.""" + if self.run_state != "running": + return + self._set_state("stopping") + self.run_load_density_thread.request_stop() + + def _set_state(self, state: str) -> None: + self.run_state = state + self.state_label.setText(self.words.get(state, state)) + active = state in {"running", "stopping"} + self.start_button.setEnabled(not active) + self.stop_button.setEnabled(state == "running") + self._settings.setEnabled(not active) + + @Slot(str) + def _worker_state(self, state: str) -> None: + if self.run_state != "stopping": + self._set_state(state) + + @Slot(dict) + def _snapshot(self, snapshot: dict) -> None: + self.last_summary = snapshot["summary"] + self.stats_panel.set_snapshot(self.last_summary) + records = snapshot.get("records", [])[-200:] + test_record_instance.test_record_list[:] = [row for row in records if not row.get("failed")] + test_record_instance.error_record_list[:] = [row for row in records if row.get("failed")] + self.results_table.setRowCount(len(records)) + for index, record in enumerate(records): + latency = record.get("response_time_ms") + values = (record.get("name", "request"), f"{latency:.1f}" if latency is not None else "", + self.words["failed"] if record.get("failed") else self.words["success"]) + for column, value in enumerate(values): + self.results_table.setItem(index, column, QTableWidgetItem(str(value))) + self.chart_panel.set_snapshot(snapshot) + + @Slot(dict) + def _completed(self, result: dict) -> None: + self.chart_panel._timer.stop() + self.last_summary = result.get("summary", self.last_summary) + self.stats_panel.set_snapshot(self.last_summary) + self._set_state(result["state"]) + self._append_log(self.words.get(result["state"], result["state"])) + + @Slot(str) + def _append_log(self, message: str) -> None: + cursor = self.log_panel.textCursor() + cursor.movePosition(QTextCursor.End) + cursor.insertText(str(message)[:1200] + "\n") + self.log_panel.setTextCursor(cursor) def add_text_to_log(self) -> None: - """ - 將日誌訊息加入到 GUI 面板 - Append log messages to GUI panel - """ - while not log_message_queue.empty(): - self.log_panel.append(log_message_queue.get_nowait()) \ No newline at end of file + """Retain the legacy queued logging entrypoint with a bounded per tick drain.""" + for _ in range(100): + if log_message_queue.empty(): + break + self._append_log(log_message_queue.get_nowait()) + + def shutdown(self) -> bool: + """Stop timers and cancel/join the worker before the widget is destroyed.""" + self.pull_log_timer.stop() + self.stats_panel._timer.stop() + self.chart_panel._timer.stop() + if self.run_load_density_thread is None: + return True + return self.run_load_density_thread.shutdown() + + def closeEvent(self, event) -> None: + """Allow destruction only after the execution thread has joined.""" + if self.shutdown(): + event.accept() + else: + event.ignore() diff --git a/je_load_density/gui/main_window.py b/je_load_density/gui/main_window.py index 8d14828..3e7e1c3 100644 --- a/je_load_density/gui/main_window.py +++ b/je_load_density/gui/main_window.py @@ -1,14 +1,14 @@ import sys from typing import Optional -from PySide6.QtWidgets import QMainWindow, QApplication, QWidget -from qt_material import QtStyleTools +from PySide6.QtWidgets import QApplication, QMainWindow, QWidget +from je_load_density.gui.dark_style import apply_dark_style from je_load_density.gui.language_wrapper.multi_language_wrapper import language_wrapper from je_load_density.gui.main_widget import LoadDensityWidget -class LoadDensityUI(QMainWindow, QtStyleTools): +class LoadDensityUI(QMainWindow): """ 負載測試主視窗 Load Test Main Window @@ -29,21 +29,23 @@ def __init__(self, parent: Optional[QWidget] = None): from ctypes import windll windll.shell32.SetCurrentProcessExplicitAppUserModelID(self.id) - # 設定字體樣式 (Set font style) - self.setStyleSheet( - "font-size: 12pt;" - "font-family: 'Lato';" - ) - - # 套用 qt-material 樣式 (Apply qt-material theme) - self.apply_stylesheet(self, "dark_amber.xml") + apply_dark_style(self) # 建立並設定主要控制元件 (Create and set main widget) self.load_density_widget = LoadDensityWidget() self.setCentralWidget(self.load_density_widget) + self.setWindowTitle(self.id) + self.resize(1200, 850) + + def closeEvent(self, event) -> None: + """Cancel and join the child supervisor before destroying the main window.""" + if self.load_density_widget.shutdown(): + event.accept() + else: + event.ignore() if __name__ == "__main__": app = QApplication(sys.argv) window = LoadDensityUI() window.show() - sys.exit(app.exec()) \ No newline at end of file + sys.exit(app.exec()) diff --git a/je_load_density/gui/run_protocol.py b/je_load_density/gui/run_protocol.py new file mode 100644 index 0000000..2deccd6 --- /dev/null +++ b/je_load_density/gui/run_protocol.py @@ -0,0 +1,73 @@ +"""Scheduler independent messages exchanged with the desktop worker.""" + +import json +from itertools import chain +from typing import Any +from urllib.parse import urlsplit, urlunsplit + +from je_load_density.utils.test_record.window_statistics import finite_number, latency_mean, latency_windows, percentile + +MAX_RECORDS = 200 +MAX_FRAME_BYTES = 131072 + + +def encode_frame(kind: str, **fields: Any) -> str: + """Retain the newest complete request rows within the reader's UTF-8 byte bound.""" + message = {"type": kind, **fields} + if kind == "snapshot": + data = dict(message["data"]) + records = data["records"] + data["records"] = [] + message["data"] = data + baseline = json.dumps(message, allow_nan=False) + "\n" + budget = MAX_FRAME_BYTES - len(baseline.encode("utf-8")) + retained = [] + for record in reversed(records): + encoded = json.dumps(record, allow_nan=False) + cost = len(encoded.encode("utf-8")) + (2 if retained else 0) + if cost > budget: + break + retained.append(record) + budget -= cost + data["records"] = list(reversed(retained)) + frame = json.dumps(message, allow_nan=False) + "\n" + if len(frame.encode("utf-8")) > MAX_FRAME_BYTES: + raise ValueError("GUI message metadata exceeds the frame byte limit") + return frame + + +def safe_name(value: object) -> str: + """Remove URL credentials, query strings and fragments from display names.""" + text = str(value or "request") + try: + parts = urlsplit(text) + if parts.netloc: + text = urlunsplit((parts.scheme, parts.netloc.rsplit("@", 1)[-1], parts.path, "", "")) + elif text.startswith(("/", "./", "../")): + text = parts.path + except ValueError: + return "request" + return text[:500] + + +def _record(record: dict, failed: bool) -> dict: + return {"name": safe_name(record.get("name") or record.get("test_url")), "failed": failed, + "start_time": finite_number(record.get("start_time")), + "response_time_ms": finite_number(record.get("response_time_ms"))} + + +def make_snapshot(success: list, failures: list, *, users: int = 0, elapsed: float = 1, + start_time: float | None = None, end_time: float | None = None) -> dict: + """Keep cumulative totals while bounding the transported, credential free request tail.""" + latencies = [value for record in chain(success, failures) + if (value := finite_number(record.get("response_time_ms"))) is not None and value >= 0] + count = len(success) + len(failures) + summary = {"requests": count, "failures": len(failures), "users": users, + "rps": count / max(elapsed, 0.001), "failure_rate": len(failures) / max(count, 1), + "mean_ms": latency_mean(latencies), "p50_ms": percentile(latencies, 50), + "p95_ms": percentile(latencies, 95), "p99_ms": percentile(latencies, 99)} + records = [_record(row, False) for row in success[-MAX_RECORDS:]] + records.extend(_record(row, True) for row in failures[-MAX_RECORDS:]) + records.sort(key=lambda row: row["start_time"] or 0) + windows = latency_windows(chain(success, failures), start=start_time, end=end_time, max_buckets=120) + return {"summary": summary, "records": records[-MAX_RECORDS:], "windows": windows} diff --git a/je_load_density/gui/run_worker.py b/je_load_density/gui/run_worker.py new file mode 100644 index 0000000..7c10af9 --- /dev/null +++ b/je_load_density/gui/run_worker.py @@ -0,0 +1,185 @@ +"""Private JSON desktop worker; engine scheduler selection precedes metrics pumping.""" + +import argparse +import contextlib +import json +import math +import sys +import time +from pathlib import Path +from typing import Any, Callable + + +class Protocol: + """Reserve original stdout for frames and consume unsafe incidental library output.""" + + def __init__(self) -> None: + self.output = sys.stdout + self._lock = contextlib.nullcontext() + self._last_output_log = float("-inf") + + def select_scheduler(self) -> None: + """Create the message lock only after Locust or native scheduler selection.""" + import threading + + self._lock = threading.Lock() + + def emit(self, kind: str, **fields) -> None: + """Write one complete JSON frame without raw action parameters.""" + from je_load_density.gui.run_protocol import encode_frame + + frame = encode_frame(kind, **fields) + with self._lock: + self.output.write(frame) + self.output.flush() + + def write(self, text: str) -> int: + """Consume incidental prints that may contain sensitive payloads.""" + now = time.monotonic() + if text.strip() and now - self._last_output_log >= 1: + self._last_output_log = now + self.emit("log", text="Test output received.") + return len(text) + + def flush(self) -> None: + """Support libraries expecting a file shaped stdout target.""" + + +class RunSession: + """Own one selected engine, cumulative records and action failure judgement.""" + + def __init__(self, config: dict, control: Path, protocol: Protocol) -> None: + self.config, self.control, self.protocol = config, control, protocol + self.environment = None + self.failed_actions = 0 + self.started = time.monotonic() + self.started_epoch = math.floor(time.time()) + self.finished = False + self.snapshot = {"summary": {}, "records": []} + + def stop_requested(self) -> bool: + """Each run has its own cancellation file; engine callbacks never touch Qt.""" + return self.control.exists() + + def on_environment(self, environment: object) -> None: + """Capture the engine handle for user count updates.""" + self.environment = environment + + def sample(self) -> None: + """Emit cumulative metrics and a bounded sanitized request tail.""" + from je_load_density.gui.run_protocol import make_snapshot + from je_load_density.utils.test_record.test_record_class import test_record_instance + + runner = getattr(self.environment, "runner", None) + users = getattr(runner, "user_count", self.config.get("users", 0)) + if self.config["engine"] == "asyncio" and self.environment is not None: + users = self.environment.snapshot()["users"] + self.snapshot = make_snapshot(test_record_instance.test_record_list, + test_record_instance.error_record_list, + users=users, elapsed=time.monotonic() - self.started, + start_time=self.started_epoch, end_time=time.time()) + self.protocol.emit("snapshot", data=self.snapshot) + + def pump(self, sleep: Callable[[float], object]) -> None: + """Use the selected scheduler, never a gevent patched Qt thread.""" + while not self.finished: + self.sample() + sleep(0.25) + + def start(self, *args: Any, **kwargs: Any) -> dict: + """Override engine and lifecycle callbacks, preserving scripted load parameters.""" + from je_load_density.engine.entrypoints import start_test + + kwargs.update(engine=self.config["engine"], stop_requested=self.stop_requested, + on_environment=self.on_environment) + return start_test(*args, **kwargs) + + def execute(self) -> None: + """Keep report actions and track swallowed failures, including nested action lists.""" + if not self.config.get("action_file"): + self.start({"user": "fast_http_user"}, self.config["users"], self.config["spawn_rate"], + self.config["duration"], + tasks={self.config["method"]: {"request_url": self.config["url"]}}) + return + from dataclasses import replace + + from je_action_core.reporting import ExecutionReporter + + from je_load_density.utils.executor.action_executor import Executor + from je_load_density.utils.package_manager.package_manager_class import PackageManager, package_manager + + session = self + + class Reporter(ExecutionReporter): + def on_failure(self, action, error): + session.failed_actions += 1 + session.protocol.emit("log", text=f"Action failed ({type(error).__name__}).") + + executor = Executor() + executor.settings = replace(executor.settings, reporter=Reporter()) + executor.event_dict["LD_start_test"] = self.start + manager = PackageManager() + manager.settings = package_manager.settings + manager.allowed_packages = set(package_manager.allowed_packages) + manager.allow_arbitrary_packages = package_manager.allow_arbitrary_packages + manager.executor = executor + executor.event_dict["LD_add_package_to_executor"] = manager.add_package_to_executor + executor.execute_files([self.config["action_file"]]) + + +def _scheduler(engine: str): + if engine == "locust": + import gevent + import locust # noqa: F401 - select scheduler before starting the metrics pump. + + return gevent.spawn, gevent.sleep + if engine != "asyncio": + raise ValueError("Unsupported engine") + import threading + + def spawn(function, *args): + thread = threading.Thread(target=function, args=args, daemon=True) + thread.start() + return thread + + return spawn, time.sleep + + +def run(config: dict, control: Path, protocol: Protocol) -> None: + """Select scheduler first and publish a terminal snapshot for every engine failure.""" + spawn, sleep = _scheduler(config["engine"]) + protocol.select_scheduler() + session = RunSession(config, control, protocol) + protocol.emit("state", state="running") + pump = spawn(session.pump, sleep) + state = "completed" + try: + session.execute() + if session.failed_actions: + state = "failed" + elif session.stop_requested(): + state = "cancelled" + except Exception as error: # Engine boundary: every failure becomes a visible terminal state. + state = "failed" + protocol.emit("log", text=f"Test failed ({type(error).__name__}). Check the action file and target.") + finally: + session.finished = True + pump.join(timeout=1) + session.sample() + protocol.emit("terminal", data={"state": state, "summary": session.snapshot["summary"]}) + + +def main() -> None: + """Read private settings and reserve stdout for bounded worker frames.""" + parser = argparse.ArgumentParser() + parser.add_argument("--config", required=True) + parser.add_argument("--control", required=True) + args = parser.parse_args() + protocol = Protocol() + config = json.loads(Path(args.config).read_text(encoding="utf-8")) + with contextlib.redirect_stdout(protocol), contextlib.redirect_stderr(protocol): + run(config, Path(args.control), protocol) + + +if __name__ == "__main__": + main() diff --git a/je_load_density/gui/stats_panel.py b/je_load_density/gui/stats_panel.py index 96f0e12..f1b6b99 100644 --- a/je_load_density/gui/stats_panel.py +++ b/je_load_density/gui/stats_panel.py @@ -2,23 +2,17 @@ from typing import List from PySide6.QtCore import QTimer -from PySide6.QtWidgets import QGroupBox, QLabel, QVBoxLayout, QWidget +from PySide6.QtWidgets import QGroupBox, QHBoxLayout, QLabel, QVBoxLayout, QWidget from je_load_density.gui.language_wrapper.multi_language_wrapper import language_wrapper from je_load_density.utils.test_record.test_record_class import test_record_instance +from je_load_density.utils.test_record.window_statistics import percentile def _percentile(values: List[float], pct: float) -> float: if not values: return 0.0 - sorted_values = sorted(values) - if len(sorted_values) == 1: - return float(sorted_values[0]) - rank = (pct / 100.0) * (len(sorted_values) - 1) - lower = int(rank) - upper = min(lower + 1, len(sorted_values) - 1) - fraction = rank - lower - return float(sorted_values[lower] + (sorted_values[upper] - sorted_values[lower]) * fraction) + return percentile(values, pct) or 0.0 class StatsPanel(QWidget): @@ -28,18 +22,24 @@ class StatsPanel(QWidget): totals plus latency percentiles. """ - def __init__(self, parent=None): + def __init__(self, parent: QWidget | None = None) -> None: super().__init__(parent) layout = QVBoxLayout() + layout.setContentsMargins(0, 0, 0, 0) words = language_wrapper.language_word_dict self.totals_label = QLabel() self.latency_label = QLabel() self.failures_label = QLabel() + self.capacity_label = QLabel() group = QGroupBox(words.get("stats_panel", "Live Stats")) group_layout = QVBoxLayout() - group_layout.addWidget(self.totals_label) + header = QHBoxLayout() + header.addWidget(self.totals_label) + header.addStretch() + header.addWidget(self.capacity_label) + group_layout.addLayout(header) group_layout.addWidget(self.latency_label) group_layout.addWidget(self.failures_label) group.setLayout(group_layout) @@ -54,6 +54,18 @@ def __init__(self, parent=None): self._timer.start() self.refresh() + def set_snapshot(self, summary: dict) -> None: + """Render cumulative child totals independently of the bounded request table.""" + self._timer.stop() + words = language_wrapper.language_word_dict + self.totals_label.setText(f"{words['stats_total']}: {summary.get('requests', 0)} " + f"{words['stats_rate']}: {summary.get('rps', 0):.1f}/s") + percentiles = " ".join(f"p{pct}: {summary.get(f'p{pct}_ms') or 0:.1f} ms" for pct in (50, 95, 99)) + self.latency_label.setText(percentiles) + self.failures_label.setText(f"{words['stats_failures']}: {summary.get('failures', 0)} " + f"{words['failure_rate']}: {summary.get('failure_rate', 0):.1%}") + self.capacity_label.setText(f"{words['user_count']}: {summary.get('users', 0)}") + def refresh(self) -> None: success_records = test_record_instance.test_record_list failure_records = test_record_instance.error_record_list diff --git a/je_load_density/utils/executor/action_executor.py b/je_load_density/utils/executor/action_executor.py index 0aeb766..62fecad 100644 --- a/je_load_density/utils/executor/action_executor.py +++ b/je_load_density/utils/executor/action_executor.py @@ -12,173 +12,12 @@ safe_builtin_commands, ) -from je_load_density.utils.action_generator.generate import ( - generate_from_curls, - generate_from_openapi, -) -from je_load_density.utils.ai.auto_baseline import calibrate_sla -from je_load_density.utils.ai.root_cause import ( - build_root_cause_prompt, - render_prompt_text, -) -from je_load_density.utils.ai.smart_shape import find_breaking_point -from je_load_density.utils.chaos.chaos_mesh import ( - apply_manifest as chaos_apply_manifest, -) -from je_load_density.utils.chaos.chaos_mesh import ( - build_network_delay as chaos_network_delay, -) -from je_load_density.utils.chaos.chaos_mesh import ( - delete_manifest as chaos_delete_manifest, -) -from je_load_density.utils.chaos.toxiproxy import ( - add_toxic as toxiproxy_add_toxic, -) -from je_load_density.utils.chaos.toxiproxy import ( - create_proxy as toxiproxy_create_proxy, -) -from je_load_density.utils.chaos.toxiproxy import ( - install_bandwidth as toxiproxy_install_bandwidth, -) -from je_load_density.utils.chaos.toxiproxy import ( - install_latency as toxiproxy_install_latency, -) -from je_load_density.utils.chaos.toxiproxy import ( - list_proxies as toxiproxy_list_proxies, -) -from je_load_density.utils.chaos.toxiproxy import ( - remove_proxies as toxiproxy_remove_proxies, -) -from je_load_density.utils.chaos.toxiproxy import ( - remove_toxic as toxiproxy_remove_toxic, -) -from je_load_density.utils.chaos.toxiproxy import ( - reset_all as toxiproxy_reset_all, -) -from je_load_density.utils.ci_annotations.canary_analysis import canary_verdict -from je_load_density.utils.ci_annotations.github_actions import emit_github_annotations -from je_load_density.utils.dashboard.live_dashboard import ( - start_dashboard, - stop_dashboard, -) -from je_load_density.utils.data.db_fixtures import apply_fixture as apply_db_fixture -from je_load_density.utils.data.db_fixtures import run_teardown as run_db_teardown -from je_load_density.utils.data.factory import build_user, build_user_pool -from je_load_density.utils.data.pii_anonymizer import scrub as pii_scrub -from je_load_density.utils.data.pii_anonymizer import scrub_string as pii_scrub_string from je_load_density.utils.exception.exception_tags import ( add_command_exception_tag, executor_data_error, executor_list_error, ) from je_load_density.utils.exception.exceptions import LoadDensityTestExecuteException -from je_load_density.utils.generate_report.generate_allure_report import ( - generate_allure_report, -) -from je_load_density.utils.generate_report.generate_chart_report import ( - generate_chart_report, -) -from je_load_density.utils.generate_report.generate_cost_report import ( - generate_cost_report, -) -from je_load_density.utils.generate_report.generate_csv_report import ( - generate_csv_report, -) -from je_load_density.utils.generate_report.generate_cyclonedx_report import ( - generate_cyclonedx_report, -) -from je_load_density.utils.generate_report.generate_excel_report import ( - generate_excel_report, -) -from je_load_density.utils.generate_report.generate_histogram_report import ( - generate_histogram_report, -) -from je_load_density.utils.generate_report.generate_html_report import ( - generate_html, - generate_html_report, -) -from je_load_density.utils.generate_report.generate_json_report import ( - generate_json, - generate_json_report, -) -from je_load_density.utils.generate_report.generate_junit_report import ( - generate_junit_report, -) -from je_load_density.utils.generate_report.generate_pdf_report import ( - generate_pdf_report, -) -from je_load_density.utils.generate_report.generate_sarif_report import ( - generate_sarif_report, -) -from je_load_density.utils.generate_report.generate_service_map import ( - generate_service_map, -) -from je_load_density.utils.generate_report.generate_summary_report import ( - build_summary, - generate_summary_report, -) -from je_load_density.utils.generate_report.generate_xml_report import ( - generate_xml, - generate_xml_report, -) -from je_load_density.utils.governance.audit_log import ( - append_audit_entry, - read_audit_log, -) -from je_load_density.utils.governance.run_tagging import ( - list_tags, - search_runs_by_tag, - tag_run, -) -from je_load_density.utils.governance.share_link import ( - issue_share_link, - verify_share_link, -) -from je_load_density.utils.governance.test_catalog import index_catalog, search_catalog -from je_load_density.utils.json.json_file.json_file import read_action_json -from je_load_density.utils.json.json_file.toml_file import ( - read_action_toml, - write_action_toml, -) -from je_load_density.utils.json.json_file.yaml_file import ( - read_action_yaml, - write_action_yaml, -) -from je_load_density.utils.linter.action_formatter import ( - format_action_document, - format_action_file, - format_action_string, -) -from je_load_density.utils.linter.action_linter import lint_action, lint_action_file -from je_load_density.utils.metrics.datadog_apm_exporter import ( - start_datadog_apm_exporter, - stop_datadog_apm_exporter, -) -from je_load_density.utils.metrics.influxdb_sink import ( - start_influxdb_sink, - stop_influxdb_sink, -) -from je_load_density.utils.metrics.opentelemetry_exporter import ( - start_opentelemetry_exporter, - stop_opentelemetry_exporter, -) -from je_load_density.utils.metrics.opentelemetry_tracing_exporter import ( - start_opentelemetry_tracing_exporter, - stop_opentelemetry_tracing_exporter, -) -from je_load_density.utils.metrics.prometheus_exporter import ( - start_prometheus_exporter, - stop_prometheus_exporter, -) -from je_load_density.utils.metrics.statsd_sink import ( - start_statsd_sink, - stop_statsd_sink, -) -from je_load_density.utils.notifier.gitlab import post_gitlab_mr_summary -from je_load_density.utils.notifier.opsgenie import post_opsgenie_alert -from je_load_density.utils.notifier.pagerduty import post_pagerduty_event -from je_load_density.utils.notifier.slack import post_slack_summary -from je_load_density.utils.notifier.teams import post_teams_summary from je_load_density.utils.package_manager.package_manager_class import package_manager from je_load_density.utils.parameterization import ( parameter_resolver, @@ -189,96 +28,211 @@ register_variable, register_variables, ) -from je_load_density.utils.recording.cdp_capture import ( - capture_cdp_session, - capture_cdp_to_har, -) -from je_load_density.utils.recording.cdp_capture import ( - discover_targets as cdp_discover_targets, -) -from je_load_density.utils.recording.curl_importer import curl_to_task -from je_load_density.utils.recording.har_importer import ( - har_to_action_json, - har_to_tasks, - load_har, -) -from je_load_density.utils.recording.jmeter_importer import ( - jmeter_to_action_json, - jmeter_to_tasks, - load_jmeter_jmx, -) -from je_load_density.utils.recording.k6_importer import ( - k6_script_to_action_json, - k6_script_to_tasks, - load_k6_script, -) -from je_load_density.utils.recording.openapi_importer import ( - load_openapi, - openapi_to_action_json, - openapi_to_tasks, -) -from je_load_density.utils.recording.postman_importer import ( - load_postman_collection, - postman_to_action_json, - postman_to_tasks, -) -from je_load_density.utils.regression.diff import diff_runs -from je_load_density.utils.regression.error_clustering import cluster_errors -from je_load_density.utils.regression.multi_run_trend import trend_runs -from je_load_density.utils.reliability.failure_budget import ( - install_failure_budget, - uninstall_failure_budget, -) -from je_load_density.utils.reliability.network_conditioner import ( - install_network_conditioner, - uninstall_network_conditioner, -) -from je_load_density.utils.schema.action_schema import export_schema -from je_load_density.utils.security.fuzz import ( - expand_task_fuzz, - mutate_json, - mutate_string, -) -from je_load_density.utils.security.graphql_checks import ( - build_alias_batching_attack, - build_depth_attack, - build_introspection_payload, - graphql_attack_pack, -) -from je_load_density.utils.security.jwt_attacks import ( - craft_alg_confusion_token, - craft_alg_none_token, - craft_expired_token, - craft_kid_traversal_token, -) -from je_load_density.utils.security.jwt_attacks import ( - craft_attack_pack as craft_jwt_attack_pack, -) -from je_load_density.utils.security.owasp_checks import run_owasp_checks -from je_load_density.utils.security.rate_limit_probe import probe_rate_limit -from je_load_density.utils.security.smuggling_checks import ( - build_cl_te, - build_te_cl, - build_te_te, - smuggling_attack_pack, -) -from je_load_density.utils.security.ssrf_checks import ( - build_ssrf_targets, - find_metadata_leak, - render_ssrf_tasks, -) -from je_load_density.utils.sla.sla_gates import assert_sla, evaluate_sla -from je_load_density.utils.stub_server.stub_server import ( - start_stub_server, - stop_stub_server, -) -from je_load_density.utils.test_record.sqlite_persistence import ( - fetch_run_records, - list_runs, - persist_records, -) from je_load_density.utils.test_record.test_record_class import test_record_instance -from je_load_density.wrapper.start_wrapper.start_test import start_test + + +def _lazy_command(module: str, attribute: str): + """Keep registry entries as functions while deferring their implementation import.""" + + def command(*args, **kwargs): + from importlib import import_module + + function = getattr(import_module(module), attribute) + return function(*args, **kwargs) + + command.__name__ = attribute + return command + + +append_audit_entry = _lazy_command("je_load_density.utils.governance.audit_log", "append_audit_entry") +apply_db_fixture = _lazy_command("je_load_density.utils.data.db_fixtures", "apply_fixture") +assert_sla = _lazy_command("je_load_density.utils.sla.sla_gates", "assert_sla") +build_alias_batching_attack = _lazy_command( + "je_load_density.utils.security.graphql_checks", "build_alias_batching_attack" +) +build_cl_te = _lazy_command("je_load_density.utils.security.smuggling_checks", "build_cl_te") +build_depth_attack = _lazy_command("je_load_density.utils.security.graphql_checks", "build_depth_attack") +build_introspection_payload = _lazy_command( + "je_load_density.utils.security.graphql_checks", "build_introspection_payload" +) +build_root_cause_prompt = _lazy_command("je_load_density.utils.ai.root_cause", "build_root_cause_prompt") +build_ssrf_targets = _lazy_command("je_load_density.utils.security.ssrf_checks", "build_ssrf_targets") +build_summary = _lazy_command("je_load_density.utils.generate_report.generate_summary_report", "build_summary") +build_te_cl = _lazy_command("je_load_density.utils.security.smuggling_checks", "build_te_cl") +build_te_te = _lazy_command("je_load_density.utils.security.smuggling_checks", "build_te_te") +build_user = _lazy_command("je_load_density.utils.data.factory", "build_user") +build_user_pool = _lazy_command("je_load_density.utils.data.factory", "build_user_pool") +calibrate_sla = _lazy_command("je_load_density.utils.ai.auto_baseline", "calibrate_sla") +canary_verdict = _lazy_command("je_load_density.utils.ci_annotations.canary_analysis", "canary_verdict") +capture_cdp_session = _lazy_command("je_load_density.utils.recording.cdp_capture", "capture_cdp_session") +capture_cdp_to_har = _lazy_command("je_load_density.utils.recording.cdp_capture", "capture_cdp_to_har") +cdp_discover_targets = _lazy_command("je_load_density.utils.recording.cdp_capture", "discover_targets") +chaos_apply_manifest = _lazy_command("je_load_density.utils.chaos.chaos_mesh", "apply_manifest") +chaos_delete_manifest = _lazy_command("je_load_density.utils.chaos.chaos_mesh", "delete_manifest") +chaos_network_delay = _lazy_command("je_load_density.utils.chaos.chaos_mesh", "build_network_delay") +cluster_errors = _lazy_command("je_load_density.utils.regression.error_clustering", "cluster_errors") +craft_alg_confusion_token = _lazy_command("je_load_density.utils.security.jwt_attacks", "craft_alg_confusion_token") +craft_alg_none_token = _lazy_command("je_load_density.utils.security.jwt_attacks", "craft_alg_none_token") +craft_expired_token = _lazy_command("je_load_density.utils.security.jwt_attacks", "craft_expired_token") +craft_jwt_attack_pack = _lazy_command("je_load_density.utils.security.jwt_attacks", "craft_attack_pack") +craft_kid_traversal_token = _lazy_command("je_load_density.utils.security.jwt_attacks", "craft_kid_traversal_token") +curl_to_task = _lazy_command("je_load_density.utils.recording.curl_importer", "curl_to_task") +diff_runs = _lazy_command("je_load_density.utils.regression.diff", "diff_runs") +emit_github_annotations = _lazy_command( + "je_load_density.utils.ci_annotations.github_actions", "emit_github_annotations" +) +evaluate_sla = _lazy_command("je_load_density.utils.sla.sla_gates", "evaluate_sla") +expand_task_fuzz = _lazy_command("je_load_density.utils.security.fuzz", "expand_task_fuzz") +export_schema = _lazy_command("je_load_density.utils.schema.action_schema", "export_schema") +fetch_run_records = _lazy_command("je_load_density.utils.test_record.sqlite_persistence", "fetch_run_records") +find_breaking_point = _lazy_command("je_load_density.utils.ai.smart_shape", "find_breaking_point") +find_metadata_leak = _lazy_command("je_load_density.utils.security.ssrf_checks", "find_metadata_leak") +format_action_document = _lazy_command("je_load_density.utils.linter.action_formatter", "format_action_document") +format_action_file = _lazy_command("je_load_density.utils.linter.action_formatter", "format_action_file") +format_action_string = _lazy_command("je_load_density.utils.linter.action_formatter", "format_action_string") +generate_allure_report = _lazy_command( + "je_load_density.utils.generate_report.generate_allure_report", "generate_allure_report" +) +generate_chart_report = _lazy_command( + "je_load_density.utils.generate_report.generate_chart_report", "generate_chart_report" +) +generate_cost_report = _lazy_command( + "je_load_density.utils.generate_report.generate_cost_report", "generate_cost_report" +) +generate_csv_report = _lazy_command("je_load_density.utils.generate_report.generate_csv_report", "generate_csv_report") +generate_cyclonedx_report = _lazy_command( + "je_load_density.utils.generate_report.generate_cyclonedx_report", "generate_cyclonedx_report" +) +generate_excel_report = _lazy_command( + "je_load_density.utils.generate_report.generate_excel_report", "generate_excel_report" +) +generate_from_curls = _lazy_command("je_load_density.utils.action_generator.generate", "generate_from_curls") +generate_from_openapi = _lazy_command("je_load_density.utils.action_generator.generate", "generate_from_openapi") +generate_histogram_report = _lazy_command( + "je_load_density.utils.generate_report.generate_histogram_report", "generate_histogram_report" +) +generate_html = _lazy_command("je_load_density.utils.generate_report.generate_html_report", "generate_html") +generate_html_report = _lazy_command( + "je_load_density.utils.generate_report.generate_html_report", "generate_html_report" +) +generate_json = _lazy_command("je_load_density.utils.generate_report.generate_json_report", "generate_json") +generate_json_report = _lazy_command( + "je_load_density.utils.generate_report.generate_json_report", "generate_json_report" +) +generate_junit_report = _lazy_command( + "je_load_density.utils.generate_report.generate_junit_report", "generate_junit_report" +) +generate_pdf_report = _lazy_command("je_load_density.utils.generate_report.generate_pdf_report", "generate_pdf_report") +generate_sarif_report = _lazy_command( + "je_load_density.utils.generate_report.generate_sarif_report", "generate_sarif_report" +) +generate_service_map = _lazy_command( + "je_load_density.utils.generate_report.generate_service_map", "generate_service_map" +) +generate_summary_report = _lazy_command( + "je_load_density.utils.generate_report.generate_summary_report", "generate_summary_report" +) +generate_xml = _lazy_command("je_load_density.utils.generate_report.generate_xml_report", "generate_xml") +generate_xml_report = _lazy_command("je_load_density.utils.generate_report.generate_xml_report", "generate_xml_report") +graphql_attack_pack = _lazy_command("je_load_density.utils.security.graphql_checks", "graphql_attack_pack") +har_to_action_json = _lazy_command("je_load_density.utils.recording.har_importer", "har_to_action_json") +har_to_tasks = _lazy_command("je_load_density.utils.recording.har_importer", "har_to_tasks") +index_catalog = _lazy_command("je_load_density.utils.governance.test_catalog", "index_catalog") +install_failure_budget = _lazy_command("je_load_density.utils.reliability.failure_budget", "install_failure_budget") +install_network_conditioner = _lazy_command( + "je_load_density.utils.reliability.network_conditioner", "install_network_conditioner" +) +issue_share_link = _lazy_command("je_load_density.utils.governance.share_link", "issue_share_link") +jmeter_to_action_json = _lazy_command("je_load_density.utils.recording.jmeter_importer", "jmeter_to_action_json") +jmeter_to_tasks = _lazy_command("je_load_density.utils.recording.jmeter_importer", "jmeter_to_tasks") +k6_script_to_action_json = _lazy_command("je_load_density.utils.recording.k6_importer", "k6_script_to_action_json") +k6_script_to_tasks = _lazy_command("je_load_density.utils.recording.k6_importer", "k6_script_to_tasks") +lint_action = _lazy_command("je_load_density.utils.linter.action_linter", "lint_action") +lint_action_file = _lazy_command("je_load_density.utils.linter.action_linter", "lint_action_file") +list_runs = _lazy_command("je_load_density.utils.test_record.sqlite_persistence", "list_runs") +list_tags = _lazy_command("je_load_density.utils.governance.run_tagging", "list_tags") +load_har = _lazy_command("je_load_density.utils.recording.har_importer", "load_har") +load_jmeter_jmx = _lazy_command("je_load_density.utils.recording.jmeter_importer", "load_jmeter_jmx") +load_k6_script = _lazy_command("je_load_density.utils.recording.k6_importer", "load_k6_script") +load_openapi = _lazy_command("je_load_density.utils.recording.openapi_importer", "load_openapi") +load_postman_collection = _lazy_command("je_load_density.utils.recording.postman_importer", "load_postman_collection") +mutate_json = _lazy_command("je_load_density.utils.security.fuzz", "mutate_json") +mutate_string = _lazy_command("je_load_density.utils.security.fuzz", "mutate_string") +openapi_to_action_json = _lazy_command("je_load_density.utils.recording.openapi_importer", "openapi_to_action_json") +openapi_to_tasks = _lazy_command("je_load_density.utils.recording.openapi_importer", "openapi_to_tasks") +persist_records = _lazy_command("je_load_density.utils.test_record.sqlite_persistence", "persist_records") +pii_scrub = _lazy_command("je_load_density.utils.data.pii_anonymizer", "scrub") +pii_scrub_string = _lazy_command("je_load_density.utils.data.pii_anonymizer", "scrub_string") +post_gitlab_mr_summary = _lazy_command("je_load_density.utils.notifier.gitlab", "post_gitlab_mr_summary") +post_opsgenie_alert = _lazy_command("je_load_density.utils.notifier.opsgenie", "post_opsgenie_alert") +post_pagerduty_event = _lazy_command("je_load_density.utils.notifier.pagerduty", "post_pagerduty_event") +post_slack_summary = _lazy_command("je_load_density.utils.notifier.slack", "post_slack_summary") +post_teams_summary = _lazy_command("je_load_density.utils.notifier.teams", "post_teams_summary") +postman_to_action_json = _lazy_command("je_load_density.utils.recording.postman_importer", "postman_to_action_json") +postman_to_tasks = _lazy_command("je_load_density.utils.recording.postman_importer", "postman_to_tasks") +probe_rate_limit = _lazy_command("je_load_density.utils.security.rate_limit_probe", "probe_rate_limit") +read_action_json = _lazy_command("je_load_density.utils.json.json_file.json_file", "read_action_json") +read_action_toml = _lazy_command("je_load_density.utils.json.json_file.toml_file", "read_action_toml") +read_action_yaml = _lazy_command("je_load_density.utils.json.json_file.yaml_file", "read_action_yaml") +read_audit_log = _lazy_command("je_load_density.utils.governance.audit_log", "read_audit_log") +render_prompt_text = _lazy_command("je_load_density.utils.ai.root_cause", "render_prompt_text") +render_ssrf_tasks = _lazy_command("je_load_density.utils.security.ssrf_checks", "render_ssrf_tasks") +run_db_teardown = _lazy_command("je_load_density.utils.data.db_fixtures", "run_teardown") +run_owasp_checks = _lazy_command("je_load_density.utils.security.owasp_checks", "run_owasp_checks") +search_catalog = _lazy_command("je_load_density.utils.governance.test_catalog", "search_catalog") +search_runs_by_tag = _lazy_command("je_load_density.utils.governance.run_tagging", "search_runs_by_tag") +smuggling_attack_pack = _lazy_command("je_load_density.utils.security.smuggling_checks", "smuggling_attack_pack") +start_dashboard = _lazy_command("je_load_density.utils.dashboard.live_dashboard", "start_dashboard") +start_datadog_apm_exporter = _lazy_command( + "je_load_density.utils.metrics.datadog_apm_exporter", "start_datadog_apm_exporter" +) +start_influxdb_sink = _lazy_command("je_load_density.utils.metrics.influxdb_sink", "start_influxdb_sink") +start_opentelemetry_exporter = _lazy_command( + "je_load_density.utils.metrics.opentelemetry_exporter", "start_opentelemetry_exporter" +) +start_opentelemetry_tracing_exporter = _lazy_command( + "je_load_density.utils.metrics.opentelemetry_tracing_exporter", "start_opentelemetry_tracing_exporter" +) +start_prometheus_exporter = _lazy_command( + "je_load_density.utils.metrics.prometheus_exporter", "start_prometheus_exporter" +) +start_statsd_sink = _lazy_command("je_load_density.utils.metrics.statsd_sink", "start_statsd_sink") +start_stub_server = _lazy_command("je_load_density.utils.stub_server.stub_server", "start_stub_server") +start_test = _lazy_command("je_load_density.engine.entrypoints", "start_test") +stop_dashboard = _lazy_command("je_load_density.utils.dashboard.live_dashboard", "stop_dashboard") +stop_datadog_apm_exporter = _lazy_command( + "je_load_density.utils.metrics.datadog_apm_exporter", "stop_datadog_apm_exporter" +) +stop_influxdb_sink = _lazy_command("je_load_density.utils.metrics.influxdb_sink", "stop_influxdb_sink") +stop_opentelemetry_exporter = _lazy_command( + "je_load_density.utils.metrics.opentelemetry_exporter", "stop_opentelemetry_exporter" +) +stop_opentelemetry_tracing_exporter = _lazy_command( + "je_load_density.utils.metrics.opentelemetry_tracing_exporter", "stop_opentelemetry_tracing_exporter" +) +stop_prometheus_exporter = _lazy_command( + "je_load_density.utils.metrics.prometheus_exporter", "stop_prometheus_exporter" +) +stop_statsd_sink = _lazy_command("je_load_density.utils.metrics.statsd_sink", "stop_statsd_sink") +stop_stub_server = _lazy_command("je_load_density.utils.stub_server.stub_server", "stop_stub_server") +tag_run = _lazy_command("je_load_density.utils.governance.run_tagging", "tag_run") +toxiproxy_add_toxic = _lazy_command("je_load_density.utils.chaos.toxiproxy", "add_toxic") +toxiproxy_create_proxy = _lazy_command("je_load_density.utils.chaos.toxiproxy", "create_proxy") +toxiproxy_install_bandwidth = _lazy_command("je_load_density.utils.chaos.toxiproxy", "install_bandwidth") +toxiproxy_install_latency = _lazy_command("je_load_density.utils.chaos.toxiproxy", "install_latency") +toxiproxy_list_proxies = _lazy_command("je_load_density.utils.chaos.toxiproxy", "list_proxies") +toxiproxy_remove_proxies = _lazy_command("je_load_density.utils.chaos.toxiproxy", "remove_proxies") +toxiproxy_remove_toxic = _lazy_command("je_load_density.utils.chaos.toxiproxy", "remove_toxic") +toxiproxy_reset_all = _lazy_command("je_load_density.utils.chaos.toxiproxy", "reset_all") +trend_runs = _lazy_command("je_load_density.utils.regression.multi_run_trend", "trend_runs") +uninstall_failure_budget = _lazy_command("je_load_density.utils.reliability.failure_budget", "uninstall_failure_budget") +uninstall_network_conditioner = _lazy_command( + "je_load_density.utils.reliability.network_conditioner", "uninstall_network_conditioner" +) +verify_share_link = _lazy_command("je_load_density.utils.governance.share_link", "verify_share_link") +write_action_toml = _lazy_command("je_load_density.utils.json.json_file.toml_file", "write_action_toml") +write_action_yaml = _lazy_command("je_load_density.utils.json.json_file.yaml_file", "write_action_yaml") + # Builtins: je_action_core's SAFE_BUILTINS allowlist, the same names every workspace framework registers # (workspace X-12). The name stays exported here. @@ -287,8 +241,13 @@ # The document key and error texts; the dispatch is je_action_core's (workspace L-6). Records are printed: # a failure's repr and action to stderr, then every record's key and value to stdout. _SETTINGS = ExecutorSettings( - rules=ActionListRules("load_density", error=LoadDensityTestExecuteException, missing_message=executor_list_error, - not_list_message=executor_list_error, empty_message=executor_list_error), + rules=ActionListRules( + "load_density", + error=LoadDensityTestExecuteException, + missing_message=executor_list_error, + not_list_message=executor_list_error, + empty_message=executor_list_error, + ), parser=LegacyActionParser(error=LoadDensityTestExecuteException, message=executor_data_error), reporter=PrintReporter(), read_json=read_action_json, @@ -309,6 +268,7 @@ def _lazy_start_socket_server(*args, **kwargs): from je_load_density.utils.socket_server.load_density_socket_server import ( start_load_density_socket_server, ) + return start_load_density_socket_server(*args, **kwargs) @@ -319,16 +279,19 @@ class Executor(ActionExecutor): """ def __init__(self) -> None: - super().__init__(_SETTINGS, CommandRegistry( - policy=CommandPolicy.FUNCTIONS_ONLY, - rejection=lambda _name: LoadDensityTestExecuteException(add_command_exception_tag))) + super().__init__( + _SETTINGS, + CommandRegistry( + policy=CommandPolicy.FUNCTIONS_ONLY, + rejection=lambda _name: LoadDensityTestExecuteException(add_command_exception_tag), + ), + ) self.event_dict = { # Core "LD_start_test": start_test, "LD_execute_action": self.execute_action, "LD_execute_files": self.execute_files, "LD_add_package_to_executor": package_manager.add_package_to_executor, - # Reports "LD_generate_html": generate_html, "LD_generate_html_report": generate_html_report, @@ -341,20 +304,17 @@ def __init__(self) -> None: "LD_generate_summary_report": generate_summary_report, "LD_generate_chart_report": generate_chart_report, "LD_summary": build_summary, - # Test record persistence "LD_persist_records": persist_records, "LD_list_runs": list_runs, "LD_fetch_run_records": fetch_run_records, "LD_clear_records": _clear_records, - # Parameter resolver "LD_register_variable": register_variable, "LD_register_variables": register_variables, "LD_register_csv_source": register_csv_source, "LD_register_csv_sources": register_csv_sources, "LD_clear_resolver": _clear_resolver, - # Recording / replay "LD_load_har": load_har, "LD_har_to_tasks": har_to_tasks, @@ -372,13 +332,11 @@ def __init__(self) -> None: "LD_load_jmeter_jmx": load_jmeter_jmx, "LD_jmeter_to_tasks": jmeter_to_tasks, "LD_jmeter_to_action_json": jmeter_to_action_json, - # Reliability "LD_install_failure_budget": install_failure_budget, "LD_uninstall_failure_budget": uninstall_failure_budget, "LD_install_network_conditioner": install_network_conditioner, "LD_uninstall_network_conditioner": uninstall_network_conditioner, - # Glue "LD_start_dashboard": start_dashboard, "LD_stop_dashboard": stop_dashboard, @@ -386,22 +344,18 @@ def __init__(self) -> None: "LD_stop_statsd_sink": stop_statsd_sink, "LD_post_slack_summary": post_slack_summary, "LD_post_teams_summary": post_teams_summary, - # Quality / DX "LD_lint_action": lint_action, "LD_lint_action_file": lint_action_file, "LD_export_schema": export_schema, "LD_emit_github_annotations": emit_github_annotations, - # SLA / regression "LD_evaluate_sla": evaluate_sla, "LD_assert_sla": assert_sla, "LD_diff_runs": diff_runs, - # Parameter resolver (DB extension) "LD_register_db_source": register_db_source, "LD_register_db_sources": register_db_sources, - # Metrics exporters "LD_start_prometheus_exporter": start_prometheus_exporter, "LD_stop_prometheus_exporter": stop_prometheus_exporter, @@ -409,53 +363,43 @@ def __init__(self) -> None: "LD_stop_influxdb_sink": stop_influxdb_sink, "LD_start_opentelemetry_exporter": start_opentelemetry_exporter, "LD_stop_opentelemetry_exporter": stop_opentelemetry_exporter, - # Control socket "LD_start_socket_server": _lazy_start_socket_server, - # New report formats "LD_generate_histogram_report": generate_histogram_report, "LD_generate_pdf_report": generate_pdf_report, "LD_generate_allure_report": generate_allure_report, - # YAML / TOML loaders "LD_read_action_yaml": read_action_yaml, "LD_write_action_yaml": write_action_yaml, "LD_read_action_toml": read_action_toml, "LD_write_action_toml": write_action_toml, - # Action formatter / generator "LD_format_action_document": format_action_document, "LD_format_action_file": format_action_file, "LD_format_action_string": format_action_string, "LD_generate_from_openapi": generate_from_openapi, "LD_generate_from_curls": generate_from_curls, - # Trend & clustering "LD_trend_runs": trend_runs, "LD_cluster_errors": cluster_errors, - # Notifiers (new) "LD_post_pagerduty_event": post_pagerduty_event, "LD_post_opsgenie_alert": post_opsgenie_alert, "LD_post_gitlab_mr_summary": post_gitlab_mr_summary, - # OTel tracing & Datadog APM "LD_start_opentelemetry_tracing_exporter": start_opentelemetry_tracing_exporter, "LD_stop_opentelemetry_tracing_exporter": stop_opentelemetry_tracing_exporter, "LD_start_datadog_apm_exporter": start_datadog_apm_exporter, "LD_stop_datadog_apm_exporter": stop_datadog_apm_exporter, - # Stub server "LD_start_stub_server": start_stub_server, "LD_stop_stub_server": stop_stub_server, - # Security "LD_mutate_string": mutate_string, "LD_mutate_json": mutate_json, "LD_expand_task_fuzz": expand_task_fuzz, "LD_run_owasp_checks": run_owasp_checks, - # Chaos engineering "LD_toxiproxy_create_proxy": toxiproxy_create_proxy, "LD_toxiproxy_list_proxies": toxiproxy_list_proxies, @@ -468,20 +412,17 @@ def __init__(self) -> None: "LD_chaos_apply_manifest": chaos_apply_manifest, "LD_chaos_delete_manifest": chaos_delete_manifest, "LD_chaos_network_delay": chaos_network_delay, - # AI "LD_build_root_cause_prompt": build_root_cause_prompt, "LD_render_prompt_text": render_prompt_text, "LD_find_breaking_point": find_breaking_point, "LD_calibrate_sla": calibrate_sla, - # New reports "LD_generate_excel_report": generate_excel_report, "LD_generate_sarif_report": generate_sarif_report, "LD_generate_cyclonedx_report": generate_cyclonedx_report, "LD_generate_service_map": generate_service_map, "LD_generate_cost_report": generate_cost_report, - # Data / state "LD_pii_scrub": pii_scrub, "LD_pii_scrub_string": pii_scrub_string, @@ -489,7 +430,6 @@ def __init__(self) -> None: "LD_build_user_pool": build_user_pool, "LD_apply_db_fixture": apply_db_fixture, "LD_run_db_teardown": run_db_teardown, - # Governance "LD_index_catalog": index_catalog, "LD_search_catalog": search_catalog, @@ -500,15 +440,12 @@ def __init__(self) -> None: "LD_read_audit_log": read_audit_log, "LD_issue_share_link": issue_share_link, "LD_verify_share_link": verify_share_link, - # Canary "LD_canary_verdict": canary_verdict, - # Recording "LD_cdp_discover_targets": cdp_discover_targets, "LD_cdp_capture_session": capture_cdp_session, "LD_cdp_capture_to_har": capture_cdp_to_har, - # Security: JWT attacks "LD_craft_alg_none_token": craft_alg_none_token, "LD_craft_alg_confusion_token": craft_alg_confusion_token, diff --git a/je_load_density/utils/generate_report/generate_summary_report.py b/je_load_density/utils/generate_report/generate_summary_report.py index a3cff02..ae63012 100644 --- a/je_load_density/utils/generate_report/generate_summary_report.py +++ b/je_load_density/utils/generate_report/generate_summary_report.py @@ -30,14 +30,14 @@ def _by_name(records: Iterable[Dict[str, Any]]) -> Dict[str, List[float]]: return grouped -def build_summary() -> Dict[str, Any]: +def build_summary(success=None, failures=None) -> Dict[str, Any]: """ 彙整成功與失敗紀錄為統計摘要。 Build a summary dict of success/failure counts and per-name latency percentiles for charting and regression checks. """ - success = test_record_instance.test_record_list - failures = test_record_instance.error_record_list + success = test_record_instance.test_record_list if success is None else success + failures = test_record_instance.error_record_list if failures is None else failures all_latencies: List[float] = [ latency diff --git a/je_load_density/utils/parameterization/__init__.py b/je_load_density/utils/parameterization/__init__.py index 1b1972c..33f777d 100644 --- a/je_load_density/utils/parameterization/__init__.py +++ b/je_load_density/utils/parameterization/__init__.py @@ -1,17 +1,22 @@ from je_load_density.utils.parameterization.parameter_resolver import ( ParameterResolver, + get_resolver, parameter_resolver, register_csv_source, register_csv_sources, register_db_source, register_db_sources, + register_session_variable, register_variable, register_variables, resolve, + use_resolver, ) __all__ = [ "ParameterResolver", + "get_resolver", + "use_resolver", "parameter_resolver", "resolve", "register_csv_source", @@ -19,5 +24,6 @@ "register_db_source", "register_db_sources", "register_variable", + "register_session_variable", "register_variables", ] diff --git a/je_load_density/utils/parameterization/parameter_resolver.py b/je_load_density/utils/parameterization/parameter_resolver.py index 50687a9..097d94c 100644 --- a/je_load_density/utils/parameterization/parameter_resolver.py +++ b/je_load_density/utils/parameterization/parameter_resolver.py @@ -1,9 +1,12 @@ +import copy import csv import itertools import os import re import threading -from typing import Any, Dict, Iterable, Iterator, List, Optional +from contextlib import contextmanager +from contextvars import ContextVar +from typing import Any, Dict, Iterable, Iterator, List, Optional, Tuple _PLACEHOLDER_PATTERN = re.compile(r"\$\{([^}]+)\}") _FUNCTION_PATTERN = re.compile(r"^([a-zA-Z_]\w*)\((.*)\)$") @@ -13,6 +16,21 @@ def _stringify(value: Any) -> Optional[str]: return None if value is None else str(value) +class _RowProvider: + """Serialize advancement across all users sharing a cached data source.""" + + def __init__(self, rows: List[Dict[str, Any]], cycle: bool) -> None: + self._rows = itertools.cycle(rows) if cycle else iter(rows) + self._lock = threading.Lock() + + def next_row(self) -> Optional[Dict[str, Any]]: + with self._lock: + return copy.deepcopy(next(self._rows, None)) + + +_ResolvedRows = Dict[Tuple[str, str], Optional[Dict[str, Any]]] + + class ParameterResolver: """ 參數解析器 @@ -21,7 +39,8 @@ class ParameterResolver: Supports: ${env.NAME} -> environment variable ${var.key} -> registered variable - ${csv.source.column} -> next row from CSV source (cycled) + ${session.key} -> per-user session variable + ${csv.source.column} -> one row per source per resolve call (cycled) ${faker.method} -> faker output (if faker installed) ${func(arg)} -> built-in helpers (uuid, now, randint(min,max)) @@ -30,31 +49,49 @@ class ParameterResolver: def __init__(self) -> None: self._variables: Dict[str, Any] = {} - self._csv_sources: Dict[str, Iterator[Dict[str, str]]] = {} - self._db_sources: Dict[str, Iterator[Dict[str, Any]]] = {} + self._session_variables: Dict[str, Any] = {} + self._csv_sources: Dict[str, _RowProvider] = {} + self._db_sources: Dict[str, _RowProvider] = {} self._lock = threading.Lock() self._faker = None def register_variable(self, name: str, value: Any) -> None: + """Set a variable in this resolver, without changing another user's state.""" with self._lock: self._variables[name] = value + def register_session_variable(self, name: str, value: Any) -> None: + """Set a value accessible through the separate ${session.NAME} namespace.""" + with self._lock: + self._session_variables[name] = value + + def fork(self) -> "ParameterResolver": + """Copy variable/session state; share synchronized CSV/DB row providers.""" + child = ParameterResolver() + with self._lock: + child._variables = copy.deepcopy(self._variables) + child._session_variables = copy.deepcopy(self._session_variables) + child._csv_sources = dict(self._csv_sources) + child._db_sources = dict(self._db_sources) + return child + def register_csv_source(self, name: str, file_path: str, cycle: bool = True) -> None: + """Cache CSV rows in a provider shared by subsequently forked users.""" rows = self._read_csv(file_path) with self._lock: - self._csv_sources[name] = itertools.cycle(rows) if cycle else iter(rows) + self._csv_sources[name] = _RowProvider(rows, cycle) def register_db_source(self, name: str, connection_string: str, query: str, cycle: bool = True) -> None: """ Register a parameter source backed by a SQL query. - Each ``${db.NAME.column}`` placeholder pulls the next row from - the cached result set. SQLAlchemy is a soft dependency. + A recursive resolve call reuses one row for every ``${db.NAME.column}`` + field. Forked resolvers share row advancement. SQLAlchemy is a soft dependency. """ rows = self._read_db(connection_string, query) with self._lock: - self._db_sources[name] = itertools.cycle(rows) if cycle else iter(rows) + self._db_sources[name] = _RowProvider(rows, cycle) @staticmethod def _read_db(connection_string: str, query: str) -> List[Dict[str, Any]]: @@ -65,10 +102,13 @@ def _read_db(connection_string: str, query: str) -> List[Dict[str, Any]]: "SQLAlchemy is required for ${db.*}; install with: pip install sqlalchemy" ) from error engine = create_engine(connection_string, future=True) - with engine.connect() as connection: - result = connection.execute(text(query)) - keys = list(result.keys()) - return [dict(zip(keys, row)) for row in result.fetchall()] + try: + with engine.connect() as connection: + result = connection.execute(text(query)) + keys = list(result.keys()) + return [dict(zip(keys, row)) for row in result.fetchall()] + finally: + engine.dispose() @staticmethod def _read_csv(file_path: str) -> List[Dict[str, str]]: @@ -79,24 +119,19 @@ def _read_csv(file_path: str) -> List[Dict[str, str]]: def _next_csv_row(self, name: str) -> Optional[Dict[str, str]]: with self._lock: source = self._csv_sources.get(name) - if source is None: - return None - try: - return next(source) - except StopIteration: - return None + return source.next_row() if source is not None else None def _next_db_row(self, name: str) -> Optional[Dict[str, Any]]: with self._lock: source = self._db_sources.get(name) - if source is None: - return None - try: - return next(source) - except StopIteration: - return None + return source.next_row() if source is not None else None - def _resolve_token(self, token: str) -> Optional[str]: + def _variable(self, name: str, session: bool = False) -> Optional[str]: + with self._lock: + values = self._session_variables if session else self._variables + return _stringify(values.get(name)) + + def _resolve_token(self, token: str, rows: _ResolvedRows) -> Optional[str]: token = token.strip() if not token: return None @@ -106,32 +141,40 @@ def _resolve_token(self, token: str) -> Optional[str]: return self._resolve_function(function_match.group(1), function_match.group(2)) if "." not in token: - return _stringify(self._variables.get(token)) + return self._variable(token) prefix, _, rest = token.partition(".") - return self._resolve_prefixed(prefix.lower(), rest) + return self._resolve_prefixed(prefix.lower(), rest, rows) - def _resolve_prefixed(self, prefix: str, rest: str) -> Optional[str]: + def _resolve_prefixed(self, prefix: str, rest: str, rows: _ResolvedRows) -> Optional[str]: if prefix == "env": return os.environ.get(rest) if prefix == "var": - return _stringify(self._variables.get(rest)) + return self._variable(rest) + if prefix == "session": + return self._variable(rest, session=True) if prefix == "csv": - return self._resolve_csv(rest) + return self._resolve_csv(rest, rows) if prefix == "db": - return self._resolve_db(rest) + return self._resolve_db(rest, rows) if prefix == "faker": return self._resolve_faker(rest) return None - def _resolve_csv(self, rest: str) -> Optional[str]: + def _resolve_csv(self, rest: str, rows: _ResolvedRows) -> Optional[str]: source_name, _, column = rest.partition(".") - row = self._next_csv_row(source_name) + key = ("csv", source_name) + if key not in rows: + rows[key] = self._next_csv_row(source_name) + row = rows[key] return None if row is None else row.get(column) - def _resolve_db(self, rest: str) -> Optional[str]: + def _resolve_db(self, rest: str, rows: _ResolvedRows) -> Optional[str]: source_name, _, column = rest.partition(".") - row = self._next_db_row(source_name) + key = ("db", source_name) + if key not in rows: + rows[key] = self._next_db_row(source_name) + row = rows[key] if row is None: return None return _stringify(row.get(column)) @@ -169,53 +212,119 @@ def _resolve_faker(self, method: str) -> Optional[str]: def resolve(self, value: Any) -> Any: """ Recursively resolve placeholders inside strings, dicts, lists, and tuples. - Non-string scalar types pass through unchanged. + One row per source is reused throughout this call. Non-string scalar + types pass through unchanged; separate calls advance to subsequent rows. """ + return self._resolve_value(value, {}) + + def _resolve_value(self, value: Any, rows: _ResolvedRows) -> Any: if isinstance(value, str): - return self._resolve_string(value) + return self._resolve_string(value, rows) if isinstance(value, dict): - return {k: self.resolve(v) for k, v in value.items()} + return {key: self._resolve_value(item, rows) for key, item in value.items()} if isinstance(value, list): - return [self.resolve(item) for item in value] + return [self._resolve_value(item, rows) for item in value] if isinstance(value, tuple): - return tuple(self.resolve(item) for item in value) + return tuple(self._resolve_value(item, rows) for item in value) return value - def _resolve_string(self, value: str) -> str: + def _resolve_string(self, value: str, rows: _ResolvedRows) -> str: def repl(match: re.Match) -> str: - resolved = self._resolve_token(match.group(1)) + resolved = self._resolve_token(match.group(1), rows) return match.group(0) if resolved is None else resolved return _PLACEHOLDER_PATTERN.sub(repl, value) def clear(self) -> None: + """Clear this resolver; already forked users retain their state and providers.""" with self._lock: self._variables.clear() + self._session_variables.clear() self._csv_sources.clear() self._db_sources.clear() -parameter_resolver = ParameterResolver() +_default_resolver = ParameterResolver() +_active_resolver: ContextVar[Optional[ParameterResolver]] = ContextVar("load_density_parameter_resolver", default=None) + + +def get_resolver() -> ParameterResolver: + """Return the explicit thread/task/greenlet resolver, or the legacy default.""" + selected = _active_resolver.get() + return _default_resolver if selected is None else selected + + +@contextmanager +def use_resolver(resolver: ParameterResolver) -> Iterator[ParameterResolver]: + """Select an explicit user's resolver for this scope and restore it on exit.""" + if not isinstance(resolver, ParameterResolver): + raise TypeError("resolver must be a ParameterResolver instance") + token = _active_resolver.set(resolver) + try: + yield resolver + finally: + _active_resolver.reset(token) + + +class _ResolverFacade: + """Retain legacy imports while selecting state when each method is called.""" + + def __getattr__(self, name: str) -> Any: + return getattr(get_resolver(), name) + + def resolve(self, value: Any) -> Any: + return get_resolver().resolve(value) + + def fork(self) -> ParameterResolver: + return get_resolver().fork() + + def register_variable(self, name: str, value: Any) -> None: + get_resolver().register_variable(name, value) + + def register_session_variable(self, name: str, value: Any) -> None: + get_resolver().register_session_variable(name, value) + + def register_csv_source(self, name: str, file_path: str, cycle: bool = True) -> None: + get_resolver().register_csv_source(name, file_path, cycle) + + def register_db_source(self, name: str, connection_string: str, query: str, cycle: bool = True) -> None: + get_resolver().register_db_source(name, connection_string, query, cycle) + + def clear(self) -> None: + get_resolver().clear() + + +parameter_resolver = _ResolverFacade() def resolve(value: Any) -> Any: + """Resolve a request using the selected resolver, falling back to legacy state.""" return parameter_resolver.resolve(value) def register_variable(name: str, value: Any) -> None: + """Register a variable in the selected resolver.""" parameter_resolver.register_variable(name, value) +def register_session_variable(name: str, value: Any) -> None: + """Register a variable in the selected resolver's session namespace.""" + parameter_resolver.register_session_variable(name, value) + + def register_csv_source(name: str, file_path: str, cycle: bool = True) -> None: + """Register cached CSV data in the selected resolver.""" parameter_resolver.register_csv_source(name, file_path, cycle) def register_variables(variables: Dict[str, Any]) -> None: + """Register a mapping of variables in the selected resolver.""" for key, value in variables.items(): parameter_resolver.register_variable(key, value) def register_csv_sources(sources: Iterable[Dict[str, Any]]) -> None: + """Register named CSV source specifications in the selected resolver.""" for source in sources: name = source.get("name") file_path = source.get("file_path") @@ -226,10 +335,12 @@ def register_csv_sources(sources: Iterable[Dict[str, Any]]) -> None: def register_db_source(name: str, connection_string: str, query: str, cycle: bool = True) -> None: + """Register cached query data in the selected resolver.""" parameter_resolver.register_db_source(name, connection_string, query, cycle) def register_db_sources(sources: Iterable[Dict[str, Any]]) -> None: + """Register named database source specifications in the selected resolver.""" for source in sources: name = source.get("name") connection_string = source.get("connection_string") diff --git a/je_load_density/utils/scenario/cookie_jar.py b/je_load_density/utils/scenario/cookie_jar.py index b4f5e90..8f80a16 100644 --- a/je_load_density/utils/scenario/cookie_jar.py +++ b/je_load_density/utils/scenario/cookie_jar.py @@ -1,29 +1,32 @@ """ Stateful per-virtual-user cookie jar. -Provides a thread-local store so concurrent Locust users keep their own -cookie state without sharing across users. Built on top of stdlib +Provides a context-local store so concurrent threads, tasks and Locust users +keep their own cookie state. Built on top of stdlib ``http.cookiejar``. """ +import asyncio import threading +from contextvars import ContextVar from http.cookiejar import CookieJar -from typing import Dict, Optional +from typing import Dict, Optional, Tuple - -_local = threading.local() +_CookieOwner = Tuple[int, Optional[int]] +_local: ContextVar[Optional[Tuple[_CookieOwner, CookieJar]]] = ContextVar("load_density_cookie_jar", default=None) _global_jars: Dict[int, CookieJar] = {} _lock = threading.Lock() def jar_for_user(user_id: Optional[int] = None) -> CookieJar: - """Return a CookieJar scoped to ``user_id`` (or current thread).""" + """Return a CookieJar scoped to ``user_id`` or the current execution context.""" if user_id is None: - jar = getattr(_local, "jar", None) - if jar is None: - jar = CookieJar() - _local.jar = jar - return jar + owner = _current_owner() + selected = _local.get() + if selected is None or selected[0] != owner: + selected = (owner, CookieJar()) + _local.set(selected) + return selected[1] with _lock: existing = _global_jars.get(user_id) @@ -33,11 +36,25 @@ def jar_for_user(user_id: Optional[int] = None) -> CookieJar: return existing +def _current_owner() -> _CookieOwner: + """Detach inherited task contexts while keeping thread and greenlet isolation.""" + try: + current_task = asyncio.current_task() + except RuntimeError: + current_task = None + return threading.get_ident(), None if current_task is None else id(current_task) + + +def _clear_current_jar() -> None: + selected = _local.get() + if selected is not None and selected[0] == _current_owner(): + selected[1].clear() + + def reset_user_jar(user_id: Optional[int] = None) -> None: - """Clear cookies for the given user (or current thread).""" + """Clear cookies for the given user or the current execution context.""" if user_id is None: - if hasattr(_local, "jar"): - _local.jar.clear() + _clear_current_jar() return with _lock: jar = _global_jars.get(user_id) @@ -51,5 +68,4 @@ def reset_all_jars() -> None: for jar in _global_jars.values(): jar.clear() _global_jars.clear() - if hasattr(_local, "jar"): - _local.jar.clear() + _clear_current_jar() diff --git a/je_load_density/utils/test_record/distributed_context.py b/je_load_density/utils/test_record/distributed_context.py new file mode 100644 index 0000000..62cebc7 --- /dev/null +++ b/je_load_density/utils/test_record/distributed_context.py @@ -0,0 +1,99 @@ +"""Explicit master aggregation composed of strict, independently owned worker contexts.""" +from __future__ import annotations + +import copy +import json +from collections.abc import Mapping, Sequence +from threading import RLock + +from je_action_core.request_context import RunContext +from je_action_core.request_record import RequestRecord, RequestRecordError, validate_request_record + + +class DistributedRunContext: + """Aggregate one run without weakening ActionCore's exact worker identity checks. + + Batch validation and global record-ID conflict checks complete before any child + context is mutated. Child contexts remain private to the aggregate; snapshots + share its lock and cannot observe a partially committed batch. + """ + + def __init__(self, source: str = "loaddensity", phase: str = "load", engine: str = "locust", + run_id: str | None = None) -> None: + validated_identity = RunContext(source, phase, engine, run_id=run_id) + self.run_id = validated_identity.run_id + self.source, self.phase, self.engine = source, phase, engine + self._workers: dict[str | None, RunContext] = {} + self._index: dict[str, tuple[str | None, str]] = {} + self._lock = RLock() + + def identity(self) -> dict[str, str]: + """Return public wire identity, independently of Core's private storage.""" + return {"run_id": self.run_id, "source": self.source, "phase": self.phase, "engine": self.engine} + + def ingest_batch(self, worker_id: str, records: Sequence[Mapping[str, object]]) -> list[RequestRecord]: + """Commit an entire authenticated-worker batch or reject it without mutation.""" + if not isinstance(worker_id, str) or not worker_id: + raise RequestRecordError("worker_id: transport identity must be a non-empty string") + return self._ingest(worker_id, records) + + def _ingest(self, worker_id: str | None, records: Sequence[Mapping[str, object]]) -> list[RequestRecord]: + validated = [validate_request_record(record) for record in records] + for record in validated: + self._check_identity(worker_id, record) + with self._lock: + new_records = self._select_new(validated) + staged = RunContext(self.source, self.phase, self.engine, worker_id=worker_id, run_id=self.run_id) + for record in new_records: + staged.append(record) + current = self._workers.get(worker_id) + if current is None: + self._workers[worker_id] = staged + else: + for record in new_records: + current.append(record) + for record in new_records: + self._index[record["record_id"]] = (worker_id, self._fingerprint(record)) + return copy.deepcopy(new_records) + + def _check_identity(self, worker_id: str | None, record: RequestRecord) -> None: + for field, expected in {**self.identity(), "worker_id": worker_id}.items(): + if record[field] != expected: + raise RequestRecordError(f"{field}: does not belong to this distributed run or worker") + + def _select_new(self, records: list[RequestRecord]) -> list[RequestRecord]: + staged = {} + for record in records: + identifier = record["record_id"] + fingerprint = self._fingerprint(record) + previous = self._index.get(identifier) + if previous is not None: + if previous[1] != fingerprint: + raise RequestRecordError("record_id: conflicting retry of an existing record") + continue + if identifier in staged and self._fingerprint(staged[identifier]) != fingerprint: + raise RequestRecordError("record_id: conflicting records in one batch") + staged[identifier] = record + return list(staged.values()) + + @staticmethod + def _fingerprint(record: RequestRecord) -> str: + return json.dumps(record, sort_keys=True, ensure_ascii=False, allow_nan=False) + + def capture(self, fields: Mapping[str, object]) -> RequestRecord: + """Capture an explicitly local master result while retaining the run-wide ID invariant.""" + local = RunContext(self.source, self.phase, self.engine, run_id=self.run_id) + record = local.capture(fields) + self._ingest(None, [record]) + return record + + def snapshot(self) -> list[RequestRecord]: + """Return a detached insertion-ordered snapshot across source workers.""" + with self._lock: + records = {record["record_id"]: record for context in self._workers.values() + for record in context.snapshot()} + return [records[identifier] for identifier in self._index] + + def to_json(self) -> str: + """Export validated canonical records without changing worker provenance.""" + return json.dumps(self.snapshot(), ensure_ascii=False, allow_nan=False) diff --git a/je_load_density/wrapper/create_locust_env/create_locust_env.py b/je_load_density/wrapper/create_locust_env/create_locust_env.py index e83afbc..d540604 100644 --- a/je_load_density/wrapper/create_locust_env/create_locust_env.py +++ b/je_load_density/wrapper/create_locust_env/create_locust_env.py @@ -1,3 +1,4 @@ +import sys from functools import partial from time import monotonic from typing import Any, Dict, Optional @@ -14,6 +15,7 @@ from je_load_density.utils.logging.loggin_instance import load_density_logger from je_load_density.wrapper.distributed_config import DistributedConfig, NativeHeartbeatScope from je_load_density.wrapper.distributed_health import DistributedHealth +from je_load_density.wrapper.event.request_hook import request_hook # noqa: F401 — register the selected Locust path setup_logging("INFO", None) @@ -148,17 +150,9 @@ def create_env( config = DistributedConfig.from_options(kwargs) shape_class = _resolve_shape(load_shape, shape_config) run_context = kwargs.pop("run_context", None) - if run_context is not None: - from je_load_density.wrapper.event.request_hook import request_hook - if another_event is not events: - raise ValueError("a canonical run context requires an isolated default event environment") - another_event = Events() - another_event.request.add_listener(partial(request_hook, record_run=run_context)) - if another_event is events and runner_mode != "local": - # Shared global event hooks would let one master's quit stop another run. - another_event = Events() - from je_load_density.wrapper.event.request_hook import request_hook - another_event.request.add_listener(request_hook) + distributed_records = kwargs.pop("distributed_records", False) + delivery = _delivery_options(runner_mode, run_context, distributed_records, kwargs) + another_event = _request_events(another_event, runner_mode, run_context, distributed_records) options = None scope = None if runner_mode != "local" or shape_class is not None: @@ -181,6 +175,9 @@ def create_env( if runner_mode == "master": env.distributed_health = DistributedHealth(env, config) env.load_density_tasks.spawn(env.distributed_health.monitor) + if delivery is not None: + from je_load_density.wrapper.distributed_records import attach_delivery + attach_delivery(env, delivery) if runner_mode != "worker": env.load_density_tasks.spawn(stats_printer(env.stats)) env.load_density_tasks.spawn(stats_history, env.runner) @@ -191,6 +188,30 @@ def create_env( return env +def _delivery_options(mode, context, enabled, options): + if type(enabled) is not bool: + raise ValueError("distributed_records must be a boolean") + if context is None and not enabled: + return None + from je_load_density.wrapper.distributed_records import delivery_options + return delivery_options(mode, context, enabled, options) + + +def _request_events(selected, mode, context, delivery): + if delivery and context is not None: + raise ValueError("workers receive canonical run identity from their master") + if context is not None and selected is not events: + raise ValueError("a canonical run context requires an isolated default event environment") + if selected is not events: + return selected + if mode == "local" and context is None: + return selected + isolated = Events() + if not delivery: + isolated.request.add_listener(partial(request_hook, record_run=context)) + return isolated + + def _resolve_shape(load_shape: Optional[str], shape_config: Optional[Dict[str, Any]]): if not load_shape: return None @@ -212,9 +233,12 @@ def _wait_for_workers(env, expected_workers: int, timeout: float = 60.0, """Wait for healthy ready workers; fail closed unless degraded startup is explicit.""" deadline = monotonic() + timeout while True: - connected = env.distributed_health.ready_count + delivery = getattr(env, "record_delivery", None) + connected = delivery.ready_count if delivery is not None else env.distributed_health.ready_count if connected >= expected_workers: return + if delivery is not None: + delivery.raise_if_failed() if stop_requested is not None and stop_requested(): return remaining = deadline - monotonic() @@ -253,6 +277,9 @@ def _stop_owned_run(env: Environment) -> None: env.cancellation_requested = True if env.startup_task is not None and not env.startup_task.dead: env.startup_task.kill(block=True) + delivery = getattr(env, "record_delivery", None) + if delivery is not None: + delivery.close() env.runner.quit() @@ -262,9 +289,13 @@ def cleanup_env(env: Environment) -> None: return env.load_density_closed = True health = getattr(env, "distributed_health", None) - if health is not None: + if health is not None and not getattr(env, "record_health_closed", False): health.close() + env.record_health_closed = True try: + delivery = getattr(env, "record_delivery", None) + if delivery is not None: + delivery.close() if env.runner is not None and env.runner.greenlet: env.runner.quit() finally: @@ -274,6 +305,8 @@ def cleanup_env(env: Environment) -> None: env.web_ui.stop() finally: _close_distributed_resources(env) + if delivery is not None and sys.exc_info()[0] is None: + delivery.raise_if_failed() def _close_distributed_resources(env: Environment) -> None: diff --git a/je_load_density/wrapper/distributed_records.py b/je_load_density/wrapper/distributed_records.py new file mode 100644 index 0000000..285b938 --- /dev/null +++ b/je_load_density/wrapper/distributed_records.py @@ -0,0 +1,650 @@ +"""Opt-in bounded canonical result delivery over Locust's native message channel.""" +from __future__ import annotations + +import copy +import json +import math +from collections import deque +from dataclasses import asdict, dataclass, field +from time import monotonic +from uuid import uuid4 + +import gevent +from je_action_core.request_record import validate_request_record +from locust.exception import RPCError + +from je_load_density.utils.test_record.contract import from_legacy_record +from je_load_density.utils.test_record.distributed_context import DistributedRunContext +from je_load_density.utils.test_record.test_record_class import test_record_instance + +HELLO = "ld_records_hello_v1" +CONFIG = "ld_records_config_v1" +READY = "ld_records_ready_v1" +BATCH = "ld_records_batch_v1" +ACK = "ld_records_ack_v1" +FLUSH = "ld_records_flush_v1" +DONE = "ld_records_done_v1" +FINISHED = "ld_records_finished_v1" +MAX_CONTROL_NONCE_BYTES = 128 +RETIRED_EPOCH_LIMIT = 128 +TERMINAL_FIELDS = frozenset(("protocol", "run_id", "worker_id", "epoch", "generation", "nonce", "pending")) + + +class RecordDeliveryError(RuntimeError): + """Canonical delivery is incomplete or violates its negotiated contract.""" + + +def _encoded(value: object) -> bytes: + return json.dumps(value, sort_keys=True, ensure_ascii=False, allow_nan=False, + separators=(",", ":")).encode("utf-8") + + +def _message_fits(data: dict, byte_limit: int) -> bool: + try: + return len(_encoded(data)) <= byte_limit + except (TypeError, ValueError): + return False + + +def _control_valid(data, limits: DeliveryLimits, fields: frozenset[str]) -> bool: + if not isinstance(data, dict) or set(data) != fields: + return False + nonce = data.get("nonce") + if not isinstance(nonce, str) or not nonce: + return False + try: + return len(nonce.encode("utf-8")) <= MAX_CONTROL_NONCE_BYTES and len(_encoded(data)) <= limits.max_batch_bytes + except (TypeError, ValueError): + return False + + +@dataclass(frozen=True) +class DeliveryLimits: + """Bound every buffered queue and serialized batch; retry interval also flushes new work.""" + + batch_size: int = 100 + max_batch_bytes: int = 262144 + max_record_bytes: int = 65536 + max_pending: int = 1000 + max_pending_bytes: int = 4194304 + interval: float = 0.1 + drain_timeout: float = 2.0 + + def __post_init__(self) -> None: + integers = (self.batch_size, self.max_batch_bytes, self.max_record_bytes, + self.max_pending, self.max_pending_bytes) + if any(type(value) is not int or value <= 0 for value in integers): + raise ValueError("record count and byte limits must be positive integers") + times = (self.interval, self.drain_timeout) + if any(type(value) not in (int, float) or not math.isfinite(value) or value <= 0 for value in times): + raise ValueError("record delivery intervals must be positive finite numbers") + if self.max_record_bytes >= self.max_batch_bytes or self.max_pending_bytes < self.max_record_bytes: + raise ValueError("record limits must fit within batch and pending byte limits") + + @classmethod + def from_options(cls, options: dict) -> DeliveryLimits: + """Read explicit bounds through the wrapper's existing keyword interface.""" + names = ("record_batch_size", "record_batch_bytes", "record_max_bytes", "record_max_pending", + "record_pending_bytes", "record_flush_interval", "record_drain_timeout") + defaults = cls() + values = (getattr(defaults, field) for field in cls.__dataclass_fields__) + return cls(*(options.get(name, default) for name, default in zip(names, values))) + + +class _Delivery: + def __init__(self, environment, limits: DeliveryLimits) -> None: + self.environment = environment + self.limits = limits + self.error: RecordDeliveryError | None = None + self.closed = False + self.send_failures = 0 + self._listeners = [] + self._drain_owner = None + + def _listen(self, hook, callback) -> None: + hook.add_listener(callback) + self._listeners.append((hook, callback)) + + def _send(self, kind: str, data: dict, worker_id: str | None = None) -> bool: + if not _message_fits(data, self.limits.max_batch_bytes): + self.send_failures += 1 + self._fail("canonical outgoing message exceeds byte limit or is not serializable") + return False + try: + with gevent.Timeout(min(self.limits.interval, self.limits.drain_timeout), False): + self.environment.runner.send_message(kind, copy.deepcopy(data), client_id=worker_id) + return True + except RPCError: + self.send_failures += 1 + return False + self.send_failures += 1 + return False + + def _fail(self, message: str, stop: bool = True) -> RecordDeliveryError: + if self.error is None: + self.error = RecordDeliveryError(message) + self.environment.process_exit_code = 1 + if self.environment.cancellation_error is None: + self.environment.cancellation_error = self.error + health = getattr(self.environment, "distributed_health", None) + if health is not None: + health.fail(message) + if stop: + self.environment.load_density_tasks.spawn(self.environment.runner.quit) + return self.error + + def raise_if_failed(self) -> None: + """Surface a delivery failure after the owning wrapper completes resource cleanup.""" + if self.error is not None: + raise self.error + + def _detach(self) -> None: + for hook, callback in self._listeners: + hook.remove_listener(callback) + + def _begin_close(self) -> bool: + if self.closed or self._drain_owner is gevent.getcurrent(): + return False + if self._drain_owner is not None: + while not self.closed: + gevent.sleep(self.limits.interval) + return False + self._drain_owner = gevent.getcurrent() + return True + + +class WorkerRecords(_Delivery): + """One bounded producer epoch; retry an identical in-flight batch until acknowledged.""" + + role = "worker" + + def __init__(self, environment, limits: DeliveryLimits) -> None: + super().__init__(environment, limits) + self.epoch = str(uuid4()) + self.worker_id = environment.runner.client_id + self.identity = None + self.generation = 0 + self.pending = deque() + self.pending_bytes = 0 + self.frame = None + self.seq = 1 + self.delivered = 0 + self.finalized = False + self._final_nonce = None + environment.runner.register_message(CONFIG, self.on_config) + environment.runner.register_message(ACK, self.on_ack) + environment.runner.register_message(FLUSH, self._on_flush) + environment.runner.register_message(FINISHED, self._on_finished) + self._listen(environment.events.test_stop, self._on_stop) + + def _base(self) -> dict: + return {"protocol": 1, "epoch": self.epoch, "worker_id": self.worker_id, + "generation": self.generation, "run_id": self.identity["run_id"] if self.identity else None} + + def on_config(self, msg, **_kwargs) -> None: + """Bind only this producer epoch to a validated master run identity.""" + if self.closed or not isinstance(msg.data, dict): + return + data = msg.data + if data.get("epoch") != self.epoch or data.get("worker_id") != self.worker_id: + return + try: + identity = {field: data[field] for field in ("run_id", "source", "phase", "engine")} + generation = data["generation"] + if type(data.get("protocol")) is not int or data["protocol"] != 1: + raise ValueError("invalid record handshake protocol") + if type(generation) is not int or generation <= 0: + raise ValueError("invalid record handshake generation") + if generation < self.generation: + return + from je_action_core.request_context import RunContext + RunContext(**identity, worker_id=self.worker_id) + if identity["engine"] != "locust": + raise ValueError("record handshake engine must be locust") + self._negotiate(data.get("limits")) + self._bind(identity, generation) + except (KeyError, TypeError, ValueError) as error: + self._fail(f"invalid canonical record handshake: {error}") + + def _negotiate(self, remote: dict | None) -> None: + if remote is None: + return + negotiated = DeliveryLimits(**remote) + local = asdict(self.limits) + self.limits = DeliveryLimits(**{key: min(value, getattr(negotiated, key)) for key, value in local.items()}) + if len(self.pending) > self.limits.max_pending or self.pending_bytes > self.limits.max_pending_bytes: + raise ValueError("pending records exceed negotiated limits") + + def _bind(self, identity: dict, generation: int) -> None: + if self.identity is not None and identity != self.identity and self.pending: + raise ValueError("master run changed with pending records") + if self.identity is not None and generation < self.generation: + return + if identity == self.identity and generation == self.generation: + self._send(READY, self._base()) + return + if self.identity is None: + rebound = [validate_request_record({**record, **identity}) for record, _size in self.pending] + sizes = [len(_encoded(record)) for record in rebound] + if any(size > self.limits.max_record_bytes for size in sizes): + raise ValueError("record exceeds negotiated record byte limit") + if sum(sizes) > self.limits.max_pending_bytes: + raise ValueError("pending records exceed negotiated byte limit") + self.pending = deque(zip(rebound, sizes)) + self.pending_bytes = sum(sizes) + self.identity = identity + self.generation = generation + self.frame = None + self.seq = 1 + self._send(READY, self._base()) + + def capture_legacy(self, entry: dict, outcome: str) -> None: + """Capture measurements into the bounded producer instead of an unbounded worker history.""" + from_legacy_record(entry, self, outcome) + + def capture(self, fields) -> dict: + """Assign a UUID once; preserve it across batching, retries and reconnection.""" + if self.closed or self.error is not None: + raise self._fail("canonical record producer is closed or failed", stop=False) + identity = self.identity or {"run_id": "awaiting-master", "source": "loaddensity", + "phase": "load", "engine": "locust"} + record = validate_request_record({"schema_version": 1, "record_id": str(uuid4()), + "scenario_id": None, "step_id": None, "assertions": [], "extensions": {}, **fields, + **identity, "worker_id": self.worker_id}) + size = len(_encoded(record)) + if size > self.limits.max_record_bytes: + raise self._fail("canonical record exceeds record byte limit") + if len(self.pending) >= self.limits.max_pending or self.pending_bytes + size > self.limits.max_pending_bytes: + raise self._fail("canonical pending queue exceeds count or byte limit") + self.pending.append((record, size)) + self.pending_bytes += size + self.finalized = False + return copy.deepcopy(record) + + def _batch(self) -> dict: + records = [] + frame = {**self._base(), "seq": self.seq, "records": records, "byte_count": 0} + for record, _size in self.pending: + if len(records) >= self.limits.batch_size: + break + records.append(record) + frame["byte_count"] = len(_encoded(records)) + if len(_encoded(frame)) > self.limits.max_batch_bytes: + records.pop() + frame["byte_count"] = len(_encoded(records)) + break + if not records: + raise self._fail("record cannot fit within the canonical batch byte limit") + return frame + + def flush(self) -> None: + """Attempt delivery without discarding unacknowledged records on transport errors.""" + if self.closed or self.error is not None or self.identity is None or not self.pending: + return + if self.frame is None: + self.frame = self._batch() + self._send(BATCH, self.frame) + + def on_ack(self, msg, **_kwargs) -> None: + """Only a matching run/epoch/generation/sequence may release the in-flight queue prefix.""" + if self.closed or self.frame is None or not isinstance(msg.data, dict): + return + data = msg.data + if type(data.get("generation")) is not int or type(data.get("seq")) is not int: + return + if any(data.get(field) != self.frame[field] for field in + ("run_id", "worker_id", "epoch", "generation", "seq")): + return + if data.get("ok") is not True: + self._fail("master rejected canonical batch: " + str(data.get("error", "unknown error"))) + return + count = len(self.frame["records"]) + for _ in range(count): + _record, size = self.pending.popleft() + self.pending_bytes -= size + self.delivered += count + self.frame = None + self.seq += 1 + + def run(self) -> None: + """Retry handshake/delivery on the native connection; the owner controls this task's lifetime.""" + next_hello = 0.0 + while not self.closed and self.error is None: + if self.identity is None or monotonic() >= next_hello: + self._send(HELLO, self._base()) + next_hello = monotonic() + 1.0 + self.flush() + gevent.sleep(self.limits.interval) + + def _on_stop(self, **_kwargs) -> None: + self.flush() + + def _on_flush(self, msg, **_kwargs) -> None: + if self.identity is None or not _control_valid(msg.data, self.limits, frozenset(("run_id", "nonce"))): + return + if msg.data.get("run_id") == self.identity["run_id"]: + self.environment.load_density_tasks.spawn(self._drain_done, msg.data.get("nonce")) + + def _drain(self) -> None: + deadline = monotonic() + self.limits.drain_timeout + while self.pending and self.error is None and monotonic() < deadline: + self.flush() + gevent.sleep(self.limits.interval) + + def _drain_done(self, nonce) -> None: + self.environment.runner.stop() + self._drain() + self._send_done(nonce) + + def _send_done(self, nonce) -> None: + self._final_nonce = nonce + self._send(DONE, {**self._base(), "nonce": nonce, "pending": len(self.pending)}) + + def _on_finished(self, msg, **_kwargs) -> None: + data = msg.data + if self.identity is None or self.pending or not _control_valid(data, self.limits, TERMINAL_FIELDS): + return + if type(data.get("protocol")) is not int or type(data.get("generation")) is not int: + return + if type(data.get("pending")) is not int or data["pending"] != 0: + return + if all(data.get(key) == value for key, value in self._base().items()): + self.finalized = data.get("nonce") == self._final_nonce + + def _finish_delivery(self) -> None: + if self.identity is None or self.finalized or self.error is not None: + return + nonce = str(uuid4()) + deadline = monotonic() + self.limits.drain_timeout + while not self.finalized and monotonic() < deadline: + self._send_done(nonce) + gevent.sleep(self.limits.interval) + if not self.finalized: + self._fail("canonical final delivery acknowledgement was not received", stop=False) + + def close(self) -> None: + """Bound the final drain and retain exact pending-loss diagnostics on failure.""" + if not self._begin_close(): + return + try: + self.environment.runner.stop() + self._drain() + if self.pending: + self._fail(f"canonical delivery incomplete: {len(self.pending)} unacknowledged records", stop=False) + self._finish_delivery() + finally: + self.closed = True + self._detach() + + def snapshot(self) -> dict: + """Return bounded queue and delivery diagnostics; counts never imply HTTP exactly-once execution.""" + return {"role": self.role, "ready": self.identity is not None, "generation": self.generation, + "pending_records": len(self.pending), "pending_bytes": self.pending_bytes, + "delivered_records": self.delivered, "send_failures": self.send_failures, + "error": str(self.error) if self.error else None} + + +@dataclass +class _Session: + epoch: str + generation: int + ready: bool = False + seq: int = 0 + fingerprint: bytes | None = None + finished: bool = False + retired_epochs: set[str] = field(default_factory=set) + + +class MasterRecords(_Delivery): + """Validate transport provenance, atomically aggregate records, and acknowledge accepted batches.""" + + role = "master" + + def __init__(self, environment, context: DistributedRunContext, limits: DeliveryLimits) -> None: + super().__init__(environment, limits) + if context.engine != "locust": + raise ValueError("distributed record aggregation requires engine=locust") + self.context = context + self.sessions: dict[str, _Session] = {} + self.accepted = 0 + self.duplicates = 0 + self.rejected = 0 + self.incomplete_workers = set() + self._nonce = None + self._awaiting = set() + environment.runner.register_message(HELLO, self.on_hello) + environment.runner.register_message(READY, self._on_ready) + environment.runner.register_message(BATCH, self.on_batch) + environment.runner.register_message(DONE, self._on_done) + self._listen(environment.events.test_stopping, self._on_stopping) + + @property + def ready_count(self) -> int: + """Count canonical-ready workers only when their native readiness and heartbeat are healthy.""" + return sum(client.state == "ready" and client.heartbeat >= 0 and + self.sessions.get(client.id) is not None and self.sessions[client.id].ready + for client in self.environment.runner.clients.all) + + def on_hello(self, msg, **_kwargs) -> None: + """Negotiate a run-bound generation for a known native worker and producer epoch.""" + if self.closed: + return + try: + data = msg.data + worker_id = msg.node_id + self._validate_hello(worker_id, data) + session = self._negotiate_session(worker_id, data) + if session is None: + return + self.sessions[worker_id] = session + self._send(CONFIG, {**self.context.identity(), "protocol": 1, "worker_id": worker_id, + "epoch": session.epoch, "generation": session.generation, + "limits": asdict(self.limits)}, worker_id) + except (KeyError, TypeError, ValueError, RecordDeliveryError) as error: + self.rejected += 1 + self._fail(f"canonical worker handshake rejected: {error}") + + def _negotiate_session(self, worker_id: str, data: dict) -> _Session | None: + current = self.sessions.get(worker_id) + if current is None: + return _Session(data["epoch"], 1) + if current.epoch == data["epoch"]: + return current + if data["epoch"] in current.retired_epochs: + self.rejected += 1 + return None + if type(data.get("generation")) is not int or data["generation"] != 0: + raise RecordDeliveryError("stale producer epoch") + if len(current.retired_epochs) >= RETIRED_EPOCH_LIMIT: + raise RecordDeliveryError("retired producer epoch history capacity reached") + return _Session(data["epoch"], current.generation + 1, + retired_epochs=current.retired_epochs | {current.epoch}) + + def _validate_hello(self, worker_id: str, data) -> None: + if not isinstance(data, dict) or type(data.get("protocol")) is not int or data["protocol"] != 1: + raise RecordDeliveryError("invalid canonical handshake protocol") + if data.get("worker_id") != worker_id or worker_id not in self.environment.runner.clients: + raise RecordDeliveryError("worker_id: handshake transport identity mismatch") + if not isinstance(data.get("epoch"), str) or not data["epoch"] or len(data["epoch"]) > 128: + raise RecordDeliveryError("invalid producer epoch") + if len(_encoded(data)) > self.limits.max_batch_bytes: + raise RecordDeliveryError("handshake exceeds byte limit") + + def _session(self, worker_id: str, data: dict) -> _Session: + if not isinstance(data, dict) or type(data.get("protocol")) is not int or data["protocol"] != 1: + raise RecordDeliveryError("invalid canonical message protocol") + session = self.sessions.get(worker_id) + if data.get("run_id") != self.context.run_id: + raise RecordDeliveryError("run_id: foreign canonical batch") + if data.get("worker_id") != worker_id or session is None: + raise RecordDeliveryError("worker_id: batch transport identity mismatch") + if type(data.get("generation")) is not int: + raise RecordDeliveryError("invalid canonical producer generation") + if data.get("epoch") != session.epoch or data.get("generation") != session.generation: + raise RecordDeliveryError("stale canonical producer generation") + return session + + def _on_ready(self, msg, **_kwargs) -> None: + try: + self._session(msg.node_id, msg.data).ready = True + except (AttributeError, TypeError, RecordDeliveryError): + self.rejected += 1 + + def on_batch(self, msg, **_kwargs) -> None: + """Reject malformed/foreign batches before mutation; acknowledge identical retries once.""" + if self.closed: + return + data = msg.data + try: + session = self._validate_batch(msg.node_id, data) + fingerprint = _encoded(data) + if data["seq"] == session.seq: + if fingerprint != session.fingerprint: + raise RecordDeliveryError("conflicting retry of canonical batch") + self.duplicates += 1 + accepted = [] + else: + accepted = self.context.ingest_batch(msg.node_id, data["records"]) + for record in accepted: + _append_legacy(record) + self.accepted += len(accepted) + session.seq = data["seq"] + session.fingerprint = fingerprint + session.finished = False + self._ack(msg.node_id, data, True, len(accepted)) + except (KeyError, TypeError, ValueError, RecordDeliveryError) as error: + self.rejected += 1 + self._ack(msg.node_id, data if isinstance(data, dict) else {}, False, 0, str(error)) + self._fail(f"canonical batch rejected: {error}") + + def _validate_batch(self, worker_id: str, data: dict) -> _Session: + session = self._session(worker_id, data) + if type(data.get("seq")) is not int or data["seq"] not in (session.seq, session.seq + 1): + raise RecordDeliveryError("unexpected canonical batch sequence") + if data["seq"] <= 0: + raise RecordDeliveryError("invalid canonical batch sequence") + self._validate_records(data) + return session + + def _validate_records(self, data: dict) -> None: + records = data.get("records") + if not isinstance(records, list) or not records or len(records) > self.limits.batch_size: + raise RecordDeliveryError("canonical batch record count exceeds limit") + if len(_encoded(data)) > self.limits.max_batch_bytes: + raise RecordDeliveryError("canonical batch byte limit exceeded") + if type(data.get("byte_count")) is not int or data["byte_count"] != len(_encoded(records)): + raise RecordDeliveryError("canonical batch byte_count mismatch") + if any(len(_encoded(record)) > self.limits.max_record_bytes for record in records): + raise RecordDeliveryError("canonical record byte limit exceeded") + + def _ack(self, worker_id: str, data: dict, ok: bool, count: int, error: str | None = None) -> None: + base = {field: data.get(field) for field in ("run_id", "worker_id", "epoch", "generation", "seq")} + self._send(ACK, {**base, "ok": ok, "accepted": count, "error": error}, worker_id) + + def _on_done(self, msg, **_kwargs) -> None: + if not _control_valid(msg.data, self.limits, TERMINAL_FIELDS): + self.rejected += 1 + return + try: + session = self._session(msg.node_id, msg.data) + if type(msg.data.get("pending")) is not int or not 0 <= msg.data["pending"] <= self.limits.max_pending: + return + session.finished = msg.data["pending"] == 0 + if session.finished: + response = {key: msg.data[key] for key in TERMINAL_FIELDS} + self._send(FINISHED, response, msg.node_id) + if msg.data.get("nonce") == self._nonce and session.finished: + self._awaiting.discard(msg.node_id) + except (AttributeError, TypeError, RecordDeliveryError): + self.rejected += 1 + + def _on_stopping(self, **_kwargs) -> None: + self.close() + + def _freeze_health(self) -> None: + health = getattr(self.environment, "distributed_health", None) + if health is not None and not getattr(self.environment, "record_health_closed", False): + health.close() + self.environment.record_health_closed = True + + def close(self) -> None: + """Ask live producers for a final drain while the native RPC listener can still acknowledge.""" + if not self._begin_close(): + return + self._freeze_health() + try: + self._drain_workers() + finally: + self.closed = True + self._detach() + + def _drain_workers(self) -> None: + self._nonce = str(uuid4()) + clients = self.environment.runner.clients + for worker_id, session in self.sessions.items(): + if session.finished: + continue + if worker_id not in clients or clients[worker_id].state == "missing": + self.incomplete_workers.add(worker_id) + else: + self._awaiting.add(worker_id) + self.incomplete_workers.update(set(clients) - set(self.sessions)) + for worker_id in self._awaiting.copy(): + self._send(FLUSH, {"run_id": self.context.run_id, "nonce": self._nonce}, worker_id) + deadline = monotonic() + self.limits.drain_timeout + while self._awaiting and monotonic() < deadline: + gevent.sleep(self.limits.interval) + self.incomplete_workers.update(self._awaiting) + if self.incomplete_workers: + self._fail("canonical delivery incomplete for workers: " + ", ".join(sorted(self.incomplete_workers)), + stop=False) + + def snapshot(self) -> dict: + """Expose aggregate delivery separately from native worker capacity health.""" + return {"role": self.role, "run_id": self.context.run_id, "accepted_records": self.accepted, + "duplicate_batches": self.duplicates, "rejected_batches": self.rejected, + "incomplete_workers": sorted(self.incomplete_workers), "send_failures": self.send_failures, + "error": str(self.error) if self.error else None} + + +def _append_legacy(record: dict) -> None: + successful = record["outcome"] == "passed" + entry = {"Method": record["request_method"], "test_url": record["request_url"], "name": record["name"], + "status_code": str(record["status_code"]) if record["status_code"] is not None else None, + "text": record.get("text"), "response_time_ms": record["response_time_ms"], + "response_length": record["response_length"], "start_time": record["start_time"], + "error": None if successful else record["error"]["message"], + "run_id": record["run_id"], "worker_id": record["worker_id"], "record_id": record["record_id"]} + if successful: + entry.update(content=record.get("content_base64"), headers=record.get("headers")) + records = test_record_instance.test_record_list if successful else test_record_instance.error_record_list + records.append(entry) + + +def delivery_options(mode: str, context, enabled: bool, options: dict): + """Validate opt-in delivery before allocating a native runner or RPC transport.""" + if enabled: + if mode != "worker" or context is not None: + raise ValueError("distributed_records is only for workers receiving identity from their master") + return ("worker", None, DeliveryLimits.from_options(options)) + if not isinstance(context, DistributedRunContext): + return None + if mode != "master" or context.engine != "locust": + raise ValueError("DistributedRunContext requires a native Locust master with engine=locust") + return ("master", context, DeliveryLimits.from_options(options)) + + +def attach_delivery(environment, options: tuple) -> None: + """Attach native message handlers and owned producer tasks after runner creation.""" + role, context, limits = options + if role == "master": + environment.record_delivery = MasterRecords(environment, context, limits) + environment.record_run_context = context + return + from functools import partial + + from je_load_density.wrapper.event.request_hook import request_hook + producer = WorkerRecords(environment, limits) + environment.record_delivery = producer + producer._listen(environment.events.request, partial(request_hook, record_sink=producer)) + environment.load_density_tasks.spawn(producer.run) diff --git a/je_load_density/wrapper/event/request_hook.py b/je_load_density/wrapper/event/request_hook.py index 3ae49ed..b681e94 100644 --- a/je_load_density/wrapper/event/request_hook.py +++ b/je_load_density/wrapper/event/request_hook.py @@ -51,9 +51,13 @@ def request_hook( entry.update(_response_fields(response, successful)) entry.update(response_time_ms=float(response_time or 0), response_length=int(response_length or 0), error=None if successful else str(exception), start_time=_epoch_seconds(start_time)) + outcome = "passed" if successful else "failed" + sink = kwargs.get("record_sink") + if sink is not None: + sink.capture_legacy(entry, outcome) + return records = test_record_instance.test_record_list if successful else test_record_instance.error_record_list records.append(entry) - outcome = "passed" if successful else "failed" selected_run = kwargs.get("record_run") if selected_run is not None: from_legacy_record(entry, selected_run, outcome) diff --git a/je_load_density/wrapper/start_wrapper/locust_start.py b/je_load_density/wrapper/start_wrapper/locust_start.py new file mode 100644 index 0000000..44433d7 --- /dev/null +++ b/je_load_density/wrapper/start_wrapper/locust_start.py @@ -0,0 +1,321 @@ +from typing import Any, Dict, Optional + +from je_load_density.utils.logging.loggin_instance import load_density_logger +from je_load_density.wrapper.create_locust_env.create_locust_env import prepare_env +from je_load_density.wrapper.user_template.amqp_user_template import ( + AmqpUserWrapper, + set_wrapper_amqp_user, +) +from je_load_density.wrapper.user_template.apns_user_template import ( + ApnsUserWrapper, + set_wrapper_apns_user, +) +from je_load_density.wrapper.user_template.async_http_user_template import ( + AsyncHttpUserWrapper, + set_wrapper_async_http_user, +) +from je_load_density.wrapper.user_template.cassandra_user_template import ( + CassandraUserWrapper, + set_wrapper_cassandra_user, +) +from je_load_density.wrapper.user_template.coap_user_template import ( + CoapUserWrapper, + set_wrapper_coap_user, +) +from je_load_density.wrapper.user_template.consul_user_template import ( + ConsulUserWrapper, + set_wrapper_consul_user, +) +from je_load_density.wrapper.user_template.couchbase_user_template import ( + CouchbaseUserWrapper, + set_wrapper_couchbase_user, +) +from je_load_density.wrapper.user_template.elasticsearch_user_template import ( + ElasticsearchUserWrapper, + set_wrapper_elasticsearch_user, +) +from je_load_density.wrapper.user_template.etcd_user_template import ( + EtcdUserWrapper, + set_wrapper_etcd_user, +) +from je_load_density.wrapper.user_template.fast_http_user_template import ( + FastHttpUserWrapper, + set_wrapper_fasthttp_user, +) +from je_load_density.wrapper.user_template.fcm_user_template import ( + FcmUserWrapper, + set_wrapper_fcm_user, +) +from je_load_density.wrapper.user_template.ftp_user_template import ( + FtpUserWrapper, + set_wrapper_ftp_user, +) +from je_load_density.wrapper.user_template.fuzz_http_user_template import ( + FuzzHttpUserWrapper, + set_wrapper_fuzz_http_user, +) +from je_load_density.wrapper.user_template.graphql_ws_user_template import ( + GraphQLWebSocketUserWrapper, + set_wrapper_graphql_ws_user, +) +from je_load_density.wrapper.user_template.grpc_user_template import ( + GrpcUserWrapper, + set_wrapper_grpc_user, +) +from je_load_density.wrapper.user_template.http3_user_template import ( + Http3UserWrapper, + set_wrapper_http3_user, +) +from je_load_density.wrapper.user_template.http_user_template import ( + HttpUserWrapper, + set_wrapper_http_user, +) +from je_load_density.wrapper.user_template.imap_user_template import ( + ImapUserWrapper, + set_wrapper_imap_user, +) +from je_load_density.wrapper.user_template.kafka_user_template import ( + KafkaUserWrapper, + set_wrapper_kafka_user, +) +from je_load_density.wrapper.user_template.ldap_user_template import ( + LdapUserWrapper, + set_wrapper_ldap_user, +) +from je_load_density.wrapper.user_template.memcached_user_template import ( + MemcachedUserWrapper, + set_wrapper_memcached_user, +) +from je_load_density.wrapper.user_template.modbus_user_template import ( + ModbusUserWrapper, + set_wrapper_modbus_user, +) +from je_load_density.wrapper.user_template.mongo_user_template import ( + MongoUserWrapper, + set_wrapper_mongo_user, +) +from je_load_density.wrapper.user_template.mqtt_user_template import ( + MqttUserWrapper, + set_wrapper_mqtt_user, +) +from je_load_density.wrapper.user_template.nats_user_template import ( + NatsUserWrapper, + set_wrapper_nats_user, +) +from je_load_density.wrapper.user_template.neo4j_user_template import ( + Neo4jUserWrapper, + set_wrapper_neo4j_user, +) +from je_load_density.wrapper.user_template.opcua_user_template import ( + OpcuaUserWrapper, + set_wrapper_opcua_user, +) +from je_load_density.wrapper.user_template.pulsar_user_template import ( + PulsarUserWrapper, + set_wrapper_pulsar_user, +) +from je_load_density.wrapper.user_template.redis_user_template import ( + RedisUserWrapper, + set_wrapper_redis_user, +) +from je_load_density.wrapper.user_template.sftp_user_template import ( + SftpUserWrapper, + set_wrapper_sftp_user, +) +from je_load_density.wrapper.user_template.smtp_user_template import ( + SmtpUserWrapper, + set_wrapper_smtp_user, +) +from je_load_density.wrapper.user_template.snmp_user_template import ( + SnmpUserWrapper, + set_wrapper_snmp_user, +) +from je_load_density.wrapper.user_template.soap_user_template import ( + SoapUserWrapper, + set_wrapper_soap_user, +) +from je_load_density.wrapper.user_template.socket_user_template import ( + SocketUserWrapper, + set_wrapper_socket_user, +) +from je_load_density.wrapper.user_template.sql_user_template import ( + SqlUserWrapper, + set_wrapper_sql_user, +) +from je_load_density.wrapper.user_template.sse_user_template import ( + SseUserWrapper, + set_wrapper_sse_user, +) +from je_load_density.wrapper.user_template.thrift_user_template import ( + ThriftUserWrapper, + set_wrapper_thrift_user, +) +from je_load_density.wrapper.user_template.vault_user_template import ( + VaultUserWrapper, + set_wrapper_vault_user, +) +from je_load_density.wrapper.user_template.webpush_user_template import ( + WebPushUserWrapper, + set_wrapper_webpush_user, +) +from je_load_density.wrapper.user_template.websocket_user_template import ( + WebSocketUserWrapper, + set_wrapper_websocket_user, +) +from je_load_density.wrapper.user_template.zmq_user_template import ( + ZmqUserWrapper, + set_wrapper_zmq_user, +) + +_USER_REGISTRY: Dict[str, Dict[str, Any]] = { + "fast_http_user": {"actually_user": FastHttpUserWrapper, "init": set_wrapper_fasthttp_user}, + "http_user": {"actually_user": HttpUserWrapper, "init": set_wrapper_http_user}, + "async_http_user": {"actually_user": AsyncHttpUserWrapper, "init": set_wrapper_async_http_user}, + "http3_user": {"actually_user": Http3UserWrapper, "init": set_wrapper_http3_user}, + "websocket_user": {"actually_user": WebSocketUserWrapper, "init": set_wrapper_websocket_user}, + "sse_user": {"actually_user": SseUserWrapper, "init": set_wrapper_sse_user}, + "graphql_ws_user": { + "actually_user": GraphQLWebSocketUserWrapper, + "init": set_wrapper_graphql_ws_user, + }, + "grpc_user": {"actually_user": GrpcUserWrapper, "init": set_wrapper_grpc_user}, + "mqtt_user": {"actually_user": MqttUserWrapper, "init": set_wrapper_mqtt_user}, + "amqp_user": {"actually_user": AmqpUserWrapper, "init": set_wrapper_amqp_user}, + "nats_user": {"actually_user": NatsUserWrapper, "init": set_wrapper_nats_user}, + "pulsar_user": {"actually_user": PulsarUserWrapper, "init": set_wrapper_pulsar_user}, + "coap_user": {"actually_user": CoapUserWrapper, "init": set_wrapper_coap_user}, + "socket_user": {"actually_user": SocketUserWrapper, "init": set_wrapper_socket_user}, + "sql_user": {"actually_user": SqlUserWrapper, "init": set_wrapper_sql_user}, + "redis_user": {"actually_user": RedisUserWrapper, "init": set_wrapper_redis_user}, + "kafka_user": {"actually_user": KafkaUserWrapper, "init": set_wrapper_kafka_user}, + "mongo_user": {"actually_user": MongoUserWrapper, "init": set_wrapper_mongo_user}, + "cassandra_user": { + "actually_user": CassandraUserWrapper, + "init": set_wrapper_cassandra_user, + }, + "elasticsearch_user": { + "actually_user": ElasticsearchUserWrapper, + "init": set_wrapper_elasticsearch_user, + }, + "smtp_user": {"actually_user": SmtpUserWrapper, "init": set_wrapper_smtp_user}, + "imap_user": {"actually_user": ImapUserWrapper, "init": set_wrapper_imap_user}, + "ftp_user": {"actually_user": FtpUserWrapper, "init": set_wrapper_ftp_user}, + "sftp_user": {"actually_user": SftpUserWrapper, "init": set_wrapper_sftp_user}, + "fuzz_http_user": { + "actually_user": FuzzHttpUserWrapper, + "init": set_wrapper_fuzz_http_user, + }, + "soap_user": {"actually_user": SoapUserWrapper, "init": set_wrapper_soap_user}, + "ldap_user": {"actually_user": LdapUserWrapper, "init": set_wrapper_ldap_user}, + "snmp_user": {"actually_user": SnmpUserWrapper, "init": set_wrapper_snmp_user}, + "modbus_user": {"actually_user": ModbusUserWrapper, "init": set_wrapper_modbus_user}, + "opcua_user": {"actually_user": OpcuaUserWrapper, "init": set_wrapper_opcua_user}, + "zmq_user": {"actually_user": ZmqUserWrapper, "init": set_wrapper_zmq_user}, + "thrift_user": {"actually_user": ThriftUserWrapper, "init": set_wrapper_thrift_user}, + "memcached_user": { + "actually_user": MemcachedUserWrapper, + "init": set_wrapper_memcached_user, + }, + "neo4j_user": {"actually_user": Neo4jUserWrapper, "init": set_wrapper_neo4j_user}, + "couchbase_user": { + "actually_user": CouchbaseUserWrapper, + "init": set_wrapper_couchbase_user, + }, + "etcd_user": {"actually_user": EtcdUserWrapper, "init": set_wrapper_etcd_user}, + "consul_user": {"actually_user": ConsulUserWrapper, "init": set_wrapper_consul_user}, + "vault_user": {"actually_user": VaultUserWrapper, "init": set_wrapper_vault_user}, + "webpush_user": {"actually_user": WebPushUserWrapper, "init": set_wrapper_webpush_user}, + "apns_user": {"actually_user": ApnsUserWrapper, "init": set_wrapper_apns_user}, + "fcm_user": {"actually_user": FcmUserWrapper, "init": set_wrapper_fcm_user}, +} + + +def _pop_distributed_config(kwargs: Dict[str, Any]) -> Dict[str, Any]: + """ + Extract Locust distributed-runner fields from ``**kwargs``. + + Keeping these in kwargs (rather than as positional parameters) lets + callers stay backwards-compatible while satisfying the public-API + parameter budget. + """ + return { + "master_bind_host": kwargs.pop("master_bind_host", "*"), + "master_bind_port": kwargs.pop("master_bind_port", 5557), + "master_host": kwargs.pop("master_host", "127.0.0.1"), + "master_port": kwargs.pop("master_port", 5557), + "expected_workers": kwargs.pop("expected_workers", 0), + "worker_startup_timeout": kwargs.pop("worker_startup_timeout", 60), + "worker_heartbeat_interval": kwargs.pop("worker_heartbeat_interval", 5), + "worker_lost_timeout": kwargs.pop("worker_lost_timeout", 15), + "worker_startup_policy": kwargs.pop("worker_startup_policy", "fail"), + } + + +def start_test( + user_detail_dict: Dict[str, Any], + user_count: int = 50, + spawn_rate: int = 10, + test_time: Optional[int] = 60, + web_ui_dict: Optional[Dict[str, Any]] = None, + runner_mode: str = "local", + load_shape: Optional[str] = None, + shape_config: Optional[Dict[str, Any]] = None, + **kwargs, +) -> Dict[str, Any]: + """ + 啟動壓力測試。Start load test. + + ``runner_mode`` is ``"local"`` | ``"master"`` | ``"worker"``. + Distributed-mode fields (``master_bind_host`` / ``master_bind_port`` / + ``master_host`` / ``master_port`` / ``expected_workers``) are + accepted via ``**kwargs`` so the signature stays under the public + API parameter budget. + + Worker timing defaults are startup=60, heartbeat=5, lost=15 seconds. + ``worker_startup_policy="fail"`` rejects startup shortfalls; explicitly + selecting ``"degraded"`` allows fewer workers while requiring at least one. + ``stop_requested`` and ``on_environment`` are cooperative lifecycle callbacks. + Master results include ``distributed_health`` and affected worker journeys; + native rebalancing recreates capacity without replaying individual requests. + """ + distributed = _pop_distributed_config(kwargs) + load_density_logger.info( + f"start_test, user_detail_dict={user_detail_dict}, user_count={user_count}, " + f"spawn_rate={spawn_rate}, test_time={test_time}, web_ui_dict={web_ui_dict}, " + f"runner_mode={runner_mode}, distributed={distributed}, params={kwargs}" + ) + + user_type = user_detail_dict.get("user", "fast_http_user") + user = _USER_REGISTRY.get(user_type) + if user is None: + raise ValueError(f"Unsupported user type: {user_type}") + + actually_user = user["actually_user"] + init_function = user["init"] + + init_function(user_detail_dict, **kwargs) + + env = prepare_env( + user_class=actually_user, + user_count=user_count, + spawn_rate=spawn_rate, + test_time=test_time, + web_ui_dict=web_ui_dict, + runner_mode=runner_mode, + load_shape=load_shape, + shape_config=shape_config, + **distributed, + **kwargs, + ) + + result = { + "user_detail": user_detail_dict, + "user_count": user_count, + "spawn_rate": spawn_rate, + "test_time": test_time, + "web_ui": web_ui_dict, + "runner_mode": runner_mode, + } + if getattr(env, "distributed_health", None) is not None: + result["distributed_health"] = env.distributed_health.snapshot() + return result diff --git a/je_load_density/wrapper/start_wrapper/start_test.py b/je_load_density/wrapper/start_wrapper/start_test.py index 44433d7..399d4b0 100644 --- a/je_load_density/wrapper/start_wrapper/start_test.py +++ b/je_load_density/wrapper/start_wrapper/start_test.py @@ -1,321 +1,115 @@ -from typing import Any, Dict, Optional +"""Compatible start-test path with explicit, lazy scheduler selection.""" -from je_load_density.utils.logging.loggin_instance import load_density_logger -from je_load_density.wrapper.create_locust_env.create_locust_env import prepare_env -from je_load_density.wrapper.user_template.amqp_user_template import ( - AmqpUserWrapper, - set_wrapper_amqp_user, -) -from je_load_density.wrapper.user_template.apns_user_template import ( - ApnsUserWrapper, - set_wrapper_apns_user, -) -from je_load_density.wrapper.user_template.async_http_user_template import ( - AsyncHttpUserWrapper, - set_wrapper_async_http_user, -) -from je_load_density.wrapper.user_template.cassandra_user_template import ( - CassandraUserWrapper, - set_wrapper_cassandra_user, -) -from je_load_density.wrapper.user_template.coap_user_template import ( - CoapUserWrapper, - set_wrapper_coap_user, -) -from je_load_density.wrapper.user_template.consul_user_template import ( - ConsulUserWrapper, - set_wrapper_consul_user, -) -from je_load_density.wrapper.user_template.couchbase_user_template import ( - CouchbaseUserWrapper, - set_wrapper_couchbase_user, -) -from je_load_density.wrapper.user_template.elasticsearch_user_template import ( - ElasticsearchUserWrapper, - set_wrapper_elasticsearch_user, -) -from je_load_density.wrapper.user_template.etcd_user_template import ( - EtcdUserWrapper, - set_wrapper_etcd_user, -) -from je_load_density.wrapper.user_template.fast_http_user_template import ( - FastHttpUserWrapper, - set_wrapper_fasthttp_user, -) -from je_load_density.wrapper.user_template.fcm_user_template import ( - FcmUserWrapper, - set_wrapper_fcm_user, -) -from je_load_density.wrapper.user_template.ftp_user_template import ( - FtpUserWrapper, - set_wrapper_ftp_user, -) -from je_load_density.wrapper.user_template.fuzz_http_user_template import ( - FuzzHttpUserWrapper, - set_wrapper_fuzz_http_user, -) -from je_load_density.wrapper.user_template.graphql_ws_user_template import ( - GraphQLWebSocketUserWrapper, - set_wrapper_graphql_ws_user, -) -from je_load_density.wrapper.user_template.grpc_user_template import ( - GrpcUserWrapper, - set_wrapper_grpc_user, -) -from je_load_density.wrapper.user_template.http3_user_template import ( - Http3UserWrapper, - set_wrapper_http3_user, -) -from je_load_density.wrapper.user_template.http_user_template import ( - HttpUserWrapper, - set_wrapper_http_user, -) -from je_load_density.wrapper.user_template.imap_user_template import ( - ImapUserWrapper, - set_wrapper_imap_user, -) -from je_load_density.wrapper.user_template.kafka_user_template import ( - KafkaUserWrapper, - set_wrapper_kafka_user, -) -from je_load_density.wrapper.user_template.ldap_user_template import ( - LdapUserWrapper, - set_wrapper_ldap_user, -) -from je_load_density.wrapper.user_template.memcached_user_template import ( - MemcachedUserWrapper, - set_wrapper_memcached_user, -) -from je_load_density.wrapper.user_template.modbus_user_template import ( - ModbusUserWrapper, - set_wrapper_modbus_user, -) -from je_load_density.wrapper.user_template.mongo_user_template import ( - MongoUserWrapper, - set_wrapper_mongo_user, -) -from je_load_density.wrapper.user_template.mqtt_user_template import ( - MqttUserWrapper, - set_wrapper_mqtt_user, -) -from je_load_density.wrapper.user_template.nats_user_template import ( - NatsUserWrapper, - set_wrapper_nats_user, -) -from je_load_density.wrapper.user_template.neo4j_user_template import ( - Neo4jUserWrapper, - set_wrapper_neo4j_user, -) -from je_load_density.wrapper.user_template.opcua_user_template import ( - OpcuaUserWrapper, - set_wrapper_opcua_user, -) -from je_load_density.wrapper.user_template.pulsar_user_template import ( - PulsarUserWrapper, - set_wrapper_pulsar_user, -) -from je_load_density.wrapper.user_template.redis_user_template import ( - RedisUserWrapper, - set_wrapper_redis_user, -) -from je_load_density.wrapper.user_template.sftp_user_template import ( - SftpUserWrapper, - set_wrapper_sftp_user, -) -from je_load_density.wrapper.user_template.smtp_user_template import ( - SmtpUserWrapper, - set_wrapper_smtp_user, -) -from je_load_density.wrapper.user_template.snmp_user_template import ( - SnmpUserWrapper, - set_wrapper_snmp_user, -) -from je_load_density.wrapper.user_template.soap_user_template import ( - SoapUserWrapper, - set_wrapper_soap_user, -) -from je_load_density.wrapper.user_template.socket_user_template import ( - SocketUserWrapper, - set_wrapper_socket_user, -) -from je_load_density.wrapper.user_template.sql_user_template import ( - SqlUserWrapper, - set_wrapper_sql_user, -) -from je_load_density.wrapper.user_template.sse_user_template import ( - SseUserWrapper, - set_wrapper_sse_user, -) -from je_load_density.wrapper.user_template.thrift_user_template import ( - ThriftUserWrapper, - set_wrapper_thrift_user, -) -from je_load_density.wrapper.user_template.vault_user_template import ( - VaultUserWrapper, - set_wrapper_vault_user, -) -from je_load_density.wrapper.user_template.webpush_user_template import ( - WebPushUserWrapper, - set_wrapper_webpush_user, -) -from je_load_density.wrapper.user_template.websocket_user_template import ( - WebSocketUserWrapper, - set_wrapper_websocket_user, -) -from je_load_density.wrapper.user_template.zmq_user_template import ( - ZmqUserWrapper, - set_wrapper_zmq_user, -) - -_USER_REGISTRY: Dict[str, Dict[str, Any]] = { - "fast_http_user": {"actually_user": FastHttpUserWrapper, "init": set_wrapper_fasthttp_user}, - "http_user": {"actually_user": HttpUserWrapper, "init": set_wrapper_http_user}, - "async_http_user": {"actually_user": AsyncHttpUserWrapper, "init": set_wrapper_async_http_user}, - "http3_user": {"actually_user": Http3UserWrapper, "init": set_wrapper_http3_user}, - "websocket_user": {"actually_user": WebSocketUserWrapper, "init": set_wrapper_websocket_user}, - "sse_user": {"actually_user": SseUserWrapper, "init": set_wrapper_sse_user}, - "graphql_ws_user": { - "actually_user": GraphQLWebSocketUserWrapper, - "init": set_wrapper_graphql_ws_user, - }, - "grpc_user": {"actually_user": GrpcUserWrapper, "init": set_wrapper_grpc_user}, - "mqtt_user": {"actually_user": MqttUserWrapper, "init": set_wrapper_mqtt_user}, - "amqp_user": {"actually_user": AmqpUserWrapper, "init": set_wrapper_amqp_user}, - "nats_user": {"actually_user": NatsUserWrapper, "init": set_wrapper_nats_user}, - "pulsar_user": {"actually_user": PulsarUserWrapper, "init": set_wrapper_pulsar_user}, - "coap_user": {"actually_user": CoapUserWrapper, "init": set_wrapper_coap_user}, - "socket_user": {"actually_user": SocketUserWrapper, "init": set_wrapper_socket_user}, - "sql_user": {"actually_user": SqlUserWrapper, "init": set_wrapper_sql_user}, - "redis_user": {"actually_user": RedisUserWrapper, "init": set_wrapper_redis_user}, - "kafka_user": {"actually_user": KafkaUserWrapper, "init": set_wrapper_kafka_user}, - "mongo_user": {"actually_user": MongoUserWrapper, "init": set_wrapper_mongo_user}, - "cassandra_user": { - "actually_user": CassandraUserWrapper, - "init": set_wrapper_cassandra_user, - }, - "elasticsearch_user": { - "actually_user": ElasticsearchUserWrapper, - "init": set_wrapper_elasticsearch_user, - }, - "smtp_user": {"actually_user": SmtpUserWrapper, "init": set_wrapper_smtp_user}, - "imap_user": {"actually_user": ImapUserWrapper, "init": set_wrapper_imap_user}, - "ftp_user": {"actually_user": FtpUserWrapper, "init": set_wrapper_ftp_user}, - "sftp_user": {"actually_user": SftpUserWrapper, "init": set_wrapper_sftp_user}, - "fuzz_http_user": { - "actually_user": FuzzHttpUserWrapper, - "init": set_wrapper_fuzz_http_user, - }, - "soap_user": {"actually_user": SoapUserWrapper, "init": set_wrapper_soap_user}, - "ldap_user": {"actually_user": LdapUserWrapper, "init": set_wrapper_ldap_user}, - "snmp_user": {"actually_user": SnmpUserWrapper, "init": set_wrapper_snmp_user}, - "modbus_user": {"actually_user": ModbusUserWrapper, "init": set_wrapper_modbus_user}, - "opcua_user": {"actually_user": OpcuaUserWrapper, "init": set_wrapper_opcua_user}, - "zmq_user": {"actually_user": ZmqUserWrapper, "init": set_wrapper_zmq_user}, - "thrift_user": {"actually_user": ThriftUserWrapper, "init": set_wrapper_thrift_user}, - "memcached_user": { - "actually_user": MemcachedUserWrapper, - "init": set_wrapper_memcached_user, - }, - "neo4j_user": {"actually_user": Neo4jUserWrapper, "init": set_wrapper_neo4j_user}, - "couchbase_user": { - "actually_user": CouchbaseUserWrapper, - "init": set_wrapper_couchbase_user, - }, - "etcd_user": {"actually_user": EtcdUserWrapper, "init": set_wrapper_etcd_user}, - "consul_user": {"actually_user": ConsulUserWrapper, "init": set_wrapper_consul_user}, - "vault_user": {"actually_user": VaultUserWrapper, "init": set_wrapper_vault_user}, - "webpush_user": {"actually_user": WebPushUserWrapper, "init": set_wrapper_webpush_user}, - "apns_user": {"actually_user": ApnsUserWrapper, "init": set_wrapper_apns_user}, - "fcm_user": {"actually_user": FcmUserWrapper, "init": set_wrapper_fcm_user}, -} +import importlib +from je_load_density.engine.entrypoints import start_test as start_test -def _pop_distributed_config(kwargs: Dict[str, Any]) -> Dict[str, Any]: - """ - Extract Locust distributed-runner fields from ``**kwargs``. - - Keeping these in kwargs (rather than as positional parameters) lets - callers stay backwards-compatible while satisfying the public-API - parameter budget. - """ - return { - "master_bind_host": kwargs.pop("master_bind_host", "*"), - "master_bind_port": kwargs.pop("master_bind_port", 5557), - "master_host": kwargs.pop("master_host", "127.0.0.1"), - "master_port": kwargs.pop("master_port", 5557), - "expected_workers": kwargs.pop("expected_workers", 0), - "worker_startup_timeout": kwargs.pop("worker_startup_timeout", 60), - "worker_heartbeat_interval": kwargs.pop("worker_heartbeat_interval", 5), - "worker_lost_timeout": kwargs.pop("worker_lost_timeout", 15), - "worker_startup_policy": kwargs.pop("worker_startup_policy", "fail"), - } - - -def start_test( - user_detail_dict: Dict[str, Any], - user_count: int = 50, - spawn_rate: int = 10, - test_time: Optional[int] = 60, - web_ui_dict: Optional[Dict[str, Any]] = None, - runner_mode: str = "local", - load_shape: Optional[str] = None, - shape_config: Optional[Dict[str, Any]] = None, - **kwargs, -) -> Dict[str, Any]: - """ - 啟動壓力測試。Start load test. - - ``runner_mode`` is ``"local"`` | ``"master"`` | ``"worker"``. - Distributed-mode fields (``master_bind_host`` / ``master_bind_port`` / - ``master_host`` / ``master_port`` / ``expected_workers``) are - accepted via ``**kwargs`` so the signature stays under the public - API parameter budget. +_LEGACY_NAMES = frozenset( + [ + "AmqpUserWrapper", + "Any", + "ApnsUserWrapper", + "AsyncHttpUserWrapper", + "CassandraUserWrapper", + "CoapUserWrapper", + "ConsulUserWrapper", + "CouchbaseUserWrapper", + "Dict", + "ElasticsearchUserWrapper", + "EtcdUserWrapper", + "FastHttpUserWrapper", + "FcmUserWrapper", + "FtpUserWrapper", + "FuzzHttpUserWrapper", + "GraphQLWebSocketUserWrapper", + "GrpcUserWrapper", + "Http3UserWrapper", + "HttpUserWrapper", + "ImapUserWrapper", + "KafkaUserWrapper", + "LdapUserWrapper", + "MemcachedUserWrapper", + "ModbusUserWrapper", + "MongoUserWrapper", + "MqttUserWrapper", + "NatsUserWrapper", + "Neo4jUserWrapper", + "OpcuaUserWrapper", + "Optional", + "PulsarUserWrapper", + "RedisUserWrapper", + "SftpUserWrapper", + "SmtpUserWrapper", + "SnmpUserWrapper", + "SoapUserWrapper", + "SocketUserWrapper", + "SqlUserWrapper", + "SseUserWrapper", + "ThriftUserWrapper", + "VaultUserWrapper", + "WebPushUserWrapper", + "WebSocketUserWrapper", + "ZmqUserWrapper", + "_USER_REGISTRY", + "_pop_distributed_config", + "load_density_logger", + "prepare_env", + "set_wrapper_amqp_user", + "set_wrapper_apns_user", + "set_wrapper_async_http_user", + "set_wrapper_cassandra_user", + "set_wrapper_coap_user", + "set_wrapper_consul_user", + "set_wrapper_couchbase_user", + "set_wrapper_elasticsearch_user", + "set_wrapper_etcd_user", + "set_wrapper_fasthttp_user", + "set_wrapper_fcm_user", + "set_wrapper_ftp_user", + "set_wrapper_fuzz_http_user", + "set_wrapper_graphql_ws_user", + "set_wrapper_grpc_user", + "set_wrapper_http3_user", + "set_wrapper_http_user", + "set_wrapper_imap_user", + "set_wrapper_kafka_user", + "set_wrapper_ldap_user", + "set_wrapper_memcached_user", + "set_wrapper_modbus_user", + "set_wrapper_mongo_user", + "set_wrapper_mqtt_user", + "set_wrapper_nats_user", + "set_wrapper_neo4j_user", + "set_wrapper_opcua_user", + "set_wrapper_pulsar_user", + "set_wrapper_redis_user", + "set_wrapper_sftp_user", + "set_wrapper_smtp_user", + "set_wrapper_snmp_user", + "set_wrapper_soap_user", + "set_wrapper_socket_user", + "set_wrapper_sql_user", + "set_wrapper_sse_user", + "set_wrapper_thrift_user", + "set_wrapper_vault_user", + "set_wrapper_webpush_user", + "set_wrapper_websocket_user", + "set_wrapper_zmq_user", + ] +) - Worker timing defaults are startup=60, heartbeat=5, lost=15 seconds. - ``worker_startup_policy="fail"`` rejects startup shortfalls; explicitly - selecting ``"degraded"`` allows fewer workers while requiring at least one. - ``stop_requested`` and ``on_environment`` are cooperative lifecycle callbacks. - Master results include ``distributed_health`` and affected worker journeys; - native rebalancing recreates capacity without replaying individual requests. - """ - distributed = _pop_distributed_config(kwargs) - load_density_logger.info( - f"start_test, user_detail_dict={user_detail_dict}, user_count={user_count}, " - f"spawn_rate={spawn_rate}, test_time={test_time}, web_ui_dict={web_ui_dict}, " - f"runner_mode={runner_mode}, distributed={distributed}, params={kwargs}" - ) - user_type = user_detail_dict.get("user", "fast_http_user") - user = _USER_REGISTRY.get(user_type) - if user is None: - raise ValueError(f"Unsupported user type: {user_type}") +def __getattr__(name: str): + if name not in _LEGACY_NAMES: + raise AttributeError(f"module {__name__!r} has no attribute {name!r}") + implementation = importlib.import_module("je_load_density.wrapper.start_wrapper.locust_start") + value = getattr(implementation, name) + globals()[name] = value + return value - actually_user = user["actually_user"] - init_function = user["init"] - init_function(user_detail_dict, **kwargs) +def __dir__() -> list[str]: + return sorted(set(globals()) | _LEGACY_NAMES) - env = prepare_env( - user_class=actually_user, - user_count=user_count, - spawn_rate=spawn_rate, - test_time=test_time, - web_ui_dict=web_ui_dict, - runner_mode=runner_mode, - load_shape=load_shape, - shape_config=shape_config, - **distributed, - **kwargs, - ) - result = { - "user_detail": user_detail_dict, - "user_count": user_count, - "spawn_rate": spawn_rate, - "test_time": test_time, - "web_ui": web_ui_dict, - "runner_mode": runner_mode, - } - if getattr(env, "distributed_health", None) is not None: - result["distributed_health"] = env.distributed_health.snapshot() - return result +__all__ = ["start_test", *(name for name in sorted(_LEGACY_NAMES) if not name.startswith("_"))] diff --git a/je_load_density/wrapper/user_template/async_http_user_template.py b/je_load_density/wrapper/user_template/async_http_user_template.py index b082e6a..f0aebb3 100644 --- a/je_load_density/wrapper/user_template/async_http_user_template.py +++ b/je_load_density/wrapper/user_template/async_http_user_template.py @@ -19,10 +19,13 @@ from je_load_density.utils.parameterization import ( parameter_resolver, register_csv_sources, + register_db_sources, register_variables, + use_resolver, ) from je_load_density.wrapper.proxy.proxy_user import locust_wrapper_proxy from je_load_density.wrapper.user_template._common import request_start_epoch +from je_load_density.wrapper.user_template.request_executor import _apply_extractors def set_wrapper_async_http_user(user_detail_dict: Dict[str, Any], **kwargs) -> type: @@ -30,6 +33,8 @@ def set_wrapper_async_http_user(user_detail_dict: Dict[str, Any], **kwargs) -> t register_variables(kwargs["variables"]) if isinstance(kwargs.get("csv_sources"), list): register_csv_sources(kwargs["csv_sources"]) + if isinstance(kwargs.get("db_sources"), list): + register_db_sources(kwargs["db_sources"]) locust_wrapper_proxy.user_dict.get("async_http_user").configure(user_detail_dict, **kwargs) return AsyncHttpUserWrapper @@ -45,6 +50,7 @@ class AsyncHttpUserWrapper(User): def __init__(self, environment): super().__init__(environment) + self._parameter_resolver = parameter_resolver.fork() self._client = None def _ensure_client(self): @@ -95,6 +101,10 @@ def _check_assertions(self, response, step: Dict[str, Any]) -> Optional[str]: return None def _do_step(self, raw_task: Dict[str, Any]) -> None: + with use_resolver(self._parameter_resolver): + self._execute_step(raw_task) + + def _execute_step(self, raw_task: Dict[str, Any]) -> None: step = parameter_resolver.resolve(raw_task) method = str(step.get("method", "")).lower() url = step.get("request_url") or step.get("url") @@ -111,6 +121,7 @@ def _do_step(self, raw_task: Dict[str, Any]) -> None: if failure is not None: self._fire(name, start, length, AssertionError(failure)) else: + _apply_extractors(response, step.get("extract") or []) self._fire(name, start, length) except Exception as error: load_density_logger.debug(f"async_http step failed: {error!r}") diff --git a/je_load_density/wrapper/user_template/fast_http_user_template.py b/je_load_density/wrapper/user_template/fast_http_user_template.py index 2dcb28e..50087df 100644 --- a/je_load_density/wrapper/user_template/fast_http_user_template.py +++ b/je_load_density/wrapper/user_template/fast_http_user_template.py @@ -3,8 +3,11 @@ from locust import FastHttpUser, between, task from je_load_density.utils.parameterization import ( + parameter_resolver, register_csv_sources, + register_db_sources, register_variables, + use_resolver, ) from je_load_density.wrapper.proxy.proxy_user import locust_wrapper_proxy from je_load_density.wrapper.user_template.scenario_runner import run_scenario @@ -19,6 +22,8 @@ def set_wrapper_fasthttp_user(user_detail_dict: Dict[str, Any], **kwargs) -> typ register_variables(kwargs["variables"]) if isinstance(kwargs.get("csv_sources"), list): register_csv_sources(kwargs["csv_sources"]) + if isinstance(kwargs.get("db_sources"), list): + register_db_sources(kwargs["db_sources"]) locust_wrapper_proxy.user_dict.get("fast_http_user").configure(user_detail_dict, **kwargs) return FastHttpUserWrapper @@ -35,6 +40,7 @@ class FastHttpUserWrapper(FastHttpUser): def __init__(self, environment): super().__init__(environment) + self._parameter_resolver = parameter_resolver.fork() self.method: Dict[str, Any] = { "get": self.client.get, "post": self.client.post, @@ -50,4 +56,5 @@ def test(self) -> None: proxy_user = locust_wrapper_proxy.user_dict.get("fast_http_user") if not proxy_user or not proxy_user.tasks: return - run_scenario(self.method, proxy_user.tasks) + with use_resolver(self._parameter_resolver): + run_scenario(self.method, proxy_user.tasks) diff --git a/je_load_density/wrapper/user_template/http_user_template.py b/je_load_density/wrapper/user_template/http_user_template.py index 9fe6e9c..6321f44 100644 --- a/je_load_density/wrapper/user_template/http_user_template.py +++ b/je_load_density/wrapper/user_template/http_user_template.py @@ -3,8 +3,11 @@ from locust import HttpUser, between, task from je_load_density.utils.parameterization import ( + parameter_resolver, register_csv_sources, + register_db_sources, register_variables, + use_resolver, ) from je_load_density.wrapper.proxy.proxy_user import locust_wrapper_proxy from je_load_density.wrapper.user_template.scenario_runner import run_scenario @@ -19,6 +22,8 @@ def set_wrapper_http_user(user_detail_dict: Dict[str, Any], **kwargs) -> type: register_variables(kwargs["variables"]) if isinstance(kwargs.get("csv_sources"), list): register_csv_sources(kwargs["csv_sources"]) + if isinstance(kwargs.get("db_sources"), list): + register_db_sources(kwargs["db_sources"]) locust_wrapper_proxy.user_dict.get("http_user").configure(user_detail_dict, **kwargs) return HttpUserWrapper @@ -35,6 +40,7 @@ class HttpUserWrapper(HttpUser): def __init__(self, environment): super().__init__(environment) + self._parameter_resolver = parameter_resolver.fork() self.method: Dict[str, Any] = { "get": self.client.get, "post": self.client.post, @@ -50,4 +56,5 @@ def test(self) -> None: proxy_user = locust_wrapper_proxy.user_dict.get("http_user") if not proxy_user or not proxy_user.tasks: return - run_scenario(self.method, proxy_user.tasks) + with use_resolver(self._parameter_resolver): + run_scenario(self.method, proxy_user.tasks) diff --git a/je_load_density/wrapper/user_template/request_executor.py b/je_load_density/wrapper/user_template/request_executor.py index 889ecce..dcc4578 100644 --- a/je_load_density/wrapper/user_template/request_executor.py +++ b/je_load_density/wrapper/user_template/request_executor.py @@ -141,7 +141,10 @@ def _apply_extractors(response: Any, extractors: Iterable[Dict[str, Any]]) -> No else: value = None if value is not None: - parameter_resolver.register_variable(var_name, value) + if extractor.get("scope", "var") == "session": + parameter_resolver.register_session_variable(var_name, value) + else: + parameter_resolver.register_variable(var_name, value) def _normalise_tasks(raw_tasks: Any) -> List[Dict[str, Any]]: diff --git a/progress.md b/progress.md index 7747706..d5e8422 100644 --- a/progress.md +++ b/progress.md @@ -7,14 +7,11 @@ Cross-repo and workspace items live in `D:\Codes\progress.md` (relevant here: X- ## Open - **#14** FSM scenario runner integration: `FsmRunner` (`je_load_density/utils/scenario/fsm.py`) is exported by `__init__.py` but `run_scenario` (`wrapper/user_template/scenario_runner.py`) does not handle `mode="fsm"`. Furthermore, `fsm.py:56` terminates on self-transitions (`target == current_state`). Fix self-transitions with a loop/retry limit, wire `FsmRunner` into `run_scenario` and user templates, and add scenario tests. -- **#15** Virtual user session context & parameter isolation: `parameter_resolver` (`je_load_density/utils/parameterization/parameter_resolver.py`) stores variables in a single global dictionary. When multiple concurrent users extract values (e.g. auth tokens, session IDs) via `task["extract"]`, they overwrite each other. Introduce session-scoped / user-local context support (`${session.VAR}` or per-user local resolution) so concurrent user journeys maintain isolated state. -- **#16** Asyncio engine as a first-class runner: `je_load_density/engine/asyncio_engine.py` is not dispatched by `wrapper/start_wrapper/start_test.py`, and package initialisation imports Locust even for async callers. Add engine selection across Python/action/CLI/GUI, lazy Locust loading, request kwargs and all assertions/extractors, scenario/retry/pacing/load shapes, per-user sessions (#15), run isolation, cancellation, reports/persistence/exporters and local/master/worker parity. Track protocol capabilities explicitly; unsupported options must fail preflight. Deliver HTTP parity first without closing full parity prematurely. Design: `docs/superpowers/specs/2026-10-02-testing-platform-roadmap.md`. +- **#16** Asyncio full parity: add engine-neutral metrics/exporter adapters, remaining protocol capabilities and native master/worker execution with recovery semantics. Finish per-run report/persistence isolation across repeated runs and scheduler isolation when switching from Locust within an embedded process. Unsupported capabilities must fail preflight. Preserve the HTTP lifecycle/scenario/session/shape contract and existing CLI/action/summary APIs. Design: `docs/superpowers/specs/2026-10-02-testing-platform-roadmap.md`. - **#17** Mixed workload multi-user support: `start_test` and `prepare_env` (`wrapper/start_wrapper/start_test.py`, `wrapper/create_locust_env/create_locust_env.py`) only instantiate a single user class from `_USER_REGISTRY`. Extend `user_detail_dict` to accept a list of user types with weights (`users: [{"user": "http_user", "weight": 8}, {"user": "websocket_user", "weight": 2}]`) and pass them to Locust's `Environment(user_classes=[...])` for realistic multi-protocol simulation. -- **#18** Finish desktop GUI redesign and shared run-state views: add action-file/target/engine/load controls, explicit Start/Stop lifecycle, retained result/log/history views and translations. Connect native cooperative callbacks and the async lifecycle from #16; expose run-state/engine/users/target context on the dashboard. The responsive browser metrics/bands and separate Qt charts are implemented. Design: `docs/superpowers/specs/2026-10-02-testing-platform-roadmap.md`. +- **#18** Dashboard run-state views: expose engine, target, active users and lifecycle context through the live dashboard/SSE, connected to the same run handles used by the desktop supervisor. Design: `docs/superpowers/specs/2026-10-02-testing-platform-roadmap.md`. - **#19** Markdown summary report generator (`LD_generate_markdown_report`): Add `generate_markdown_report.py` under `utils/generate_report/` to render GFM summary tables (RPS, latencies, SLA breaches, error clusters) suitable for GitHub Actions `$GITHUB_STEP_SUMMARY` and GitLab MR comments. Register the command in `action_executor.py`, export in `__init__.py`, and add tests. - **#20** Pre-flight health check gate: Add a target health probe before ramping load in `start_test` / `bench`. If the target endpoint is unreachable, DNS fails, or TLS handshake fails, abort early with a descriptive error instead of spawning hundreds of workers that flood logs with connection refused errors. - **#23** [BLOCKED: two releases must ship the warning first] Flip the package gate's default to refuse packages outside the allowlist: set `self.allow_arbitrary_packages = False` in `PackageManager.__init__` (`je_load_density/utils/package_manager/package_manager_class.py`); the warning branch is je_action_core's `_check_allowed` and stays for the other projects. Update the "Package gate" paragraph in the three READMEs and the "Package Gate" section of `docs/source/{En,Zh}/doc/package_manager/package_manager_doc.rst`. The warning is first released in the version after 0.0.77 (`origin/main`); flip once two releases after that one have shipped it. Decide first how a user who only runs action files (`loaddensity`, `-e`, the socket server) allows a package without a Python host to call `executor.allow_packages(...)`. -- **#24** Installed-wheel GUI smoke lifecycle: extend `test/smoke/` and the Docker GUI capability probe with offscreen execution and cancellation after the UI lifecycle redesign (#18). Keep client/thread cleanup observable. - **#28** Shared record release activation: coordinate the actual ActionCore request-record release, then activate consumer dependency floors and cross-project installed-wheel fixtures. Current opt-in APIs must retain compatibility with older core installations until that release. Design: `docs/superpowers/specs/2026-10-02-shared-record-design.md`. -- **#29** Distributed canonical record aggregation and finite-work recovery: health/startup/native virtual-user rebalancing is implemented. Add an explicit master ingestion contract that preserves source worker identity, bounded worker batches/final flush and run-scoped retry deduplication. ActionCore `RunContext.append` currently requires matching worker identity, so aggregation cannot silently rewrite it. Define lease/reassignment semantics only when introducing actual finite-work shards; native ongoing load does not replay HTTP side effects or migrate sessions. Target health is separately tracked by #20. - **#30** One script for functional/load/synthetic modes: extend APITestka's existing `integrations/load_density*.py` bridge and WebRunner's `utils/synthetic_monitoring/monitor.py` so common request/assertion steps are authored once and execution profiles choose functional correctness, capacity/SLA or periodic monitoring. Unsupported conversions must fail preflight and failed checks must reach monitor transitions. Depends on #28 and async parity #16; preserve existing CLI/action/summary contracts. diff --git a/test/smoke/gui_probe.py b/test/smoke/gui_probe.py new file mode 100644 index 0000000..988e7ca --- /dev/null +++ b/test/smoke/gui_probe.py @@ -0,0 +1,112 @@ +"""Required installed wheel smoke: real HTTP execution and cancellation through desktop controls.""" + +import os +import sys +import time + +_HTTP_SERVER = """ +from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer +class Handler(BaseHTTPRequestHandler): + def do_GET(self): + body = b'gui-smoke' + self.send_response(200) + self.send_header('Content-Length', str(len(body))) + self.end_headers() + self.wfile.write(body) + def log_message(self, *args): + pass +server = ThreadingHTTPServer(('127.0.0.1', 0), Handler) +print(server.server_port, flush=True) +server.serve_forever() +""" + + +def _until(application, predicate, timeout: float = 25) -> None: + from PySide6.QtCore import QThread + + deadline = time.monotonic() + timeout + while time.monotonic() < deadline: + application.processEvents() + if predicate(): + return + QThread.msleep(20) + raise RuntimeError("GUI lifecycle did not reach the expected state") + + +def _run(application, widget, engine: str, *, cancel: bool) -> None: + widget.engine_combobox.setCurrentText(engine) + widget.test_time_input.setText("30" if cancel else "1") + widget.start_button.click() + worker = widget.run_load_density_thread + if widget.run_state != "running": + raise RuntimeError("GUI did not start") + widget.run_load_density() + if widget.run_load_density_thread is not worker: + raise RuntimeError("GUI accepted a duplicate run") + if cancel: + _until(application, lambda: widget.last_summary.get("requests", 0) > 0) + widget.stop_button.click() + if widget.run_state != "stopping": + raise RuntimeError("GUI cancellation was not visible") + _until(application, lambda: widget.run_state not in {"running", "stopping"}) + expected = "cancelled" if cancel else "completed" + if widget.run_state != expected: + details = widget.log_panel.toPlainText() + raise RuntimeError(f"GUI {engine} expected {expected}, got {widget.run_state}: {details}") + if widget.last_summary.get("requests", 0) <= 0 or not widget.results_table.rowCount(): + raise RuntimeError("GUI failed to retain actual HTTP results") + if widget.last_summary.get("failures", 0): + raise RuntimeError("GUI HTTP smoke recorded failed requests") + _until(application, lambda: not worker.isRunning()) + if worker._process is None or worker._process.poll() is None: + raise RuntimeError("GUI worker process leaked") + + +def run_gui_probe() -> None: + """Fail when GUI dependencies or either engine's actual execution/cancel path is broken.""" + os.environ["QT_QPA_PLATFORM"] = "offscreen" + from PySide6.QtWidgets import QApplication + + from je_load_density.gui.child_process import ChildProcess + from je_load_density.gui.main_window import LoadDensityUI + + application = QApplication.instance() or QApplication([]) + server = ChildProcess([sys.executable, "-u", "-c", _HTTP_SERVER]) + window = None + try: + ready = [] + + def server_ready(): + ready.extend(server.read_lines()) + return bool(ready) + + _until(application, server_ready, timeout=5) + port = int(ready[0]) + window = LoadDensityUI() + window.show() + application.processEvents() + widget = window.load_density_widget + widget.url_input.setText(f"http://127.0.0.1:{port}/") + widget.user_count_input.setText("1") + widget.spawn_rate_input.setText("1") + for engine in ("Locust", "asyncio"): + for cancel in (False, True): + _run(application, widget, engine, cancel=cancel) + widget.test_time_input.setText("30") + widget.run_load_density() + worker = widget.run_load_density_thread + _until(application, lambda: widget.last_summary.get("requests", 0) > 0) + window.close() + application.processEvents() + if worker.isRunning() or worker._process.poll() is None: + raise RuntimeError("Closing the GUI leaked its active child") + finally: + if window is not None: + window.close() + application.processEvents() + server.kill() + server.wait(timeout=3) + + +if __name__ == "__main__": + run_gui_probe() diff --git a/test/smoke/platform_probes.py b/test/smoke/platform_probes.py index de36e73..d3c6fe7 100644 --- a/test/smoke/platform_probes.py +++ b/test/smoke/platform_probes.py @@ -102,21 +102,9 @@ def faker() -> None: def gui() -> None: - os.environ["QT_QPA_PLATFORM"] = "offscreen" - from PySide6.QtWidgets import QApplication + from gui_probe import run_gui_probe - from je_load_density.gui.main_window import LoadDensityUI - - application = QApplication.instance() or QApplication([]) - window = LoadDensityUI() - try: - window.show() - application.processEvents() - if not window.isVisible(): - raise RuntimeError("GUI offscreen launch failed") - finally: - window.close() - application.processEvents() + run_gui_probe() def k8s() -> None: diff --git a/test/test_asyncio_engine.py b/test/test_asyncio_engine.py index a4006c0..6e15c13 100644 --- a/test/test_asyncio_engine.py +++ b/test/test_asyncio_engine.py @@ -1,10 +1,10 @@ -"""The asyncio engine against a local HTTP server (progress.md #1). +"""The asyncio engine against a local HTTP server. -The server runs in its own interpreter: importing LoadDensity imports locust, whose gevent -patching turns a server thread in this process into a greenlet that never gets scheduled while -asyncio holds the main thread -- an in-process stub would only ever time out. No external network. +The server and native client run in fresh interpreters because other tests select Locust, +whose gevent patching changes this process's socket and thread scheduling. No external network. """ -import asyncio +import json +import socket import subprocess # nosec B404 - the test server is a child process import sys import time @@ -14,7 +14,6 @@ pytest.importorskip("httpx") -from je_load_density.engine.asyncio_engine import run_async_load # noqa: E402 from je_load_density.utils.test_record.test_record_class import test_record_instance # noqa: E402 _SERVER = """ @@ -61,7 +60,25 @@ def _clean_records(): def _run(tasks, **kwargs): - return asyncio.run(run_async_load(tasks=tasks, users=2, duration_seconds=1.0, **kwargs)) + # The suite also selects Locust. Exercise actual native I/O in an unpatched + # interpreter, just like bench and the isolated desktop worker. + source = """ +import asyncio, json, sys +from je_load_density.engine.asyncio_engine import run_async_load +from je_load_density.utils.test_record.test_record_class import test_record_instance +options = json.loads(sys.argv[1]) +result = asyncio.run(run_async_load(**options)) +print(json.dumps({'result': result, 'success': test_record_instance.test_record_list, + 'failure': test_record_instance.error_record_list})) +""" + options = {"tasks": tasks, "users": 2, "duration_seconds": 1.0, **kwargs} + completed = subprocess.run([sys.executable, "-c", source, json.dumps(options)], + capture_output=True, text=True, check=False, timeout=20) + assert completed.returncode == 0, completed.stdout + completed.stderr + measured = json.loads(completed.stdout.splitlines()[-1]) + test_record_instance.test_record_list[:] = measured["success"] + test_record_instance.error_record_list[:] = measured["failure"] + return measured["result"] def test_successful_requests_are_recorded(base_url): @@ -84,8 +101,11 @@ def test_server_errors_count_as_failures_like_locust(base_url): def test_connection_failures_are_recorded_with_status_zero(): - # Port 9 (discard) on loopback refuses the connection. - _run([{"method": "get", "request_url": "http://127.0.0.1:9/nothing", "timeout": 0.5}]) + # Reserve a random port without listening, avoiding host-specific discard services/firewalls. + with socket.socket() as unused: + unused.bind(("127.0.0.1", 0)) + url = f"http://127.0.0.1:{unused.getsockname()[1]}/nothing" + _run([{"method": "get", "request_url": url, "timeout": 0.2}], users=1, duration_seconds=2.0) assert test_record_instance.error_record_list assert test_record_instance.error_record_list[0]["status_code"] == "0" diff --git a/test/test_distributed_health.py b/test/test_distributed_health.py index 29ee9e9..cc81174 100644 --- a/test/test_distributed_health.py +++ b/test/test_distributed_health.py @@ -195,6 +195,7 @@ def test_native_loss_during_ramp_retains_total_and_spawn_constraint(master, monk _until(lambda: master.runner.user_count >= 2) active.remove("second") ramp.get(timeout=5) + _until(lambda: master.runner.spawning_completed and master.runner.user_count == 6, timeout=5) assert master.runner.user_count == 6 assert master.runner.target_user_count == 6 assert master.runner.spawn_rate == 2 @@ -237,7 +238,8 @@ def created(env): monkeypatch.setattr(env.shape_class, "get_run_time", lambda: next(shape_time)) env.events.spawning_complete.add_listener(lambda **_kw: seen.append(env.runner.target_user_count)) with gevent.Timeout(5): - env = wrapper.prepare_env(IdleUser, runner_mode="master", expected_workers=2, test_time=None, + env = wrapper.prepare_env(IdleUser, runner_mode="master", master_bind_port=0, + expected_workers=2, test_time=None, worker_heartbeat_interval=0.02, worker_lost_timeout=0.06, load_shape="stages", shape_config={"stages": [ {"duration": 1, "users": 2, "spawn_rate": 100}, @@ -355,7 +357,7 @@ def broken_stop(): return False with gevent.Timeout(2): with pytest.raises(RuntimeError, match="master stop callback failed"): - wrapper.prepare_env(IdleUser, runner_mode="master", expected_workers=2, + wrapper.prepare_env(IdleUser, runner_mode="master", master_bind_port=0, expected_workers=2, user_count=2, spawn_rate=100, test_time=None, on_environment=created, stop_requested=broken_stop) assert seen[0].runner.server.socket.closed @@ -375,7 +377,7 @@ def broken_stop(): return False with gevent.Timeout(2): with pytest.raises(RuntimeError, match="ramp callback failed"): - wrapper.prepare_env(IdleUser, runner_mode="master", expected_workers=2, + wrapper.prepare_env(IdleUser, runner_mode="master", master_bind_port=0, expected_workers=2, user_count=10, spawn_rate=1, test_time=None, on_environment=created, stop_requested=broken_stop) assert len(seen[0].runner.greenlet) == 0 diff --git a/test/test_distributed_records.py b/test/test_distributed_records.py new file mode 100644 index 0000000..2754921 --- /dev/null +++ b/test/test_distributed_records.py @@ -0,0 +1,710 @@ +"""Canonical aggregation and native worker delivery contracts.""" +import importlib +import json + +import pytest + +pytest.importorskip("je_action_core.request_context", reason="canonical records require the new ActionCore API") + +from je_load_density.utils.test_record.run_context import RunContext # noqa: E402 + + +def delivery_module(): + spec = importlib.util.find_spec("je_load_density.wrapper.distributed_records") + assert spec is not None, "distributed delivery is not implemented" + return importlib.import_module("je_load_density.wrapper.distributed_records") + + +def fake_environment(worker_id="worker-a"): + from types import SimpleNamespace + + from gevent.pool import Group + from locust.event import Events + + sent = [] + handlers = {} + runner = SimpleNamespace(client_id=worker_id, clients={}, + register_message=lambda name, handler: handlers.update({name: handler}), + send_message=lambda name, data=None, client_id=None: sent.append((name, data, client_id)), + quit=lambda: None, stop=lambda: None) + env = SimpleNamespace(runner=runner, events=Events(), load_density_tasks=Group(), + cancellation_error=None, process_exit_code=None) + return env, sent, handlers + + +def configure(producer, generation=1): + from types import SimpleNamespace + producer.on_config(msg=SimpleNamespace(data={"protocol": 1, "run_id": "run-one", "source": "loaddensity", + "phase": "load", "engine": "locust", "worker_id": "worker-a", "generation": generation, + "epoch": producer.epoch})) + + +def legacy_entry(): + return {"Method": "GET", "test_url": "http://localhost/", "name": "home", "status_code": "200", + "response_time_ms": 25, "response_length": 2, "start_time": 100.0} + + +def aggregate(): + spec = importlib.util.find_spec("je_load_density.utils.test_record.distributed_context") + assert spec is not None, "distributed aggregate is not implemented" + module = importlib.import_module("je_load_density.utils.test_record.distributed_context") + return module.DistributedRunContext(run_id="run-one") + + +def measured(worker="worker-a", identifier="record-one", **overrides): + context = RunContext("loaddensity", "load", "locust", worker_id=worker, run_id="run-one") + return context.capture({"record_id": identifier, "protocol": "http", "request_method": "GET", + "request_url": "http://localhost/", "name": "home", "status_code": 200, + "start_time": 100.0, "end_time": 100.025, "response_time_ms": 25, + "response_length": 2, "outcome": "passed", "error": None, **overrides}) + + +def test_aggregate_preserves_workers_and_globally_deduplicates_retries(): + run = aggregate() + first = measured() + second = measured("worker-b", "record-two") + assert len(run.ingest_batch("worker-a", [first])) == 1 + assert run.ingest_batch("worker-a", [dict(reversed(list(first.items())))]) == [] + assert len(run.ingest_batch("worker-b", [second])) == 1 + assert [item["worker_id"] for item in run.snapshot()] == ["worker-a", "worker-b"] + assert [item["record_id"] for item in json.loads(run.to_json())] == ["record-one", "record-two"] + + +@pytest.mark.parametrize("field,value", [("run_id", "foreign"), ("engine", "asyncio"), + ("source", "apitestka"), ("phase", "functional"), + ("worker_id", "worker-b")]) +def test_foreign_identity_rejects_whole_batch(field, value): + run = aggregate() + bad = measured(identifier="record-two") + bad[field] = value + with pytest.raises(ValueError, match=field): + run.ingest_batch("worker-a", [measured(), bad]) + assert run.snapshot() == [] + + +def test_conflicting_retry_rejects_new_records_in_same_batch(): + run = aggregate() + original = measured() + run.ingest_batch("worker-a", [original]) + conflicting = measured(response_time_ms=25.0) + with pytest.raises(ValueError, match="conflicting"): + run.ingest_batch("worker-a", [measured(identifier="record-new"), conflicting]) + assert run.snapshot() == [original] + + +def test_record_identifier_cannot_be_reused_by_another_worker(): + run = aggregate() + run.ingest_batch("worker-a", [measured()]) + with pytest.raises(ValueError, match="record_id"): + run.ingest_batch("worker-b", [measured("worker-b")]) + assert len(run.snapshot()) == 1 + + +def test_malformed_record_rejects_entire_batch(): + run = aggregate() + malformed = measured(identifier="record-two") + malformed["schema_version"] = 2 + with pytest.raises(ValueError, match="schema_version"): + run.ingest_batch("worker-a", [measured(), malformed]) + assert run.snapshot() == [] + + +def test_aggregate_snapshot_is_detached(): + run = aggregate() + run.ingest_batch("worker-a", [measured()]) + snapshot = run.snapshot() + snapshot[0]["name"] = "changed" + assert run.snapshot()[0]["name"] == "home" + + +def test_producer_retries_identical_bounded_batch_until_matching_ack(): + module = delivery_module() + env, sent, _ = fake_environment() + producer = module.WorkerRecords(env, module.DeliveryLimits(batch_size=2)) + configure(producer) + for _ in range(3): + producer.capture_legacy(legacy_entry(), "passed") + producer.flush() + first = sent[-1][1] + producer.flush() + assert sent[-1][1] == first + assert len(first["records"]) == 2 + assert len({record["record_id"] for record in first["records"]}) == 2 + producer.on_ack(msg=type("Message", (), {"data": {**first, "ok": True, "records": None}})()) + producer.flush() + assert sent[-1][1]["seq"] == 2 + assert len(sent[-1][1]["records"]) == 1 + assert producer.snapshot()["pending_records"] == 1 + + +def test_stale_ack_cannot_release_pending_records(): + module = delivery_module() + env, _, _ = fake_environment() + producer = module.WorkerRecords(env, module.DeliveryLimits()) + configure(producer) + producer.capture_legacy(legacy_entry(), "passed") + producer.flush() + producer.on_ack(msg=type("Message", (), {"data": {"run_id": "run-one", "worker_id": "worker-a", + "epoch": producer.epoch, "generation": 0, "seq": 1, "ok": True}})()) + assert producer.snapshot()["pending_records"] == 1 + + +def test_pending_overflow_fails_without_unbounded_queue_growth(): + module = delivery_module() + env, _, _ = fake_environment() + producer = module.WorkerRecords(env, module.DeliveryLimits(max_pending=2)) + configure(producer) + producer.capture_legacy(legacy_entry(), "passed") + producer.capture_legacy(legacy_entry(), "passed") + with pytest.raises(module.RecordDeliveryError, match="pending"): + producer.capture_legacy(legacy_entry(), "passed") + assert producer.snapshot()["pending_records"] == 2 + assert env.process_exit_code == 1 + env.load_density_tasks.kill() + + +def test_record_byte_limit_rejects_oversized_record_before_enqueuing(): + module = delivery_module() + env, _, _ = fake_environment() + producer = module.WorkerRecords(env, module.DeliveryLimits(max_record_bytes=1024)) + configure(producer) + oversized = {**legacy_entry(), "name": "x" * 2000} + with pytest.raises(module.RecordDeliveryError, match="record"): + producer.capture_legacy(oversized, "passed") + assert producer.snapshot()["pending_records"] == 0 + env.load_density_tasks.kill() + + +def frame(records, **overrides): + size = len(json.dumps(records, sort_keys=True, ensure_ascii=False, allow_nan=False, + separators=(",", ":")).encode("utf-8")) + return {"protocol": 1, "run_id": "run-one", "worker_id": "worker-a", "epoch": "producer-a", + "generation": 1, "seq": 1, "records": records, "byte_count": size, **overrides} + + +def receiver(): + from types import SimpleNamespace + module = delivery_module() + assert hasattr(module, "MasterRecords"), "master record receiver is not implemented" + env, sent, handlers = fake_environment() + env.runner.clients["worker-a"] = SimpleNamespace(state="ready", heartbeat=1) + run = aggregate() + bridge = module.MasterRecords(env, run, module.DeliveryLimits()) + bridge.on_hello(msg=SimpleNamespace(node_id="worker-a", data={"protocol": 1, "epoch": "producer-a", + "worker_id": "worker-a", "generation": 0, "run_id": None})) + return bridge, run, env, sent, handlers + + +def test_master_retries_ack_without_duplicating_canonical_or_legacy_results(): + from types import SimpleNamespace + + from je_load_density.utils.test_record.test_record_class import test_record_instance + test_record_instance.clear_records() + bridge, run, _env, sent, _ = receiver() + message = SimpleNamespace(node_id="worker-a", data=frame([measured()])) + bridge.on_batch(msg=message) + bridge.on_batch(msg=message) + assert len(run.snapshot()) == 1 + assert len(test_record_instance.test_record_list) == 1 + assert sent[-1][1]["ok"] is True + assert bridge.snapshot()["duplicate_batches"] == 1 + assert test_record_instance.test_record_list[0]["worker_id"] == "worker-a" + test_record_instance.clear_records() + + +@pytest.mark.parametrize("overrides", [{"worker_id": "spoofed"}, {"generation": 0}, {"seq": 2}, + {"run_id": "foreign"}, {"byte_count": 1}]) +def test_master_invalid_envelope_cannot_partially_ingest(overrides): + from types import SimpleNamespace + bridge, run, env, sent, _ = receiver() + bridge.on_batch(msg=SimpleNamespace(node_id="worker-a", data=frame([measured()], **overrides))) + assert run.snapshot() == [] + assert sent[-1][1]["ok"] is False + assert env.process_exit_code == 1 + env.load_density_tasks.kill() + + +def test_same_generation_handshake_does_not_change_inflight_batch(): + module = delivery_module() + env, sent, _ = fake_environment() + producer = module.WorkerRecords(env, module.DeliveryLimits()) + configure(producer) + producer.capture_legacy(legacy_entry(), "passed") + producer.flush() + original = sent[-1][1] + producer.capture_legacy(legacy_entry(), "passed") + configure(producer) + producer.flush() + assert sent[-1][1] == original + + +def until(predicate, timeout=5): + import gevent + with gevent.Timeout(timeout): + while not predicate(): + gevent.sleep(0.005) + + +@pytest.fixture +def native_pair(): + import gevent + from locust import User, task + + from je_load_density.utils.test_record.test_record_class import test_record_instance + from je_load_density.wrapper.create_locust_env import create_locust_env as wrapper + + class IdleUser(User): + @task + def idle(self): + gevent.sleep(0.01) + + test_record_instance.clear_records() + run = aggregate() + master = wrapper.create_env(IdleUser, runner_mode="master", master_bind_host="127.0.0.1", + master_bind_port=0, run_context=run, record_flush_interval=0.01, + record_drain_timeout=2) + worker = None + try: + worker = wrapper.create_env(IdleUser, runner_mode="worker", master_host="127.0.0.1", + master_port=master.runner.server.port, distributed_records=True, + record_flush_interval=0.01, record_drain_timeout=2) + until(lambda: master.record_delivery.ready_count == 1) + yield master, worker, run + finally: + wrapper.cleanup_env(master) + if worker is not None: + wrapper.cleanup_env(worker) + test_record_instance.clear_records() + + +def fire_request(env, *, exception=None): + env.events.request.fire(start_time=100, url="http://localhost/", request_type="GET", name="home", + context={}, response=None, exception=exception, response_length=2, response_time=25) + + +def test_native_rpc_binds_identity_and_aggregates_worker_request_hook(native_pair): + from je_load_density.utils.test_record.test_record_class import test_record_instance + master, worker, run = native_pair + fire_request(worker) + fire_request(worker, exception=ValueError("HTTP failed")) + until(lambda: len(run.snapshot()) == 2 and worker.record_delivery.snapshot()["pending_records"] == 0) + assert {item["worker_id"] for item in run.snapshot()} == {worker.runner.client_id} + assert {item["run_id"] for item in run.snapshot()} == {run.run_id} + assert len(test_record_instance.test_record_list) == 1 + assert len(test_record_instance.error_record_list) == 1 + assert master.record_delivery.snapshot()["accepted_records"] == 2 + + +def test_native_final_drain_delivers_batch_before_runner_and_rpc_cleanup(native_pair): + from je_load_density.wrapper.create_locust_env import create_locust_env as wrapper + master, worker, run = native_pair + master.runner.start(1, spawn_rate=1) + fire_request(worker) + wrapper.cleanup_env(master) + assert len(run.snapshot()) == 1 + assert master.record_delivery.snapshot()["error"] is None + assert worker.record_delivery.snapshot()["pending_records"] == 0 + assert master.runner.server.socket.closed + assert len(master.load_density_tasks) == 0 + + +def test_native_lost_ack_retries_without_duplicate_results(native_pair, monkeypatch): + module = delivery_module() + master, worker, run = native_pair + send = master.runner.send_message + dropped = [] + def drop_first_ack(kind, data=None, client_id=None): + if kind == module.ACK and not dropped: + dropped.append(data) + return + send(kind, data, client_id=client_id) + monkeypatch.setattr(master.runner, "send_message", drop_first_ack) + fire_request(worker) + until(lambda: worker.record_delivery.delivered == 1) + assert len(run.snapshot()) == 1 + assert master.record_delivery.duplicates >= 1 + + +def test_native_connection_reconnect_preserves_identity_and_pending_retry(native_pair): + import gevent + master, worker, run = native_pair + producer = worker.record_delivery + generation = producer.generation + address = f"tcp://127.0.0.1:{master.runner.server.port}" + worker.runner.client.socket.disconnect(address) + fire_request(worker) + gevent.sleep(0.02) + worker.runner.client.socket.connect(address) + until(lambda: producer.delivered == 1) + assert producer.generation == generation + assert run.snapshot()[0]["worker_id"] == worker.runner.client_id + assert master.record_delivery.error is None + + +def test_native_cooperative_cancel_drains_and_cleans_owned_tasks(native_pair, monkeypatch): + master, worker, run = native_pair + from je_load_density.wrapper.create_locust_env import create_locust_env as wrapper + monkeypatch.setattr(wrapper, "create_env", lambda *_args, **_kwargs: master) + def cancel(): + if worker.runner.user_count: + fire_request(worker) + return True + return False + env = wrapper.prepare_env(worker.user_classes[0], runner_mode="master", user_count=1, + test_time=None, stop_requested=cancel) + assert env.cancellation_requested + assert len(run.snapshot()) == 1 + assert len(env.load_density_tasks) == 0 + assert len(env.runner.greenlet) == 0 + assert env.record_delivery.error is None + + +def test_transport_failure_retains_pending_and_final_drain_reports_loss(monkeypatch): + from locust.exception import RPCError + module = delivery_module() + env, _, _ = fake_environment() + producer = module.WorkerRecords(env, module.DeliveryLimits(interval=0.001, drain_timeout=0.01)) + configure(producer) + producer.capture_legacy(legacy_entry(), "passed") + def disconnect(*_args, **_kwargs): + raise RPCError("disconnected") + monkeypatch.setattr(env.runner, "send_message", disconnect) + producer.flush() + assert producer.snapshot()["pending_records"] == 1 + producer.close() + assert producer.snapshot()["send_failures"] > 0 + assert "unacknowledged" in producer.snapshot()["error"] + assert env.process_exit_code == 1 + + +def test_late_ack_after_failed_close_preserves_loss_diagnostics(): + from types import SimpleNamespace + module = delivery_module() + env, sent, _ = fake_environment() + producer = module.WorkerRecords(env, module.DeliveryLimits(interval=0.001, drain_timeout=0.01)) + configure(producer) + producer.capture_legacy(legacy_entry(), "passed") + producer.flush() + original = sent[-1][1] + producer.close() + producer.on_ack(msg=SimpleNamespace(data={**original, "ok": True})) + assert producer.snapshot()["pending_records"] == 1 + + +def test_pending_byte_limit_is_independent_of_record_count(): + module = delivery_module() + env, _, _ = fake_environment() + producer = module.WorkerRecords(env, module.DeliveryLimits(max_record_bytes=1024, max_pending_bytes=1024)) + configure(producer) + entry = {**legacy_entry(), "name": "x" * 300} + producer.capture_legacy(entry, "passed") + with pytest.raises(module.RecordDeliveryError, match="pending"): + producer.capture_legacy(entry, "passed") + assert producer.snapshot()["pending_bytes"] <= 1024 + assert producer.snapshot()["pending_records"] == 1 + env.load_density_tasks.kill() + + +def test_batch_byte_boundary_retains_remaining_records_until_next_ack(): + from types import SimpleNamespace + module = delivery_module() + env, sent, _ = fake_environment() + producer = module.WorkerRecords(env, module.DeliveryLimits(max_record_bytes=1024, max_batch_bytes=1400)) + configure(producer) + for _ in range(3): + producer.capture_legacy(legacy_entry(), "passed") + producer.flush() + first = sent[-1][1] + assert len(module._encoded(first)) <= 1400 + assert len(first["records"]) < 3 + producer.on_ack(msg=SimpleNamespace(data={**first, "ok": True})) + producer.flush() + assert producer.frame["seq"] == 2 + assert producer.snapshot()["pending_records"] > 0 + + +def test_master_conflicting_batch_retry_preserves_previously_accepted_results(): + from types import SimpleNamespace + bridge, run, env, sent, _ = receiver() + bridge.on_batch(msg=SimpleNamespace(node_id="worker-a", data=frame([measured()]))) + bridge.on_batch(msg=SimpleNamespace(node_id="worker-a", data=frame([measured(name="changed")]))) + assert run.snapshot()[0]["name"] == "home" + assert sent[-1][1]["ok"] is False + env.load_density_tasks.kill() + + +def test_master_missing_worker_is_explicit_in_final_delivery_diagnostics(): + bridge, _run, env, _sent, _ = receiver() + env.runner.clients["worker-a"].state = "missing" + bridge.close() + assert bridge.snapshot()["incomplete_workers"] == ["worker-a"] + assert env.process_exit_code == 1 + + +def test_new_epoch_rejects_late_batch_from_previous_generation(): + from types import SimpleNamespace + bridge, run, env, sent, _ = receiver() + bridge.on_hello(msg=SimpleNamespace(node_id="worker-a", data={"protocol": 1, "epoch": "producer-b", + "worker_id": "worker-a", "generation": 0, "run_id": None})) + assert sent[-1][1]["generation"] == 2 + bridge.on_batch(msg=SimpleNamespace(node_id="worker-a", data=frame([measured()]))) + assert run.snapshot() == [] + assert sent[-1][1]["ok"] is False + env.load_density_tasks.kill() + + +def test_native_two_workers_final_flush_preserves_source_and_global_summary(native_pair): + from je_load_density.utils.test_record.test_record_class import test_record_instance + from je_load_density.wrapper.create_locust_env import create_locust_env as wrapper + master, first, run = native_pair + second = wrapper.create_env(first.user_classes[0], runner_mode="worker", master_host="127.0.0.1", + master_port=master.runner.server.port, distributed_records=True, + record_flush_interval=0.01, record_drain_timeout=2) + try: + until(lambda: master.record_delivery.ready_count == 2) + fire_request(first) + fire_request(second) + wrapper.cleanup_env(master) + assert len(run.snapshot()) == 2 + assert {record["worker_id"] for record in run.snapshot()} == {first.runner.client_id, second.runner.client_id} + assert len(test_record_instance.test_record_list) == 2 + assert master.record_delivery.snapshot()["incomplete_workers"] == [] + finally: + wrapper.cleanup_env(second) + + +def test_native_worker_local_stop_flushes_before_its_transport_closes(native_pair): + from je_load_density.wrapper.create_locust_env import create_locust_env as wrapper + master, worker, run = native_pair + master.runner.start(1, spawn_rate=1) + fire_request(worker) + wrapper.cleanup_env(worker) + until(lambda: len(run.snapshot()) == 1) + assert worker.record_delivery.delivered == 1 + assert worker.runner.client.socket.closed + assert len(worker.load_density_tasks) == 0 + + +@pytest.mark.parametrize("options", [{"batch_size": 0}, {"max_pending": True}, {"interval": float("nan")}, + {"max_record_bytes": 262144}, {"max_pending_bytes": 1}]) +def test_delivery_limits_reject_invalid_or_inconsistent_bounds(options): + module = delivery_module() + with pytest.raises(ValueError): + module.DeliveryLimits(**options) + + +def test_workers_cannot_override_master_run_identity_before_connecting(): + from locust import User + + from je_load_density.wrapper.create_locust_env import create_locust_env as wrapper + context = RunContext("loaddensity", "load", "locust") + with pytest.raises(ValueError, match="receiving identity"): + wrapper.create_env(User, runner_mode="worker", distributed_records=True, run_context=context) + + +def test_master_negotiated_bounds_limit_prehandshake_queue(): + from types import SimpleNamespace + module = delivery_module() + env, _, _ = fake_environment() + producer = module.WorkerRecords(env, module.DeliveryLimits()) + producer.capture_legacy(legacy_entry(), "passed") + producer.capture_legacy(legacy_entry(), "passed") + producer.on_config(msg=SimpleNamespace(data={"protocol": 1, "run_id": "run-one", "source": "loaddensity", + "phase": "load", "engine": "locust", "worker_id": "worker-a", "generation": 1, + "epoch": producer.epoch, "limits": {**module.asdict(module.DeliveryLimits()), "max_pending": 1}})) + assert producer.error is not None + assert producer.identity is None + assert producer.snapshot()["pending_records"] == 2 + env.load_density_tasks.kill() + + +def test_record_retry_in_new_sequence_does_not_duplicate_global_summary(): + from types import SimpleNamespace + + from je_load_density.utils.test_record.test_record_class import test_record_instance + test_record_instance.clear_records() + bridge, run, _env, sent, _ = receiver() + bridge.on_batch(msg=SimpleNamespace(node_id="worker-a", data=frame([measured()]))) + bridge.on_batch(msg=SimpleNamespace(node_id="worker-a", data=frame([measured()], seq=2))) + assert len(run.snapshot()) == 1 + assert len(test_record_instance.test_record_list) == 1 + assert sent[-1][1]["accepted"] == 0 + assert sent[-1][1]["ok"] is True + test_record_instance.clear_records() + + +def test_master_malformed_record_rejects_valid_prefix_without_mutating(): + from types import SimpleNamespace + bridge, run, env, sent, _ = receiver() + records = [measured(), measured(identifier="record-two", schema_version=1)] + records[1]["schema_version"] = 2 + bridge.on_batch(msg=SimpleNamespace(node_id="worker-a", data=frame(records))) + assert run.snapshot() == [] + assert sent[-1][1]["ok"] is False + env.load_density_tasks.kill() + + +def test_master_batch_count_limit_rejects_before_atomic_ingest(): + from types import SimpleNamespace + module = delivery_module() + bridge, run, env, sent, _ = receiver() + bridge.limits = module.DeliveryLimits(batch_size=1) + records = [measured(), measured(identifier="record-two")] + bridge.on_batch(msg=SimpleNamespace(node_id="worker-a", data=frame(records))) + assert run.snapshot() == [] + assert sent[-1][1]["ok"] is False + env.load_density_tasks.kill() + + +def test_aggregate_sqlite_roundtrip_preserves_worker_provenance_and_deduplication(tmp_path): + from je_load_density.utils.test_record.sqlite_persistence import fetch_canonical_records, persist_canonical_records + run = aggregate() + run.ingest_batch("worker-a", [measured()]) + run.ingest_batch("worker-b", [measured("worker-b", "record-two")]) + database = str(tmp_path / "distributed.sqlite") + assert persist_canonical_records(database, run) == run.run_id + persist_canonical_records(database, run) + assert fetch_canonical_records(database, run.run_id) == run.snapshot() + + +def test_stalled_native_send_does_not_exceed_final_drain_budget(monkeypatch): + import gevent + module = delivery_module() + env, _, _ = fake_environment() + producer = module.WorkerRecords(env, module.DeliveryLimits(interval=0.001, drain_timeout=0.01)) + configure(producer) + producer.capture_legacy(legacy_entry(), "passed") + def stalled_send(*_args, **_kwargs): + gevent.sleep(1) + monkeypatch.setattr(env.runner, "send_message", stalled_send) + with gevent.Timeout(0.1): + producer.close() + assert producer.snapshot()["pending_records"] == 1 + assert producer.snapshot()["error"] is not None + assert producer.snapshot()["send_failures"] > 0 + + +def test_stale_config_cannot_mutate_bounds_or_fail_pending_queue(): + from types import SimpleNamespace + module = delivery_module() + env, _, _ = fake_environment() + producer = module.WorkerRecords(env, module.DeliveryLimits()) + configure(producer, generation=2) + producer.capture_legacy(legacy_entry(), "passed") + producer.capture_legacy(legacy_entry(), "passed") + original = producer.limits + producer.on_config(msg=SimpleNamespace(data={"protocol": 1, "run_id": "run-one", "source": "loaddensity", + "phase": "load", "engine": "locust", "worker_id": "worker-a", "generation": 1, + "epoch": producer.epoch, "limits": {**module.asdict(module.DeliveryLimits()), "max_pending": 1}})) + assert producer.limits == original + assert producer.error is None + assert producer.generation == 2 + assert producer.snapshot()["pending_records"] == 2 + env.load_density_tasks.kill() + + +def test_cleanup_delivery_error_preserves_original_environment_callback_failure(monkeypatch): + from locust import User, runners + + from je_load_density.wrapper.create_locust_env import create_locust_env as wrapper + env = wrapper.create_env(User, runner_mode="master", master_bind_port=0, run_context=aggregate()) + env.runner.clients["unbound-worker"] = runners.WorkerNode("unbound-worker") + monkeypatch.setattr(wrapper, "create_env", lambda *_args, **_kwargs: env) + def rejected(_env): + raise RuntimeError("environment callback failed") + with pytest.raises(RuntimeError, match="environment callback failed"): + wrapper.prepare_env(User, runner_mode="master", on_environment=rejected) + assert env.record_delivery.error is not None + assert env.runner.server.socket.closed + assert len(env.load_density_tasks) == 0 + + +@pytest.mark.parametrize("extra", ["oversized_nonce", "unknown_field"]) +def test_invalid_done_control_cannot_finish_session_or_reflect_oversized_payload(extra): + from types import SimpleNamespace + module = delivery_module() + bridge, _run, _env, sent, _ = receiver() + data = {"protocol": 1, "run_id": "run-one", "worker_id": "worker-a", "epoch": "producer-a", + "generation": 1, "nonce": "terminal", "pending": 0} + if extra == "oversized_nonce": + data["nonce"] = "x" * (bridge.limits.max_batch_bytes + 1) + else: + data["extra"] = "x" * (bridge.limits.max_batch_bytes + 1) + bridge._on_done(msg=SimpleNamespace(node_id="worker-a", data=data)) + assert bridge.sessions["worker-a"].finished is False + assert bridge.rejected == 1 + assert all(kind != module.FINISHED for kind, _payload, _target in sent) + + +def test_valid_done_control_acknowledges_only_expected_bounded_fields(): + from types import SimpleNamespace + module = delivery_module() + bridge, _run, _env, sent, _ = receiver() + data = {"protocol": 1, "run_id": "run-one", "worker_id": "worker-a", "epoch": "producer-a", + "generation": 1, "nonce": "terminal", "pending": 0} + bridge._on_done(msg=SimpleNamespace(node_id="worker-a", data=data)) + assert bridge.sessions["worker-a"].finished is True + assert sent[-1][0] == module.FINISHED + assert sent[-1][1] == data + assert len(module._encoded(sent[-1][1])) <= bridge.limits.max_batch_bytes + + +def test_invalid_flush_control_does_not_spawn_drain_or_echo_nonce(): + from types import SimpleNamespace + module = delivery_module() + env, sent, _ = fake_environment() + producer = module.WorkerRecords(env, module.DeliveryLimits()) + configure(producer) + producer._on_flush(msg=SimpleNamespace(data={"run_id": "run-one", "nonce": "x" * 129})) + assert len(env.load_density_tasks) == 0 + assert all(kind != module.DONE for kind, _payload, _target in sent) + + +def test_invalid_finished_control_does_not_finalize_producer(): + from types import SimpleNamespace + module = delivery_module() + env, _sent, _ = fake_environment() + producer = module.WorkerRecords(env, module.DeliveryLimits()) + configure(producer) + producer._final_nonce = "terminal" + producer._on_finished(msg=SimpleNamespace(data={**producer._base(), "nonce": "terminal", "pending": 0, + "extra": "x" * 129})) + assert producer.finalized is False + + +def test_retired_bootstrap_hello_cannot_resurrect_epoch_or_poison_current_producer(): + from types import SimpleNamespace + module = delivery_module() + module.test_record_instance.clear_records() + bridge, run, env, sent, _ = receiver() + bridge.on_hello(msg=SimpleNamespace(node_id="worker-a", data={"protocol": 1, "epoch": "producer-b", + "worker_id": "worker-a", "generation": 0, "run_id": None})) + configured = sent[-1][1] + bridge.on_hello(msg=SimpleNamespace(node_id="worker-a", data={"protocol": 1, "epoch": "producer-a", + "worker_id": "worker-a", "generation": 0, "run_id": None})) + assert bridge.sessions["worker-a"].epoch == "producer-b" + assert bridge.sessions["worker-a"].generation == 2 + assert bridge.error is None + bridge.on_batch(msg=SimpleNamespace(node_id="worker-a", data=frame([measured()], epoch="producer-b", + generation=configured["generation"]))) + assert len(run.snapshot()) == 1 + assert sent[-1][1]["ok"] is True + module.test_record_instance.clear_records() + env.load_density_tasks.kill() + + +def test_retired_epoch_capacity_fails_without_eviction_or_session_mutation(monkeypatch): + from types import SimpleNamespace + module = delivery_module() + monkeypatch.setattr(module, "RETIRED_EPOCH_LIMIT", 1) + bridge, _run, env, _sent, _ = receiver() + for epoch in ("producer-b", "producer-c"): + bridge.on_hello(msg=SimpleNamespace(node_id="worker-a", data={"protocol": 1, "epoch": epoch, + "worker_id": "worker-a", "generation": 0, "run_id": None})) + assert bridge.error is not None + assert "retired" in str(bridge.error) + assert bridge.sessions["worker-a"].epoch == "producer-b" + assert bridge.sessions["worker-a"].generation == 2 + assert bridge.sessions["worker-a"].retired_epochs == {"producer-a"} + assert env.process_exit_code == 1 + env.load_density_tasks.kill() diff --git a/test/test_expansion2_smoke.py b/test/test_expansion2_smoke.py index 8d7ef0b..52b4128 100644 --- a/test/test_expansion2_smoke.py +++ b/test/test_expansion2_smoke.py @@ -18,7 +18,6 @@ import pytest - # ---------- Security: JWT --------------------------------------------------- @@ -52,7 +51,8 @@ def test_graphql_depth_attack_nests_correctly(): def test_ssrf_targets_include_metadata(): from je_load_density.utils.security.ssrf_checks import ( - build_ssrf_targets, find_metadata_leak, + build_ssrf_targets, + find_metadata_leak, ) targets = build_ssrf_targets() assert any("169.254.169.254" in t for t in targets) @@ -170,7 +170,9 @@ def test_factory_builds_pool(): def test_run_tagging_round_trip(tmp_path): from je_load_density.utils.governance.run_tagging import ( - list_tags, search_runs_by_tag, tag_run, + list_tags, + search_runs_by_tag, + tag_run, ) db_path = tmp_path / "runs.db" @@ -196,7 +198,8 @@ def test_run_tagging_round_trip(tmp_path): def test_audit_log_appends(tmp_path): from je_load_density.utils.governance.audit_log import ( - append_audit_entry, read_audit_log, + append_audit_entry, + read_audit_log, ) log_path = tmp_path / "audit.jsonl" @@ -208,7 +211,8 @@ def test_audit_log_appends(tmp_path): def test_share_link_round_trip(): from je_load_density.utils.governance.share_link import ( - issue_share_link, verify_share_link, + issue_share_link, + verify_share_link, ) secret = "topsecret" @@ -221,7 +225,8 @@ def test_share_link_round_trip(): def test_share_link_rejects_bad_signature(): from je_load_density.utils.governance.share_link import ( - issue_share_link, verify_share_link, + issue_share_link, + verify_share_link, ) url = issue_share_link("https://reports.example", "/r/123.html", "secret", @@ -295,26 +300,28 @@ def test_canary_verdict_promotes_when_steady(): # ---------- Asyncio engine + bench CLI -------------------------------------- -def test_asyncio_engine_returns_summary(): +def test_asyncio_engine_returns_summary(monkeypatch): """ Verify the engine wiring — httpx invocation, error-record path, and - summary-dict return shape. Uses a non-routable port so every attempt - fails fast; we're asserting plumbing, not network behaviour. + summary-dict return shape with a deterministic connection failure. Other + tests verify native socket I/O in interpreters unpatched by Locust. """ from je_load_density.engine.asyncio_engine import run_async_load from je_load_density.utils.test_record.test_record_class import test_record_instance + httpx = pytest.importorskip("httpx") + original = httpx.AsyncClient + + def disconnected(request): + raise httpx.ConnectError("connection refused", request=request) + + monkeypatch.setattr(httpx, "AsyncClient", + lambda **kwargs: original(transport=httpx.MockTransport(disconnected), **kwargs)) test_record_instance.clear_records() - try: - result = asyncio.run(run_async_load( - tasks=[{"method": "get", - "request_url": "http://127.0.0.1:1/never", - "timeout": 0.2}], - # Long enough that a loaded machine still sends at least one request before the deadline. - users=1, duration_seconds=2.0, - )) - except RuntimeError as error: - pytest.skip(f"httpx not installed: {error}") + result = asyncio.run(run_async_load( + tasks=[{"method": "get", "request_url": "http://127.0.0.1:1/never", "timeout": 0.2}], + users=1, duration_seconds=0.05, + )) assert "requests" in result and "failures" in result assert result["failures"] >= 1 @@ -323,7 +330,7 @@ def test_asyncio_engine_returns_summary(): def test_i18n_translates_known_key(): - from je_load_density.utils.dx.i18n import t, available_locales + from je_load_density.utils.dx.i18n import available_locales, t assert t("missing_locust", "zh-TW") != t("missing_locust", "en") assert "zh-TW" in available_locales() diff --git a/test/test_gui_lifecycle.py b/test/test_gui_lifecycle.py new file mode 100644 index 0000000..917b02c --- /dev/null +++ b/test/test_gui_lifecycle.py @@ -0,0 +1,331 @@ +"""Desktop lifecycle contracts without starting external services.""" + +import io +import json +import os +import subprocess +import sys +import time + +os.environ.setdefault("QT_QPA_PLATFORM", "offscreen") + +import pytest + +pytest.importorskip("PySide6") +from PySide6.QtCore import QObject, Signal +from PySide6.QtWidgets import QApplication + + +@pytest.fixture(scope="module") +def application(): + return QApplication.instance() or QApplication([]) + + +class ControlledWorker(QObject): + state_changed = Signal(str) + snapshot_ready = Signal(dict) + log_message = Signal(str) + completed = Signal(dict) + + def __init__(self, **kwargs): + super().__init__() + self.config = kwargs + self.started = False + self.stopped = False + + def start(self): + self.started = True + + def isRunning(self): + return self.started + + def request_stop(self): + self.stopped = True + + def shutdown(self, timeout_ms=6000): + self.stopped = True + self.started = False + return True + + +@pytest.fixture +def widget(application, monkeypatch): + from je_load_density.gui import main_widget + + monkeypatch.setattr(main_widget, "LoadDensityGUIThread", ControlledWorker) + view = main_widget.LoadDensityWidget() + view.url_input.setText("http://127.0.0.1:8080/") + view.test_time_input.setText("1") + view.user_count_input.setText("2") + view.spawn_rate_input.setText("2") + yield view + view.close() + + +def test_parent_import_does_not_patch_native_threads(): + source = "import sys; import je_load_density.gui.main_widget; assert 'locust' not in sys.modules" + result = subprocess.run([sys.executable, "-c", source], capture_output=True, text=True, timeout=20) + assert result.returncode == 0, result.stderr + + +def test_duplicate_start_and_visible_stop(widget): + widget.run_load_density() + worker = widget.run_load_density_thread + widget.run_load_density() + assert widget.run_load_density_thread is worker + assert widget.run_state == "running" + assert not widget.start_button.isEnabled() + widget.stop_load_density() + assert widget.run_state == "stopping" + assert worker.stopped + + +def test_new_run_clears_previous_summary(widget): + widget.last_summary = {"requests": 10} + widget.run_load_density() + assert widget.last_summary == {} + + +def test_embedded_widget_has_readable_dark_palette(widget, application): + from PySide6.QtGui import QPalette + + widget.resize(1200, 850) + widget.show() + application.processEvents() + foreground = widget.url_input.palette().color(QPalette.Text) + background = widget.url_input.palette().color(QPalette.Base) + assert foreground.lightnessF() > 0.8 + assert background.lightnessF() < 0.25 + assert widget.chart_panel._rps_view.height() >= 210 + assert "p50" in widget.stats_panel.latency_label.text() + assert "p99" in widget.stats_panel.latency_label.text() + + +def test_failure_keeps_results_and_close_stops_worker(widget): + widget.run_load_density() + worker = widget.run_load_density_thread + snapshot = {"summary": {"requests": 12, "failures": 2, "p95_ms": 40}, + "records": [{"name": "request", "response_time_ms": 40, "failed": False}]} + worker.snapshot_ready.emit(snapshot) + worker.completed.emit({"state": "failed", "summary": snapshot["summary"]}) + assert widget.run_state == "failed" + assert widget.results_table.rowCount() == 1 + assert widget.results_table.item(0, 0).text() == "request" + assert "12" in widget.stats_panel.totals_label.text() + widget.close() + assert worker.stopped + + +def test_protocol_bounds_and_redacts_records(): + from je_load_density.gui.run_protocol import make_snapshot + + records = [{"test_url": "https://user:password@example.org/a?token=secret", "start_time": 10, + "response_time_ms": 20, "request_headers": {"Authorization": "secret"}}] * 250 + snapshot = make_snapshot(records, [], users=2, elapsed=1) + assert snapshot["summary"]["requests"] == 250 + assert len(snapshot["records"]) == 200 + assert snapshot["summary"]["p95_ms"] == 20 + serialized = str(snapshot) + assert "secret" not in serialized + assert "password" not in serialized + assert "Authorization" not in serialized + + +@pytest.mark.parametrize("value, expected", [ + ("/api?token=super-secret#secret", "/api"), + ("../api?token=super-secret", "../api"), + ("//user:password@example.org/api?token=super-secret", "//example.org/api"), + ("https://" + "sensitive" * 100 + ":password@example.org/api?token=super-secret", "https://example.org/api"), + ("search?cache", "search?cache"), +]) +def test_protocol_parses_full_target_before_bounding_names(value, expected): + from je_load_density.gui.run_protocol import safe_name + + assert safe_name(value) == expected + + +def test_serialized_snapshot_stays_within_reader_byte_limit_and_keeps_latest(monkeypatch): + from je_load_density.gui.run_protocol import make_snapshot + from je_load_density.gui.run_worker import Protocol + + records = [{"name": "測" * 490 + str(index), "start_time": index, + "response_time_ms": index} for index in range(200)] + snapshot = make_snapshot(records, [], users=2) + output = io.StringIO() + monkeypatch.setattr(sys, "stdout", output) + protocol = Protocol() + protocol.emit("snapshot", data=snapshot) + frame = output.getvalue().encode("utf-8") + assert len(frame) <= 131072 + message = json.loads(frame) + assert message["data"]["summary"]["requests"] == 200 + assert message["data"]["windows"] == snapshot["windows"] + retained = message["data"]["records"] + assert retained + assert retained[-1]["name"].endswith("199") + assert [row["start_time"] for row in retained] == list(range(200 - len(retained), 200)) + assert len(snapshot["records"]) == 200 + + +def test_chart_uses_complete_child_windows_after_request_tail_truncation(application): + from je_load_density.gui.chart_panel import LiveChartPanel + from je_load_density.gui.run_protocol import make_snapshot + + records = [{"name": "first", "start_time": 100.5, "response_time_ms": 10}] * 250 + records += [{"name": "second", "start_time": 101.5, "response_time_ms": 100}] * 250 + snapshot = make_snapshot(records, [], start_time=100, end_time=102) + assert snapshot["summary"]["requests"] == 500 + assert len(snapshot["records"]) == 200 + assert [window["count"] for window in snapshot["windows"]] == [250, 250] + assert [window["p95_ms"] for window in snapshot["windows"]] == [10, 100] + chart = LiveChartPanel() + try: + chart.reset_history(100) + chart.set_snapshot(snapshot) + chart.refresh() + assert chart._rps_series.count() == 2 + assert [chart._rps_series.at(index).y() for index in range(2)] == [250, 250] + assert [chart._latency_lines[0].at(index).y() for index in range(2)] == [10, 100] + finally: + chart._timer.stop() + chart.close() + + +def test_action_failure_remains_visible_and_reports_continue(tmp_path): + from je_load_density.gui.run_worker import RunSession + + class ProtocolSink: + def emit(self, *args, **kwargs): + pass + + action_file = tmp_path / "actions.json" + report = tmp_path / "summary" + actions = {"load_density": [["missing_command"], ["LD_generate_summary_report", [str(report)]]]} + action_file.write_text(json.dumps(actions), encoding="utf-8") + session = RunSession({"engine": "asyncio", "action_file": str(action_file)}, tmp_path / "stop", ProtocolSink()) + session.execute() + assert session.failed_actions == 1 + assert report.with_suffix(".json").is_file() + + +def test_session_package_commands_are_registered_on_private_executor(tmp_path, monkeypatch): + from je_load_density.gui.run_worker import RunSession + from je_load_density.utils.executor.action_executor import executor + from je_load_density.utils.package_manager.package_manager_class import package_manager + + monkeypatch.setattr(package_manager, "allowed_packages", {"json"}) + monkeypatch.setattr(package_manager, "allow_arbitrary_packages", False) + original_commands = dict(executor.event_dict) + + class ProtocolSink: + def emit(self, *args, **kwargs): + pass + + action_file = tmp_path / "packages.json" + actions = {"load_density": [["LD_add_package_to_executor", ["json"]], ["dumps", [{"ok": True}]]]} + action_file.write_text(json.dumps(actions), encoding="utf-8") + session = RunSession({"engine": "asyncio", "action_file": str(action_file)}, tmp_path / "stop", ProtocolSink()) + session.execute() + assert session.failed_actions == 0 + assert executor.event_dict == original_commands + actions["load_density"].append(["LD_add_package_to_executor", ["pickle"]]) + action_file.write_text(json.dumps(actions), encoding="utf-8") + session.execute() + assert session.failed_actions == 1 + + +def test_cleanup_failure_still_notifies_failed_with_retained_summary(application, monkeypatch): + from je_load_density.gui.load_density_gui_thread import LoadDensityGUIThread + + class FailedCleanup: + def poll(self): + return 0 + + def wait(self, timeout): + raise OSError("password=secret") + + worker = LoadDensityGUIThread() + worker._process = FailedCleanup() + worker._last_snapshot = {"summary": {"requests": 12}, "records": []} + worker._terminal = {"state": "completed"} + monkeypatch.setattr(worker, "_launch", lambda config: None) + monkeypatch.setattr(worker, "_consume", lambda: None) + outcomes, logs = [], [] + worker.completed.connect(outcomes.append) + worker.log_message.connect(logs.append) + worker.run() + assert outcomes == [{"state": "failed", "summary": {"requests": 12}}] + assert logs + assert "secret" not in " ".join(logs) + + +def test_scripted_load_preserved_with_selected_engine_and_callbacks(tmp_path, monkeypatch): + from je_load_density.engine import entrypoints + from je_load_density.gui.run_worker import RunSession + + calls = [] + monkeypatch.setattr(entrypoints, "start_test", lambda *args, **kwargs: calls.append((args, kwargs))) + session = RunSession({"engine": "asyncio"}, tmp_path / "stop", None) + tasks = {"get": {"request_url": "http://localhost/example"}} + session.start({"user": "http_user"}, 7, 3, 11, tasks=tasks, engine="locust") + args, kwargs = calls[0] + assert args == ({"user": "http_user"}, 7, 3, 11) + assert kwargs["tasks"] is tasks + assert kwargs["engine"] == "asyncio" + assert not kwargs["stop_requested"]() + session.control.touch() + assert kwargs["stop_requested"]() + + +def test_child_incidental_output_is_visible_without_credentials(monkeypatch): + from je_load_density.gui.run_worker import Protocol + + output = io.StringIO() + monkeypatch.setattr(sys, "stdout", output) + protocol = Protocol() + protocol.write("password=secret\n") + message = json.loads(output.getvalue()) + assert message["type"] == "log" + assert "secret" not in message["text"] + + +def test_unresponsive_child_is_killed_and_thread_joined(application, monkeypatch, tmp_path): + from je_load_density.gui import load_density_gui_thread + + monkeypatch.setattr(load_density_gui_thread, "STOP_GRACE_SECONDS", 0.1) + pid_file = tmp_path / "actual-child.pid" + + class UnresponsiveWorker(load_density_gui_thread.LoadDensityGUIThread): + def _launch(self, config): + script = (f"from pathlib import Path; import os,time; " + f"Path({str(pid_file)!r}).write_text(str(os.getpid())); time.sleep(60)") + self._process = load_density_gui_thread.ChildProcess( + [sys.executable, "-c", script]) + + worker = UnresponsiveWorker() + worker.start() + started = time.monotonic() + try: + deadline = started + 2 + while not pid_file.exists() and time.monotonic() < deadline: + load_density_gui_thread.QThread.msleep(20) + assert pid_file.exists() + assert worker.shutdown(timeout_ms=1500) + assert not worker.isRunning() + assert worker._process.poll() is not None + assert time.monotonic() - started < 3 + if os.name == "nt": + import ctypes + + kernel = ctypes.WinDLL("kernel32", use_last_error=True) + kernel.OpenProcess.restype = ctypes.c_void_p + kernel.WaitForSingleObject.argtypes = [ctypes.c_void_p, ctypes.c_ulong] + kernel.CloseHandle.argtypes = [ctypes.c_void_p] + handle = kernel.OpenProcess(0x00100000, False, int(pid_file.read_text())) + if handle: + try: + assert kernel.WaitForSingleObject(handle, 1000) == 0 + finally: + kernel.CloseHandle(handle) + finally: + worker.shutdown(timeout_ms=1500) diff --git a/test/test_legacy_cli_contract.py b/test/test_legacy_cli_contract.py index b2a54ab..3840f5e 100644 --- a/test/test_legacy_cli_contract.py +++ b/test/test_legacy_cli_contract.py @@ -91,3 +91,31 @@ def test_no_flag_exits_non_zero(tmp_path): def test_create_project_dir_is_exported(): assert callable(je_load_density.create_project_dir) + + +def test_cli_executes_with_reporter_api_without_core_collector(monkeypatch): + from types import SimpleNamespace + + from je_action_core.reporting import PrintReporter + + from je_load_density import __main__ as cli + from je_load_density.utils.exception.exceptions import LoadDensityTestExecuteException + from je_load_density.utils.executor.action_executor import Executor + + settings = Executor().settings + settings_before = settings + reported = [] + monkeypatch.setattr(PrintReporter, "on_failure", lambda *args: reported.append("failure")) + monkeypatch.setattr(PrintReporter, "on_records", lambda *args: reported.append("records")) + + def execute(actions): + assert actions == [["failing action", []]] + legacy.settings.reporter.on_failure(actions[0], RuntimeError("failed")) + legacy.settings.reporter.on_records({"failure": "failed"}) + + legacy = SimpleNamespace(settings=settings, execute_action=execute) + monkeypatch.setattr(cli, "executor", legacy) + with pytest.raises(LoadDensityTestExecuteException, match="1 action"): + cli._execute_cli_actions([["failing action", []]]) + assert legacy.settings is settings_before + assert reported == ["failure", "records"] diff --git a/test/test_native_async_runner.py b/test/test_native_async_runner.py new file mode 100644 index 0000000..e58ee8b --- /dev/null +++ b/test/test_native_async_runner.py @@ -0,0 +1,437 @@ +"""Native HTTP behavior, session ownership and lifecycle contracts.""" + +import asyncio + +import httpx +import pytest + +from je_load_density.engine.asyncio_engine import run_async_load +from je_load_density.engine.entrypoints import start_test +from je_load_density.utils.parameterization import get_resolver, use_resolver +from je_load_density.utils.test_record.test_record_class import test_record_instance + + +@pytest.fixture +def transport_factory(monkeypatch): + original = httpx.AsyncClient + clients = [] + + def install(handler): + def factory(**kwargs): + client = original(transport=httpx.MockTransport(handler), **kwargs) + clients.append(client) + return client + + monkeypatch.setattr(httpx, "AsyncClient", factory) + return clients + + return install + + +def test_dispatch_uses_native_http_assertions_and_named_kwargs(transport_factory): + received = [] + + def handler(request): + received.append(request) + return httpx.Response(200, json={"ok": True}, headers={"X-State": "ready"}) + + clients = transport_factory(handler) + result = start_test( + {"user": "http_user"}, + engine="asyncio", + user_count=1, + spawn_rate=100, + test_time=0.08, + tasks=[ + { + "method": "post", + "request_url": "https://local/check", + "name": "check", + "params": {"tenant": "one"}, + "json": {"hello": "world"}, + "auth": {"type": "basic", "username": "u", "password": "p"}, + "assertions": [ + {"type": "json_path", "path": "ok", "value": True}, + {"type": "header", "name": "X-State", "value": "ready"}, + ], + } + ], + ) + assert result["summary"]["totals"]["successes"] > 0 + assert result["summary"]["per_name"]["check"]["count"] > 0 + assert received[0].url.params["tenant"] == "one" + assert received[0].headers["Authorization"].startswith("Basic ") + assert all(client.is_closed for client in clients) + + +def test_virtual_users_keep_cookie_and_extracted_token_paired(transport_factory): + issued = [] + checked = [] + + def handler(request): + if request.url.path == "/login": + token = str(len(issued) + 1) + issued.append(token) + return httpx.Response(200, json={"token": token}, headers={"Set-Cookie": f"token={token}; Path=/"}) + checked.append((request.headers["Authorization"], request.headers["Cookie"])) + return httpx.Response(200) + + transport_factory(handler) + with use_resolver(get_resolver().fork()): + result = asyncio.run( + run_async_load( + [ + { + "method": "get", + "request_url": "https://local/login", + "extract": [{"from": "json_path", "path": "token", "var": "token", "scope": "session"}], + }, + { + "method": "get", + "request_url": "https://local/check", + "headers": {"Authorization": "${session.token}"}, + }, + ], + users=3, + duration_seconds=0.08, + ) + ) + assert result["requests"] > 0 + assert checked and all(cookie == f"token={token}" for token, cookie in checked) + + +def test_assertion_failure_is_measured_and_retried(transport_factory): + counter = 0 + + def handler(_request): + nonlocal counter + counter += 1 + return httpx.Response(200, json={"ok": counter > 1}) + + transport_factory(handler) + result = asyncio.run( + run_async_load( + [ + { + "method": "get", + "request_url": "https://local/check", + "assertions": [{"type": "json_path", "path": "ok", "value": True}], + "retry": {"flaky": 1, "base_delay": 0, "jitter": 0}, + "think_time": 0.02, + } + ], + users=1, + duration_seconds=0.08, + ) + ) + assert result["failures"] == 1 + assert result["requests"] >= 1 + assert test_record_instance.error_record_list[-1]["status_code"] == "200" + + +@pytest.mark.parametrize( + "options", + [ + {"users": 0}, + {"users": True}, + {"duration_seconds": float("nan")}, + {"max_in_flight": 0}, + {"runner_mode": "master"}, + ], +) +def test_invalid_capabilities_fail_before_opening_clients(transport_factory, options): + clients = transport_factory(lambda _request: httpx.Response(200)) + with pytest.raises((ValueError, TypeError)): + asyncio.run(run_async_load([{"request_url": "https://local", "method": "get"}], **options)) + assert clients == [] + + +@pytest.mark.parametrize( + "task", + [ + {"assertions": [{"type": "unknown"}]}, + {"extract": [{"from": "unknown", "var": "x"}]}, + {"network_conditioner": {}}, + {"retry": {"transient": -1}}, + ], +) +def test_unsupported_steps_fail_preflight(transport_factory, task): + clients = transport_factory(lambda _request: httpx.Response(200)) + with pytest.raises(ValueError): + asyncio.run(run_async_load([{"method": "get", "request_url": "https://local", **task}])) + assert not clients + + +def test_stop_during_io_closes_clients_without_failure_records(transport_factory): + from je_load_density.engine.asyncio_engine import AsyncRunHandle + + entered = asyncio.Event() + + async def handler(_request): + entered.set() + await asyncio.sleep(30) + return httpx.Response(200) + + clients = transport_factory(handler) + + async def run(): + handle = AsyncRunHandle([{"method": "get", "request_url": "https://local"}], users=2, duration_seconds=20) + await handle.start() + await asyncio.wait_for(entered.wait(), 2) + handle.stop() + result = await asyncio.wait_for(handle.wait(), 2) + assert result["state"] == "cancelled" + assert result["failures"] == 0 + assert handle.snapshot()["users"] == 0 + + asyncio.run(run()) + assert clients and all(client.is_closed for client in clients) + + +def test_repeated_run_summary_does_not_include_legacy_history(transport_factory): + transport_factory(lambda _request: httpx.Response(200)) + test_record_instance.error_record_list.append({"response_time_ms": 900, "name": "previous"}) + result = asyncio.run( + run_async_load([{"request_url": "https://local", "method": "get"}], users=1, duration_seconds=0.03) + ) + assert result["summary"]["totals"]["failures"] == 0 + assert "previous" not in result["summary"]["per_name"] + + +def test_completed_worker_error_is_observed_before_forced_deadline(monkeypatch): + from je_load_density.engine.asyncio_engine import AsyncRunHandle + + async def run(): + entered = asyncio.Event() + handle = AsyncRunHandle([{"method": "get", "url": "https://local"}], users=1, duration_seconds=60) + resize = handle._resize + + async def failed_user(_httpx, _resolver): + entered.set() + raise RuntimeError("worker failed at deadline") + + async def resize_at_deadline(httpx_module, count, available): + await resize(httpx_module, count, available) + await entered.wait() + await asyncio.sleep(0) + handle.duration = 0 + + monkeypatch.setattr(handle, "_user", failed_user) + monkeypatch.setattr(handle, "_resize", resize_at_deadline) + await handle.start() + with pytest.raises(RuntimeError, match="worker failed at deadline"): + await handle.wait() + assert entered.is_set() + assert handle.state == "failed" + assert all(worker.done() for worker in handle.workers) + + asyncio.run(run()) + + +def test_worker_failure_at_run_deadline_propagates(transport_factory): + transport_factory(lambda _request: (_ for _ in ()).throw(ValueError("invalid transport"))) + with pytest.raises(ValueError, match="invalid transport"): + asyncio.run(run_async_load([{"request_url": "https://local"}], users=1, duration_seconds=0.005)) + + +def test_expected_error_status_can_pass_assertions(transport_factory): + transport_factory(lambda _request: httpx.Response(404, text="missing")) + result = asyncio.run( + run_async_load( + [ + { + "request_url": "https://local", + "assertions": [{"type": "status_code", "value": 404}], + } + ], + users=1, + duration_seconds=0.03, + ) + ) + assert result["requests"] > 0 + assert result["failures"] == 0 + + +def test_httpx_connection_failure_uses_transient_retry_budget(transport_factory): + calls = 0 + + def handler(request): + nonlocal calls + calls += 1 + if calls == 1: + raise httpx.ConnectError("refused", request=request) + return httpx.Response(200) + + transport_factory(handler) + result = asyncio.run( + run_async_load( + [ + { + "request_url": "https://local", + "retry": {"transient": 1, "base_delay": 0, "jitter": 0}, + "think_time": 30, + } + ], + users=1, + duration_seconds=0.05, + ) + ) + assert result["failures"] == 1 + assert result["requests"] == 1 + + +@pytest.mark.parametrize( + "task", + [ + {"headers": 7}, + {"run_if": {"unsupported": True}}, + {"think_time": {"minimum": 1}}, + {"auth": {"type": "basic", "usernmae": "u"}}, + ], +) +def test_static_invalid_second_step_fails_before_any_io(transport_factory, task): + clients = transport_factory(lambda _request: httpx.Response(200)) + with pytest.raises(ValueError): + asyncio.run( + run_async_load( + [{"request_url": "https://local"}, {"request_url": "https://local", **task}], + users=1, + duration_seconds=0.03, + ) + ) + assert clients == [] + + +@pytest.mark.parametrize("additional_options", [{}, {"verify": "${verify_tls}"}]) +def test_invalid_static_transport_fails_before_earlier_step_io(transport_factory, monkeypatch, additional_options): + received = [] + clients = transport_factory(lambda request: received.append(request) or httpx.Response(200)) + original = httpx.AsyncClient + + def client(**options): + if options.get("cert") == "invalid.pem": + raise ValueError("invalid certificate configuration") + return original(**options) + + monkeypatch.setattr(httpx, "AsyncClient", client) + with pytest.raises(ValueError, match="invalid certificate"): + asyncio.run( + run_async_load( + [{"request_url": "https://local"}, + {"request_url": "https://local", "cert": "invalid.pem", **additional_options}], + users=1, + duration_seconds=0.1, + variables={"verify_tls": False}, + ) + ) + assert received == [] + assert all(client.is_closed for client in clients) + + +def test_unknown_shape_option_fails_before_any_io(transport_factory): + clients = transport_factory(lambda _request: httpx.Response(200)) + with pytest.raises(ValueError): + asyncio.run( + run_async_load( + [{"request_url": "https://local"}], + users=1, + duration_seconds=0.03, + load_shape="spike", + shape_config={"baseline_users": 1, "spwan_rate": 100}, + ) + ) + assert clients == [] + + +@pytest.mark.parametrize("limit,expected", [(None, 3), (1, 1)]) +def test_shape_concurrency_uses_peak_users_unless_explicitly_limited(transport_factory, limit, expected): + active = peak = 0 + + async def handler(_request): + nonlocal active, peak + active += 1 + peak = max(peak, active) + try: + await asyncio.sleep(0.04) + return httpx.Response(200) + finally: + active -= 1 + + transport_factory(handler) + asyncio.run( + run_async_load( + [{"request_url": "https://local"}], + users=1, + duration_seconds=0.12, + load_shape="stages", + shape_config={"stages": [{"duration": 1, "users": 3, "spawn_rate": 1000}]}, + max_in_flight=limit, + ) + ) + assert peak == expected + assert active == 0 + + +def test_certificate_clients_do_not_mutate_shared_trust_context(transport_factory): + import ssl + + from je_load_density.engine.async_http import UserClientPool + + seen = [] + + class Client: + def __init__(self, **kwargs): + seen.append(kwargs) + self.cookies = httpx.Cookies() + + async def __aenter__(self): + return self + + async def __aexit__(self, *_args): + return False + + async def run(): + trust = ssl.create_default_context() + module = type("Http", (), {"Cookies": httpx.Cookies, "AsyncClient": Client}) + async with UserClientPool(module, False, trust) as pool: + await pool.client({"cert": "one.pem"}) + await pool.client({"client_cert": "two.pem"}) + await pool.client({}) + assert seen[0]["verify"] is not trust + assert seen[1]["verify"] is not trust + assert seen[2]["verify"] is trust + + asyncio.run(run()) + + +@pytest.mark.parametrize("operation", ["stop", "scale_down"]) +def test_worker_cleanup_error_propagates_on_stop_and_scale_down(operation): + from je_load_density.engine.asyncio_engine import AsyncRunHandle + + async def run(): + entered = asyncio.Event() + handle = AsyncRunHandle([{"request_url": "https://local"}], users=1, duration_seconds=20) + + async def user(_httpx, _resolver): + entered.set() + try: + await asyncio.sleep(30) + finally: + raise RuntimeError("client cleanup failed") + + handle._user = user + await handle.start() + await asyncio.wait_for(entered.wait(), 2) + try: + if operation == "scale_down": + with pytest.raises(RuntimeError, match="client cleanup failed"): + await handle._resize(None, 0, 0) + handle.stop() + with pytest.raises(RuntimeError, match="client cleanup failed"): + await handle.wait() + finally: + if not handle._task.done(): + handle.stop() + await asyncio.gather(handle._task, return_exceptions=True) + + asyncio.run(run()) diff --git a/test/test_native_imports.py b/test/test_native_imports.py new file mode 100644 index 0000000..4b98996 --- /dev/null +++ b/test/test_native_imports.py @@ -0,0 +1,73 @@ +"""Native engine entry points preserve the interpreter's I/O scheduling.""" + +import subprocess +import sys + +import pytest + + +@pytest.mark.parametrize("name", ["task", "TaskSet", "SequentialTaskSet"]) +def test_selecting_locust_exports_registers_legacy_request_records(name): + source = ( + "from je_load_density import " + + name + + """, test_record_instance +from locust import events +events.request.fire(start_time=100, url='http://local', request_type='GET', name='one', + context={}, response=None, exception=None, response_length=0, response_time=1) +assert len(test_record_instance.test_record_list) == 1 +""" + ) + result = subprocess.run([sys.executable, "-c", source], capture_output=True, text=True, timeout=20, check=False) + assert result.returncode == 0, result.stdout + result.stderr + + +@pytest.mark.parametrize( + "statement", + [ + "import je_load_density", + "from je_load_density import run_async_load, build_summary, start_test", + "from je_load_density.engine.asyncio_engine import run_async_load", + "from je_load_density.wrapper.start_wrapper.start_test import start_test", + "from je_load_density import execute_action; execute_action({'load_density': [['LD_summary']]})", + "from je_load_density.__main__ import _build_parser; _build_parser()", + ], +) +def test_native_entry_points_do_not_import_locust_or_patch_io(statement): + source = ( + """ +import socket, ssl, threading, time, sys +original = (socket.socket, ssl.SSLSocket, threading.Thread, time.sleep) +""" + + statement + + """ +assert 'locust' not in sys.modules, 'native import eagerly loads Locust' +assert 'gevent.monkey' not in sys.modules, 'native import loads gevent monkey patching' +assert original == (socket.socket, ssl.SSLSocket, threading.Thread, time.sleep) +""" + ) + result = subprocess.run([sys.executable, "-c", source], text=True, capture_output=True, timeout=20, check=False) + assert result.returncode == 0, result.stdout + result.stderr + + +def test_package_inspection_does_not_register_unrestricted_import_helper(): + source = """ +from je_load_density.utils.exception.exceptions import LoadDensityTestExecuteException +from je_load_density.utils.executor.action_executor import Executor +from je_load_density.utils.package_manager.package_manager_class import PackageManager +manager = PackageManager() +manager.executor = Executor() +manager.set_allow_arbitrary_packages(False) +manager.allow_packages('je_load_density') +manager.add_package_to_executor('je_load_density') +assert callable(manager.executor.event_dict['resolve']) +try: + manager.add_package_to_executor('os') +except LoadDensityTestExecuteException: + pass +else: + raise AssertionError('host package gate did not refuse os') +assert 'import_module' not in manager.executor.event_dict, 'public inspection registered an unrestricted module loader' +""" + result = subprocess.run([sys.executable, "-c", source], text=True, capture_output=True, timeout=20, check=False) + assert result.returncode == 0, result.stdout + result.stderr diff --git a/test/test_parameter_context.py b/test/test_parameter_context.py new file mode 100644 index 0000000..1c99e46 --- /dev/null +++ b/test/test_parameter_context.py @@ -0,0 +1,379 @@ +"""Parameter state follows an explicit user while data rows remain shared.""" + +import asyncio +import subprocess +import sys +from contextlib import contextmanager +from types import SimpleNamespace + +import gevent +import pytest + +from je_load_density.utils import parameterization as parameters +from je_load_density.utils.parameterization import ( + ParameterResolver, + parameter_resolver, + register_variable, +) +from je_load_density.wrapper.user_template.request_executor import _apply_extractors + + +@pytest.fixture(autouse=True) +def isolated_default(): + parameter_resolver.clear() + yield + parameter_resolver.clear() + + +def test_fork_detaches_nested_variable_values_and_future_registration(): + base = ParameterResolver() + nested = {"roles": ["reader"]} + base.register_variable("claims", nested) + first, second = base.fork(), base.fork() + nested["roles"].append("admin") + first.register_variable("claims", {"roles": ["writer"]}) + assert first.resolve("${var.claims}") == "{'roles': ['writer']}" + assert second.resolve("${var.claims}") == "{'roles': ['reader']}" + assert base.resolve("${var.claims}") == "{'roles': ['reader', 'admin']}" + + +def test_scoped_facade_restores_nested_context_and_keeps_default_state(): + register_variable("token", "default") + first, second = parameter_resolver.fork(), parameter_resolver.fork() + saved_resolve = parameter_resolver.resolve + with parameters.use_resolver(first): + register_variable("token", "first") + assert saved_resolve("${var.token}") == "first" + with pytest.raises(RuntimeError), parameters.use_resolver(second): + register_variable("token", "second") + assert parameters.resolve("${token}") == "second" + raise RuntimeError("scope failed") + assert parameters.get_resolver() is first + assert parameters.resolve("${token}") == "first" + assert saved_resolve("${token}") == "default" + + +def test_async_users_keep_interleaved_extractions_local(): + async def user(token): + local = parameter_resolver.fork() + with parameters.use_resolver(local): + _apply_extractors(SimpleNamespace(json=lambda: {"token": token}), + [{"var": "token", "path": "token"}]) + await asyncio.sleep(0) + return parameter_resolver.resolve({"Authorization": "Bearer ${var.token}"}) + + async def execute(): + return await asyncio.gather(user("first"), user("second")) + + assert asyncio.run(execute()) == [{"Authorization": "Bearer first"}, {"Authorization": "Bearer second"}] + assert parameter_resolver.resolve("${var.token}") == "${var.token}" + + +def test_greenlet_scopes_restore_and_do_not_share_tokens(): + def user(token): + with parameters.use_resolver(parameter_resolver.fork()): + register_variable("token", token) + gevent.sleep(0) + return parameters.resolve("${var.token}") + + users = [gevent.spawn(user, token) for token in ("first", "second")] + assert [user.get() for user in users] == ["first", "second"] + assert parameters.resolve("${var.token}") == "${var.token}" + + +def test_session_extraction_is_distinct_and_user_local(): + register_variable("token", "base") + first, second = parameter_resolver.fork(), parameter_resolver.fork() + with parameters.use_resolver(first): + _apply_extractors(SimpleNamespace(json=lambda: {"token": "session-first"}), + [{"var": "token", "path": "token", "scope": "session"}]) + assert parameters.resolve("${session.token}/${var.token}") == "session-first/base" + with parameters.use_resolver(second): + assert parameters.resolve("${session.token}/${var.token}") == "${session.token}/base" + + +def test_csv_request_fields_reuse_one_row_and_forks_share_provider(tmp_path): + path = tmp_path / "users.csv" + path.write_text("user,password\nalice,alice-secret\nbob,bob-secret\n", encoding="utf-8") + base = ParameterResolver() + base.register_csv_source("accounts", str(path), cycle=False) + first, second = base.fork(), base.fork() + task = {"url": "https://example.test/${csv.accounts.user}", + "json": {"user": "${csv.accounts.user}", "password": "${csv.accounts.password}"}} + assert first.resolve(task) == {"url": "https://example.test/alice", + "json": {"user": "alice", "password": "alice-secret"}} + assert second.resolve(task) == {"url": "https://example.test/bob", + "json": {"user": "bob", "password": "bob-secret"}} + assert base.resolve(task) == task + + +def test_db_request_fields_reuse_one_row_across_nested_collections(monkeypatch): + monkeypatch.setattr(ParameterResolver, "_read_db", staticmethod(lambda *_args: [ + {"id": 1, "name": "Alice"}, {"id": 2, "name": "Bob"}])) + base = ParameterResolver() + base.register_db_source("accounts", "sqlite://", "SELECT id, name FROM accounts") + task = ("${db.accounts.id}", [{"name": "${db.accounts.name}"}]) + assert base.fork().resolve(task) == ("1", [{"name": "Alice"}]) + assert base.fork().resolve(task) == ("2", [{"name": "Bob"}]) + assert base.resolve(task) == ("1", [{"name": "Alice"}]) + + +def test_source_registration_and_clear_within_scope_do_not_change_base(tmp_path): + path = tmp_path / "accounts.csv" + path.write_text("name\nAlice\n", encoding="utf-8") + register_variable("token", "base") + with parameters.use_resolver(parameter_resolver.fork()): + parameters.register_csv_source("local", str(path)) + assert parameters.resolve("${csv.local.name}") == "Alice" + parameter_resolver.clear() + assert parameters.resolve("${token}") == "${token}" + assert parameters.resolve("${token}") == "base" + assert parameters.resolve("${csv.local.name}") == "${csv.local.name}" + + +class _LocustHttpResponse: + status_code, text, content = 200, "ok", b"ok" + + def __init__(self, token): + self.token = token + + def json(self): + return {"token": self.token} + + def __enter__(self): + return self + + def __exit__(self, *_args): + gevent.sleep(0) + + def success(self): + pass + + def failure(self, reason): + pytest.fail(reason) + + +def _setup_locust_http_users(monkeypatch, backend): + from locust.env import Environment + + from je_load_density.wrapper.proxy.proxy_user import locust_wrapper_proxy + from je_load_density.wrapper.user_template import ( + async_http_user_template as async_template, + ) + from je_load_density.wrapper.user_template import ( + fast_http_user_template as fast_template, + ) + from je_load_density.wrapper.user_template import ( + http_user_template as http_template, + ) + + templates = {"http": (http_template.HttpUserWrapper, "http_user"), + "fast_http": (fast_template.FastHttpUserWrapper, "fast_http_user"), + "async_http": (async_template.AsyncHttpUserWrapper, "async_http_user")} + user_type, proxy_name = templates[backend] + login = {"method": "get", "url": "/login", "extract": [{"var": "token", "path": "token"}]} + request = {"method": "get", "url": "/data", "headers": {"Authorization": "Bearer ${token}"}} + proxy = SimpleNamespace(tasks=[login, request]) + monkeypatch.setitem(locust_wrapper_proxy.user_dict, proxy_name, proxy) + env = Environment() + env.events.request.add_listener(lambda **_kwargs: gevent.sleep(0)) + + users, calls = [], {"first": [], "second": []} + for token in calls: + user = user_type(env) + + def get(url, _token=token, **kwargs): + if url == "/data": + calls[_token].append(kwargs["headers"]["Authorization"]) + return _LocustHttpResponse(_token) + + if backend == "async_http": + user._client = SimpleNamespace(request=lambda _method, url, _get=get, **kwargs: _get(url, **kwargs)) + else: + user.method = {"get": get} + users.append(user) + return users, calls, proxy, request + + +@pytest.mark.parametrize("backend", ["http", "fast_http", "async_http"]) +def test_locust_http_users_keep_extracted_tokens_across_scenario_calls(monkeypatch, backend): + users, calls, proxy, request = _setup_locust_http_users(monkeypatch, backend) + invoke = (lambda user: user.run_tasks()) if backend == "async_http" else (lambda user: user.test()) + gevent.joinall([gevent.spawn(invoke, user) for user in users], raise_error=True) + proxy.tasks = [request] + gevent.joinall([gevent.spawn(invoke, user) for user in users], raise_error=True) + assert calls == {"first": ["Bearer first", "Bearer first"], "second": ["Bearer second", "Bearer second"]} + assert parameters.resolve("${token}") == "${token}" + + +@pytest.mark.parametrize("module_name,setter,proxy_name", [ + ("http_user_template", "set_wrapper_http_user", "http_user"), + ("fast_http_user_template", "set_wrapper_fasthttp_user", "fast_http_user"), + ("async_http_user_template", "set_wrapper_async_http_user", "async_http_user"), +]) +def test_http_template_setup_registers_database_sources(monkeypatch, module_name, setter, proxy_name): + import importlib + + from je_load_density.wrapper.proxy.proxy_user import locust_wrapper_proxy + + module = importlib.import_module(f"je_load_density.wrapper.user_template.{module_name}") + monkeypatch.setattr(ParameterResolver, "_read_db", staticmethod(lambda *_args: [{"name": "Alice"}])) + monkeypatch.setitem(locust_wrapper_proxy.user_dict, proxy_name, SimpleNamespace(configure=lambda *_a, **_k: None)) + getattr(module, setter)({}, db_sources=[{"name": "accounts", "connection_string": "sqlite://", + "query": "SELECT name FROM accounts"}]) + assert parameters.resolve("${db.accounts.name}") == "Alice" + + +def test_shared_provider_assigns_complete_rows_once_across_concurrent_users(tmp_path): + path = tmp_path / "accounts.csv" + path.write_text("id,secret\n" + "".join(f"{index},secret-{index}\n" for index in range(64)), encoding="utf-8") + base = ParameterResolver() + base.register_csv_source("accounts", str(path), cycle=False) + + def resolve_user(local): + with parameters.use_resolver(local): + gevent.sleep(0) + return parameters.resolve({"id": "${csv.accounts.id}", "secret": "${csv.accounts.secret}"}) + + users = [gevent.spawn(resolve_user, base.fork()) for _index in range(64)] + rows = [user.get() for user in users] + assert {row["id"] for row in rows} == {str(index) for index in range(64)} + assert all(row["secret"] == f"secret-{row['id']}" for row in rows) + assert base.resolve("${csv.accounts.id}") == "${csv.accounts.id}" + + +def test_cancelled_async_scope_restores_parent_selection(): + parent, child = ParameterResolver(), ParameterResolver() + parent.register_variable("token", "parent") + + async def execute(): + entered = asyncio.Event() + + async def user(): + with parameters.use_resolver(child): + register_variable("token", "child") + entered.set() + await asyncio.Event().wait() + + with parameters.use_resolver(parent): + task = asyncio.create_task(user()) + await entered.wait() + task.cancel() + with pytest.raises(asyncio.CancelledError): + await task + assert parameters.get_resolver() is parent + assert parameters.resolve("${token}") == "parent" + + asyncio.run(execute()) + assert parameters.resolve("${token}") == "${token}" + + +@pytest.mark.parametrize("fail_query", [False, True]) +def test_database_source_closes_connection_and_disposes_pool_after_cache(monkeypatch, fail_query): + closed = [] + result = SimpleNamespace(keys=lambda: ["name"], fetchall=lambda: [("Alice",)]) + + def execute(_query): + if fail_query: + raise ValueError("query failed") + return result + + @contextmanager + def connect(): + try: + yield SimpleNamespace(execute=execute) + finally: + closed.append("connection") + + engine = SimpleNamespace(connect=connect, dispose=lambda: closed.append("pool")) + monkeypatch.setitem(sys.modules, "sqlalchemy", SimpleNamespace(create_engine=lambda *_a, **_k: engine, + text=lambda query: query)) + base = ParameterResolver() + if fail_query: + with pytest.raises(ValueError, match="query failed"): + base.register_db_source("accounts", "sqlite://", "SELECT name") + else: + base.register_db_source("accounts", "sqlite://", "SELECT name") + assert base.fork().resolve("${db.accounts.name}") == "Alice" + assert closed == ["connection", "pool"] + + +def test_cookie_jars_created_before_locust_patch_remain_user_local(): + script = """ +from je_load_density.utils.scenario.cookie_jar import jar_for_user +parent = jar_for_user() +import locust +import threading +seen = [] +thread = threading.Thread(target=lambda: seen.append(jar_for_user())) +thread.start() +thread.join() +assert seen[0] is not jar_for_user() +""" + completed = subprocess.run([sys.executable, "-c", script], capture_output=True, text=True, timeout=20) + assert completed.returncode == 0, completed.stderr + + +def test_async_cookie_jars_do_not_inherit_parent_or_sibling_state(): + from je_load_density.utils.scenario.cookie_jar import jar_for_user + + async def execute(): + parent = jar_for_user() + + async def user(): + first = jar_for_user() + await asyncio.sleep(0) + assert jar_for_user() is first + return first + + first, second = await asyncio.gather(user(), user()) + assert first is not parent + assert second is not parent + assert first is not second + assert jar_for_user() is parent + + asyncio.run(execute()) + + +def test_parameter_only_import_leaves_locust_and_gevent_unloaded(): + script = """ +import sys +from je_load_density.utils.parameterization import ParameterResolver, use_resolver, resolve +with use_resolver(ParameterResolver()): + assert resolve('plain') == 'plain' +assert 'locust' not in sys.modules +assert 'gevent' not in sys.modules +""" + completed = subprocess.run([sys.executable, "-c", script], capture_output=True, text=True, timeout=20) + assert completed.returncode == 0, completed.stderr + + +def test_native_threads_keep_local_tokens_and_allocate_shared_rows(tmp_path): + path = tmp_path / "accounts.csv" + path.write_text("id,secret\n" + "".join(f"{index},secret-{index}\n" for index in range(64)), encoding="utf-8") + script = """ +import sys +import threading +from concurrent.futures import ThreadPoolExecutor +from je_load_density.utils.parameterization import ParameterResolver, use_resolver, resolve, register_variable +base = ParameterResolver() +base.register_csv_source('accounts', sys.argv[1], cycle=False) +register_variable('token', 'default') +barrier = threading.Barrier(8) +def user(index): + with use_resolver(base.fork()): + register_variable('token', f'user-{index}') + barrier.wait(timeout=10) + row = resolve({'id': '${csv.accounts.id}', 'secret': '${csv.accounts.secret}', 'token': '${token}'}) + assert row['token'] == f'user-{index}' + assert row['secret'] == f"secret-{row['id']}" + return row['id'] +with ThreadPoolExecutor(max_workers=8) as pool: + ids = list(pool.map(user, range(64))) +assert set(ids) == {str(index) for index in range(64)} +assert resolve('${token}') == 'default' +assert 'locust' not in sys.modules +assert 'gevent' not in sys.modules +""" + completed = subprocess.run([sys.executable, "-c", script, str(path)], capture_output=True, text=True, timeout=20) + assert completed.returncode == 0, completed.stderr diff --git a/test/test_record_engine_integration.py b/test/test_record_engine_integration.py index 4bd7fdd..4b0475d 100644 --- a/test/test_record_engine_integration.py +++ b/test/test_record_engine_integration.py @@ -5,13 +5,24 @@ pytest.importorskip("je_action_core.request_context", reason="coordinated ActionCore request-record release") -from je_load_density.engine.asyncio_engine import _send_one, run_async_load +import httpx +import locust # noqa: F401 - select the Locust scheduler before importing requests in this mixed-engine suite. +import requests + +from je_load_density.engine.async_http import measure, record_attempt +from je_load_density.engine.asyncio_engine import run_async_load from je_load_density.utils.test_record.run_context import RunContext, use_run_context from je_load_density.utils.test_record.test_record_class import test_record_instance from je_load_density.wrapper.event.request_hook import request_hook -import httpx -import requests + +async def send_measured(client, task): + async def send(resolved, _base): + return await client.request(resolved.get("method", "get"), resolved["request_url"]) + + pool = SimpleNamespace(send=send, httpx=httpx) + attempt = await measure(pool, task, "") + record_attempt(attempt, SimpleNamespace(successes=[], failures=[])) @pytest.fixture(autouse=True) @@ -41,7 +52,7 @@ async def execute(): transport = httpx.MockTransport(lambda request: httpx.Response(status, content=b"ok")) with use_run_context(run): async with httpx.AsyncClient(transport=transport) as client: - await _send_one(client, {"method": "get", "request_url": "http://localhost/"}) + await send_measured(client, {"method": "get", "request_url": "http://localhost/"}) asyncio.run(execute()) assert run.snapshot()[0]["outcome"] == outcome @@ -50,10 +61,10 @@ async def execute(): def test_async_timeout_records_measured_elapsed_and_nullable_canonical_status(monkeypatch): - from je_load_density.engine import asyncio_engine + from je_load_density.engine import async_http clock = iter((10.0, 10.025)) - monkeypatch.setattr(asyncio_engine, "time", SimpleNamespace(time=lambda: 100.0, monotonic=lambda: next(clock))) + monkeypatch.setattr(async_http, "time", SimpleNamespace(time=lambda: 100.0, monotonic=lambda: next(clock))) run = RunContext(source="loaddensity", phase="load", engine="asyncio") async def handle(request): @@ -63,7 +74,7 @@ async def handle(request): async def execute(): with use_run_context(run): async with httpx.AsyncClient(transport=httpx.MockTransport(handle)) as client: - await _send_one(client, {"request_url": "http://localhost/"}) + await send_measured(client, {"request_url": "http://localhost/"}) asyncio.run(execute()) assert test_record_instance.error_record_list[0]["response_time_ms"] == pytest.approx(25.0) @@ -73,16 +84,32 @@ async def execute(): assert record["error"]["kind"] == "timeout" -def test_new_async_run_summary_excludes_old_global_records(): +def test_new_async_run_summary_excludes_old_global_records(monkeypatch): test_record_instance.test_record_list.append({"name": "old"}) - result = asyncio.run(run_async_load(tasks=[{"request_url": "http://localhost/"}], users=0)) - assert result == {"requests": 0, "failures": 0} - - -def test_async_run_propagates_explicit_context_to_workers(): + original = httpx.AsyncClient + transport = httpx.MockTransport(lambda _request: httpx.Response(200)) + monkeypatch.setattr(httpx, "AsyncClient", lambda **kwargs: original(transport=transport, **kwargs)) + result = asyncio.run(run_async_load(tasks=[{"request_url": "http://localhost/"}], users=1, duration_seconds=0.03)) + assert result["requests"] > 0 + assert result["failures"] == 0 + assert "old" not in result["summary"]["per_name"] + + +def test_async_run_propagates_explicit_context_to_workers(monkeypatch): + async def timeout(request): + raise httpx.ReadTimeout("controlled timeout", request=request) + original = httpx.AsyncClient + transport = httpx.MockTransport(timeout) + monkeypatch.setattr(httpx, "AsyncClient", lambda **kwargs: original(transport=transport, **kwargs)) run = RunContext(source="loaddensity", phase="load", engine="asyncio") - result = asyncio.run(run_async_load(tasks=[{"request_url": "http://127.0.0.1:9/", "timeout": 0.1}], - users=1, duration_seconds=0.2, run_context=run)) + result = asyncio.run( + run_async_load( + tasks=[{"request_url": "http://127.0.0.1:9/", "timeout": 0.1}], + users=1, + duration_seconds=0.2, + run_context=run, + ) + ) assert result["failures"] >= 1 assert len(run.snapshot()) == result["failures"] @@ -90,6 +117,7 @@ def test_async_run_propagates_explicit_context_to_workers(): def test_locust_environment_captures_context_across_greenlet_boundary(): import gevent from locust import HttpUser + from je_load_density.wrapper.create_locust_env.create_locust_env import create_env run = RunContext(source="loaddensity", phase="load", engine="locust") @@ -98,9 +126,18 @@ def test_locust_environment_captures_context_across_greenlet_boundary(): response.status_code = 200 response._content = b"ok" try: - greenlet = gevent.spawn(env.events.request.fire, start_time=100.0, url="http://localhost/", - request_type="GET", name="home", context={}, response=response, - exception=None, response_length=2, response_time=25.0) + greenlet = gevent.spawn( + env.events.request.fire, + start_time=100.0, + url="http://localhost/", + request_type="GET", + name="home", + context={}, + response=response, + exception=None, + response_length=2, + response_time=25.0, + ) greenlet.get(timeout=2) assert len(run.snapshot()) == 1 assert run.snapshot()[0]["status_code"] == 200 From 17d784348d128e57e8dedbf174560f9a8e64d8e4 Mon Sep 17 00:00:00 2001 From: JeffreyChen Date: Sat, 3 Oct 2026 11:11:02 +0800 Subject: [PATCH 14/18] fix: isolate capability probes and cancel owned startup tasks --- docs/updates/2026-10.md | 10 + docs/updates/README.md | 1 + je_load_density/__init__.py | 544 ++++++++++++++---- je_load_density/engine/async_config.py | 40 +- je_load_density/engine/asyncio_engine.py | 15 +- je_load_density/gui/run_worker.py | 32 +- .../utils/executor/action_executor.py | 135 +++-- test/smoke/capability_probe.py | 9 +- test/smoke/server.py | 2 +- test/test_distributed_records.py | 15 +- test/test_extra_capabilities.py | 16 +- test/test_gui_lifecycle.py | 83 ++- test/test_native_async_runner.py | 94 ++- test/test_parameter_context.py | 12 +- 14 files changed, 762 insertions(+), 246 deletions(-) diff --git a/docs/updates/2026-10.md b/docs/updates/2026-10.md index ceafac2..a10717b 100644 --- a/docs/updates/2026-10.md +++ b/docs/updates/2026-10.md @@ -267,3 +267,13 @@ Added a stdlib smoke harness that exercises local HTTP requests in isolated proc - **Verification**: Final full suite 1590 passed/14 optional skips/four existing warnings; changed Python Ruff E/F/W/C 90/I and whitespace checks passed. Six rebuilt installed-wheel smoke tests passed against published ActionCore 0.0.2. The final installed-wheel GUI probe with a Locust-prepatched parent completed/cancelled both engines and closed an active window without leaked child/QThread. Published-core focused native/parameter checks 61 passed; GUI 19 passed with canonical tests skipped; legacy CLI plus checkout smoke 16 passed. Sphinx dummy parsing succeeded with 576 autosummary/mock/reference/static warnings; no claim of a warning-free documentation build. - **Evidence**: test/test_native_async_runner.py, test/test_native_imports.py, test/test_parameter_context.py, test/test_distributed_records.py, test/test_gui_lifecycle.py, test/test_legacy_cli_contract.py and test/smoke/gui_probe.py; four 2026-10-03 plans. Independent review findings were reproduced and corrected. Actual desktop 1200 x 850 rendering was inspected. Three README languages, En/Zh Sphinx and architecture section 6 describe the implemented boundary. - **Open items**: See progress.md #16/#18/#28/#30. Native exporters/non-HTTP/master-worker parity, embedded scheduler switching, shared-record release activation and a unified functional/load/synthetic script remain open. Native I/O after selecting Locust uses a fresh interpreter. Delivery is bounded in-memory transport without finite-work leases, request replay or session migration. Docker is unavailable locally; updated remote CI follows the feature-branch push. + +## U-20261003-03 · 2026-10-03 · Isolate extras probes and clean up cancelled async startup · #done #ci #quality #async #gui + +- **What**: completed the pending PR #118 follow-up. Every `all` extra probe runs in its own interpreter, retaining required capability checks and accumulated failures. The previous CI run's etcd failure came from late Locust TLS monkey patching after other SDK imports. +- **Quality**: queried SonarCloud and Codacy APIs; added static declarations for lazy exports, shared repeated module literals, split native validation helpers and exception/assertion tests, renamed spawn credits, matched the HTTP smoke callback signature and ordered imports. Runtime export/command mappings and native import isolation remain unchanged. +- **GUI**: validate fixed `run.json` and `stop` filenames in the same private temporary directory before reading settings. Ten regression cases cover valid control paths, unrelated filenames, traversal, mismatched directories and resolved escapes. +- **Cancellation**: independent review reproduced cancellation during `AsyncRunHandle.start()` leaving its owned run active. The new regression failed first; startup now cancels and awaits its owned task before propagating cancellation. Native runner tests: 31 passed. +- **Verification**: final complete suite 1601 passed, 14 optional skips and four existing warnings; changed-file Ruff E/F/W/C90/I at 120 columns and whitespace checks passed. A rebuilt installed wheel against published ActionCore 0.0.2 passed six CLI/report/SLA/dashboard smoke tests and the Locust-prepatched GUI completion/cancellation/active-window-close probe. +- **Review**: no additional important findings. Type-checker visibility for the dynamically exported `request_hook` remains a nonblocking omission; runtime access is preserved. The earlier single full-suite connection-test subprocess timeout did not reproduce in native/prepatched focused runs or the two subsequent complete runs; no timeout/assertion was weakened. +- **Boundary**: user-facing commands and configuration are unchanged, so the existing three README languages, Sphinx and architecture contracts still apply. Remaining async/dashboard/shared-record/script capabilities stay in `progress.md`; this stage does not close them. Remote extras and quality gates rerun after the feature-branch push; no scan-rule bypass or audit suppression was added. diff --git a/docs/updates/README.md b/docs/updates/README.md index 5e1a46d..d667c0f 100644 --- a/docs/updates/README.md +++ b/docs/updates/README.md @@ -58,6 +58,7 @@ In the same commit: delete the item from `progress.md`, add a `#done` entry here | ID | Date | Title | Tags | Batch | |---|---|---|---|---| +| U-20261003-03 | 2026-10-03 | Isolate extras probes and clean up cancelled async startup | #done #ci #quality #async #gui | [2026-10](2026-10.md) | | U-20261003-02 | 2026-10-03 | Add isolated runners and canonical worker delivery | #done #async #gui #distributed #parameters #15 #24 #29 | [2026-10](2026-10.md) | | U-20261003-01 | 2026-10-03 | Verify extras matrix and resolve quality findings | #done #ci #quality #25 | [2026-10](2026-10.md) | | U-20261002-11 | 2026-10-02 | Share percentile bands and redesign live dashboard | #done #charts #ui #26 | [2026-10](2026-10.md) | diff --git a/je_load_density/__init__.py b/je_load_density/__init__.py index df0d87f..0da1395 100644 --- a/je_load_density/__init__.py +++ b/je_load_density/__init__.py @@ -1,93 +1,391 @@ """Public APIs are loaded on demand; native asyncio imports do not patch process I/O.""" import importlib +from typing import TYPE_CHECKING + +if TYPE_CHECKING: + from locust import SequentialTaskSet as SequentialTaskSet + from locust import TaskSet as TaskSet + from locust import task as task + + from je_load_density.cloud.aws_fargate import launch_fargate_workers as launch_fargate_workers + from je_load_density.cloud.aws_lambda import invoke_lambda_workers as invoke_lambda_workers + from je_load_density.cloud.aws_lambda import lambda_worker_handler as lambda_worker_handler + from je_load_density.cloud.azure_aci import launch_aci_workers as launch_aci_workers + from je_load_density.cloud.gcp_cloud_run import run_cloud_run_job as run_cloud_run_job + from je_load_density.engine.asyncio_engine import AsyncRunHandle as AsyncRunHandle + from je_load_density.engine.asyncio_engine import run_async_load as run_async_load + from je_load_density.engine.entrypoints import start_test as start_test + from je_load_density.utils.action_generator.generate import generate_from_curls as generate_from_curls + from je_load_density.utils.action_generator.generate import generate_from_openapi as generate_from_openapi + from je_load_density.utils.action_generator.generate import merge_actions as merge_actions + from je_load_density.utils.ai.auto_baseline import calibrate_sla as calibrate_sla + from je_load_density.utils.ai.auto_tune import AutoTuner as AutoTuner + from je_load_density.utils.ai.root_cause import build_root_cause_prompt as build_root_cause_prompt + from je_load_density.utils.ai.root_cause import render_prompt_text as render_prompt_text + from je_load_density.utils.ai.smart_shape import find_breaking_point as find_breaking_point + from je_load_density.utils.auth.aws_sigv4 import sign_aws_request as sign_aws_request + from je_load_density.utils.auth.jwt_signer import decode_jwt as decode_jwt + from je_load_density.utils.auth.jwt_signer import sign_jwt as sign_jwt + from je_load_density.utils.auth.oauth2 import OAuth2Client as OAuth2Client + from je_load_density.utils.auth.oauth2 import fetch_client_credentials_token as fetch_client_credentials_token + from je_load_density.utils.auth.oauth2 import fetch_password_token as fetch_password_token + from je_load_density.utils.auth.oauth2 import refresh_token as refresh_token + from je_load_density.utils.callback.callback_function_executor import callback_executor as callback_executor + from je_load_density.utils.chaos.chaos_mesh import apply_manifest as chaos_apply_manifest + from je_load_density.utils.chaos.chaos_mesh import build_network_delay as chaos_build_network_delay + from je_load_density.utils.chaos.chaos_mesh import delete_manifest as chaos_delete_manifest + from je_load_density.utils.chaos.toxiproxy import add_toxic as toxiproxy_add_toxic + from je_load_density.utils.chaos.toxiproxy import create_proxy as toxiproxy_create_proxy + from je_load_density.utils.chaos.toxiproxy import install_bandwidth as toxiproxy_install_bandwidth + from je_load_density.utils.chaos.toxiproxy import install_latency as toxiproxy_install_latency + from je_load_density.utils.chaos.toxiproxy import list_proxies as toxiproxy_list_proxies + from je_load_density.utils.chaos.toxiproxy import remove_proxies as toxiproxy_remove_proxies + from je_load_density.utils.chaos.toxiproxy import remove_toxic as toxiproxy_remove_toxic + from je_load_density.utils.chaos.toxiproxy import reset_all as toxiproxy_reset_all + from je_load_density.utils.ci_annotations.canary_analysis import canary_verdict as canary_verdict + from je_load_density.utils.ci_annotations.github_actions import emit_github_annotations as emit_github_annotations + from je_load_density.utils.ci_annotations.github_actions import format_github_annotation as format_github_annotation + from je_load_density.utils.dashboard.live_dashboard import snapshot_metrics as snapshot_metrics + from je_load_density.utils.dashboard.live_dashboard import start_dashboard as start_dashboard + from je_load_density.utils.dashboard.live_dashboard import stop_dashboard as stop_dashboard + from je_load_density.utils.data.db_fixtures import apply_fixture as apply_fixture + from je_load_density.utils.data.db_fixtures import run_teardown as run_teardown + from je_load_density.utils.data.factory import build_user as build_user + from je_load_density.utils.data.factory import build_user_pool as build_user_pool + from je_load_density.utils.data.pii_anonymizer import find_pii as find_pii + from je_load_density.utils.data.pii_anonymizer import scrub as pii_scrub + from je_load_density.utils.data.pii_anonymizer import scrub_string as pii_scrub_string + from je_load_density.utils.dx.i18n import available_locales as available_locales + from je_load_density.utils.dx.i18n import get_current_locale as get_current_locale + from je_load_density.utils.dx.i18n import t as translate + from je_load_density.utils.dx.leak_detector import detect_growing_allocations as detect_growing_allocations + from je_load_density.utils.dx.leak_detector import start_leak_detector as start_leak_detector + from je_load_density.utils.dx.leak_detector import stop_leak_detector as stop_leak_detector + from je_load_density.utils.dx.profiler import memory_snapshot as memory_snapshot + from je_load_density.utils.dx.profiler import profile_call as profile_call + from je_load_density.utils.dx.repl import start_repl as start_repl + from je_load_density.utils.executor.action_executor import add_command_to_executor as add_command_to_executor + from je_load_density.utils.executor.action_executor import execute_action as execute_action + from je_load_density.utils.executor.action_executor import execute_files as execute_files + from je_load_density.utils.executor.action_executor import executor as executor + from je_load_density.utils.file_process.get_dir_file_list import get_dir_files_as_list as get_dir_files_as_list + from je_load_density.utils.generate_report.generate_allure_report import ( + generate_allure_report as generate_allure_report, + ) + from je_load_density.utils.generate_report.generate_chart_report import ( + generate_chart_report as generate_chart_report, + ) + from je_load_density.utils.generate_report.generate_cost_report import estimate_run_cost as estimate_run_cost + from je_load_density.utils.generate_report.generate_cost_report import generate_cost_report as generate_cost_report + from je_load_density.utils.generate_report.generate_csv_report import generate_csv_report as generate_csv_report + from je_load_density.utils.generate_report.generate_cyclonedx_report import ( + generate_cyclonedx_report as generate_cyclonedx_report, + ) + from je_load_density.utils.generate_report.generate_excel_report import ( + generate_excel_report as generate_excel_report, + ) + from je_load_density.utils.generate_report.generate_histogram_report import ( + generate_histogram_report as generate_histogram_report, + ) + from je_load_density.utils.generate_report.generate_html_report import generate_html as generate_html + from je_load_density.utils.generate_report.generate_html_report import generate_html_report as generate_html_report + from je_load_density.utils.generate_report.generate_json_report import generate_json as generate_json + from je_load_density.utils.generate_report.generate_json_report import generate_json_report as generate_json_report + from je_load_density.utils.generate_report.generate_junit_report import ( + generate_junit_report as generate_junit_report, + ) + from je_load_density.utils.generate_report.generate_pdf_report import generate_pdf_report as generate_pdf_report + from je_load_density.utils.generate_report.generate_sarif_report import ( + generate_sarif_report as generate_sarif_report, + ) + from je_load_density.utils.generate_report.generate_service_map import build_service_map as build_service_map + from je_load_density.utils.generate_report.generate_service_map import generate_service_map as generate_service_map + from je_load_density.utils.generate_report.generate_summary_report import build_summary as build_summary + from je_load_density.utils.generate_report.generate_summary_report import ( + generate_summary_report as generate_summary_report, + ) + from je_load_density.utils.generate_report.generate_xml_report import generate_xml as generate_xml + from je_load_density.utils.generate_report.generate_xml_report import generate_xml_report as generate_xml_report + from je_load_density.utils.governance.audit_log import append_audit_entry as append_audit_entry + from je_load_density.utils.governance.audit_log import read_audit_log as read_audit_log + from je_load_density.utils.governance.run_tagging import list_tags as list_tags + from je_load_density.utils.governance.run_tagging import search_runs_by_tag as search_runs_by_tag + from je_load_density.utils.governance.run_tagging import tag_run as tag_run + from je_load_density.utils.governance.share_link import issue_share_link as issue_share_link + from je_load_density.utils.governance.share_link import verify_share_link as verify_share_link + from je_load_density.utils.governance.test_catalog import index_catalog as index_catalog + from je_load_density.utils.governance.test_catalog import search_catalog as search_catalog + from je_load_density.utils.graphql.graphql_task import extract_field as extract_field + from je_load_density.utils.graphql.graphql_task import graphql_to_http_task as graphql_to_http_task + from je_load_density.utils.json.json_file.json_file import read_action_json as read_action_json + from je_load_density.utils.json.json_file.toml_file import read_action_toml as read_action_toml + from je_load_density.utils.json.json_file.toml_file import write_action_toml as write_action_toml + from je_load_density.utils.json.json_file.yaml_file import read_action_yaml as read_action_yaml + from je_load_density.utils.json.json_file.yaml_file import write_action_yaml as write_action_yaml + from je_load_density.utils.linter.action_formatter import format_action_document as format_action_document + from je_load_density.utils.linter.action_formatter import format_action_file as format_action_file + from je_load_density.utils.linter.action_formatter import format_action_string as format_action_string + from je_load_density.utils.linter.action_linter import lint_action as lint_action + from je_load_density.utils.linter.action_linter import lint_action_file as lint_action_file + from je_load_density.utils.load_shapes.shapes import SoakShape as SoakShape + from je_load_density.utils.load_shapes.shapes import SpikeShape as SpikeShape + from je_load_density.utils.load_shapes.shapes import StagesShape as StagesShape + from je_load_density.utils.load_shapes.shapes import build_load_shape as build_load_shape + from je_load_density.utils.metrics import start_influxdb_sink as start_influxdb_sink + from je_load_density.utils.metrics import start_opentelemetry_exporter as start_opentelemetry_exporter + from je_load_density.utils.metrics import start_prometheus_exporter as start_prometheus_exporter + from je_load_density.utils.metrics import stop_influxdb_sink as stop_influxdb_sink + from je_load_density.utils.metrics import stop_opentelemetry_exporter as stop_opentelemetry_exporter + from je_load_density.utils.metrics import stop_prometheus_exporter as stop_prometheus_exporter + from je_load_density.utils.metrics.datadog_apm_exporter import ( + start_datadog_apm_exporter as start_datadog_apm_exporter, + ) + from je_load_density.utils.metrics.datadog_apm_exporter import ( + stop_datadog_apm_exporter as stop_datadog_apm_exporter, + ) + from je_load_density.utils.metrics.opentelemetry_tracing_exporter import ( + start_opentelemetry_tracing_exporter as start_opentelemetry_tracing_exporter, + ) + from je_load_density.utils.metrics.opentelemetry_tracing_exporter import ( + stop_opentelemetry_tracing_exporter as stop_opentelemetry_tracing_exporter, + ) + from je_load_density.utils.metrics.statsd_sink import start_statsd_sink as start_statsd_sink + from je_load_density.utils.metrics.statsd_sink import stop_statsd_sink as stop_statsd_sink + from je_load_density.utils.notifier.gitlab import build_gitlab_mr_note as build_gitlab_mr_note + from je_load_density.utils.notifier.gitlab import post_gitlab_mr_summary as post_gitlab_mr_summary + from je_load_density.utils.notifier.opsgenie import build_opsgenie_alert as build_opsgenie_alert + from je_load_density.utils.notifier.opsgenie import post_opsgenie_alert as post_opsgenie_alert + from je_load_density.utils.notifier.pagerduty import build_pagerduty_event as build_pagerduty_event + from je_load_density.utils.notifier.pagerduty import post_pagerduty_event as post_pagerduty_event + from je_load_density.utils.notifier.slack import build_slack_summary as build_slack_summary + from je_load_density.utils.notifier.slack import post_slack_summary as post_slack_summary + from je_load_density.utils.notifier.teams import build_teams_summary as build_teams_summary + from je_load_density.utils.notifier.teams import post_teams_summary as post_teams_summary + from je_load_density.utils.parameterization import get_resolver as get_resolver + from je_load_density.utils.parameterization import parameter_resolver as parameter_resolver + from je_load_density.utils.parameterization import register_csv_source as register_csv_source + from je_load_density.utils.parameterization import register_csv_sources as register_csv_sources + from je_load_density.utils.parameterization import register_db_source as register_db_source + from je_load_density.utils.parameterization import register_db_sources as register_db_sources + from je_load_density.utils.parameterization import register_session_variable as register_session_variable + from je_load_density.utils.parameterization import register_variable as register_variable + from je_load_density.utils.parameterization import register_variables as register_variables + from je_load_density.utils.parameterization import resolve as resolve + from je_load_density.utils.parameterization import use_resolver as use_resolver + from je_load_density.utils.project.create_project_structure import create_project_dir as create_project_dir + from je_load_density.utils.recording.cdp_capture import capture_cdp_session as capture_cdp_session + from je_load_density.utils.recording.cdp_capture import capture_cdp_to_har as capture_cdp_to_har + from je_load_density.utils.recording.cdp_capture import discover_targets as cdp_discover_targets + from je_load_density.utils.recording.curl_importer import curl_to_task as curl_to_task + from je_load_density.utils.recording.har_importer import har_to_action_json as har_to_action_json + from je_load_density.utils.recording.har_importer import har_to_tasks as har_to_tasks + from je_load_density.utils.recording.har_importer import load_har as load_har + from je_load_density.utils.recording.jmeter_importer import jmeter_to_action_json as jmeter_to_action_json + from je_load_density.utils.recording.jmeter_importer import jmeter_to_tasks as jmeter_to_tasks + from je_load_density.utils.recording.jmeter_importer import load_jmeter_jmx as load_jmeter_jmx + from je_load_density.utils.recording.k6_importer import k6_script_to_action_json as k6_script_to_action_json + from je_load_density.utils.recording.k6_importer import k6_script_to_tasks as k6_script_to_tasks + from je_load_density.utils.recording.k6_importer import load_k6_script as load_k6_script + from je_load_density.utils.recording.openapi_importer import load_openapi as load_openapi + from je_load_density.utils.recording.openapi_importer import openapi_to_action_json as openapi_to_action_json + from je_load_density.utils.recording.openapi_importer import openapi_to_tasks as openapi_to_tasks + from je_load_density.utils.recording.postman_importer import load_postman_collection as load_postman_collection + from je_load_density.utils.recording.postman_importer import postman_to_action_json as postman_to_action_json + from je_load_density.utils.recording.postman_importer import postman_to_tasks as postman_to_tasks + from je_load_density.utils.regression.diff import diff_runs as diff_runs + from je_load_density.utils.regression.diff import summarise_records as summarise_records + from je_load_density.utils.regression.error_clustering import cluster_errors as cluster_errors + from je_load_density.utils.regression.multi_run_trend import trend_runs as trend_runs + from je_load_density.utils.reliability.adaptive_retry import AdaptiveRetryPolicy as AdaptiveRetryPolicy + from je_load_density.utils.reliability.adaptive_retry import classify_error as classify_error + from je_load_density.utils.reliability.adaptive_retry import run_with_retry as run_with_retry + from je_load_density.utils.reliability.failure_budget import CircuitOpenError as CircuitOpenError + from je_load_density.utils.reliability.failure_budget import FailureBudget as FailureBudget + from je_load_density.utils.reliability.failure_budget import install_failure_budget as install_failure_budget + from je_load_density.utils.reliability.failure_budget import uninstall_failure_budget as uninstall_failure_budget + from je_load_density.utils.reliability.network_conditioner import NetworkConditioner as NetworkConditioner + from je_load_density.utils.reliability.network_conditioner import ( + install_network_conditioner as install_network_conditioner, + ) + from je_load_density.utils.reliability.network_conditioner import ( + uninstall_network_conditioner as uninstall_network_conditioner, + ) + from je_load_density.utils.reliability.process_supervisor import ProcessSupervisor as ProcessSupervisor + from je_load_density.utils.reliability.process_supervisor import with_watchdog as with_watchdog + from je_load_density.utils.scenario.cookie_jar import jar_for_user as jar_for_user + from je_load_density.utils.scenario.cookie_jar import reset_all_jars as reset_all_jars + from je_load_density.utils.scenario.cookie_jar import reset_user_jar as reset_user_jar + from je_load_density.utils.scenario.fsm import FsmRunner as FsmRunner + from je_load_density.utils.schema.action_schema import action_json_schema as action_json_schema + from je_load_density.utils.schema.action_schema import export_schema as export_schema + from je_load_density.utils.security.fuzz import expand_task_fuzz as expand_task_fuzz + from je_load_density.utils.security.fuzz import fuzz_query_string as fuzz_query_string + from je_load_density.utils.security.fuzz import mutate_json as mutate_json + from je_load_density.utils.security.fuzz import mutate_string as mutate_string + from je_load_density.utils.security.graphql_checks import build_alias_batching_attack as build_alias_batching_attack + from je_load_density.utils.security.graphql_checks import build_depth_attack as build_depth_attack + from je_load_density.utils.security.graphql_checks import build_introspection_payload as build_introspection_payload + from je_load_density.utils.security.graphql_checks import graphql_attack_pack as graphql_attack_pack + from je_load_density.utils.security.jwt_attacks import craft_alg_confusion_token as craft_alg_confusion_token + from je_load_density.utils.security.jwt_attacks import craft_alg_none_token as craft_alg_none_token + from je_load_density.utils.security.jwt_attacks import craft_attack_pack as craft_jwt_attack_pack + from je_load_density.utils.security.jwt_attacks import craft_expired_token as craft_expired_token + from je_load_density.utils.security.jwt_attacks import craft_kid_traversal_token as craft_kid_traversal_token + from je_load_density.utils.security.owasp_checks import ( + check_broken_object_level_auth as check_broken_object_level_auth, + ) + from je_load_density.utils.security.owasp_checks import ( + check_excessive_data_exposure as check_excessive_data_exposure, + ) + from je_load_density.utils.security.owasp_checks import check_security_headers as check_security_headers + from je_load_density.utils.security.owasp_checks import check_sensitive_token_leak as check_sensitive_token_leak + from je_load_density.utils.security.owasp_checks import run_owasp_checks as run_owasp_checks + from je_load_density.utils.security.rate_limit_probe import probe_rate_limit as probe_rate_limit + from je_load_density.utils.security.smuggling_checks import build_cl_te as build_cl_te + from je_load_density.utils.security.smuggling_checks import build_te_cl as build_te_cl + from je_load_density.utils.security.smuggling_checks import build_te_te as build_te_te + from je_load_density.utils.security.smuggling_checks import smuggling_attack_pack as smuggling_attack_pack + from je_load_density.utils.security.ssrf_checks import build_ssrf_targets as build_ssrf_targets + from je_load_density.utils.security.ssrf_checks import find_metadata_leak as find_metadata_leak + from je_load_density.utils.security.ssrf_checks import render_ssrf_tasks as render_ssrf_tasks + from je_load_density.utils.sla.sla_gates import assert_sla as assert_sla + from je_load_density.utils.sla.sla_gates import evaluate_sla as evaluate_sla + from je_load_density.utils.socket_server.load_density_socket_server import ( + start_load_density_socket_server as start_load_density_socket_server, + ) + from je_load_density.utils.stub_server.stub_server import start_stub_server as start_stub_server + from je_load_density.utils.stub_server.stub_server import stop_stub_server as stop_stub_server + from je_load_density.utils.test_record.sqlite_persistence import fetch_run_records as fetch_run_records + from je_load_density.utils.test_record.sqlite_persistence import list_runs as list_runs + from je_load_density.utils.test_record.sqlite_persistence import persist_records as persist_records + from je_load_density.utils.test_record.test_record_class import test_record_instance as test_record_instance + from je_load_density.utils.throttle.rps_throttle import RpsThrottle as RpsThrottle + from je_load_density.utils.throttle.rps_throttle import get_throttle as get_throttle + from je_load_density.utils.throttle.rps_throttle import reset_throttles as reset_throttles + from je_load_density.wrapper.create_locust_env.create_locust_env import create_env as create_env + from je_load_density.wrapper.create_locust_env.create_locust_env import prepare_env as prepare_env + from je_load_density.wrapper.proxy.proxy_user import locust_wrapper_proxy as locust_wrapper_proxy + +_MODULE_UTILS_RELIABILITY_ADAPTIVE_RETRY = "je_load_density.utils.reliability.adaptive_retry" +_MODULE_UTILS_RELIABILITY_FAILURE_BUDGET = "je_load_density.utils.reliability.failure_budget" +_MODULE_UTILS_RELIABILITY_NETWORK_CONDITIONER = "je_load_density.utils.reliability.network_conditioner" +_MODULE_UTILS_AUTH_OAUTH2 = "je_load_density.utils.auth.oauth2" +_MODULE_UTILS_THROTTLE_RPS_THROTTLE = "je_load_density.utils.throttle.rps_throttle" +_MODULE_UTILS_LOAD_SHAPES_SHAPES = "je_load_density.utils.load_shapes.shapes" +_MODULE_UTILS_EXECUTOR_ACTION_EXECUTOR = "je_load_density.utils.executor.action_executor" +_MODULE_UTILS_DX_I18N = "je_load_density.utils.dx.i18n" +_MODULE_UTILS_SECURITY_GRAPHQL_CHECKS = "je_load_density.utils.security.graphql_checks" +_MODULE_UTILS_SECURITY_SMUGGLING_CHECKS = "je_load_density.utils.security.smuggling_checks" +_MODULE_UTILS_SECURITY_SSRF_CHECKS = "je_load_density.utils.security.ssrf_checks" +_MODULE_UTILS_RECORDING_CDP_CAPTURE = "je_load_density.utils.recording.cdp_capture" +_MODULE_UTILS_CHAOS_CHAOS_MESH = "je_load_density.utils.chaos.chaos_mesh" +_MODULE_UTILS_SECURITY_OWASP_CHECKS = "je_load_density.utils.security.owasp_checks" +_MODULE_UTILS_SECURITY_JWT_ATTACKS = "je_load_density.utils.security.jwt_attacks" +_MODULE_UTILS_DX_LEAK_DETECTOR = "je_load_density.utils.dx.leak_detector" +_MODULE_UTILS_SECURITY_FUZZ = "je_load_density.utils.security.fuzz" +_MODULE_UTILS_TEST_RECORD_SQLITE_PERSISTENCE = "je_load_density.utils.test_record.sqlite_persistence" +_MODULE_UTILS_DATA_PII_ANONYMIZER = "je_load_density.utils.data.pii_anonymizer" +_MODULE_UTILS_LINTER_ACTION_FORMATTER = "je_load_density.utils.linter.action_formatter" +_MODULE_UTILS_ACTION_GENERATOR_GENERATE = "je_load_density.utils.action_generator.generate" +_MODULE_UTILS_RECORDING_HAR_IMPORTER = "je_load_density.utils.recording.har_importer" +_MODULE_UTILS_SCENARIO_COOKIE_JAR = "je_load_density.utils.scenario.cookie_jar" +_MODULE_UTILS_RECORDING_JMETER_IMPORTER = "je_load_density.utils.recording.jmeter_importer" +_MODULE_UTILS_RECORDING_K6_IMPORTER = "je_load_density.utils.recording.k6_importer" +_MODULE_UTILS_GOVERNANCE_RUN_TAGGING = "je_load_density.utils.governance.run_tagging" +_MODULE_UTILS_RECORDING_OPENAPI_IMPORTER = "je_load_density.utils.recording.openapi_importer" +_MODULE_UTILS_RECORDING_POSTMAN_IMPORTER = "je_load_density.utils.recording.postman_importer" +_MODULE_UTILS_PARAMETERIZATION = "je_load_density.utils.parameterization" +_MODULE_UTILS_DASHBOARD_LIVE_DASHBOARD = "je_load_density.utils.dashboard.live_dashboard" +_MODULE_UTILS_METRICS = "je_load_density.utils.metrics" +_MODULE_UTILS_CHAOS_TOXIPROXY = "je_load_density.utils.chaos.toxiproxy" _EXPORTS = { "AsyncRunHandle": ("je_load_density.engine.asyncio_engine", "AsyncRunHandle"), - "AdaptiveRetryPolicy": ("je_load_density.utils.reliability.adaptive_retry", "AdaptiveRetryPolicy"), + "AdaptiveRetryPolicy": (_MODULE_UTILS_RELIABILITY_ADAPTIVE_RETRY, "AdaptiveRetryPolicy"), "AutoTuner": ("je_load_density.utils.ai.auto_tune", "AutoTuner"), - "CircuitOpenError": ("je_load_density.utils.reliability.failure_budget", "CircuitOpenError"), - "FailureBudget": ("je_load_density.utils.reliability.failure_budget", "FailureBudget"), + "CircuitOpenError": (_MODULE_UTILS_RELIABILITY_FAILURE_BUDGET, "CircuitOpenError"), + "FailureBudget": (_MODULE_UTILS_RELIABILITY_FAILURE_BUDGET, "FailureBudget"), "FsmRunner": ("je_load_density.utils.scenario.fsm", "FsmRunner"), - "NetworkConditioner": ("je_load_density.utils.reliability.network_conditioner", "NetworkConditioner"), - "OAuth2Client": ("je_load_density.utils.auth.oauth2", "OAuth2Client"), + "NetworkConditioner": (_MODULE_UTILS_RELIABILITY_NETWORK_CONDITIONER, "NetworkConditioner"), + "OAuth2Client": (_MODULE_UTILS_AUTH_OAUTH2, "OAuth2Client"), "ProcessSupervisor": ("je_load_density.utils.reliability.process_supervisor", "ProcessSupervisor"), - "RpsThrottle": ("je_load_density.utils.throttle.rps_throttle", "RpsThrottle"), + "RpsThrottle": (_MODULE_UTILS_THROTTLE_RPS_THROTTLE, "RpsThrottle"), "SequentialTaskSet": ("locust", "SequentialTaskSet"), - "SoakShape": ("je_load_density.utils.load_shapes.shapes", "SoakShape"), - "SpikeShape": ("je_load_density.utils.load_shapes.shapes", "SpikeShape"), - "StagesShape": ("je_load_density.utils.load_shapes.shapes", "StagesShape"), + "SoakShape": (_MODULE_UTILS_LOAD_SHAPES_SHAPES, "SoakShape"), + "SpikeShape": (_MODULE_UTILS_LOAD_SHAPES_SHAPES, "SpikeShape"), + "StagesShape": (_MODULE_UTILS_LOAD_SHAPES_SHAPES, "StagesShape"), "TaskSet": ("locust", "TaskSet"), "action_json_schema": ("je_load_density.utils.schema.action_schema", "action_json_schema"), - "add_command_to_executor": ("je_load_density.utils.executor.action_executor", "add_command_to_executor"), + "add_command_to_executor": (_MODULE_UTILS_EXECUTOR_ACTION_EXECUTOR, "add_command_to_executor"), "append_audit_entry": ("je_load_density.utils.governance.audit_log", "append_audit_entry"), "apply_fixture": ("je_load_density.utils.data.db_fixtures", "apply_fixture"), "assert_sla": ("je_load_density.utils.sla.sla_gates", "assert_sla"), - "available_locales": ("je_load_density.utils.dx.i18n", "available_locales"), - "build_alias_batching_attack": ("je_load_density.utils.security.graphql_checks", "build_alias_batching_attack"), - "build_cl_te": ("je_load_density.utils.security.smuggling_checks", "build_cl_te"), - "build_depth_attack": ("je_load_density.utils.security.graphql_checks", "build_depth_attack"), + "available_locales": (_MODULE_UTILS_DX_I18N, "available_locales"), + "build_alias_batching_attack": (_MODULE_UTILS_SECURITY_GRAPHQL_CHECKS, "build_alias_batching_attack"), + "build_cl_te": (_MODULE_UTILS_SECURITY_SMUGGLING_CHECKS, "build_cl_te"), + "build_depth_attack": (_MODULE_UTILS_SECURITY_GRAPHQL_CHECKS, "build_depth_attack"), "build_gitlab_mr_note": ("je_load_density.utils.notifier.gitlab", "build_gitlab_mr_note"), - "build_introspection_payload": ("je_load_density.utils.security.graphql_checks", "build_introspection_payload"), - "build_load_shape": ("je_load_density.utils.load_shapes.shapes", "build_load_shape"), + "build_introspection_payload": (_MODULE_UTILS_SECURITY_GRAPHQL_CHECKS, "build_introspection_payload"), + "build_load_shape": (_MODULE_UTILS_LOAD_SHAPES_SHAPES, "build_load_shape"), "build_opsgenie_alert": ("je_load_density.utils.notifier.opsgenie", "build_opsgenie_alert"), "build_pagerduty_event": ("je_load_density.utils.notifier.pagerduty", "build_pagerduty_event"), "build_root_cause_prompt": ("je_load_density.utils.ai.root_cause", "build_root_cause_prompt"), "build_service_map": ("je_load_density.utils.generate_report.generate_service_map", "build_service_map"), "build_slack_summary": ("je_load_density.utils.notifier.slack", "build_slack_summary"), - "build_ssrf_targets": ("je_load_density.utils.security.ssrf_checks", "build_ssrf_targets"), + "build_ssrf_targets": (_MODULE_UTILS_SECURITY_SSRF_CHECKS, "build_ssrf_targets"), "build_summary": ("je_load_density.utils.generate_report.generate_summary_report", "build_summary"), - "build_te_cl": ("je_load_density.utils.security.smuggling_checks", "build_te_cl"), - "build_te_te": ("je_load_density.utils.security.smuggling_checks", "build_te_te"), + "build_te_cl": (_MODULE_UTILS_SECURITY_SMUGGLING_CHECKS, "build_te_cl"), + "build_te_te": (_MODULE_UTILS_SECURITY_SMUGGLING_CHECKS, "build_te_te"), "build_teams_summary": ("je_load_density.utils.notifier.teams", "build_teams_summary"), "build_user": ("je_load_density.utils.data.factory", "build_user"), "build_user_pool": ("je_load_density.utils.data.factory", "build_user_pool"), "calibrate_sla": ("je_load_density.utils.ai.auto_baseline", "calibrate_sla"), "callback_executor": ("je_load_density.utils.callback.callback_function_executor", "callback_executor"), "canary_verdict": ("je_load_density.utils.ci_annotations.canary_analysis", "canary_verdict"), - "capture_cdp_session": ("je_load_density.utils.recording.cdp_capture", "capture_cdp_session"), - "capture_cdp_to_har": ("je_load_density.utils.recording.cdp_capture", "capture_cdp_to_har"), - "cdp_discover_targets": ("je_load_density.utils.recording.cdp_capture", "discover_targets"), - "chaos_apply_manifest": ("je_load_density.utils.chaos.chaos_mesh", "apply_manifest"), - "chaos_build_network_delay": ("je_load_density.utils.chaos.chaos_mesh", "build_network_delay"), - "chaos_delete_manifest": ("je_load_density.utils.chaos.chaos_mesh", "delete_manifest"), - "check_broken_object_level_auth": ("je_load_density.utils.security.owasp_checks", "check_broken_object_level_auth"), - "check_excessive_data_exposure": ("je_load_density.utils.security.owasp_checks", "check_excessive_data_exposure"), - "check_security_headers": ("je_load_density.utils.security.owasp_checks", "check_security_headers"), - "check_sensitive_token_leak": ("je_load_density.utils.security.owasp_checks", "check_sensitive_token_leak"), - "classify_error": ("je_load_density.utils.reliability.adaptive_retry", "classify_error"), + "capture_cdp_session": (_MODULE_UTILS_RECORDING_CDP_CAPTURE, "capture_cdp_session"), + "capture_cdp_to_har": (_MODULE_UTILS_RECORDING_CDP_CAPTURE, "capture_cdp_to_har"), + "cdp_discover_targets": (_MODULE_UTILS_RECORDING_CDP_CAPTURE, "discover_targets"), + "chaos_apply_manifest": (_MODULE_UTILS_CHAOS_CHAOS_MESH, "apply_manifest"), + "chaos_build_network_delay": (_MODULE_UTILS_CHAOS_CHAOS_MESH, "build_network_delay"), + "chaos_delete_manifest": (_MODULE_UTILS_CHAOS_CHAOS_MESH, "delete_manifest"), + "check_broken_object_level_auth": (_MODULE_UTILS_SECURITY_OWASP_CHECKS, "check_broken_object_level_auth"), + "check_excessive_data_exposure": (_MODULE_UTILS_SECURITY_OWASP_CHECKS, "check_excessive_data_exposure"), + "check_security_headers": (_MODULE_UTILS_SECURITY_OWASP_CHECKS, "check_security_headers"), + "check_sensitive_token_leak": (_MODULE_UTILS_SECURITY_OWASP_CHECKS, "check_sensitive_token_leak"), + "classify_error": (_MODULE_UTILS_RELIABILITY_ADAPTIVE_RETRY, "classify_error"), "cluster_errors": ("je_load_density.utils.regression.error_clustering", "cluster_errors"), - "craft_alg_confusion_token": ("je_load_density.utils.security.jwt_attacks", "craft_alg_confusion_token"), - "craft_alg_none_token": ("je_load_density.utils.security.jwt_attacks", "craft_alg_none_token"), - "craft_expired_token": ("je_load_density.utils.security.jwt_attacks", "craft_expired_token"), - "craft_jwt_attack_pack": ("je_load_density.utils.security.jwt_attacks", "craft_attack_pack"), - "craft_kid_traversal_token": ("je_load_density.utils.security.jwt_attacks", "craft_kid_traversal_token"), + "craft_alg_confusion_token": (_MODULE_UTILS_SECURITY_JWT_ATTACKS, "craft_alg_confusion_token"), + "craft_alg_none_token": (_MODULE_UTILS_SECURITY_JWT_ATTACKS, "craft_alg_none_token"), + "craft_expired_token": (_MODULE_UTILS_SECURITY_JWT_ATTACKS, "craft_expired_token"), + "craft_jwt_attack_pack": (_MODULE_UTILS_SECURITY_JWT_ATTACKS, "craft_attack_pack"), + "craft_kid_traversal_token": (_MODULE_UTILS_SECURITY_JWT_ATTACKS, "craft_kid_traversal_token"), "create_env": ("je_load_density.wrapper.create_locust_env.create_locust_env", "create_env"), "create_project_dir": ("je_load_density.utils.project.create_project_structure", "create_project_dir"), "curl_to_task": ("je_load_density.utils.recording.curl_importer", "curl_to_task"), "decode_jwt": ("je_load_density.utils.auth.jwt_signer", "decode_jwt"), - "detect_growing_allocations": ("je_load_density.utils.dx.leak_detector", "detect_growing_allocations"), + "detect_growing_allocations": (_MODULE_UTILS_DX_LEAK_DETECTOR, "detect_growing_allocations"), "diff_runs": ("je_load_density.utils.regression.diff", "diff_runs"), "emit_github_annotations": ("je_load_density.utils.ci_annotations.github_actions", "emit_github_annotations"), "estimate_run_cost": ("je_load_density.utils.generate_report.generate_cost_report", "estimate_run_cost"), "evaluate_sla": ("je_load_density.utils.sla.sla_gates", "evaluate_sla"), - "execute_action": ("je_load_density.utils.executor.action_executor", "execute_action"), - "execute_files": ("je_load_density.utils.executor.action_executor", "execute_files"), - "executor": ("je_load_density.utils.executor.action_executor", "executor"), - "expand_task_fuzz": ("je_load_density.utils.security.fuzz", "expand_task_fuzz"), + "execute_action": (_MODULE_UTILS_EXECUTOR_ACTION_EXECUTOR, "execute_action"), + "execute_files": (_MODULE_UTILS_EXECUTOR_ACTION_EXECUTOR, "execute_files"), + "executor": (_MODULE_UTILS_EXECUTOR_ACTION_EXECUTOR, "executor"), + "expand_task_fuzz": (_MODULE_UTILS_SECURITY_FUZZ, "expand_task_fuzz"), "export_schema": ("je_load_density.utils.schema.action_schema", "export_schema"), "extract_field": ("je_load_density.utils.graphql.graphql_task", "extract_field"), - "fetch_client_credentials_token": ("je_load_density.utils.auth.oauth2", "fetch_client_credentials_token"), - "fetch_password_token": ("je_load_density.utils.auth.oauth2", "fetch_password_token"), - "fetch_run_records": ("je_load_density.utils.test_record.sqlite_persistence", "fetch_run_records"), + "fetch_client_credentials_token": (_MODULE_UTILS_AUTH_OAUTH2, "fetch_client_credentials_token"), + "fetch_password_token": (_MODULE_UTILS_AUTH_OAUTH2, "fetch_password_token"), + "fetch_run_records": (_MODULE_UTILS_TEST_RECORD_SQLITE_PERSISTENCE, "fetch_run_records"), "find_breaking_point": ("je_load_density.utils.ai.smart_shape", "find_breaking_point"), - "find_metadata_leak": ("je_load_density.utils.security.ssrf_checks", "find_metadata_leak"), - "find_pii": ("je_load_density.utils.data.pii_anonymizer", "find_pii"), - "format_action_document": ("je_load_density.utils.linter.action_formatter", "format_action_document"), - "format_action_file": ("je_load_density.utils.linter.action_formatter", "format_action_file"), - "format_action_string": ("je_load_density.utils.linter.action_formatter", "format_action_string"), + "find_metadata_leak": (_MODULE_UTILS_SECURITY_SSRF_CHECKS, "find_metadata_leak"), + "find_pii": (_MODULE_UTILS_DATA_PII_ANONYMIZER, "find_pii"), + "format_action_document": (_MODULE_UTILS_LINTER_ACTION_FORMATTER, "format_action_document"), + "format_action_file": (_MODULE_UTILS_LINTER_ACTION_FORMATTER, "format_action_file"), + "format_action_string": (_MODULE_UTILS_LINTER_ACTION_FORMATTER, "format_action_string"), "format_github_annotation": ("je_load_density.utils.ci_annotations.github_actions", "format_github_annotation"), - "fuzz_query_string": ("je_load_density.utils.security.fuzz", "fuzz_query_string"), + "fuzz_query_string": (_MODULE_UTILS_SECURITY_FUZZ, "fuzz_query_string"), "generate_allure_report": ( "je_load_density.utils.generate_report.generate_allure_report", "generate_allure_report", @@ -100,8 +398,8 @@ "generate_cyclonedx_report", ), "generate_excel_report": ("je_load_density.utils.generate_report.generate_excel_report", "generate_excel_report"), - "generate_from_curls": ("je_load_density.utils.action_generator.generate", "generate_from_curls"), - "generate_from_openapi": ("je_load_density.utils.action_generator.generate", "generate_from_openapi"), + "generate_from_curls": (_MODULE_UTILS_ACTION_GENERATOR_GENERATE, "generate_from_curls"), + "generate_from_openapi": (_MODULE_UTILS_ACTION_GENERATOR_GENERATE, "generate_from_openapi"), "generate_histogram_report": ( "je_load_density.utils.generate_report.generate_histogram_report", "generate_histogram_report", @@ -120,56 +418,56 @@ ), "generate_xml": ("je_load_density.utils.generate_report.generate_xml_report", "generate_xml"), "generate_xml_report": ("je_load_density.utils.generate_report.generate_xml_report", "generate_xml_report"), - "get_current_locale": ("je_load_density.utils.dx.i18n", "get_current_locale"), + "get_current_locale": (_MODULE_UTILS_DX_I18N, "get_current_locale"), "get_dir_files_as_list": ("je_load_density.utils.file_process.get_dir_file_list", "get_dir_files_as_list"), - "get_throttle": ("je_load_density.utils.throttle.rps_throttle", "get_throttle"), - "graphql_attack_pack": ("je_load_density.utils.security.graphql_checks", "graphql_attack_pack"), + "get_throttle": (_MODULE_UTILS_THROTTLE_RPS_THROTTLE, "get_throttle"), + "graphql_attack_pack": (_MODULE_UTILS_SECURITY_GRAPHQL_CHECKS, "graphql_attack_pack"), "graphql_to_http_task": ("je_load_density.utils.graphql.graphql_task", "graphql_to_http_task"), - "har_to_action_json": ("je_load_density.utils.recording.har_importer", "har_to_action_json"), - "har_to_tasks": ("je_load_density.utils.recording.har_importer", "har_to_tasks"), + "har_to_action_json": (_MODULE_UTILS_RECORDING_HAR_IMPORTER, "har_to_action_json"), + "har_to_tasks": (_MODULE_UTILS_RECORDING_HAR_IMPORTER, "har_to_tasks"), "index_catalog": ("je_load_density.utils.governance.test_catalog", "index_catalog"), - "install_failure_budget": ("je_load_density.utils.reliability.failure_budget", "install_failure_budget"), + "install_failure_budget": (_MODULE_UTILS_RELIABILITY_FAILURE_BUDGET, "install_failure_budget"), "install_network_conditioner": ( - "je_load_density.utils.reliability.network_conditioner", + _MODULE_UTILS_RELIABILITY_NETWORK_CONDITIONER, "install_network_conditioner", ), "invoke_lambda_workers": ("je_load_density.cloud.aws_lambda", "invoke_lambda_workers"), "issue_share_link": ("je_load_density.utils.governance.share_link", "issue_share_link"), - "jar_for_user": ("je_load_density.utils.scenario.cookie_jar", "jar_for_user"), - "jmeter_to_action_json": ("je_load_density.utils.recording.jmeter_importer", "jmeter_to_action_json"), - "jmeter_to_tasks": ("je_load_density.utils.recording.jmeter_importer", "jmeter_to_tasks"), - "k6_script_to_action_json": ("je_load_density.utils.recording.k6_importer", "k6_script_to_action_json"), - "k6_script_to_tasks": ("je_load_density.utils.recording.k6_importer", "k6_script_to_tasks"), + "jar_for_user": (_MODULE_UTILS_SCENARIO_COOKIE_JAR, "jar_for_user"), + "jmeter_to_action_json": (_MODULE_UTILS_RECORDING_JMETER_IMPORTER, "jmeter_to_action_json"), + "jmeter_to_tasks": (_MODULE_UTILS_RECORDING_JMETER_IMPORTER, "jmeter_to_tasks"), + "k6_script_to_action_json": (_MODULE_UTILS_RECORDING_K6_IMPORTER, "k6_script_to_action_json"), + "k6_script_to_tasks": (_MODULE_UTILS_RECORDING_K6_IMPORTER, "k6_script_to_tasks"), "lambda_worker_handler": ("je_load_density.cloud.aws_lambda", "lambda_worker_handler"), "launch_aci_workers": ("je_load_density.cloud.azure_aci", "launch_aci_workers"), "launch_fargate_workers": ("je_load_density.cloud.aws_fargate", "launch_fargate_workers"), "lint_action": ("je_load_density.utils.linter.action_linter", "lint_action"), "lint_action_file": ("je_load_density.utils.linter.action_linter", "lint_action_file"), - "list_runs": ("je_load_density.utils.test_record.sqlite_persistence", "list_runs"), - "list_tags": ("je_load_density.utils.governance.run_tagging", "list_tags"), - "load_har": ("je_load_density.utils.recording.har_importer", "load_har"), - "load_jmeter_jmx": ("je_load_density.utils.recording.jmeter_importer", "load_jmeter_jmx"), - "load_k6_script": ("je_load_density.utils.recording.k6_importer", "load_k6_script"), - "load_openapi": ("je_load_density.utils.recording.openapi_importer", "load_openapi"), - "load_postman_collection": ("je_load_density.utils.recording.postman_importer", "load_postman_collection"), + "list_runs": (_MODULE_UTILS_TEST_RECORD_SQLITE_PERSISTENCE, "list_runs"), + "list_tags": (_MODULE_UTILS_GOVERNANCE_RUN_TAGGING, "list_tags"), + "load_har": (_MODULE_UTILS_RECORDING_HAR_IMPORTER, "load_har"), + "load_jmeter_jmx": (_MODULE_UTILS_RECORDING_JMETER_IMPORTER, "load_jmeter_jmx"), + "load_k6_script": (_MODULE_UTILS_RECORDING_K6_IMPORTER, "load_k6_script"), + "load_openapi": (_MODULE_UTILS_RECORDING_OPENAPI_IMPORTER, "load_openapi"), + "load_postman_collection": (_MODULE_UTILS_RECORDING_POSTMAN_IMPORTER, "load_postman_collection"), "locust_wrapper_proxy": ("je_load_density.wrapper.proxy.proxy_user", "locust_wrapper_proxy"), "memory_snapshot": ("je_load_density.utils.dx.profiler", "memory_snapshot"), - "merge_actions": ("je_load_density.utils.action_generator.generate", "merge_actions"), - "mutate_json": ("je_load_density.utils.security.fuzz", "mutate_json"), - "mutate_string": ("je_load_density.utils.security.fuzz", "mutate_string"), - "openapi_to_action_json": ("je_load_density.utils.recording.openapi_importer", "openapi_to_action_json"), - "openapi_to_tasks": ("je_load_density.utils.recording.openapi_importer", "openapi_to_tasks"), - "parameter_resolver": ("je_load_density.utils.parameterization", "parameter_resolver"), - "persist_records": ("je_load_density.utils.test_record.sqlite_persistence", "persist_records"), - "pii_scrub": ("je_load_density.utils.data.pii_anonymizer", "scrub"), - "pii_scrub_string": ("je_load_density.utils.data.pii_anonymizer", "scrub_string"), + "merge_actions": (_MODULE_UTILS_ACTION_GENERATOR_GENERATE, "merge_actions"), + "mutate_json": (_MODULE_UTILS_SECURITY_FUZZ, "mutate_json"), + "mutate_string": (_MODULE_UTILS_SECURITY_FUZZ, "mutate_string"), + "openapi_to_action_json": (_MODULE_UTILS_RECORDING_OPENAPI_IMPORTER, "openapi_to_action_json"), + "openapi_to_tasks": (_MODULE_UTILS_RECORDING_OPENAPI_IMPORTER, "openapi_to_tasks"), + "parameter_resolver": (_MODULE_UTILS_PARAMETERIZATION, "parameter_resolver"), + "persist_records": (_MODULE_UTILS_TEST_RECORD_SQLITE_PERSISTENCE, "persist_records"), + "pii_scrub": (_MODULE_UTILS_DATA_PII_ANONYMIZER, "scrub"), + "pii_scrub_string": (_MODULE_UTILS_DATA_PII_ANONYMIZER, "scrub_string"), "post_gitlab_mr_summary": ("je_load_density.utils.notifier.gitlab", "post_gitlab_mr_summary"), "post_opsgenie_alert": ("je_load_density.utils.notifier.opsgenie", "post_opsgenie_alert"), "post_pagerduty_event": ("je_load_density.utils.notifier.pagerduty", "post_pagerduty_event"), "post_slack_summary": ("je_load_density.utils.notifier.slack", "post_slack_summary"), "post_teams_summary": ("je_load_density.utils.notifier.teams", "post_teams_summary"), - "postman_to_action_json": ("je_load_density.utils.recording.postman_importer", "postman_to_action_json"), - "postman_to_tasks": ("je_load_density.utils.recording.postman_importer", "postman_to_tasks"), + "postman_to_action_json": (_MODULE_UTILS_RECORDING_POSTMAN_IMPORTER, "postman_to_action_json"), + "postman_to_tasks": (_MODULE_UTILS_RECORDING_POSTMAN_IMPORTER, "postman_to_tasks"), "prepare_env": ("je_load_density.wrapper.create_locust_env.create_locust_env", "prepare_env"), "probe_rate_limit": ("je_load_density.utils.security.rate_limit_probe", "probe_rate_limit"), "profile_call": ("je_load_density.utils.dx.profiler", "profile_call"), @@ -177,81 +475,81 @@ "read_action_toml": ("je_load_density.utils.json.json_file.toml_file", "read_action_toml"), "read_action_yaml": ("je_load_density.utils.json.json_file.yaml_file", "read_action_yaml"), "read_audit_log": ("je_load_density.utils.governance.audit_log", "read_audit_log"), - "refresh_token": ("je_load_density.utils.auth.oauth2", "refresh_token"), - "register_csv_source": ("je_load_density.utils.parameterization", "register_csv_source"), - "register_csv_sources": ("je_load_density.utils.parameterization", "register_csv_sources"), - "register_db_source": ("je_load_density.utils.parameterization", "register_db_source"), - "register_db_sources": ("je_load_density.utils.parameterization", "register_db_sources"), - "register_variable": ("je_load_density.utils.parameterization", "register_variable"), - "get_resolver": ("je_load_density.utils.parameterization", "get_resolver"), - "use_resolver": ("je_load_density.utils.parameterization", "use_resolver"), - "register_session_variable": ("je_load_density.utils.parameterization", "register_session_variable"), - "register_variables": ("je_load_density.utils.parameterization", "register_variables"), + "refresh_token": (_MODULE_UTILS_AUTH_OAUTH2, "refresh_token"), + "register_csv_source": (_MODULE_UTILS_PARAMETERIZATION, "register_csv_source"), + "register_csv_sources": (_MODULE_UTILS_PARAMETERIZATION, "register_csv_sources"), + "register_db_source": (_MODULE_UTILS_PARAMETERIZATION, "register_db_source"), + "register_db_sources": (_MODULE_UTILS_PARAMETERIZATION, "register_db_sources"), + "register_variable": (_MODULE_UTILS_PARAMETERIZATION, "register_variable"), + "get_resolver": (_MODULE_UTILS_PARAMETERIZATION, "get_resolver"), + "use_resolver": (_MODULE_UTILS_PARAMETERIZATION, "use_resolver"), + "register_session_variable": (_MODULE_UTILS_PARAMETERIZATION, "register_session_variable"), + "register_variables": (_MODULE_UTILS_PARAMETERIZATION, "register_variables"), "render_prompt_text": ("je_load_density.utils.ai.root_cause", "render_prompt_text"), - "render_ssrf_tasks": ("je_load_density.utils.security.ssrf_checks", "render_ssrf_tasks"), + "render_ssrf_tasks": (_MODULE_UTILS_SECURITY_SSRF_CHECKS, "render_ssrf_tasks"), "request_hook": ("je_load_density.wrapper.event.request_hook", "request_hook"), - "reset_all_jars": ("je_load_density.utils.scenario.cookie_jar", "reset_all_jars"), - "reset_throttles": ("je_load_density.utils.throttle.rps_throttle", "reset_throttles"), - "reset_user_jar": ("je_load_density.utils.scenario.cookie_jar", "reset_user_jar"), - "resolve": ("je_load_density.utils.parameterization", "resolve"), + "reset_all_jars": (_MODULE_UTILS_SCENARIO_COOKIE_JAR, "reset_all_jars"), + "reset_throttles": (_MODULE_UTILS_THROTTLE_RPS_THROTTLE, "reset_throttles"), + "reset_user_jar": (_MODULE_UTILS_SCENARIO_COOKIE_JAR, "reset_user_jar"), + "resolve": (_MODULE_UTILS_PARAMETERIZATION, "resolve"), "run_async_load": ("je_load_density.engine.asyncio_engine", "run_async_load"), "run_cloud_run_job": ("je_load_density.cloud.gcp_cloud_run", "run_cloud_run_job"), - "run_owasp_checks": ("je_load_density.utils.security.owasp_checks", "run_owasp_checks"), + "run_owasp_checks": (_MODULE_UTILS_SECURITY_OWASP_CHECKS, "run_owasp_checks"), "run_teardown": ("je_load_density.utils.data.db_fixtures", "run_teardown"), - "run_with_retry": ("je_load_density.utils.reliability.adaptive_retry", "run_with_retry"), + "run_with_retry": (_MODULE_UTILS_RELIABILITY_ADAPTIVE_RETRY, "run_with_retry"), "search_catalog": ("je_load_density.utils.governance.test_catalog", "search_catalog"), - "search_runs_by_tag": ("je_load_density.utils.governance.run_tagging", "search_runs_by_tag"), + "search_runs_by_tag": (_MODULE_UTILS_GOVERNANCE_RUN_TAGGING, "search_runs_by_tag"), "sign_aws_request": ("je_load_density.utils.auth.aws_sigv4", "sign_aws_request"), "sign_jwt": ("je_load_density.utils.auth.jwt_signer", "sign_jwt"), - "smuggling_attack_pack": ("je_load_density.utils.security.smuggling_checks", "smuggling_attack_pack"), - "snapshot_metrics": ("je_load_density.utils.dashboard.live_dashboard", "snapshot_metrics"), - "start_dashboard": ("je_load_density.utils.dashboard.live_dashboard", "start_dashboard"), + "smuggling_attack_pack": (_MODULE_UTILS_SECURITY_SMUGGLING_CHECKS, "smuggling_attack_pack"), + "snapshot_metrics": (_MODULE_UTILS_DASHBOARD_LIVE_DASHBOARD, "snapshot_metrics"), + "start_dashboard": (_MODULE_UTILS_DASHBOARD_LIVE_DASHBOARD, "start_dashboard"), "start_datadog_apm_exporter": ("je_load_density.utils.metrics.datadog_apm_exporter", "start_datadog_apm_exporter"), - "start_influxdb_sink": ("je_load_density.utils.metrics", "start_influxdb_sink"), - "start_leak_detector": ("je_load_density.utils.dx.leak_detector", "start_leak_detector"), + "start_influxdb_sink": (_MODULE_UTILS_METRICS, "start_influxdb_sink"), + "start_leak_detector": (_MODULE_UTILS_DX_LEAK_DETECTOR, "start_leak_detector"), "start_load_density_socket_server": ( "je_load_density.utils.socket_server.load_density_socket_server", "start_load_density_socket_server", ), - "start_opentelemetry_exporter": ("je_load_density.utils.metrics", "start_opentelemetry_exporter"), + "start_opentelemetry_exporter": (_MODULE_UTILS_METRICS, "start_opentelemetry_exporter"), "start_opentelemetry_tracing_exporter": ( "je_load_density.utils.metrics.opentelemetry_tracing_exporter", "start_opentelemetry_tracing_exporter", ), - "start_prometheus_exporter": ("je_load_density.utils.metrics", "start_prometheus_exporter"), + "start_prometheus_exporter": (_MODULE_UTILS_METRICS, "start_prometheus_exporter"), "start_repl": ("je_load_density.utils.dx.repl", "start_repl"), "start_statsd_sink": ("je_load_density.utils.metrics.statsd_sink", "start_statsd_sink"), "start_stub_server": ("je_load_density.utils.stub_server.stub_server", "start_stub_server"), "start_test": ("je_load_density.engine.entrypoints", "start_test"), - "stop_dashboard": ("je_load_density.utils.dashboard.live_dashboard", "stop_dashboard"), + "stop_dashboard": (_MODULE_UTILS_DASHBOARD_LIVE_DASHBOARD, "stop_dashboard"), "stop_datadog_apm_exporter": ("je_load_density.utils.metrics.datadog_apm_exporter", "stop_datadog_apm_exporter"), - "stop_influxdb_sink": ("je_load_density.utils.metrics", "stop_influxdb_sink"), - "stop_leak_detector": ("je_load_density.utils.dx.leak_detector", "stop_leak_detector"), - "stop_opentelemetry_exporter": ("je_load_density.utils.metrics", "stop_opentelemetry_exporter"), + "stop_influxdb_sink": (_MODULE_UTILS_METRICS, "stop_influxdb_sink"), + "stop_leak_detector": (_MODULE_UTILS_DX_LEAK_DETECTOR, "stop_leak_detector"), + "stop_opentelemetry_exporter": (_MODULE_UTILS_METRICS, "stop_opentelemetry_exporter"), "stop_opentelemetry_tracing_exporter": ( "je_load_density.utils.metrics.opentelemetry_tracing_exporter", "stop_opentelemetry_tracing_exporter", ), - "stop_prometheus_exporter": ("je_load_density.utils.metrics", "stop_prometheus_exporter"), + "stop_prometheus_exporter": (_MODULE_UTILS_METRICS, "stop_prometheus_exporter"), "stop_statsd_sink": ("je_load_density.utils.metrics.statsd_sink", "stop_statsd_sink"), "stop_stub_server": ("je_load_density.utils.stub_server.stub_server", "stop_stub_server"), "summarise_records": ("je_load_density.utils.regression.diff", "summarise_records"), - "tag_run": ("je_load_density.utils.governance.run_tagging", "tag_run"), + "tag_run": (_MODULE_UTILS_GOVERNANCE_RUN_TAGGING, "tag_run"), "task": ("locust", "task"), "test_record_instance": ("je_load_density.utils.test_record.test_record_class", "test_record_instance"), - "toxiproxy_add_toxic": ("je_load_density.utils.chaos.toxiproxy", "add_toxic"), - "toxiproxy_create_proxy": ("je_load_density.utils.chaos.toxiproxy", "create_proxy"), - "toxiproxy_install_bandwidth": ("je_load_density.utils.chaos.toxiproxy", "install_bandwidth"), - "toxiproxy_install_latency": ("je_load_density.utils.chaos.toxiproxy", "install_latency"), - "toxiproxy_list_proxies": ("je_load_density.utils.chaos.toxiproxy", "list_proxies"), - "toxiproxy_remove_proxies": ("je_load_density.utils.chaos.toxiproxy", "remove_proxies"), - "toxiproxy_remove_toxic": ("je_load_density.utils.chaos.toxiproxy", "remove_toxic"), - "toxiproxy_reset_all": ("je_load_density.utils.chaos.toxiproxy", "reset_all"), - "translate": ("je_load_density.utils.dx.i18n", "t"), + "toxiproxy_add_toxic": (_MODULE_UTILS_CHAOS_TOXIPROXY, "add_toxic"), + "toxiproxy_create_proxy": (_MODULE_UTILS_CHAOS_TOXIPROXY, "create_proxy"), + "toxiproxy_install_bandwidth": (_MODULE_UTILS_CHAOS_TOXIPROXY, "install_bandwidth"), + "toxiproxy_install_latency": (_MODULE_UTILS_CHAOS_TOXIPROXY, "install_latency"), + "toxiproxy_list_proxies": (_MODULE_UTILS_CHAOS_TOXIPROXY, "list_proxies"), + "toxiproxy_remove_proxies": (_MODULE_UTILS_CHAOS_TOXIPROXY, "remove_proxies"), + "toxiproxy_remove_toxic": (_MODULE_UTILS_CHAOS_TOXIPROXY, "remove_toxic"), + "toxiproxy_reset_all": (_MODULE_UTILS_CHAOS_TOXIPROXY, "reset_all"), + "translate": (_MODULE_UTILS_DX_I18N, "t"), "trend_runs": ("je_load_density.utils.regression.multi_run_trend", "trend_runs"), - "uninstall_failure_budget": ("je_load_density.utils.reliability.failure_budget", "uninstall_failure_budget"), + "uninstall_failure_budget": (_MODULE_UTILS_RELIABILITY_FAILURE_BUDGET, "uninstall_failure_budget"), "uninstall_network_conditioner": ( - "je_load_density.utils.reliability.network_conditioner", + _MODULE_UTILS_RELIABILITY_NETWORK_CONDITIONER, "uninstall_network_conditioner", ), "verify_share_link": ("je_load_density.utils.governance.share_link", "verify_share_link"), diff --git a/je_load_density/engine/async_config.py b/je_load_density/engine/async_config.py index ab657c5..809c2e3 100644 --- a/je_load_density/engine/async_config.py +++ b/je_load_density/engine/async_config.py @@ -105,7 +105,12 @@ def number(value: Any, name: str, *, integer: bool = False, minimum: float = 0) def _validate_rules(task: dict) -> None: - for rule in task.get("assertions") or []: + _validate_assertions(task.get("assertions") or []) + _validate_extractors(task.get("extract") or []) + + +def _validate_assertions(rules: list) -> None: + for rule in rules: if not isinstance(rule, dict) or rule.get("type") not in _ASSERTIONS: raise ValueError("Unsupported native assertion") _rule_fields(rule, {"type", "value", "path", "name"}) @@ -113,7 +118,10 @@ def _validate_rules(task: dict) -> None: raise ValueError("Assertions require value") if rule["type"] == "status_code": number(rule.get("value"), "assertion status_code", integer=True, minimum=100) - for rule in task.get("extract") or []: + + +def _validate_extractors(rules: list) -> None: + for rule in rules: if not isinstance(rule, dict) or rule.get("from", "json_path") not in _EXTRACTORS: raise ValueError("Unsupported native extractor") _rule_fields(rule, {"from", "var", "scope", "path", "name"}) @@ -130,13 +138,20 @@ def _rule_fields(rule: dict, fields: set) -> None: def _validate_timing(task: dict) -> None: - retry = task.get("retry") + _validate_retry(task.get("retry")) + _validate_think_time(task.get("think_time", 0)) + _validate_throttle(task.get("throttle")) + + +def _validate_retry(retry: Any) -> None: if retry is not None: if not isinstance(retry, dict) or set(retry) - _RETRY_FIELDS.keys(): raise ValueError("Unsupported retry configuration") for key, value in retry.items(): number(value, f"retry.{key}", integer=key in {"transient", "flaky", "permanent"}) - think = task.get("think_time", 0) + + +def _validate_think_time(think: Any) -> None: if isinstance(think, dict): if set(think) - {"min", "max"}: raise ValueError("Unsupported think_time configuration") @@ -144,7 +159,9 @@ def _validate_timing(task: dict) -> None: number(think.get("max", think.get("min", 0)), "think_time.max") else: number(think, "think_time") - throttle = task.get("throttle") + + +def _validate_throttle(throttle: Any) -> None: if throttle is not None: if not isinstance(throttle, dict) or set(throttle) - {"key", "rps", "burst"}: raise ValueError("Unsupported throttle configuration") @@ -193,17 +210,20 @@ def _validate_task(task: dict) -> None: raise ValueError("Only HTTP(S) URLs are supported") if "timeout" in task and "${" not in str(task["timeout"]): number(task["timeout"], "timeout", minimum=0.000001) - auth = task.get("auth") + _validate_auth(task.get("auth")) + _request_types(task) + number(task.get("weight", 1), "weight", integer=True) + _validate_rules(task) + _validate_timing(task) + + +def _validate_auth(auth: Any) -> None: if auth is not None and (not isinstance(auth, dict) or auth.get("type") not in {"basic", "bearer"}): raise ValueError("Native auth supports basic and bearer") if auth is not None: names = {"type", "username", "password"} if auth["type"] == "basic" else {"type", "token"} if set(auth) - names: raise ValueError("Unsupported auth configuration") - _request_types(task) - number(task.get("weight", 1), "weight", integer=True) - _validate_rules(task) - _validate_timing(task) def validate_shape(name: str | None, config: dict | None) -> None: diff --git a/je_load_density/engine/asyncio_engine.py b/je_load_density/engine/asyncio_engine.py index 4d5121d..eef0108 100644 --- a/je_load_density/engine/asyncio_engine.py +++ b/je_load_density/engine/asyncio_engine.py @@ -108,6 +108,13 @@ async def start(self): self._loop = asyncio.get_running_loop() self.semaphore = asyncio.Semaphore(self.limit) self._task = asyncio.create_task(self._execute()) + # Let the owned run begin before returning control to the caller. + try: + await asyncio.sleep(0) + except asyncio.CancelledError: + self._task.cancel() + await asyncio.gather(self._task, return_exceptions=True) + raise return self def stop(self) -> None: @@ -223,7 +230,7 @@ def _check_workers(self) -> None: self.workers = [task for task in self.workers if not task.done()] async def _schedule(self, httpx) -> None: - credits, previous = 1.0, self.started + spawn_credits, previous = 1.0, self.started while not self._stop.is_set(): self._check_workers() now = time.monotonic() @@ -238,10 +245,10 @@ async def _schedule(self, httpx) -> None: if target is None: return count, rate = target - credits = min(max(count, 1), credits + (now - previous) * rate) + spawn_credits = min(max(count, 1), spawn_credits + (now - previous) * rate) previous = now - await self._resize(httpx, count, int(credits)) - credits -= self._spawned + await self._resize(httpx, count, int(spawn_credits)) + spawn_credits -= self._spawned await asyncio.sleep(min(0.02, max(0, self.duration - elapsed))) async def _resize(self, httpx, count: int, available: int) -> None: diff --git a/je_load_density/gui/run_worker.py b/je_load_density/gui/run_worker.py index 7c10af9..5816d52 100644 --- a/je_load_density/gui/run_worker.py +++ b/je_load_density/gui/run_worker.py @@ -4,7 +4,9 @@ import contextlib import json import math +import re import sys +import tempfile import time from pathlib import Path from typing import Any, Callable @@ -169,16 +171,42 @@ def run(config: dict, control: Path, protocol: Protocol) -> None: protocol.emit("terminal", data={"state": state, "summary": session.snapshot["summary"]}) +def _private_worker_path(argument: str, filename: str) -> Path: + """Rebuild a fixed worker filename inside one GUI-created temporary directory.""" + supplied = Path(argument) + if not supplied.is_absolute() or ".." in supplied.parts: + raise ValueError("Invalid private worker path") + directory = supplied.parent.name + if re.fullmatch(r"loaddensity-gui-[a-z0-9_]{8}", directory, flags=re.ASCII) is None: + raise ValueError("Invalid private worker directory") + temporary_root = Path(tempfile.gettempdir()).absolute() + expected = temporary_root / directory / filename + canonical = temporary_root.resolve() / directory / filename + if supplied != expected or supplied.resolve() != canonical: + raise ValueError("Invalid private worker path") + return canonical + + +def _private_worker_paths(config: str, control: str) -> tuple[Path, Path]: + """Reject unrelated configuration and cancellation files before reading either.""" + config_path = _private_worker_path(config, "run.json") + control_path = _private_worker_path(control, "stop") + if config_path.parent != control_path.parent: + raise ValueError("Mismatched private worker paths") + return config_path, control_path + + def main() -> None: """Read private settings and reserve stdout for bounded worker frames.""" parser = argparse.ArgumentParser() parser.add_argument("--config", required=True) parser.add_argument("--control", required=True) args = parser.parse_args() + config_path, control_path = _private_worker_paths(args.config, args.control) protocol = Protocol() - config = json.loads(Path(args.config).read_text(encoding="utf-8")) + config = json.loads(config_path.read_text(encoding="utf-8")) with contextlib.redirect_stdout(protocol), contextlib.redirect_stderr(protocol): - run(config, Path(args.control), protocol) + run(config, control_path, protocol) if __name__ == "__main__": diff --git a/je_load_density/utils/executor/action_executor.py b/je_load_density/utils/executor/action_executor.py index 62fecad..3d23f6b 100644 --- a/je_load_density/utils/executor/action_executor.py +++ b/je_load_density/utils/executor/action_executor.py @@ -44,52 +44,65 @@ def command(*args, **kwargs): return command +_MODULE_UTILS_SECURITY_GRAPHQL_CHECKS = "je_load_density.utils.security.graphql_checks" +_MODULE_UTILS_SECURITY_SMUGGLING_CHECKS = "je_load_density.utils.security.smuggling_checks" +_MODULE_UTILS_SECURITY_SSRF_CHECKS = "je_load_density.utils.security.ssrf_checks" +_MODULE_UTILS_RECORDING_CDP_CAPTURE = "je_load_density.utils.recording.cdp_capture" +_MODULE_UTILS_CHAOS_CHAOS_MESH = "je_load_density.utils.chaos.chaos_mesh" +_MODULE_UTILS_SECURITY_JWT_ATTACKS = "je_load_density.utils.security.jwt_attacks" +_MODULE_UTILS_SECURITY_FUZZ = "je_load_density.utils.security.fuzz" +_MODULE_UTILS_TEST_RECORD_SQLITE_PERSISTENCE = "je_load_density.utils.test_record.sqlite_persistence" +_MODULE_UTILS_LINTER_ACTION_FORMATTER = "je_load_density.utils.linter.action_formatter" +_MODULE_UTILS_RECORDING_HAR_IMPORTER = "je_load_density.utils.recording.har_importer" +_MODULE_UTILS_RECORDING_JMETER_IMPORTER = "je_load_density.utils.recording.jmeter_importer" +_MODULE_UTILS_RECORDING_K6_IMPORTER = "je_load_density.utils.recording.k6_importer" +_MODULE_UTILS_GOVERNANCE_RUN_TAGGING = "je_load_density.utils.governance.run_tagging" +_MODULE_UTILS_RECORDING_OPENAPI_IMPORTER = "je_load_density.utils.recording.openapi_importer" +_MODULE_UTILS_RECORDING_POSTMAN_IMPORTER = "je_load_density.utils.recording.postman_importer" +_MODULE_UTILS_CHAOS_TOXIPROXY = "je_load_density.utils.chaos.toxiproxy" + append_audit_entry = _lazy_command("je_load_density.utils.governance.audit_log", "append_audit_entry") apply_db_fixture = _lazy_command("je_load_density.utils.data.db_fixtures", "apply_fixture") assert_sla = _lazy_command("je_load_density.utils.sla.sla_gates", "assert_sla") -build_alias_batching_attack = _lazy_command( - "je_load_density.utils.security.graphql_checks", "build_alias_batching_attack" -) -build_cl_te = _lazy_command("je_load_density.utils.security.smuggling_checks", "build_cl_te") -build_depth_attack = _lazy_command("je_load_density.utils.security.graphql_checks", "build_depth_attack") -build_introspection_payload = _lazy_command( - "je_load_density.utils.security.graphql_checks", "build_introspection_payload" -) +build_alias_batching_attack = _lazy_command(_MODULE_UTILS_SECURITY_GRAPHQL_CHECKS, "build_alias_batching_attack") +build_cl_te = _lazy_command(_MODULE_UTILS_SECURITY_SMUGGLING_CHECKS, "build_cl_te") +build_depth_attack = _lazy_command(_MODULE_UTILS_SECURITY_GRAPHQL_CHECKS, "build_depth_attack") +build_introspection_payload = _lazy_command(_MODULE_UTILS_SECURITY_GRAPHQL_CHECKS, "build_introspection_payload") build_root_cause_prompt = _lazy_command("je_load_density.utils.ai.root_cause", "build_root_cause_prompt") -build_ssrf_targets = _lazy_command("je_load_density.utils.security.ssrf_checks", "build_ssrf_targets") +build_ssrf_targets = _lazy_command(_MODULE_UTILS_SECURITY_SSRF_CHECKS, "build_ssrf_targets") build_summary = _lazy_command("je_load_density.utils.generate_report.generate_summary_report", "build_summary") -build_te_cl = _lazy_command("je_load_density.utils.security.smuggling_checks", "build_te_cl") -build_te_te = _lazy_command("je_load_density.utils.security.smuggling_checks", "build_te_te") +build_te_cl = _lazy_command(_MODULE_UTILS_SECURITY_SMUGGLING_CHECKS, "build_te_cl") +build_te_te = _lazy_command(_MODULE_UTILS_SECURITY_SMUGGLING_CHECKS, "build_te_te") build_user = _lazy_command("je_load_density.utils.data.factory", "build_user") build_user_pool = _lazy_command("je_load_density.utils.data.factory", "build_user_pool") calibrate_sla = _lazy_command("je_load_density.utils.ai.auto_baseline", "calibrate_sla") canary_verdict = _lazy_command("je_load_density.utils.ci_annotations.canary_analysis", "canary_verdict") -capture_cdp_session = _lazy_command("je_load_density.utils.recording.cdp_capture", "capture_cdp_session") -capture_cdp_to_har = _lazy_command("je_load_density.utils.recording.cdp_capture", "capture_cdp_to_har") -cdp_discover_targets = _lazy_command("je_load_density.utils.recording.cdp_capture", "discover_targets") -chaos_apply_manifest = _lazy_command("je_load_density.utils.chaos.chaos_mesh", "apply_manifest") -chaos_delete_manifest = _lazy_command("je_load_density.utils.chaos.chaos_mesh", "delete_manifest") -chaos_network_delay = _lazy_command("je_load_density.utils.chaos.chaos_mesh", "build_network_delay") +capture_cdp_session = _lazy_command(_MODULE_UTILS_RECORDING_CDP_CAPTURE, "capture_cdp_session") +capture_cdp_to_har = _lazy_command(_MODULE_UTILS_RECORDING_CDP_CAPTURE, "capture_cdp_to_har") +cdp_discover_targets = _lazy_command(_MODULE_UTILS_RECORDING_CDP_CAPTURE, "discover_targets") +chaos_apply_manifest = _lazy_command(_MODULE_UTILS_CHAOS_CHAOS_MESH, "apply_manifest") +chaos_delete_manifest = _lazy_command(_MODULE_UTILS_CHAOS_CHAOS_MESH, "delete_manifest") +chaos_network_delay = _lazy_command(_MODULE_UTILS_CHAOS_CHAOS_MESH, "build_network_delay") cluster_errors = _lazy_command("je_load_density.utils.regression.error_clustering", "cluster_errors") -craft_alg_confusion_token = _lazy_command("je_load_density.utils.security.jwt_attacks", "craft_alg_confusion_token") -craft_alg_none_token = _lazy_command("je_load_density.utils.security.jwt_attacks", "craft_alg_none_token") -craft_expired_token = _lazy_command("je_load_density.utils.security.jwt_attacks", "craft_expired_token") -craft_jwt_attack_pack = _lazy_command("je_load_density.utils.security.jwt_attacks", "craft_attack_pack") -craft_kid_traversal_token = _lazy_command("je_load_density.utils.security.jwt_attacks", "craft_kid_traversal_token") +craft_alg_confusion_token = _lazy_command(_MODULE_UTILS_SECURITY_JWT_ATTACKS, "craft_alg_confusion_token") +craft_alg_none_token = _lazy_command(_MODULE_UTILS_SECURITY_JWT_ATTACKS, "craft_alg_none_token") +craft_expired_token = _lazy_command(_MODULE_UTILS_SECURITY_JWT_ATTACKS, "craft_expired_token") +craft_jwt_attack_pack = _lazy_command(_MODULE_UTILS_SECURITY_JWT_ATTACKS, "craft_attack_pack") +craft_kid_traversal_token = _lazy_command(_MODULE_UTILS_SECURITY_JWT_ATTACKS, "craft_kid_traversal_token") curl_to_task = _lazy_command("je_load_density.utils.recording.curl_importer", "curl_to_task") diff_runs = _lazy_command("je_load_density.utils.regression.diff", "diff_runs") emit_github_annotations = _lazy_command( "je_load_density.utils.ci_annotations.github_actions", "emit_github_annotations" ) evaluate_sla = _lazy_command("je_load_density.utils.sla.sla_gates", "evaluate_sla") -expand_task_fuzz = _lazy_command("je_load_density.utils.security.fuzz", "expand_task_fuzz") +expand_task_fuzz = _lazy_command(_MODULE_UTILS_SECURITY_FUZZ, "expand_task_fuzz") export_schema = _lazy_command("je_load_density.utils.schema.action_schema", "export_schema") -fetch_run_records = _lazy_command("je_load_density.utils.test_record.sqlite_persistence", "fetch_run_records") +fetch_run_records = _lazy_command(_MODULE_UTILS_TEST_RECORD_SQLITE_PERSISTENCE, "fetch_run_records") find_breaking_point = _lazy_command("je_load_density.utils.ai.smart_shape", "find_breaking_point") -find_metadata_leak = _lazy_command("je_load_density.utils.security.ssrf_checks", "find_metadata_leak") -format_action_document = _lazy_command("je_load_density.utils.linter.action_formatter", "format_action_document") -format_action_file = _lazy_command("je_load_density.utils.linter.action_formatter", "format_action_file") -format_action_string = _lazy_command("je_load_density.utils.linter.action_formatter", "format_action_string") +find_metadata_leak = _lazy_command(_MODULE_UTILS_SECURITY_SSRF_CHECKS, "find_metadata_leak") +format_action_document = _lazy_command(_MODULE_UTILS_LINTER_ACTION_FORMATTER, "format_action_document") +format_action_file = _lazy_command(_MODULE_UTILS_LINTER_ACTION_FORMATTER, "format_action_file") +format_action_string = _lazy_command(_MODULE_UTILS_LINTER_ACTION_FORMATTER, "format_action_string") generate_allure_report = _lazy_command( "je_load_density.utils.generate_report.generate_allure_report", "generate_allure_report" ) @@ -134,33 +147,33 @@ def command(*args, **kwargs): ) generate_xml = _lazy_command("je_load_density.utils.generate_report.generate_xml_report", "generate_xml") generate_xml_report = _lazy_command("je_load_density.utils.generate_report.generate_xml_report", "generate_xml_report") -graphql_attack_pack = _lazy_command("je_load_density.utils.security.graphql_checks", "graphql_attack_pack") -har_to_action_json = _lazy_command("je_load_density.utils.recording.har_importer", "har_to_action_json") -har_to_tasks = _lazy_command("je_load_density.utils.recording.har_importer", "har_to_tasks") +graphql_attack_pack = _lazy_command(_MODULE_UTILS_SECURITY_GRAPHQL_CHECKS, "graphql_attack_pack") +har_to_action_json = _lazy_command(_MODULE_UTILS_RECORDING_HAR_IMPORTER, "har_to_action_json") +har_to_tasks = _lazy_command(_MODULE_UTILS_RECORDING_HAR_IMPORTER, "har_to_tasks") index_catalog = _lazy_command("je_load_density.utils.governance.test_catalog", "index_catalog") install_failure_budget = _lazy_command("je_load_density.utils.reliability.failure_budget", "install_failure_budget") install_network_conditioner = _lazy_command( "je_load_density.utils.reliability.network_conditioner", "install_network_conditioner" ) issue_share_link = _lazy_command("je_load_density.utils.governance.share_link", "issue_share_link") -jmeter_to_action_json = _lazy_command("je_load_density.utils.recording.jmeter_importer", "jmeter_to_action_json") -jmeter_to_tasks = _lazy_command("je_load_density.utils.recording.jmeter_importer", "jmeter_to_tasks") -k6_script_to_action_json = _lazy_command("je_load_density.utils.recording.k6_importer", "k6_script_to_action_json") -k6_script_to_tasks = _lazy_command("je_load_density.utils.recording.k6_importer", "k6_script_to_tasks") +jmeter_to_action_json = _lazy_command(_MODULE_UTILS_RECORDING_JMETER_IMPORTER, "jmeter_to_action_json") +jmeter_to_tasks = _lazy_command(_MODULE_UTILS_RECORDING_JMETER_IMPORTER, "jmeter_to_tasks") +k6_script_to_action_json = _lazy_command(_MODULE_UTILS_RECORDING_K6_IMPORTER, "k6_script_to_action_json") +k6_script_to_tasks = _lazy_command(_MODULE_UTILS_RECORDING_K6_IMPORTER, "k6_script_to_tasks") lint_action = _lazy_command("je_load_density.utils.linter.action_linter", "lint_action") lint_action_file = _lazy_command("je_load_density.utils.linter.action_linter", "lint_action_file") -list_runs = _lazy_command("je_load_density.utils.test_record.sqlite_persistence", "list_runs") -list_tags = _lazy_command("je_load_density.utils.governance.run_tagging", "list_tags") -load_har = _lazy_command("je_load_density.utils.recording.har_importer", "load_har") -load_jmeter_jmx = _lazy_command("je_load_density.utils.recording.jmeter_importer", "load_jmeter_jmx") -load_k6_script = _lazy_command("je_load_density.utils.recording.k6_importer", "load_k6_script") -load_openapi = _lazy_command("je_load_density.utils.recording.openapi_importer", "load_openapi") -load_postman_collection = _lazy_command("je_load_density.utils.recording.postman_importer", "load_postman_collection") -mutate_json = _lazy_command("je_load_density.utils.security.fuzz", "mutate_json") -mutate_string = _lazy_command("je_load_density.utils.security.fuzz", "mutate_string") -openapi_to_action_json = _lazy_command("je_load_density.utils.recording.openapi_importer", "openapi_to_action_json") -openapi_to_tasks = _lazy_command("je_load_density.utils.recording.openapi_importer", "openapi_to_tasks") -persist_records = _lazy_command("je_load_density.utils.test_record.sqlite_persistence", "persist_records") +list_runs = _lazy_command(_MODULE_UTILS_TEST_RECORD_SQLITE_PERSISTENCE, "list_runs") +list_tags = _lazy_command(_MODULE_UTILS_GOVERNANCE_RUN_TAGGING, "list_tags") +load_har = _lazy_command(_MODULE_UTILS_RECORDING_HAR_IMPORTER, "load_har") +load_jmeter_jmx = _lazy_command(_MODULE_UTILS_RECORDING_JMETER_IMPORTER, "load_jmeter_jmx") +load_k6_script = _lazy_command(_MODULE_UTILS_RECORDING_K6_IMPORTER, "load_k6_script") +load_openapi = _lazy_command(_MODULE_UTILS_RECORDING_OPENAPI_IMPORTER, "load_openapi") +load_postman_collection = _lazy_command(_MODULE_UTILS_RECORDING_POSTMAN_IMPORTER, "load_postman_collection") +mutate_json = _lazy_command(_MODULE_UTILS_SECURITY_FUZZ, "mutate_json") +mutate_string = _lazy_command(_MODULE_UTILS_SECURITY_FUZZ, "mutate_string") +openapi_to_action_json = _lazy_command(_MODULE_UTILS_RECORDING_OPENAPI_IMPORTER, "openapi_to_action_json") +openapi_to_tasks = _lazy_command(_MODULE_UTILS_RECORDING_OPENAPI_IMPORTER, "openapi_to_tasks") +persist_records = _lazy_command(_MODULE_UTILS_TEST_RECORD_SQLITE_PERSISTENCE, "persist_records") pii_scrub = _lazy_command("je_load_density.utils.data.pii_anonymizer", "scrub") pii_scrub_string = _lazy_command("je_load_density.utils.data.pii_anonymizer", "scrub_string") post_gitlab_mr_summary = _lazy_command("je_load_density.utils.notifier.gitlab", "post_gitlab_mr_summary") @@ -168,20 +181,20 @@ def command(*args, **kwargs): post_pagerduty_event = _lazy_command("je_load_density.utils.notifier.pagerduty", "post_pagerduty_event") post_slack_summary = _lazy_command("je_load_density.utils.notifier.slack", "post_slack_summary") post_teams_summary = _lazy_command("je_load_density.utils.notifier.teams", "post_teams_summary") -postman_to_action_json = _lazy_command("je_load_density.utils.recording.postman_importer", "postman_to_action_json") -postman_to_tasks = _lazy_command("je_load_density.utils.recording.postman_importer", "postman_to_tasks") +postman_to_action_json = _lazy_command(_MODULE_UTILS_RECORDING_POSTMAN_IMPORTER, "postman_to_action_json") +postman_to_tasks = _lazy_command(_MODULE_UTILS_RECORDING_POSTMAN_IMPORTER, "postman_to_tasks") probe_rate_limit = _lazy_command("je_load_density.utils.security.rate_limit_probe", "probe_rate_limit") read_action_json = _lazy_command("je_load_density.utils.json.json_file.json_file", "read_action_json") read_action_toml = _lazy_command("je_load_density.utils.json.json_file.toml_file", "read_action_toml") read_action_yaml = _lazy_command("je_load_density.utils.json.json_file.yaml_file", "read_action_yaml") read_audit_log = _lazy_command("je_load_density.utils.governance.audit_log", "read_audit_log") render_prompt_text = _lazy_command("je_load_density.utils.ai.root_cause", "render_prompt_text") -render_ssrf_tasks = _lazy_command("je_load_density.utils.security.ssrf_checks", "render_ssrf_tasks") +render_ssrf_tasks = _lazy_command(_MODULE_UTILS_SECURITY_SSRF_CHECKS, "render_ssrf_tasks") run_db_teardown = _lazy_command("je_load_density.utils.data.db_fixtures", "run_teardown") run_owasp_checks = _lazy_command("je_load_density.utils.security.owasp_checks", "run_owasp_checks") search_catalog = _lazy_command("je_load_density.utils.governance.test_catalog", "search_catalog") -search_runs_by_tag = _lazy_command("je_load_density.utils.governance.run_tagging", "search_runs_by_tag") -smuggling_attack_pack = _lazy_command("je_load_density.utils.security.smuggling_checks", "smuggling_attack_pack") +search_runs_by_tag = _lazy_command(_MODULE_UTILS_GOVERNANCE_RUN_TAGGING, "search_runs_by_tag") +smuggling_attack_pack = _lazy_command(_MODULE_UTILS_SECURITY_SMUGGLING_CHECKS, "smuggling_attack_pack") start_dashboard = _lazy_command("je_load_density.utils.dashboard.live_dashboard", "start_dashboard") start_datadog_apm_exporter = _lazy_command( "je_load_density.utils.metrics.datadog_apm_exporter", "start_datadog_apm_exporter" @@ -215,15 +228,15 @@ def command(*args, **kwargs): ) stop_statsd_sink = _lazy_command("je_load_density.utils.metrics.statsd_sink", "stop_statsd_sink") stop_stub_server = _lazy_command("je_load_density.utils.stub_server.stub_server", "stop_stub_server") -tag_run = _lazy_command("je_load_density.utils.governance.run_tagging", "tag_run") -toxiproxy_add_toxic = _lazy_command("je_load_density.utils.chaos.toxiproxy", "add_toxic") -toxiproxy_create_proxy = _lazy_command("je_load_density.utils.chaos.toxiproxy", "create_proxy") -toxiproxy_install_bandwidth = _lazy_command("je_load_density.utils.chaos.toxiproxy", "install_bandwidth") -toxiproxy_install_latency = _lazy_command("je_load_density.utils.chaos.toxiproxy", "install_latency") -toxiproxy_list_proxies = _lazy_command("je_load_density.utils.chaos.toxiproxy", "list_proxies") -toxiproxy_remove_proxies = _lazy_command("je_load_density.utils.chaos.toxiproxy", "remove_proxies") -toxiproxy_remove_toxic = _lazy_command("je_load_density.utils.chaos.toxiproxy", "remove_toxic") -toxiproxy_reset_all = _lazy_command("je_load_density.utils.chaos.toxiproxy", "reset_all") +tag_run = _lazy_command(_MODULE_UTILS_GOVERNANCE_RUN_TAGGING, "tag_run") +toxiproxy_add_toxic = _lazy_command(_MODULE_UTILS_CHAOS_TOXIPROXY, "add_toxic") +toxiproxy_create_proxy = _lazy_command(_MODULE_UTILS_CHAOS_TOXIPROXY, "create_proxy") +toxiproxy_install_bandwidth = _lazy_command(_MODULE_UTILS_CHAOS_TOXIPROXY, "install_bandwidth") +toxiproxy_install_latency = _lazy_command(_MODULE_UTILS_CHAOS_TOXIPROXY, "install_latency") +toxiproxy_list_proxies = _lazy_command(_MODULE_UTILS_CHAOS_TOXIPROXY, "list_proxies") +toxiproxy_remove_proxies = _lazy_command(_MODULE_UTILS_CHAOS_TOXIPROXY, "remove_proxies") +toxiproxy_remove_toxic = _lazy_command(_MODULE_UTILS_CHAOS_TOXIPROXY, "remove_toxic") +toxiproxy_reset_all = _lazy_command(_MODULE_UTILS_CHAOS_TOXIPROXY, "reset_all") trend_runs = _lazy_command("je_load_density.utils.regression.multi_run_trend", "trend_runs") uninstall_failure_budget = _lazy_command("je_load_density.utils.reliability.failure_budget", "uninstall_failure_budget") uninstall_network_conditioner = _lazy_command( diff --git a/test/smoke/capability_probe.py b/test/smoke/capability_probe.py index 88b3783..88fa818 100644 --- a/test/smoke/capability_probe.py +++ b/test/smoke/capability_probe.py @@ -1,8 +1,11 @@ """Non-skipping installation probes for every declared package extra.""" import argparse +import subprocess +import sys import traceback from collections.abc import Callable +from pathlib import Path import platform_probes import protocol_probes @@ -43,7 +46,11 @@ def run_probe(extra: str) -> None: if name not in PROBES: raise ValueError(f"extra: missing capability probe for {name}") try: - PROBES[name]() + if extra == "all": + # Each capability chooses its scheduler before SDK/TLS imports. + subprocess.run([sys.executable, str(Path(__file__).resolve()), name], check=True, timeout=120) + else: + PROBES[name]() print(f"capability passed: {name}", flush=True) except Exception: if extra != "all": diff --git a/test/smoke/server.py b/test/smoke/server.py index 3bec794..00f98cf 100644 --- a/test/smoke/server.py +++ b/test/smoke/server.py @@ -25,7 +25,7 @@ def do_POST(self) -> None: self.end_headers() Thread(target=self.server.shutdown, daemon=True).start() - def log_message(self, _format: str, *_args: object) -> None: + def log_message(self, format: str, *args: object) -> None: pass diff --git a/test/test_distributed_records.py b/test/test_distributed_records.py index 2754921..75206de 100644 --- a/test/test_distributed_records.py +++ b/test/test_distributed_records.py @@ -77,8 +77,9 @@ def test_foreign_identity_rejects_whole_batch(field, value): run = aggregate() bad = measured(identifier="record-two") bad[field] = value + records = [measured(), bad] with pytest.raises(ValueError, match=field): - run.ingest_batch("worker-a", [measured(), bad]) + run.ingest_batch("worker-a", records) assert run.snapshot() == [] @@ -87,16 +88,18 @@ def test_conflicting_retry_rejects_new_records_in_same_batch(): original = measured() run.ingest_batch("worker-a", [original]) conflicting = measured(response_time_ms=25.0) + records = [measured(identifier="record-new"), conflicting] with pytest.raises(ValueError, match="conflicting"): - run.ingest_batch("worker-a", [measured(identifier="record-new"), conflicting]) + run.ingest_batch("worker-a", records) assert run.snapshot() == [original] def test_record_identifier_cannot_be_reused_by_another_worker(): run = aggregate() run.ingest_batch("worker-a", [measured()]) + records = [measured("worker-b")] with pytest.raises(ValueError, match="record_id"): - run.ingest_batch("worker-b", [measured("worker-b")]) + run.ingest_batch("worker-b", records) assert len(run.snapshot()) == 1 @@ -104,8 +107,9 @@ def test_malformed_record_rejects_entire_batch(): run = aggregate() malformed = measured(identifier="record-two") malformed["schema_version"] = 2 + records = [measured(), malformed] with pytest.raises(ValueError, match="schema_version"): - run.ingest_batch("worker-a", [measured(), malformed]) + run.ingest_batch("worker-a", records) assert run.snapshot() == [] @@ -156,8 +160,9 @@ def test_pending_overflow_fails_without_unbounded_queue_growth(): configure(producer) producer.capture_legacy(legacy_entry(), "passed") producer.capture_legacy(legacy_entry(), "passed") + entry = legacy_entry() with pytest.raises(module.RecordDeliveryError, match="pending"): - producer.capture_legacy(legacy_entry(), "passed") + producer.capture_legacy(entry, "passed") assert producer.snapshot()["pending_records"] == 2 assert env.process_exit_code == 1 env.load_density_tasks.kill() diff --git a/test/test_extra_capabilities.py b/test/test_extra_capabilities.py index c2c0c53..d31cad0 100644 --- a/test/test_extra_capabilities.py +++ b/test/test_extra_capabilities.py @@ -45,13 +45,25 @@ def missing_redis(name, *args, **kwargs): def test_all_probe_reports_failures_and_still_exercises_other_capabilities(monkeypatch): + import subprocess + probe_module = probes(monkeypatch) calls = [] + local_calls = [] def broken(): raise ImportError("missing dependency") - monkeypatch.setattr(probe_module, "PROBES", {"broken": broken, "healthy": lambda: calls.append("healthy")}) + def child(arguments, **options): + calls.append(arguments[-1]) + assert options["check"] is True + assert options["timeout"] > 0 + if arguments[-1] == "broken": + raise subprocess.CalledProcessError(1, arguments) + + monkeypatch.setattr(probe_module, "PROBES", {"broken": broken, "healthy": lambda: local_calls.append("healthy")}) + monkeypatch.setattr(subprocess, "run", child) with pytest.raises(RuntimeError, match="broken"): probe_module.run_probe("all") - assert calls == ["healthy"] + assert calls == ["broken", "healthy"] + assert local_calls == [] diff --git a/test/test_gui_lifecycle.py b/test/test_gui_lifecycle.py index 917b02c..39641fb 100644 --- a/test/test_gui_lifecycle.py +++ b/test/test_gui_lifecycle.py @@ -5,6 +5,7 @@ import os import subprocess import sys +import tempfile import time os.environ.setdefault("QT_QPA_PLATFORM", "offscreen") @@ -168,9 +169,10 @@ def test_serialized_snapshot_stays_within_reader_byte_limit_and_keeps_latest(mon def test_chart_uses_complete_child_windows_after_request_tail_truncation(application): - from je_load_density.gui.chart_panel import LiveChartPanel from je_load_density.gui.run_protocol import make_snapshot + from je_load_density.gui.chart_panel import LiveChartPanel + records = [{"name": "first", "start_time": 100.5, "response_time_ms": 10}] * 250 records += [{"name": "second", "start_time": 101.5, "response_time_ms": 100}] * 250 snapshot = make_snapshot(records, [], start_time=100, end_time=102) @@ -210,6 +212,7 @@ def emit(self, *args, **kwargs): def test_session_package_commands_are_registered_on_private_executor(tmp_path, monkeypatch): from je_load_density.gui.run_worker import RunSession + from je_load_density.utils.executor.action_executor import executor from je_load_density.utils.package_manager.package_manager_class import package_manager @@ -260,9 +263,10 @@ def wait(self, timeout): def test_scripted_load_preserved_with_selected_engine_and_callbacks(tmp_path, monkeypatch): - from je_load_density.engine import entrypoints from je_load_density.gui.run_worker import RunSession + from je_load_density.engine import entrypoints + calls = [] monkeypatch.setattr(entrypoints, "start_test", lambda *args, **kwargs: calls.append((args, kwargs))) session = RunSession({"engine": "asyncio"}, tmp_path / "stop", None) @@ -289,6 +293,81 @@ def test_child_incidental_output_is_visible_without_credentials(monkeypatch): assert "secret" not in message["text"] +@pytest.fixture +def worker_cli(tmp_path, monkeypatch): + from je_load_density.gui import run_worker + + monkeypatch.setattr(tempfile, "gettempdir", lambda: str(tmp_path)) + directory = tmp_path / "loaddensity-gui-abcdefgh" + directory.mkdir() + config = directory / "run.json" + config.write_text('{"engine":"asyncio"}', encoding="utf-8") + control = directory / "stop" + calls = [] + monkeypatch.setattr(run_worker, "run", lambda *args: calls.append(args)) + monkeypatch.setattr(sys, "argv", ["worker", "--config", str(config), "--control", str(control)]) + return run_worker, config, control, calls + + +@pytest.mark.parametrize("cancel_requested", [False, True]) +def test_worker_cli_accepts_private_settings_and_matching_cancel_file(worker_cli, cancel_requested): + worker, config, control, calls = worker_cli + if cancel_requested: + control.touch() + worker.main() + assert calls[0][0] == {"engine": "asyncio"} + assert calls[0][1] == control + assert len(calls) == 1 + + +@pytest.mark.parametrize("invalid_path", ["outside", "filename", "control", "mismatch", "traversal", "nested"]) +def test_worker_cli_rejects_untrusted_paths_before_reading(worker_cli, monkeypatch, invalid_path): + from pathlib import Path + + worker, config, control, calls = worker_cli + if invalid_path == "outside": + config = config.parent.parent / "run.json" + elif invalid_path == "filename": + config = config.with_name("credentials.json") + elif invalid_path == "control": + control = control.with_name("credentials.json") + elif invalid_path == "mismatch": + control = control.parent.parent / "loaddensity-gui-ijklmnop" / "stop" + elif invalid_path == "traversal": + config = config.parent / ".." / config.parent.name / "run.json" + else: + config = config.parent / "nested" / "run.json" + control = config.with_name("stop") + monkeypatch.setattr(sys, "argv", ["worker", "--config", str(config), "--control", str(control)]) + + def forbidden_read(*args, **kwargs): + raise AssertionError("Untrusted configuration was read") + + monkeypatch.setattr(Path, "read_text", forbidden_read) + with pytest.raises(ValueError, match="private worker"): + worker.main() + assert calls == [] + + +@pytest.mark.parametrize("escaped_file", ["config", "control"]) +def test_worker_cli_rejects_resolved_escape_before_reading(worker_cli, monkeypatch, escaped_file): + from pathlib import Path + + worker, config, control, calls = worker_cli + resolve = Path.resolve + escaped_path = config if escaped_file == "config" else control + + def escaped_resolve(path, *args, **kwargs): + if path == escaped_path: + return config.parent.parent / "credentials.json" + return resolve(path, *args, **kwargs) + + monkeypatch.setattr(Path, "resolve", escaped_resolve) + with pytest.raises(ValueError, match="private worker"): + worker.main() + assert calls == [] + + def test_unresponsive_child_is_killed_and_thread_joined(application, monkeypatch, tmp_path): from je_load_density.gui import load_density_gui_thread diff --git a/test/test_native_async_runner.py b/test/test_native_async_runner.py index e58ee8b..7258379 100644 --- a/test/test_native_async_runner.py +++ b/test/test_native_async_runner.py @@ -4,9 +4,9 @@ import httpx import pytest +from je_load_density.engine.entrypoints import start_test from je_load_density.engine.asyncio_engine import run_async_load -from je_load_density.engine.entrypoints import start_test from je_load_density.utils.parameterization import get_resolver, use_resolver from je_load_density.utils.test_record.test_record_class import test_record_instance @@ -97,7 +97,8 @@ def handler(request): ) ) assert result["requests"] > 0 - assert checked and all(cookie == f"token={token}" for token, cookie in checked) + assert checked + assert all(cookie == f"token={token}" for token, cookie in checked) def test_assertion_failure_is_measured_and_retried(transport_factory): @@ -141,8 +142,9 @@ def handler(_request): ) def test_invalid_capabilities_fail_before_opening_clients(transport_factory, options): clients = transport_factory(lambda _request: httpx.Response(200)) + run = run_async_load([{"request_url": "https://local", "method": "get"}], **options) with pytest.raises((ValueError, TypeError)): - asyncio.run(run_async_load([{"request_url": "https://local", "method": "get"}], **options)) + asyncio.run(run) assert clients == [] @@ -157,8 +159,9 @@ def test_invalid_capabilities_fail_before_opening_clients(transport_factory, opt ) def test_unsupported_steps_fail_preflight(transport_factory, task): clients = transport_factory(lambda _request: httpx.Response(200)) + run = run_async_load([{"method": "get", "request_url": "https://local", **task}]) with pytest.raises(ValueError): - asyncio.run(run_async_load([{"method": "get", "request_url": "https://local", **task}])) + asyncio.run(run) assert not clients @@ -185,7 +188,8 @@ async def run(): assert handle.snapshot()["users"] == 0 asyncio.run(run()) - assert clients and all(client.is_closed for client in clients) + assert clients + assert all(client.is_closed for client in clients) def test_repeated_run_summary_does_not_include_legacy_history(transport_factory): @@ -230,8 +234,9 @@ async def resize_at_deadline(httpx_module, count, available): def test_worker_failure_at_run_deadline_propagates(transport_factory): transport_factory(lambda _request: (_ for _ in ()).throw(ValueError("invalid transport"))) + run = run_async_load([{"request_url": "https://local"}], users=1, duration_seconds=0.005) with pytest.raises(ValueError, match="invalid transport"): - asyncio.run(run_async_load([{"request_url": "https://local"}], users=1, duration_seconds=0.005)) + asyncio.run(run) def test_expected_error_status_can_pass_assertions(transport_factory): @@ -291,14 +296,13 @@ def handler(request): ) def test_static_invalid_second_step_fails_before_any_io(transport_factory, task): clients = transport_factory(lambda _request: httpx.Response(200)) + run = run_async_load( + [{"request_url": "https://local"}, {"request_url": "https://local", **task}], + users=1, + duration_seconds=0.03, + ) with pytest.raises(ValueError): - asyncio.run( - run_async_load( - [{"request_url": "https://local"}, {"request_url": "https://local", **task}], - users=1, - duration_seconds=0.03, - ) - ) + asyncio.run(run) assert clients == [] @@ -314,32 +318,30 @@ def client(**options): return original(**options) monkeypatch.setattr(httpx, "AsyncClient", client) + run = run_async_load( + [{"request_url": "https://local"}, + {"request_url": "https://local", "cert": "invalid.pem", **additional_options}], + users=1, + duration_seconds=0.1, + variables={"verify_tls": False}, + ) with pytest.raises(ValueError, match="invalid certificate"): - asyncio.run( - run_async_load( - [{"request_url": "https://local"}, - {"request_url": "https://local", "cert": "invalid.pem", **additional_options}], - users=1, - duration_seconds=0.1, - variables={"verify_tls": False}, - ) - ) + asyncio.run(run) assert received == [] assert all(client.is_closed for client in clients) def test_unknown_shape_option_fails_before_any_io(transport_factory): clients = transport_factory(lambda _request: httpx.Response(200)) + run = run_async_load( + [{"request_url": "https://local"}], + users=1, + duration_seconds=0.03, + load_shape="spike", + shape_config={"baseline_users": 1, "spwan_rate": 100}, + ) with pytest.raises(ValueError): - asyncio.run( - run_async_load( - [{"request_url": "https://local"}], - users=1, - duration_seconds=0.03, - load_shape="spike", - shape_config={"baseline_users": 1, "spwan_rate": 100}, - ) - ) + asyncio.run(run) assert clients == [] @@ -435,3 +437,33 @@ async def user(_httpx, _resolver): await asyncio.gather(handle._task, return_exceptions=True) asyncio.run(run()) + + +def test_cancellation_during_start_closes_owned_run_and_clients(transport_factory, monkeypatch): + from je_load_density.engine import asyncio_engine + + clients = transport_factory(lambda _request: httpx.Response(200)) + handles = [] + + class TrackedHandle(asyncio_engine.AsyncRunHandle): + def __init__(self, *args, **kwargs): + super().__init__(*args, **kwargs) + handles.append(self) + + monkeypatch.setattr(asyncio_engine, "AsyncRunHandle", TrackedHandle) + + async def run(): + caller = asyncio.create_task( + asyncio_engine.run_async_load([{"request_url": "https://local"}], duration_seconds=20) + ) + asyncio.get_running_loop().call_soon(caller.cancel) + with pytest.raises(asyncio.CancelledError): + await caller + assert len(handles) == 1 + assert handles[0]._task.done() + assert handles[0].state == "cancelled" + assert all(worker.done() for worker in handles[0].workers) + assert clients + assert all(client.is_closed for client in clients) + + asyncio.run(run()) diff --git a/test/test_parameter_context.py b/test/test_parameter_context.py index 1c99e46..02ca16f 100644 --- a/test/test_parameter_context.py +++ b/test/test_parameter_context.py @@ -38,16 +38,20 @@ def test_fork_detaches_nested_variable_values_and_future_registration(): def test_scoped_facade_restores_nested_context_and_keeps_default_state(): + def fail_in_scope(resolver): + with parameters.use_resolver(resolver): + register_variable("token", "second") + assert parameters.resolve("${token}") == "second" + raise RuntimeError("scope failed") + register_variable("token", "default") first, second = parameter_resolver.fork(), parameter_resolver.fork() saved_resolve = parameter_resolver.resolve with parameters.use_resolver(first): register_variable("token", "first") assert saved_resolve("${var.token}") == "first" - with pytest.raises(RuntimeError), parameters.use_resolver(second): - register_variable("token", "second") - assert parameters.resolve("${token}") == "second" - raise RuntimeError("scope failed") + with pytest.raises(RuntimeError): + fail_in_scope(second) assert parameters.get_resolver() is first assert parameters.resolve("${token}") == "first" assert saved_resolve("${token}") == "default" From b387260ce093ce9b9d95a79760f21e5a493396e1 Mon Sep 17 00:00:00 2001 From: JeffreyChen Date: Sat, 3 Oct 2026 12:48:13 +0800 Subject: [PATCH 15/18] fix: retain async worker stop intent through cancellation --- architecture.md | 2 + docs/updates/2026-10.md | 9 +++ docs/updates/README.md | 1 + je_load_density/engine/async_http.py | 9 ++- je_load_density/engine/asyncio_engine.py | 27 ++++++-- test/smoke/server.py | 2 +- test/test_asyncio_engine.py | 33 ++++++++-- test/test_native_async_runner.py | 80 ++++++++++++++++++++++++ 8 files changed, 149 insertions(+), 14 deletions(-) diff --git a/architecture.md b/architecture.md index d1d7400..de238af 100644 --- a/architecture.md +++ b/architecture.md @@ -226,6 +226,8 @@ report callbacks and settings restoration do not require the unreleased core col The public start_test signature remains compatible and accepts engine through kwargs: locust defaults to the existing environment path, asyncio dispatches local native HTTP. AsyncRunHandle owns clients/tasks, user resolvers, run summary and cooperative callbacks; +retiring workers retain explicit stop intent through transport cancellation scopes, +including load-shape downsizing, retry and journey-step boundaries. The existing legacy record/report and canonical opt-in paths remain available. Unsupported protocols, distributed options and exporter integrations are retained as outstanding capabilities. The desktop Qt supervisor communicates with a fresh engine interpreter through bounded diff --git a/docs/updates/2026-10.md b/docs/updates/2026-10.md index a10717b..aa5f459 100644 --- a/docs/updates/2026-10.md +++ b/docs/updates/2026-10.md @@ -277,3 +277,12 @@ Added a stdlib smoke harness that exercises local HTTP requests in isolated proc - **Verification**: final complete suite 1601 passed, 14 optional skips and four existing warnings; changed-file Ruff E/F/W/C90/I at 120 columns and whitespace checks passed. A rebuilt installed wheel against published ActionCore 0.0.2 passed six CLI/report/SLA/dashboard smoke tests and the Locust-prepatched GUI completion/cancellation/active-window-close probe. - **Review**: no additional important findings. Type-checker visibility for the dynamically exported `request_hook` remains a nonblocking omission; runtime access is preserved. The earlier single full-suite connection-test subprocess timeout did not reproduce in native/prepatched focused runs or the two subsequent complete runs; no timeout/assertion was weakened. - **Boundary**: user-facing commands and configuration are unchanged, so the existing three README languages, Sphinx and architecture contracts still apply. Remaining async/dashboard/shared-record/script capabilities stay in `progress.md`; this stage does not close them. Remote extras and quality gates rerun after the feature-branch push; no scan-rule bypass or audit suppression was added. + +## U-20261003-04 · 2026-10-03 · Retain async stop intent through transport cancellation · #done #fix #async #ci + +- **Evidence**: CI run 37092441662 exposed native HTTP timeouts on Ubuntu Python 3.12/3.14 and the all-extras GUI probe. File-backed diagnostics reproduced a live child blocked in run cleanup, with a worker still issuing HTTP requests after cancellation. The earlier suspicion of pipe communication alone was insufficient: AnyIO transport cancellation scopes can absorb the external task cancellation while the worker continues. +- **Fix**: preserve stop intent independently of exceptions. Final cleanup signals the run stop; retiring tasks retain a per-worker marker through load-shape shrinking. Journey steps, semaphore admission, retries and pacing observe those markers. Completed attempts remain recorded; surviving users keep running during downsizing, and clients close without repeated forced cancellation. +- **Regressions**: three new cases failed first, then passed: a swallowed cancellation followed by long pacing, the same with a failed response/retry budget, and load shrink that stops only the retiring worker. Native runner/real HTTP focused suite: 38 passed. Eight real connection-failure diagnostic runs complete normally after the fix. +- **Harness/quality**: native child results use regular temporary files and bounded process polling instead of gevent pipe readers; the same 20-second bound remains, with stack diagnostics on stalled children. The smoke logging override accepts compatible variadic arguments, verified against Pylint's inherited-signature and builtin-name rules. +- **Verification**: final full suite 1604 passed, 14 optional skips and four existing warnings; scoped Ruff E/F/W/C90/I, Pylint callback checks and whitespace checks passed. Rebuilt installed wheel against published ActionCore 0.0.2 passes six CLI/report/SLA/dashboard smoke tests and the Locust-prepatched GUI completion/cancel/active-close probe. +- **Boundary**: architecture section 6 describes stop intent; public commands/configuration and README/Sphinx contracts remain unchanged. The first follow-up head resolved all 68 reported SonarCloud issues; Codacy subprocess/import/XML audits remain explicit review findings, without scan-rule bypass. Remote CI reruns on the follow-up commit. diff --git a/docs/updates/README.md b/docs/updates/README.md index d667c0f..4c8913f 100644 --- a/docs/updates/README.md +++ b/docs/updates/README.md @@ -58,6 +58,7 @@ In the same commit: delete the item from `progress.md`, add a `#done` entry here | ID | Date | Title | Tags | Batch | |---|---|---|---|---| +| U-20261003-04 | 2026-10-03 | Retain async stop intent through transport cancellation | #done #fix #async #ci | [2026-10](2026-10.md) | | U-20261003-03 | 2026-10-03 | Isolate extras probes and clean up cancelled async startup | #done #ci #quality #async #gui | [2026-10](2026-10.md) | | U-20261003-02 | 2026-10-03 | Add isolated runners and canonical worker delivery | #done #async #gui #distributed #parameters #15 #24 #29 | [2026-10](2026-10.md) | | U-20261003-01 | 2026-10-03 | Verify extras matrix and resolve quality findings | #done #ci #quality #25 | [2026-10](2026-10.md) | diff --git a/je_load_density/engine/async_http.py b/je_load_density/engine/async_http.py index aa7e1f9..80d1866 100644 --- a/je_load_density/engine/async_http.py +++ b/je_load_density/engine/async_http.py @@ -173,12 +173,16 @@ async def execute_step(pool: UserClientPool, raw_task: dict, run) -> None: throttle = task.get("throttle") policy = retry_policy(task["retry"]) if "retry" in task else None attempt_number = 0 - while True: + while not run._worker_stopping(): if throttle: await run.throttle(throttle).acquire() async with run.semaphore: + if run._worker_stopping(): + return attempt = await measure(pool, task, run.base_url) record_attempt(attempt, run) + if run._worker_stopping(): + return attempt_number += 1 if attempt.error is None or policy is None: break @@ -186,4 +190,5 @@ async def execute_step(pool: UserClientPool, raw_task: dict, run) -> None: if not decision.will_retry: break await asyncio.sleep(decision.delay_seconds) - await asyncio.sleep(_think_time_seconds(task.get("think_time"))) + if not run._worker_stopping(): + await asyncio.sleep(_think_time_seconds(task.get("think_time"))) diff --git a/je_load_density/engine/asyncio_engine.py b/je_load_density/engine/asyncio_engine.py index eef0108..3ef1a60 100644 --- a/je_load_density/engine/asyncio_engine.py +++ b/je_load_density/engine/asyncio_engine.py @@ -60,6 +60,7 @@ def __init__(self, tasks, users=10, duration_seconds=10.0, **options) -> None: ) self.successes, self.failures = [], [] self.workers = [] + self._retiring = set() self.state = "created" self.started = None self.finished = None @@ -211,11 +212,23 @@ async def _execute(self) -> dict: self.finished = time.monotonic() return self.snapshot() - async def _cancel_workers(self) -> list: - for task in self.workers: + def _stop_workers(self, workers: list) -> None: + self._retiring.update(workers) + for task in workers: if not task.done(): task.cancel() - return await asyncio.gather(*self.workers, return_exceptions=True) + + def _worker_stopping(self) -> bool: + """Keep stop intent even when transport cancellation scopes absorb the exception.""" + return self._stop.is_set() or asyncio.current_task() in self._retiring + + async def _cancel_workers(self) -> list: + self._stop.set() + self._stop_workers(self.workers) + try: + return await asyncio.gather(*self.workers, return_exceptions=True) + finally: + self._retiring.clear() def _raise_worker_errors(self, results: list) -> None: for result in results: @@ -253,10 +266,10 @@ async def _schedule(self, httpx) -> None: async def _resize(self, httpx, count: int, available: int) -> None: excess = self.workers[count:] - for task in excess: - task.cancel() if excess: + self._stop_workers(excess) results = await asyncio.gather(*excess, return_exceptions=True) + self._retiring.difference_update(excess) self._raise_worker_errors(results) self.workers = self.workers[:count] self._spawned = min(max(count - len(self.workers), 0), available) @@ -270,12 +283,14 @@ async def _user(self, httpx, resolver) -> None: pool = UserClientPool(httpx, self.http2, self.verify_context) with use_resolver(resolver): async with pool: - while not self._stop.is_set(): + while not self._worker_stopping(): tasks = self.payload["tasks"] if self.payload["mode"] == "weighted": chosen = _pick_weighted(tasks) tasks = [] if chosen is None else [chosen] for task in tasks: + if self._worker_stopping(): + return if _condition_passes(task): await execute_step(pool, task, self) await asyncio.sleep(0.001) diff --git a/test/smoke/server.py b/test/smoke/server.py index 00f98cf..d272e97 100644 --- a/test/smoke/server.py +++ b/test/smoke/server.py @@ -25,7 +25,7 @@ def do_POST(self) -> None: self.end_headers() Thread(target=self.server.shutdown, daemon=True).start() - def log_message(self, format: str, *args: object) -> None: + def log_message(self, *args: object, **kwargs: object) -> None: pass diff --git a/test/test_asyncio_engine.py b/test/test_asyncio_engine.py index 6e15c13..de56d6f 100644 --- a/test/test_asyncio_engine.py +++ b/test/test_asyncio_engine.py @@ -7,6 +7,7 @@ import socket import subprocess # nosec B404 - the test server is a child process import sys +import tempfile import time import urllib.request @@ -63,7 +64,8 @@ def _run(tasks, **kwargs): # The suite also selects Locust. Exercise actual native I/O in an unpatched # interpreter, just like bench and the isolated desktop worker. source = """ -import asyncio, json, sys +import asyncio, faulthandler, json, sys +faulthandler.dump_traceback_later(8, repeat=True) from je_load_density.engine.asyncio_engine import run_async_load from je_load_density.utils.test_record.test_record_class import test_record_instance options = json.loads(sys.argv[1]) @@ -72,10 +74,31 @@ def _run(tasks, **kwargs): 'failure': test_record_instance.error_record_list})) """ options = {"tasks": tasks, "users": 2, "duration_seconds": 1.0, **kwargs} - completed = subprocess.run([sys.executable, "-c", source, json.dumps(options)], - capture_output=True, text=True, check=False, timeout=20) - assert completed.returncode == 0, completed.stdout + completed.stderr - measured = json.loads(completed.stdout.splitlines()[-1]) + arguments = [sys.executable, "-c", source, json.dumps(options)] + # Locust patches subprocess communication in this parent; regular files avoid + # gevent pipe-reader joins while keeping the same native child and timeout. + with ( + tempfile.TemporaryFile(mode="w+", encoding="utf-8") as output, + tempfile.TemporaryFile(mode="w+", encoding="utf-8") as errors, + ): + child = subprocess.Popen(arguments, stdout=output, stderr=errors, text=True) + try: + deadline = time.monotonic() + 20 + while child.poll() is None and time.monotonic() < deadline: + time.sleep(0.02) + if child.poll() is None: + output.seek(0) + errors.seek(0) + raise subprocess.TimeoutExpired(arguments, 20, output.read(), errors.read()) + finally: + if child.poll() is None: + child.kill() + child.wait(timeout=5) + output.seek(0) + errors.seek(0) + stdout, stderr = output.read(), errors.read() + assert child.returncode == 0, stdout + stderr + measured = json.loads(stdout.splitlines()[-1]) test_record_instance.test_record_list[:] = measured["success"] test_record_instance.error_record_list[:] = measured["failure"] return measured["result"] diff --git a/test/test_native_async_runner.py b/test/test_native_async_runner.py index 7258379..abf357f 100644 --- a/test/test_native_async_runner.py +++ b/test/test_native_async_runner.py @@ -467,3 +467,83 @@ async def run(): assert all(client.is_closed for client in clients) asyncio.run(run()) + + +@pytest.mark.parametrize("status", [200, 503]) +def test_deadline_stops_steps_and_retries_after_transport_swallows_cancel(transport_factory, status): + from je_load_density.engine.asyncio_engine import AsyncRunHandle + + calls = [] + + async def handler(request): + calls.append(request.url.path) + try: + await asyncio.Event().wait() + except asyncio.CancelledError: + if len(calls) != 1: + raise + return httpx.Response(status) + + clients = transport_factory(handler) + + async def run(): + handle = AsyncRunHandle([ + {"request_url": "https://local/first", "think_time": 30, + "retry": {"permanent": 2, "base_delay": 30, "jitter": 0}}, + {"request_url": "https://local/next"}, + ], users=1, duration_seconds=0.05) + await handle.start() + result = await asyncio.wait_for(handle.wait(), 1) + assert result["state"] == "completed" + assert calls == ["/first"] + assert all(worker.done() for worker in handle.workers) + + asyncio.run(run()) + assert clients + assert all(client.is_closed for client in clients) + + +def test_shrinking_load_stops_only_retiring_worker_when_transport_swallows_cancel(transport_factory, monkeypatch): + from je_load_density.engine.asyncio_engine import AsyncRunHandle + + entered = asyncio.Event() + counts = {} + + async def handler(_request): + worker = asyncio.current_task() + counts[worker] = counts.get(worker, 0) + 1 + if len(counts) == 2: + entered.set() + try: + await asyncio.Event().wait() + except asyncio.CancelledError: + if counts[worker] != 1: + raise + return httpx.Response(200) + + clients = transport_factory(handler) + + async def run(): + handle = AsyncRunHandle([{"request_url": "https://local"}], users=2, duration_seconds=20) + + async def schedule(httpx_module): + await handle._resize(httpx_module, 2, 2) + await handle._stop.wait() + + monkeypatch.setattr(handle, "_schedule", schedule) + await handle.start() + try: + await asyncio.wait_for(entered.wait(), 1) + survivor, retiring = handle.workers + await asyncio.wait_for(handle._resize(None, 1, 0), 1) + assert retiring.done() + assert not survivor.done() + assert not handle._stop.is_set() + assert counts[retiring] == 1 + finally: + handle.stop() + await asyncio.wait_for(handle.wait(), 1) + + asyncio.run(run()) + assert clients + assert all(client.is_closed for client in clients) From 3a6bb1fa7064aa05e29a2201a1aaa6435ae1818e Mon Sep 17 00:00:00 2001 From: JeffreyChen Date: Sat, 3 Oct 2026 12:54:42 +0800 Subject: [PATCH 16/18] refactor: expose the shared async worker stop query --- docs/updates/2026-10.md | 6 ++++++ docs/updates/README.md | 1 + je_load_density/engine/async_http.py | 8 ++++---- je_load_density/engine/asyncio_engine.py | 6 +++--- 4 files changed, 14 insertions(+), 7 deletions(-) diff --git a/docs/updates/2026-10.md b/docs/updates/2026-10.md index aa5f459..fa5b165 100644 --- a/docs/updates/2026-10.md +++ b/docs/updates/2026-10.md @@ -286,3 +286,9 @@ Added a stdlib smoke harness that exercises local HTTP requests in isolated proc - **Harness/quality**: native child results use regular temporary files and bounded process polling instead of gevent pipe readers; the same 20-second bound remains, with stack diagnostics on stalled children. The smoke logging override accepts compatible variadic arguments, verified against Pylint's inherited-signature and builtin-name rules. - **Verification**: final full suite 1604 passed, 14 optional skips and four existing warnings; scoped Ruff E/F/W/C90/I, Pylint callback checks and whitespace checks passed. Rebuilt installed wheel against published ActionCore 0.0.2 passes six CLI/report/SLA/dashboard smoke tests and the Locust-prepatched GUI completion/cancel/active-close probe. - **Boundary**: architecture section 6 describes stop intent; public commands/configuration and README/Sphinx contracts remain unchanged. The first follow-up head resolved all 68 reported SonarCloud issues; Codacy subprocess/import/XML audits remain explicit review findings, without scan-rule bypass. Remote CI reruns on the follow-up commit. + +## U-20261003-05 · 2026-10-03 · Expose the shared worker stop query without private access · #done #quality #async + +- **What**: rename the new shared stop-query method to `worker_stop_requested`. The HTTP step executor consumes an explicit component interface instead of another class's protected member; stopping behavior stays the same. +- **Verification**: 38 native/real-HTTP tests passed after the rename; Ruff E/F/W/C90/I and Pylint protected-access/inherited-signature/builtin-name checks passed. The preceding runtime fix passed the complete 1604-test suite and rebuilt installed-wheel CLI/GUI probes. +- **Scope**: no CLI/action/configuration changes or scan suppressions. Remaining capabilities stay in `progress.md`. diff --git a/docs/updates/README.md b/docs/updates/README.md index 4c8913f..6af66a1 100644 --- a/docs/updates/README.md +++ b/docs/updates/README.md @@ -58,6 +58,7 @@ In the same commit: delete the item from `progress.md`, add a `#done` entry here | ID | Date | Title | Tags | Batch | |---|---|---|---|---| +| U-20261003-05 | 2026-10-03 | Expose the shared worker stop query without private access | #done #quality #async | [2026-10](2026-10.md) | | U-20261003-04 | 2026-10-03 | Retain async stop intent through transport cancellation | #done #fix #async #ci | [2026-10](2026-10.md) | | U-20261003-03 | 2026-10-03 | Isolate extras probes and clean up cancelled async startup | #done #ci #quality #async #gui | [2026-10](2026-10.md) | | U-20261003-02 | 2026-10-03 | Add isolated runners and canonical worker delivery | #done #async #gui #distributed #parameters #15 #24 #29 | [2026-10](2026-10.md) | diff --git a/je_load_density/engine/async_http.py b/je_load_density/engine/async_http.py index 80d1866..ef71c10 100644 --- a/je_load_density/engine/async_http.py +++ b/je_load_density/engine/async_http.py @@ -173,15 +173,15 @@ async def execute_step(pool: UserClientPool, raw_task: dict, run) -> None: throttle = task.get("throttle") policy = retry_policy(task["retry"]) if "retry" in task else None attempt_number = 0 - while not run._worker_stopping(): + while not run.worker_stop_requested(): if throttle: await run.throttle(throttle).acquire() async with run.semaphore: - if run._worker_stopping(): + if run.worker_stop_requested(): return attempt = await measure(pool, task, run.base_url) record_attempt(attempt, run) - if run._worker_stopping(): + if run.worker_stop_requested(): return attempt_number += 1 if attempt.error is None or policy is None: @@ -190,5 +190,5 @@ async def execute_step(pool: UserClientPool, raw_task: dict, run) -> None: if not decision.will_retry: break await asyncio.sleep(decision.delay_seconds) - if not run._worker_stopping(): + if not run.worker_stop_requested(): await asyncio.sleep(_think_time_seconds(task.get("think_time"))) diff --git a/je_load_density/engine/asyncio_engine.py b/je_load_density/engine/asyncio_engine.py index 3ef1a60..755a2b6 100644 --- a/je_load_density/engine/asyncio_engine.py +++ b/je_load_density/engine/asyncio_engine.py @@ -218,7 +218,7 @@ def _stop_workers(self, workers: list) -> None: if not task.done(): task.cancel() - def _worker_stopping(self) -> bool: + def worker_stop_requested(self) -> bool: """Keep stop intent even when transport cancellation scopes absorb the exception.""" return self._stop.is_set() or asyncio.current_task() in self._retiring @@ -283,13 +283,13 @@ async def _user(self, httpx, resolver) -> None: pool = UserClientPool(httpx, self.http2, self.verify_context) with use_resolver(resolver): async with pool: - while not self._worker_stopping(): + while not self.worker_stop_requested(): tasks = self.payload["tasks"] if self.payload["mode"] == "weighted": chosen = _pick_weighted(tasks) tasks = [] if chosen is None else [chosen] for task in tasks: - if self._worker_stopping(): + if self.worker_stop_requested(): return if _condition_passes(task): await execute_step(pool, task, self) From 34b11371c792f71598b853e56675df25d924d78a Mon Sep 17 00:00:00 2001 From: JeffreyChen Date: Sat, 3 Oct 2026 13:40:40 +0800 Subject: [PATCH 17/18] fix: harden smoke XML and scope reviewed security exemptions --- docs/updates/2026-10.md | 37 +++++++++++++++++++ docs/updates/README.md | 3 +- je_load_density/__init__.py | 2 + .../utils/executor/action_executor.py | 3 ++ test/smoke/capability_probe.py | 2 + test/smoke/install_extra.py | 9 ++++- test/smoke/platform_probes.py | 2 + test/smoke/run_cell.py | 8 +++- test/smoke/test_wheel_smoke.py | 21 ++++++++++- test/test_asyncio_engine.py | 6 +++ test/test_dashboard_ui.py | 3 ++ test/test_extras_matrix.py | 2 + test/test_gui_lifecycle.py | 13 ++++--- test/test_native_imports.py | 4 ++ test/test_parameter_context.py | 6 +++ test/test_protocol_event_timing.py | 2 + 16 files changed, 112 insertions(+), 11 deletions(-) diff --git a/docs/updates/2026-10.md b/docs/updates/2026-10.md index fa5b165..e026110 100644 --- a/docs/updates/2026-10.md +++ b/docs/updates/2026-10.md @@ -292,3 +292,40 @@ Added a stdlib smoke harness that exercises local HTTP requests in isolated proc - **What**: rename the new shared stop-query method to `worker_stop_requested`. The HTTP step executor consumes an explicit component interface instead of another class's protected member; stopping behavior stays the same. - **Verification**: 38 native/real-HTTP tests passed after the rename; Ruff E/F/W/C90/I and Pylint protected-access/inherited-signature/builtin-name checks passed. The preceding runtime fix passed the complete 1604-test suite and rebuilt installed-wheel CLI/GUI probes. - **Scope**: no CLI/action/configuration changes or scan suppressions. Remaining capabilities stay in `progress.md`. + +## U-20261003-06 · 2026-10-03 · Harden smoke XML and document reviewed Codacy false positives · #done #security #quality #ci + +- **XML**: the installed-wheel JUnit smoke parser uses the already-required `defusedxml` dependency. The new entity-expansion regression failed against the stdlib parser first, then passed; valid report assertions remain intact. No runtime/install requirement changed. +- **Triage**: reviewed all 26 Codacy findings from PR #118 at head `3a6bb1f`. Two XML findings are repaired. The remaining 24 audit findings at 22 sites have exact-rule `nosemgrep` annotations and adjacent trust-boundary explanations. Fixed argv subprocesses do not use a shell; variable paths/options are argv/JSON data, probe names are checked by the fixed registry or regex. Import targets come from constant registries or literal pytest parameters. `TimeoutExpired(...)` constructs an exception and launches no process. +- **Scope**: all security rules and files remain scanned. No global pattern disabling, file exclusion, account-level dismissal or unqualified suppression was added. Long full rule IDs receive only a local Ruff line-length exception. GUI test import order was cleaned up without behavior changes. +- **Verification**: complete suite 1605 passed, 14 optional skips and four existing warnings; rebuilt installed wheel against published ActionCore 0.0.2 passed all seven CLI/report/SLA/dashboard/MCP/native/XML smoke tests. Scoped Ruff E/F/W/C90/I and whitespace checks pass. Independent review found no blocking issues and verified Opengrep combines the preceding and inline rule IDs without suppressing other rules. Remote Codacy/Sonar/CI analysis follows the push. +- **Evidence**: reviewed original issue identities below; subsequent line movement does not change the recorded input snapshot. New capabilities remain in `progress.md`. + +| Original site | Rule | Resolution | Codacy issue ID | +|---|---|---|---| +| `je_load_density/__init__.py:817` | `non-literal-import` | reviewed exact-rule exemption | `9df02cd3f2a7777c9ec7f31adc441fe3` | +| `je_load_density/utils/executor/action_executor.py:40` | `non-literal-import` | reviewed exact-rule exemption | `31c5ef6d27c718a9136d93f95fbf252c` | +| `test/smoke/capability_probe.py:51` | `dangerous-subprocess-use-audit` | reviewed exact-rule exemption | `7b1e97810088a66b19c82a06ddd6a79f` | +| `test/smoke/install_extra.py:19` | `dangerous-subprocess-use-audit` | reviewed exact-rule exemption | `976d85ee7a92019b9b95049edce9bbd5` | +| `test/smoke/install_extra.py:19` | `dangerous-subprocess-use-tainted-env-args` | reviewed exact-rule exemption | `c8d39f84e2e90b8d527493d7db124175` | +| `test/smoke/install_extra.py:22` | `dangerous-subprocess-use-audit` | reviewed exact-rule exemption | `2bf6a6572abaa20c8b3a7ac20d7a5dac` | +| `test/smoke/install_extra.py:23` | `dangerous-subprocess-use-audit` | reviewed exact-rule exemption | `37afbf423b1e8f2b1c22c5f14bc00557` | +| `test/smoke/platform_probes.py:191` | `dangerous-subprocess-use-audit` | reviewed exact-rule exemption | `759f11128237f6d839fb827141eb4e8e` | +| `test/smoke/run_cell.py:12` | `dangerous-subprocess-use-audit` | reviewed exact-rule exemption | `acd70906c66bee864317a3b521aac8d1` | +| `test/smoke/run_cell.py:12` | `dangerous-subprocess-use-tainted-env-args` | reviewed exact-rule exemption | `7646a33312b5dc1f9b33d4cac9de6b26` | +| `test/smoke/run_cell.py:14` | `dangerous-subprocess-use-audit` | reviewed exact-rule exemption | `8b8a1395be42e76c83ca39e2a1b57f52` | +| `test/smoke/test_wheel_smoke.py:15` | `use-defused-xml` | defusedxml parser | `c3d20bfa3700e6b28c45c8d7ff4815ef` | +| `test/smoke/test_wheel_smoke.py:31` | `dangerous-subprocess-use-audit` | reviewed exact-rule exemption | `851967d8a075c02190850626feabebf1` | +| `test/smoke/test_wheel_smoke.py:61` | `dangerous-subprocess-use-audit` | reviewed exact-rule exemption | `f2310fb4f6802988616238cf5616e8ca` | +| `test/smoke/test_wheel_smoke.py:96` | `use-defused-xml-parse` | defusedxml parser | `b5134b9f2e4f0ff81869435b05b4658c` | +| `test/test_asyncio_engine.py:84` | `dangerous-subprocess-use-audit` | reviewed exact-rule exemption | `56bfb1851f03738b2bd13b103de72808` | +| `test/test_asyncio_engine.py:92` | `dangerous-subprocess-use-audit` | reviewed exact-rule exemption | `cc4e8d26b4949e55a787fdec1335d963` | +| `test/test_dashboard_ui.py:48` | `dangerous-subprocess-use-audit` | reviewed exact-rule exemption | `bb3e7e7780bd7c31af5db172fd1dc8e` | +| `test/test_extras_matrix.py:53` | `dangerous-subprocess-use-audit` | reviewed exact-rule exemption | `c554bb3d17fc703495e40e96134a7bab` | +| `test/test_gui_lifecycle.py:68` | `dangerous-subprocess-use-audit` | reviewed exact-rule exemption | `a34e54aebf1a261feff0dccafcfe5bb` | +| `test/test_native_imports.py:21` | `dangerous-subprocess-use-audit` | reviewed exact-rule exemption | `a416d8b40f9717d8e2fd73ed9732aa78` | +| `test/test_native_imports.py:49` | `dangerous-subprocess-use-audit` | reviewed exact-rule exemption | `842faf9de75ca2c708e9ffca7586358c` | +| `test/test_parameter_context.py:223` | `non-literal-import` | reviewed exact-rule exemption | `89898db66924c5c97baf0f9e1fc6da2` | +| `test/test_parameter_context.py:317` | `dangerous-subprocess-use-audit` | reviewed exact-rule exemption | `6dbba09e54cf028095340bbabf4fec5e` | +| `test/test_parameter_context.py:382` | `dangerous-subprocess-use-audit` | reviewed exact-rule exemption | `b460bc99819abcd0e93ec85f9e537812` | +| `test/test_protocol_event_timing.py:24` | `non-literal-import` | reviewed exact-rule exemption | `f1a7aa05499cdb341ed86867c14557c3` | diff --git a/docs/updates/README.md b/docs/updates/README.md index 6af66a1..3a68ac1 100644 --- a/docs/updates/README.md +++ b/docs/updates/README.md @@ -58,6 +58,7 @@ In the same commit: delete the item from `progress.md`, add a `#done` entry here | ID | Date | Title | Tags | Batch | |---|---|---|---|---| +| U-20261003-06 | 2026-10-03 | Harden smoke XML and document reviewed Codacy false positives | #done #security #quality #ci | [2026-10](2026-10.md) | | U-20261003-05 | 2026-10-03 | Expose the shared worker stop query without private access | #done #quality #async | [2026-10](2026-10.md) | | U-20261003-04 | 2026-10-03 | Retain async stop intent through transport cancellation | #done #fix #async #ci | [2026-10](2026-10.md) | | U-20261003-03 | 2026-10-03 | Isolate extras probes and clean up cancelled async startup | #done #ci #quality #async #gui | [2026-10](2026-10.md) | @@ -136,5 +137,5 @@ In the same commit: delete the item from `progress.md`, add a `#done` entry here | File | Period | Entries | |---|---|---:| -| [2026-10.md](2026-10.md) | 2026-10 | 12 | +| [2026-10.md](2026-10.md) | 2026-10 | 29 | | [2026-09.md](2026-09.md) | 2026-09 | 45 | diff --git a/je_load_density/__init__.py b/je_load_density/__init__.py index 0da1395..b4ebf39 100644 --- a/je_load_density/__init__.py +++ b/je_load_density/__init__.py @@ -814,6 +814,8 @@ def __getattr__(name: str): if target is None: raise AttributeError(f"module {__name__!r} has no attribute {name!r}") module, attribute = target + # Security audit: Only targets from the fixed _EXPORTS map reach this import; unknown names raise before import. + # nosemgrep: python.lang.security.audit.non-literal-import.non-literal-import value = getattr(importlib.import_module(module), attribute) if module == "locust": importlib.import_module("je_load_density.wrapper.event.request_hook") diff --git a/je_load_density/utils/executor/action_executor.py b/je_load_density/utils/executor/action_executor.py index 3d23f6b..572e941 100644 --- a/je_load_density/utils/executor/action_executor.py +++ b/je_load_density/utils/executor/action_executor.py @@ -37,6 +37,9 @@ def _lazy_command(module: str, attribute: str): def command(*args, **kwargs): from importlib import import_module + # Security audit: Private factory captures only module/attribute literals declared below, never action-file + # data. + # nosemgrep: python.lang.security.audit.non-literal-import.non-literal-import function = getattr(import_module(module), attribute) return function(*args, **kwargs) diff --git a/test/smoke/capability_probe.py b/test/smoke/capability_probe.py index 88fa818..45b1da0 100644 --- a/test/smoke/capability_probe.py +++ b/test/smoke/capability_probe.py @@ -48,6 +48,8 @@ def run_probe(extra: str) -> None: try: if extra == "all": # Each capability chooses its scheduler before SDK/TLS imports. + # Security audit: Fixed current script and PROBES-validated capability name run as argv without a shell. + # nosemgrep: python.lang.security.audit.dangerous-subprocess-use-audit.dangerous-subprocess-use-audit subprocess.run([sys.executable, str(Path(__file__).resolve()), name], check=True, timeout=120) else: PROBES[name]() diff --git a/test/smoke/install_extra.py b/test/smoke/install_extra.py index d7b2bf5..de11a1f 100644 --- a/test/smoke/install_extra.py +++ b/test/smoke/install_extra.py @@ -16,10 +16,17 @@ def main() -> None: raise ValueError("wheel: expected exactly one checkout-built wheel") requirement = str(wheels[0]) + (f"[{extra}]" if extra != "base" else "") # pywebpush's http-ece dependency publishes source archives; all other dependencies require wheels. - subprocess.run([sys.executable, "-m", "pip", "install", "--only-binary", ":all:", + # Security audit: CI-built wheel path and regex-validated extra are argv data; no shell evaluates them. + # nosemgrep: python.lang.security.audit.dangerous-subprocess-use-tainted-env-args.dangerous-subprocess-use-tainted-env-args # noqa: E501 + subprocess.run( # nosemgrep: python.lang.security.audit.dangerous-subprocess-use-audit.dangerous-subprocess-use-audit # noqa: E501 + [sys.executable, "-m", "pip", "install", "--only-binary", ":all:", "--no-binary", "http-ece", requirement], check=True, timeout=600) + # Security audit: Current interpreter runs a fixed pip module/command with no shell. + # nosemgrep: python.lang.security.audit.dangerous-subprocess-use-audit.dangerous-subprocess-use-audit subprocess.run([sys.executable, "-m", "pip", "check"], check=True, timeout=30) + # Security audit: Current interpreter runs the fixed framework help command with no shell. + # nosemgrep: python.lang.security.audit.dangerous-subprocess-use-audit.dangerous-subprocess-use-audit subprocess.run([sys.executable, "-m", "je_load_density", "--help"], check=True, timeout=30) diff --git a/test/smoke/platform_probes.py b/test/smoke/platform_probes.py index d3c6fe7..81f68be 100644 --- a/test/smoke/platform_probes.py +++ b/test/smoke/platform_probes.py @@ -188,6 +188,8 @@ def mcp() -> None: request = {"jsonrpc": "2.0", "id": 1, "method": "initialize", "params": {"protocolVersion": "2025-06-18", "capabilities": {}, "clientInfo": {"name": "extras-probe", "version": "1"}}} + # Security audit: Current interpreter runs the fixed MCP module; the JSON request is stdin data, no shell. + # nosemgrep: python.lang.security.audit.dangerous-subprocess-use-audit.dangerous-subprocess-use-audit response = subprocess.run([sys.executable, "-m", "je_load_density.mcp_server"], input=json.dumps(request) + "\n", text=True, capture_output=True, timeout=30, check=True) diff --git a/test/smoke/run_cell.py b/test/smoke/run_cell.py index b79a125..cb40b19 100644 --- a/test/smoke/run_cell.py +++ b/test/smoke/run_cell.py @@ -9,8 +9,14 @@ def main() -> None: """Keep capability results separate from the base runtime smoke result.""" directory = Path(__file__).resolve().parent - subprocess.run([sys.executable, str(directory / "capability_probe.py"), os.environ.get("LD_EXTRA", "base")], + # Security audit: Fixed sibling script receives LD_EXTRA as one argv value; its probe registry rejects unknown + # names. + # nosemgrep: python.lang.security.audit.dangerous-subprocess-use-tainted-env-args.dangerous-subprocess-use-tainted-env-args # noqa: E501 + subprocess.run( # nosemgrep: python.lang.security.audit.dangerous-subprocess-use-audit.dangerous-subprocess-use-audit # noqa: E501 + [sys.executable, str(directory / "capability_probe.py"), os.environ.get("LD_EXTRA", "base")], check=True, timeout=120) + # Security audit: Current interpreter runs the fixed sibling smoke script with no shell. + # nosemgrep: python.lang.security.audit.dangerous-subprocess-use-audit.dangerous-subprocess-use-audit subprocess.run([sys.executable, str(directory / "test_wheel_smoke.py"), "-v"], check=True, timeout=240) diff --git a/test/smoke/test_wheel_smoke.py b/test/smoke/test_wheel_smoke.py index 4799b31..64b0f95 100644 --- a/test/smoke/test_wheel_smoke.py +++ b/test/smoke/test_wheel_smoke.py @@ -1,4 +1,4 @@ -"""Stdlib smoke harness; Docker copies it outside the checkout to test the installed wheel.""" +"""Installed-wheel smoke harness; Docker copies it outside the checkout to test the installed wheel.""" import json import os @@ -12,12 +12,24 @@ from pathlib import Path from urllib.error import URLError from urllib.request import Request, urlopen -from xml.etree import ElementTree + +from defusedxml import ElementTree +from defusedxml.common import DefusedXmlException SMOKE_DIRECTORY = Path(__file__).resolve().parent PROCESS_TIMEOUT_SECONDS = 45 +class ReportXmlSecurity(unittest.TestCase): + def test_junit_parser_rejects_entity_expansion(self) -> None: + payload = ']>&count;' + with tempfile.TemporaryDirectory(prefix="loaddensity-xml-") as directory: + path = Path(directory) / "junit.xml" + path.write_text(payload, encoding="utf-8") + with self.assertRaises(DefusedXmlException): + ElementTree.parse(path) + + class WheelSmoke(unittest.TestCase): @classmethod def setUpClass(cls) -> None: @@ -28,6 +40,9 @@ def setUpClass(cls) -> None: if (source / "je_load_density").is_dir(): cls.environment["PYTHONPATH"] = str(source) + os.pathsep + cls.environment.get("PYTHONPATH", "") ready = cls.directory / "ready.json" + # Security audit: Current interpreter runs our sibling loopback server; temporary path is one argv value, no + # shell. + # nosemgrep: python.lang.security.audit.dangerous-subprocess-use-audit.dangerous-subprocess-use-audit cls.server = subprocess.Popen( [sys.executable, str(SMOKE_DIRECTORY / "server.py"), str(ready)], stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL, cwd=cls.directory, @@ -58,6 +73,8 @@ def tearDownClass(cls) -> None: cls.temporary.cleanup() def command(self, *arguments: str, input_text: str | None = None) -> subprocess.CompletedProcess: + # Security audit: Callers supply fixed module/script flags; dynamic URLs/paths stay argv data, never shell code. + # nosemgrep: python.lang.security.audit.dangerous-subprocess-use-audit.dangerous-subprocess-use-audit return subprocess.run( [sys.executable, *arguments], input=input_text, capture_output=True, text=True, encoding="utf-8", errors="replace", env=self.environment, cwd=self.directory, diff --git a/test/test_asyncio_engine.py b/test/test_asyncio_engine.py index de56d6f..0449b6d 100644 --- a/test/test_asyncio_engine.py +++ b/test/test_asyncio_engine.py @@ -81,6 +81,9 @@ def _run(tasks, **kwargs): tempfile.TemporaryFile(mode="w+", encoding="utf-8") as output, tempfile.TemporaryFile(mode="w+", encoding="utf-8") as errors, ): + # Security audit: Fixed interpreter/-c program; task options are JSON in sys.argv[1], not interpolated Python or + # shell. + # nosemgrep: python.lang.security.audit.dangerous-subprocess-use-audit.dangerous-subprocess-use-audit child = subprocess.Popen(arguments, stdout=output, stderr=errors, text=True) try: deadline = time.monotonic() + 20 @@ -89,6 +92,9 @@ def _run(tasks, **kwargs): if child.poll() is None: output.seek(0) errors.seek(0) + # Security audit: Constructing TimeoutExpired only describes a timed-out process; it does not execute + # arguments. + # nosemgrep: python.lang.security.audit.dangerous-subprocess-use-audit.dangerous-subprocess-use-audit raise subprocess.TimeoutExpired(arguments, 20, output.read(), errors.read()) finally: if child.poll() is None: diff --git a/test/test_dashboard_ui.py b/test/test_dashboard_ui.py index 3edc621..c655a2b 100644 --- a/test/test_dashboard_ui.py +++ b/test/test_dashboard_ui.py @@ -45,6 +45,9 @@ class EventSource { constructor() { source=this; } } "latency_windows": [{**window, "start_time": 0}, {**window, "start_time": 1, "p50_ms": None, "p95_ms": None, "p99_ms": None}, {**window, "start_time": 2}]} + # Security audit: PATH-resolved Node runs fixed harness code; report/name payloads arrive on stdin, never shell + # code. + # nosemgrep: python.lang.security.audit.dangerous-subprocess-use-audit.dangerous-subprocess-use-audit completed = subprocess.run([node, "-e", harness], input=json.dumps({"script": script, "data": data}), text=True, encoding="utf-8", capture_output=True, timeout=10, check=False) assert completed.returncode == 0, completed.stderr diff --git a/test/test_extras_matrix.py b/test/test_extras_matrix.py index ff97b82..529e209 100644 --- a/test/test_extras_matrix.py +++ b/test/test_extras_matrix.py @@ -50,6 +50,8 @@ def test_unsafe_extra_names_are_rejected_before_shell_parameters(tmp_path): def test_cli_writes_checkout_matrix_independently_of_current_directory(tmp_path): destination = tmp_path / "github-output" (tmp_path / "pyproject.toml").write_text('[project.optional-dependencies]\nattacker = []\n', encoding="utf-8") + # Security audit: Fixed checkout matrix script and temporary output path are separate argv values; no shell. + # nosemgrep: python.lang.security.audit.dangerous-subprocess-use-audit.dangerous-subprocess-use-audit completed = subprocess.run([sys.executable, str(ROOT / "scripts/extras_matrix.py"), "--output", str(destination)], cwd=tmp_path, capture_output=True, text=True, timeout=10, check=False) diff --git a/test/test_gui_lifecycle.py b/test/test_gui_lifecycle.py index 39641fb..c6d0eb5 100644 --- a/test/test_gui_lifecycle.py +++ b/test/test_gui_lifecycle.py @@ -65,6 +65,8 @@ def widget(application, monkeypatch): def test_parent_import_does_not_patch_native_threads(): source = "import sys; import je_load_density.gui.main_widget; assert 'locust' not in sys.modules" + # Security audit: Current interpreter executes a fixed literal import-isolation program without a shell. + # nosemgrep: python.lang.security.audit.dangerous-subprocess-use-audit.dangerous-subprocess-use-audit result = subprocess.run([sys.executable, "-c", source], capture_output=True, text=True, timeout=20) assert result.returncode == 0, result.stderr @@ -169,9 +171,8 @@ def test_serialized_snapshot_stays_within_reader_byte_limit_and_keeps_latest(mon def test_chart_uses_complete_child_windows_after_request_tail_truncation(application): - from je_load_density.gui.run_protocol import make_snapshot - from je_load_density.gui.chart_panel import LiveChartPanel + from je_load_density.gui.run_protocol import make_snapshot records = [{"name": "first", "start_time": 100.5, "response_time_ms": 10}] * 250 records += [{"name": "second", "start_time": 101.5, "response_time_ms": 100}] * 250 @@ -212,9 +213,10 @@ def emit(self, *args, **kwargs): def test_session_package_commands_are_registered_on_private_executor(tmp_path, monkeypatch): from je_load_density.gui.run_worker import RunSession - from je_load_density.utils.executor.action_executor import executor - from je_load_density.utils.package_manager.package_manager_class import package_manager + from je_load_density.utils.package_manager.package_manager_class import ( + package_manager, + ) monkeypatch.setattr(package_manager, "allowed_packages", {"json"}) monkeypatch.setattr(package_manager, "allow_arbitrary_packages", False) @@ -263,9 +265,8 @@ def wait(self, timeout): def test_scripted_load_preserved_with_selected_engine_and_callbacks(tmp_path, monkeypatch): - from je_load_density.gui.run_worker import RunSession - from je_load_density.engine import entrypoints + from je_load_density.gui.run_worker import RunSession calls = [] monkeypatch.setattr(entrypoints, "start_test", lambda *args, **kwargs: calls.append((args, kwargs))) diff --git a/test/test_native_imports.py b/test/test_native_imports.py index 4b98996..b9948d1 100644 --- a/test/test_native_imports.py +++ b/test/test_native_imports.py @@ -18,6 +18,8 @@ def test_selecting_locust_exports_registers_legacy_request_records(name): assert len(test_record_instance.test_record_list) == 1 """ ) + # Security audit: Source is composed only from literal test code and three fixed pytest export names; no shell. + # nosemgrep: python.lang.security.audit.dangerous-subprocess-use-audit.dangerous-subprocess-use-audit result = subprocess.run([sys.executable, "-c", source], capture_output=True, text=True, timeout=20, check=False) assert result.returncode == 0, result.stdout + result.stderr @@ -46,6 +48,8 @@ def test_native_entry_points_do_not_import_locust_or_patch_io(statement): assert original == (socket.socket, ssl.SSLSocket, threading.Thread, time.sleep) """ ) + # Security audit: Source is composed only from literal test code and fixed pytest import statements; no shell. + # nosemgrep: python.lang.security.audit.dangerous-subprocess-use-audit.dangerous-subprocess-use-audit result = subprocess.run([sys.executable, "-c", source], text=True, capture_output=True, timeout=20, check=False) assert result.returncode == 0, result.stdout + result.stderr diff --git a/test/test_parameter_context.py b/test/test_parameter_context.py index 02ca16f..b7cf109 100644 --- a/test/test_parameter_context.py +++ b/test/test_parameter_context.py @@ -220,6 +220,8 @@ def test_http_template_setup_registers_database_sources(monkeypatch, module_name from je_load_density.wrapper.proxy.proxy_user import locust_wrapper_proxy + # Security audit: Module suffix comes only from the three literal pytest parameter cases above. + # nosemgrep: python.lang.security.audit.non-literal-import.non-literal-import module = importlib.import_module(f"je_load_density.wrapper.user_template.{module_name}") monkeypatch.setattr(ParameterResolver, "_read_db", staticmethod(lambda *_args: [{"name": "Alice"}])) monkeypatch.setitem(locust_wrapper_proxy.user_dict, proxy_name, SimpleNamespace(configure=lambda *_a, **_k: None)) @@ -314,6 +316,8 @@ def test_cookie_jars_created_before_locust_patch_remain_user_local(): thread.join() assert seen[0] is not jar_for_user() """ + # Security audit: Current interpreter executes the fixed test program without a shell. + # nosemgrep: python.lang.security.audit.dangerous-subprocess-use-audit.dangerous-subprocess-use-audit completed = subprocess.run([sys.executable, "-c", script], capture_output=True, text=True, timeout=20) assert completed.returncode == 0, completed.stderr @@ -379,5 +383,7 @@ def user(index): assert 'locust' not in sys.modules assert 'gevent' not in sys.modules """ + # Security audit: Fixed test program reads the temporary action path from sys.argv; no Python/shell interpolation. + # nosemgrep: python.lang.security.audit.dangerous-subprocess-use-audit.dangerous-subprocess-use-audit completed = subprocess.run([sys.executable, "-c", script, str(path)], capture_output=True, text=True, timeout=20) assert completed.returncode == 0, completed.stderr diff --git a/test/test_protocol_event_timing.py b/test/test_protocol_event_timing.py index b66491f..d4ef759 100644 --- a/test/test_protocol_event_timing.py +++ b/test/test_protocol_event_timing.py @@ -21,6 +21,8 @@ ("websocket", "WebSocketUserWrapper", ("health", 42, 3)), ]) def test_protocol_events_report_epoch_start_and_monotonic_duration(monkeypatch, module_name, class_name, args): + # Security audit: Module suffix comes only from the ten literal pytest parameter cases above. + # nosemgrep: python.lang.security.audit.non-literal-import.non-literal-import module = importlib.import_module(f"je_load_density.wrapper.user_template.{module_name}_user_template") clock = SimpleNamespace(time=lambda: 1000, monotonic=lambda: 44) monkeypatch.setattr(module, "time", clock) From 7d7b090a0a745e8702d04c01f99fbbf1379deecc Mon Sep 17 00:00:00 2001 From: JeffreyChen Date: Sat, 3 Oct 2026 13:46:01 +0800 Subject: [PATCH 18/18] fix: align audit annotations with scanned argument expressions --- docs/updates/2026-10.md | 13 +++++++++++++ docs/updates/README.md | 3 ++- test/smoke/install_extra.py | 2 +- test/smoke/run_cell.py | 2 +- test/test_native_imports.py | 2 ++ test/test_parameter_context.py | 2 ++ 6 files changed, 21 insertions(+), 3 deletions(-) diff --git a/docs/updates/2026-10.md b/docs/updates/2026-10.md index e026110..cac4caf 100644 --- a/docs/updates/2026-10.md +++ b/docs/updates/2026-10.md @@ -329,3 +329,16 @@ Added a stdlib smoke harness that exercises local HTTP requests in isolated proc | `test/test_parameter_context.py:317` | `dangerous-subprocess-use-audit` | reviewed exact-rule exemption | `6dbba09e54cf028095340bbabf4fec5e` | | `test/test_parameter_context.py:382` | `dangerous-subprocess-use-audit` | reviewed exact-rule exemption | `b460bc99819abcd0e93ec85f9e537812` | | `test/test_protocol_event_timing.py:24` | `non-literal-import` | reviewed exact-rule exemption | `f1a7aa05499cdb341ed86867c14557c3` | + +## U-20261003-07 · 2026-10-03 · Align security annotations with argument-expression scan locations · #done #quality #security #ci + +- **Remote evidence**: Codacy completed analysis of `34b1137`; findings fell from 26 to four. Two additional fixed test subprocesses surfaced; two taint matches point to the argv expressions rather than the call's first line. +- **Fix**: move each exact taint-rule annotation immediately above its matched argv expression, keeping the distinct audit rule inline on the call. Add exact audit annotations for the literal package-gate and parameter-import isolation programs. Trust explanations, arguments, timeouts, cleanup and assertions remain intact. No executable behavior or dependency changed. +- **Verification**: 32 native import/parameter tests passed; scoped Ruff E/F/W/C90/I and whitespace checks passed. Independent follow-up review found no issues. The preceding source revision passed the full 1605-test suite and seven rebuilt installed-wheel smoke tests. The final remote analysis follows the push. + +| Original site at 34b1137 | Codacy issue ID | +|---|---| +| `test/test_native_imports.py:76` | `842faf9de75ca2c708e9ffca7586358c` | +| `test/test_parameter_context.py:355` | `6dbba09e54cf028095340bbabf4fec5e` | +| `test/smoke/install_extra.py:22` | `21c71abada62748a6f66033481b743e5` | +| `test/smoke/run_cell.py:16` | `314e2c58a5430731933f6e0481342758` | diff --git a/docs/updates/README.md b/docs/updates/README.md index 3a68ac1..fde6a9e 100644 --- a/docs/updates/README.md +++ b/docs/updates/README.md @@ -58,6 +58,7 @@ In the same commit: delete the item from `progress.md`, add a `#done` entry here | ID | Date | Title | Tags | Batch | |---|---|---|---|---| +| U-20261003-07 | 2026-10-03 | Align security annotations with argument-expression scan locations | #done #quality #security #ci | [2026-10](2026-10.md) | | U-20261003-06 | 2026-10-03 | Harden smoke XML and document reviewed Codacy false positives | #done #security #quality #ci | [2026-10](2026-10.md) | | U-20261003-05 | 2026-10-03 | Expose the shared worker stop query without private access | #done #quality #async | [2026-10](2026-10.md) | | U-20261003-04 | 2026-10-03 | Retain async stop intent through transport cancellation | #done #fix #async #ci | [2026-10](2026-10.md) | @@ -137,5 +138,5 @@ In the same commit: delete the item from `progress.md`, add a `#done` entry here | File | Period | Entries | |---|---|---:| -| [2026-10.md](2026-10.md) | 2026-10 | 29 | +| [2026-10.md](2026-10.md) | 2026-10 | 30 | | [2026-09.md](2026-09.md) | 2026-09 | 45 | diff --git a/test/smoke/install_extra.py b/test/smoke/install_extra.py index de11a1f..4e60d9d 100644 --- a/test/smoke/install_extra.py +++ b/test/smoke/install_extra.py @@ -17,8 +17,8 @@ def main() -> None: requirement = str(wheels[0]) + (f"[{extra}]" if extra != "base" else "") # pywebpush's http-ece dependency publishes source archives; all other dependencies require wheels. # Security audit: CI-built wheel path and regex-validated extra are argv data; no shell evaluates them. - # nosemgrep: python.lang.security.audit.dangerous-subprocess-use-tainted-env-args.dangerous-subprocess-use-tainted-env-args # noqa: E501 subprocess.run( # nosemgrep: python.lang.security.audit.dangerous-subprocess-use-audit.dangerous-subprocess-use-audit # noqa: E501 + # nosemgrep: python.lang.security.audit.dangerous-subprocess-use-tainted-env-args.dangerous-subprocess-use-tainted-env-args # noqa: E501 [sys.executable, "-m", "pip", "install", "--only-binary", ":all:", "--no-binary", "http-ece", requirement], check=True, timeout=600) diff --git a/test/smoke/run_cell.py b/test/smoke/run_cell.py index cb40b19..0c4378e 100644 --- a/test/smoke/run_cell.py +++ b/test/smoke/run_cell.py @@ -11,8 +11,8 @@ def main() -> None: directory = Path(__file__).resolve().parent # Security audit: Fixed sibling script receives LD_EXTRA as one argv value; its probe registry rejects unknown # names. - # nosemgrep: python.lang.security.audit.dangerous-subprocess-use-tainted-env-args.dangerous-subprocess-use-tainted-env-args # noqa: E501 subprocess.run( # nosemgrep: python.lang.security.audit.dangerous-subprocess-use-audit.dangerous-subprocess-use-audit # noqa: E501 + # nosemgrep: python.lang.security.audit.dangerous-subprocess-use-tainted-env-args.dangerous-subprocess-use-tainted-env-args # noqa: E501 [sys.executable, str(directory / "capability_probe.py"), os.environ.get("LD_EXTRA", "base")], check=True, timeout=120) # Security audit: Current interpreter runs the fixed sibling smoke script with no shell. diff --git a/test/test_native_imports.py b/test/test_native_imports.py index b9948d1..fa04e38 100644 --- a/test/test_native_imports.py +++ b/test/test_native_imports.py @@ -73,5 +73,7 @@ def test_package_inspection_does_not_register_unrestricted_import_helper(): raise AssertionError('host package gate did not refuse os') assert 'import_module' not in manager.executor.event_dict, 'public inspection registered an unrestricted module loader' """ + # Security audit: Fixed literal package-gate regression program runs in the current interpreter without a shell. + # nosemgrep: python.lang.security.audit.dangerous-subprocess-use-audit.dangerous-subprocess-use-audit result = subprocess.run([sys.executable, "-c", source], text=True, capture_output=True, timeout=20, check=False) assert result.returncode == 0, result.stdout + result.stderr diff --git a/test/test_parameter_context.py b/test/test_parameter_context.py index b7cf109..3f657b9 100644 --- a/test/test_parameter_context.py +++ b/test/test_parameter_context.py @@ -352,6 +352,8 @@ def test_parameter_only_import_leaves_locust_and_gevent_unloaded(): assert 'locust' not in sys.modules assert 'gevent' not in sys.modules """ + # Security audit: Fixed parameter-import regression program runs in the current interpreter without a shell. + # nosemgrep: python.lang.security.audit.dangerous-subprocess-use-audit.dangerous-subprocess-use-audit completed = subprocess.run([sys.executable, "-c", script], capture_output=True, text=True, timeout=20) assert completed.returncode == 0, completed.stderr