From 7a91906ece5cd599175a2e08ebcc9c8bc667a6f6 Mon Sep 17 00:00:00 2001 From: Richard McDaniel <1130888+rmcdaniel@users.noreply.github.com> Date: Wed, 4 Nov 2026 05:43:45 -0600 Subject: [PATCH] Define Server-measured schedule-to-start recorder contract --- sdk-metrics/README.md | 76 +++++++++++++++++++++++++----- sdk-metrics/contract.json | 10 +++- tests/test_sdk_metrics_contract.py | 11 +++++ 3 files changed, 84 insertions(+), 13 deletions(-) diff --git a/sdk-metrics/README.md b/sdk-metrics/README.md index 61e1bc3..f1f00d3 100644 --- a/sdk-metrics/README.md +++ b/sdk-metrics/README.md @@ -1,9 +1,10 @@ # SDK client and worker metrics [`contract.json`](contract.json) is the shared list of SDK recorder names, -instrument types, units and tag keys for PHP, Python and Rust. It captures the -inspected Python recorder surface. Implementation, installed acceptance and -exporter-guide delivery remain separate from publishing this list. +instrument types, units and tag keys for PHP, Python and Rust. The original six +instruments capture the inspected Python recorder surface. Revision 2 adds the +schedule-to-start histogram defined below. Implementation, installed acceptance +and exporter-guide delivery remain separate from publishing this list. Rust's recorder and optional Prometheus adapter have an [installed-package receipt](https://github.com/durable-workflow/sdk-rust/pull/131#issuecomment-6364178456) @@ -20,8 +21,9 @@ and compare observations from their actual client and worker recorder hooks. Offline tests may vendor its exact JSON bytes with the upstream commit and SHA-256 recorded and verified. Update that copy from the shared contract, without a second hand-maintained language-specific vocabulary. -The reference commit identifies the inspected implementation, not a required -package version or release qualification. +The reference commit identifies the inspected implementation of the original +six instruments, not a required package version, implementation of later +additions, or release qualification. ## Recorder observations @@ -30,9 +32,11 @@ and `record(name, value, tags)` operations. Language-specific method spelling may differ. The default recorder discards observations. The worker uses its explicit recorder, otherwise its client's recorder, otherwise the no-op. -Each observed operation increments its counter by one and records one -nonnegative monotonic duration in seconds with the same tag map. Unit `1` is a -count, and `s` is seconds. Tags are strings and tag ordering is immaterial. +The client-request, worker-poll and worker-task observations each increment +their counter by one and record one nonnegative monotonic duration in seconds +with the same tag map. The schedule-to-start instrument is a separate histogram +without a paired counter. Unit `1` is a count, and `s` is seconds. Tags are +strings and tag ordering is immaterial. Only the listed tag keys belong to each core metric. Exporters may add resource labels separately. Recorders should be fast and must handle their own exporter failures without throwing into the client or worker. @@ -42,12 +46,51 @@ failures without throwing into the client or worker. | `client_request` | One instrumented client request, including its transport retries, backoff and response handling. This is not a count of individual HTTP attempts. | | `worker_poll` | One observed poll result and its elapsed polling time. Empty polls, backpressure and errors remain distinct from a delivered task. | | `worker_task` | One dispatch of a claimed workflow, activity, query or supported update-validation task and its processing time. A completed task does not establish a completed workflow. Replay and retries can produce more dispatches. | +| `worker_task_dispatch` | A valid Server-measured eligibility-to-lease duration for a workflow or activity task that this worker actually dispatches. This excludes delivery and processing time. | These counters describe process observations. They are not durable execution counts, exactly-once receipts, billing units or schedule-to-start latency. Missing observations are not zero or healthy. Abrupt process loss can prevent the final observation. +## Schedule-to-start observations + +Revision 2 adds `durable_workflow_worker_task_schedule_to_start_seconds`, a +histogram recorded with `record` in seconds. It has exactly `task_kind` and +`task_queue` tags. Only `workflow` and `activity` dispatches belong to this +instrument. It has no outcome tag or paired counter. Existing instruments and +their meanings are unchanged. + +The [accepted field and clock decision](https://github.com/durable-workflow/sdk-python/issues/188#issuecomment-6629640808) +requires `server_capabilities.claimed_task_ready_age` to be Boolean `true` and +optional, non-null `task.ready_age_ms`. The SDK accepts an actual integer from `0` through +`9007199254740991`, excludes booleans and coercion, and records the value divided +by `1000` on actual dispatch. It does not observe transport retries or empty, +denied, stale, query or update-validation answers. A replayed poll receipt keeps +the original lease's value. A genuinely new lease has its own ready origin. +These remain worker dispatch observations, not exactly-once execution counts. + +Server measures from the claimed attempt's known eligibility instant to the +same committed lease timestamp used by its claim transaction. Scheduled delay +and retry backoff are excluded. New creation, retry, release and expired-lease +transitions must retain the appropriate origin. An old creation timestamp must +not stand in for an unknown later ready transition. Neither worker wall-clock +subtraction nor aggregate backlog age measures this boundary. Server clock +assumptions still apply across nodes. + +An unsupported capability, missing value or unknown legacy origin produces no +sample. Malformed, negative, non-integral or out-of-range values produce no +sample and only a bounded diagnostic. Server checks a signed duration before +flooring valid nonnegative values to milliseconds. A real zero, including a +valid sub-millisecond duration quantized to zero, is recorded. Never replace +missing or invalid data with zero or clamp a negative duration. No workflow, +run, task, attempt, worker or session identity belongs in the metric tags. + +Publishing this contract does not add the claim field to a Server, implement an +SDK hook, or qualify an installed package. Those gates remain on the owning +Server and SDK issues. SDKs pinned to revision 1 retain their existing six +instruments until they explicitly adopt a later contract commit. + ## Tag values Client metrics have `method`, `route`, `plane`, `status_code` and `outcome`: @@ -67,7 +110,8 @@ Client metrics have `method`, `route`, `plane`, `status_code` and `outcome`: error taxonomy. Dashboards should not assume those categories match between languages. -Worker metrics have `task_kind`, `task_queue` and `outcome`: +Worker poll and processing metrics have `task_kind`, `task_queue` and `outcome`. +The schedule-to-start histogram uses only the first two: - `task_kind` identifies `workflow`, `activity`, `query` or supported `update_validation` work. Unsupported task kinds do not create new SDK @@ -116,9 +160,9 @@ entries. Python's current exception-type outcomes remain language-specific. Each SDK must exercise its actual recorder on successful and failed requests, transport retry, normalized routes, empty/delivered/error/backpressure polls, and supported task success/failure paths. Check the names, recorder operations, -tag keys and string values against the pinned contract. Require a counter -increment of one and a matching nonnegative duration observation for each -completed observed operation. Compare deterministic outcomes between SDKs, +tag keys and string values against the pinned contract. For the original three +scopes, require a counter increment of one and a matching nonnegative duration +observation for each completed observed operation. Compare deterministic outcomes between SDKs, while keeping language-specific exceptions explicit. Exercise default no-op, client-to-worker inheritance and an explicit worker override. @@ -131,3 +175,11 @@ Use installed-package checks for the published adapters and examples. Record each SDK's package and guide results on #233, preserving the completed Rust and Python receipts and PHP's remaining work. Source inspection and this contract's tests do not provide that runtime evidence. + +The schedule-to-start addition also needs actual recorder and installed-package +checks for both task kinds, delayed/retried/released/expired attempts, valid zero, +unknown legacy origins, malformed and clock-invalid values, and receipt replay +versus a new lease. Verify exact values and the two tag keys, absence of samples +when unsupported, and no duplicate observation from HTTP retries. Retain normal +old-client behavior. Use the exact published Server and installed SDK before +calling this addition delivered. diff --git a/sdk-metrics/contract.json b/sdk-metrics/contract.json index 0726392..f6b675e 100644 --- a/sdk-metrics/contract.json +++ b/sdk-metrics/contract.json @@ -1,6 +1,6 @@ { "schema": "durable-workflow.sdk-metrics/v1", - "revision": 1, + "revision": 2, "reference": { "repository": "durable-workflow/sdk-python", "commit": "2da02a7ac8a68a7cd4267e86ef10e0198a309592", @@ -58,6 +58,14 @@ "recorder_method": "record", "scope": "worker_task", "tags": ["task_kind", "task_queue", "outcome"] + }, + { + "name": "durable_workflow_worker_task_schedule_to_start_seconds", + "type": "histogram", + "unit": "s", + "recorder_method": "record", + "scope": "worker_task_dispatch", + "tags": ["task_kind", "task_queue"] } ] } diff --git a/tests/test_sdk_metrics_contract.py b/tests/test_sdk_metrics_contract.py index 85b6aa4..080371d 100644 --- a/tests/test_sdk_metrics_contract.py +++ b/tests/test_sdk_metrics_contract.py @@ -71,6 +71,17 @@ def test_counts_and_durations_share_exact_tag_keys(self) -> None: for metric in scopes[scope]: self.assertEqual(tags, set(metric["tags"])) + def test_schedule_to_start_is_a_separate_dispatch_histogram(self) -> None: + self.assertGreaterEqual(self.contract["revision"], 2) + dispatch_metrics = [metric for metric in self.metrics if metric["scope"] == "worker_task_dispatch"] + self.assertEqual(1, len(dispatch_metrics)) + metric = dispatch_metrics[0] + self.assertEqual("durable_workflow_worker_task_schedule_to_start_seconds", metric["name"]) + self.assertEqual("histogram", metric["type"]) + self.assertEqual("record", metric["recorder_method"]) + self.assertEqual("s", metric["unit"]) + self.assertEqual({"task_kind", "task_queue"}, set(metric["tags"])) + if __name__ == "__main__": unittest.main()