diff --git a/docs/interop/AAC_OTEL_TIER2_CONFORMANCE.md b/docs/interop/AAC_OTEL_TIER2_CONFORMANCE.md new file mode 100644 index 00000000..fdaf7caa --- /dev/null +++ b/docs/interop/AAC_OTEL_TIER2_CONFORMANCE.md @@ -0,0 +1,51 @@ +# AAC OpenTelemetry tier-2 independent consumer experiment + +Status: experimental. + +Target: `draft-palanisamy-scitt-aac-otel-00`, published 2026-10-02. + +The draft currently lists one known implementation for this extension: +`action-state-group/capsule-emit`. It also says the registry structure for +tier-2 conformance artifacts is not yet specified, while publishing the +positive and must-fail vectors needed for such a profile. + +CounterProof therefore implements a separate consumer from the draft text +instead of importing `capsule-emit`. + +## What is checked + +- both currently permitted JSON placements are recognized; +- trace/span identifiers are lower-case hex with exact lengths; +- clear `tracestate` is refused for extension-level trust; +- semconv values are default-deny and the draft's content-bearing values are + never admitted; +- `semconv.source` must pin the GenAI semantic-conventions source; +- clear resource attributes are restricted to the draft's closed subset; +- invalid extension data becomes `INFORMATIONAL_ONLY`, and never changes the + surrounding Agent Action Capsule base verdict; +- correlated same-producer telemetry is never counted as independent + corroboration. + +## Why this is a higher-value interop node + +This extension joins two still-moving ecosystems: SCITT Agent Action Capsules +and OpenTelemetry GenAI semantic conventions. A second independently written +consumer can expose ambiguous wording or accidental producer assumptions while +the tier-2 conformance registry is still explicitly TBD. + +## Independent-consumer finding + +The draft's Section 9 says its positive vector must be accepted and must reproduce the same `capsule_id` under JCS binding. The published vector currently contains only the extension block, without a complete base Capsule or an expected `capsule_id`. + +That means this experiment can independently test the extension's admission semantics, but it cannot claim that the published vector alone makes the `capsule_id` reproducibility requirement executable. A complete frozen Capsule + expected identifier, or an explicit statement that binding is exercised only by tier-1 CPB vectors, would close that gap. + +An attempt to file this narrowly against the reference implementation repository was blocked by the current GitHub integration permission (HTTP 403). This is therefore **handoff-ready but not externally delivered** and must not be counted as author feedback. + +## Non-claims + +This is not an IETF Working Group adoption claim, not OpenTelemetry +registration of `aac.capsule_id`, and not evidence that Action State Group +depends on CounterProof. + +Promotion requires an external author/maintainer to review, consume, cite, or +request the conformance result. diff --git a/examples/interop/aac-otel-draft-00-vectors.json b/examples/interop/aac-otel-draft-00-vectors.json new file mode 100644 index 00000000..7dbfab55 --- /dev/null +++ b/examples/interop/aac-otel-draft-00-vectors.json @@ -0,0 +1,51 @@ +{ + "draft": "draft-palanisamy-scitt-aac-otel-00", + "published": "2026-10-02", + "positive": { + "org.agentactioncapsule.otel": { + "trace_id": "4bf92f3577b34da6a3ce929d0e0e4736", + "span_id": "00f067aa0ba902b7", + "trace_flags": "01", + "resource": {"service.name": "support-agent"}, + "semconv": { + "source": "open-telemetry/semantic-conventions-genai@8c1b98a", + "gen_ai.provider.name": "local", + "gen_ai.operation.name": "execute_tool" + } + } + }, + "must_fail": [ + { + "name": "malformed-trace-id", + "org.agentactioncapsule.otel": { + "trace_id": "4BF92F3577b34da6a3ce929d0e0e473", + "span_id": "00f067aa0ba902b7" + } + }, + { + "name": "clear-tracestate", + "org.agentactioncapsule.otel": { + "trace_id": "4bf92f3577b34da6a3ce929d0e0e4736", + "span_id": "00f067aa0ba902b7", + "tracestate": "congo=t61rcWkgMzE" + } + }, + { + "name": "non-allowlisted-content", + "org.agentactioncapsule.otel": { + "trace_id": "4bf92f3577b34da6a3ce929d0e0e4736", + "span_id": "00f067aa0ba902b7", + "semconv": { + "gen_ai.input.messages": [ + {"role": "user", "content": "…"} + ] + } + } + } + ], + "claim_boundary": [ + "This fixture transcribes the draft's published tier-2 vectors for independent consumption.", + "Passing these vectors is not IETF endorsement and does not change Agent Action Capsule base verification.", + "A correlated span remains same-producer telemetry and is not independent corroboration." + ] +} diff --git a/skill_factory/evolution/aac_otel_extension.py b/skill_factory/evolution/aac_otel_extension.py new file mode 100644 index 00000000..332cc22e --- /dev/null +++ b/skill_factory/evolution/aac_otel_extension.py @@ -0,0 +1,259 @@ +"""Independent consumer for draft-palanisamy-scitt-aac-otel-00. + +This module implements only the extension-level admission semantics needed to +consume the draft's positive and must-fail vectors. It does not import +capsule-emit and does not change Agent Action Capsule base verification. + +Core rule: correlation metadata may help join a Capsule to telemetry, but it +must never be upgraded into independent corroboration. +""" + +from __future__ import annotations + +from dataclasses import dataclass +import re +from typing import Any + + +BLOCK_KEY = "org.agentactioncapsule.otel" +_SEMCONV_SOURCE_PREFIX = "open-telemetry/semantic-conventions-genai@" +_CACHE_INPUT_TOKENS = re.compile(r"^gen_ai\.usage\.cache_[A-Za-z0-9_.-]+\.input_tokens$") + +_HEX = { + "trace_id": re.compile(r"^[0-9a-f]{32}$"), + "span_id": re.compile(r"^[0-9a-f]{16}$"), + "parent_span_id": re.compile(r"^[0-9a-f]{16}$"), + "trace_flags": re.compile(r"^[0-9a-f]{2}$"), + "tracestate_digest": re.compile(r"^[0-9a-f]{64}$"), +} + +_RESOURCE_ALLOWED = { + "service.name", + "service.version", + "service.namespace", + "deployment.environment.name", + "telemetry.sdk.name", + "telemetry.sdk.version", +} + +_SEMCONV_EXACT_ALLOWED = { + "source", + "gen_ai.provider.name", + "gen_ai.request.model", + "gen_ai.response.model", + "gen_ai.operation.name", + "gen_ai.agent.id", + "gen_ai.agent.name", + "gen_ai.agent.version", + "gen_ai.workflow.name", + "gen_ai.tool.name", + "gen_ai.tool.type", + "gen_ai.tool.call.id", + "gen_ai.usage.input_tokens", + "gen_ai.usage.output_tokens", + "gen_ai.usage.reasoning.output_tokens", + "gen_ai.request.temperature", + "gen_ai.request.top_p", + "gen_ai.request.top_k", + "gen_ai.request.max_tokens", + "gen_ai.request.seed", + "gen_ai.request.stop_sequences", + "gen_ai.request.reasoning.level", + "gen_ai.request.choice.count", + "gen_ai.response.finish_reasons", + "gen_ai.response.status", + "gen_ai.output.type", + "gen_ai.response.id", + "gen_ai.request.previous_response.id", + "gen_ai.data_source.id", + "gen_ai.memory.store.id", + "gen_ai.prompt.name", + "gen_ai.prompt.version", +} + +_NEVER_ENTER_PREFIXES = ( + "gen_ai.input.messages", + "gen_ai.output.messages", + "gen_ai.system_instructions", + "gen_ai.tool.call.arguments", + "gen_ai.tool.call.result", + "gen_ai.tool.definitions", + "gen_ai.memory.records", + "gen_ai.memory.query.text", + "gen_ai.retrieval.query.text", + "gen_ai.retrieval.documents", + "gen_ai.prompt.variable.", + "gen_ai.conversation.id", + "mcp.session.id", + "session_id", + "enduser.", + "user.", + "gen_ai.evaluation.", +) + +_OUT_OF_SCOPE_PREFIXES = ( + "gen_ai.client.", + "gen_ai.server.", + "mcp.", +) + +_KNOWN_BLOCK_FIELDS = { + "trace_id", + "span_id", + "parent_span_id", + "trace_flags", + "tracestate_digest", + "span_name", + "resource", + "semconv", +} + + +@dataclass(frozen=True) +class OTelExtensionVerdict: + status: str + reasons: tuple[str, ...] + placement: str + independent_corroboration: bool + + +class OTelExtensionError(ValueError): + """Raised only for structurally unusable outer input.""" + + +def _is_namespaced(name: str) -> bool: + return "." in name or "://" in name + + +def _extract_block(capsule: dict[str, Any]) -> tuple[dict[str, Any] | None, str]: + if not isinstance(capsule, dict): + raise OTelExtensionError("capsule must be a JSON object") + + top = capsule.get(BLOCK_KEY) + nested = ( + ((capsule.get("model_attestation") or {}).get("compute_attestation") or {}).get( + BLOCK_KEY + ) + if isinstance(capsule.get("model_attestation"), dict) + else None + ) + + if top is not None and nested is not None: + if top != nested: + return None, "conflicting-dual-placement" + return top, "both-identical" + if top is not None: + return top, "top-level" + if nested is not None: + return nested, "model_attestation.compute_attestation" + return None, "absent" + + +def _semconv_name_allowed(name: str) -> bool: + return name in _SEMCONV_EXACT_ALLOWED or bool(_CACHE_INPUT_TOKENS.fullmatch(name)) + + +def _validate_semconv(semconv: Any, reasons: list[str]) -> None: + if semconv is None: + return + if not isinstance(semconv, dict): + reasons.append("semconv-not-object") + return + + source = semconv.get("source") + source_suffix = ( + source[len(_SEMCONV_SOURCE_PREFIX):] + if isinstance(source, str) and source.startswith(_SEMCONV_SOURCE_PREFIX) + else "" + ) + if not source_suffix.strip(): + reasons.append("semconv-source-missing-or-unpinned") + + for name in semconv: + if _semconv_name_allowed(name): + continue + if any(name.startswith(prefix) for prefix in _NEVER_ENTER_PREFIXES): + reasons.append(f"semconv-never-enters:{name}") + continue + if any(name.startswith(prefix) for prefix in _OUT_OF_SCOPE_PREFIXES): + reasons.append(f"semconv-out-of-scope:{name}") + continue + reasons.append(f"semconv-not-allowlisted:{name}") + + +def _validate_resource(resource: Any, reasons: list[str]) -> None: + if resource is None: + return + if not isinstance(resource, dict): + reasons.append("resource-not-object") + return + for name in resource: + if name not in _RESOURCE_ALLOWED: + reasons.append(f"resource-not-clear-safe:{name}") + + +def validate_aac_otel_extension(capsule: dict[str, Any]) -> OTelExtensionVerdict: + """Conservatively validate the AAC OpenTelemetry correlation extension. + + Invalid extension data is classified INFORMATIONAL_ONLY instead of changing + the surrounding Capsule's base verification verdict, matching the draft's + extension boundary. + """ + block, placement = _extract_block(capsule) + if placement == "conflicting-dual-placement": + return OTelExtensionVerdict( + status="INFORMATIONAL_ONLY", + reasons=("conflicting-dual-placement",), + placement=placement, + independent_corroboration=False, + ) + if block is None: + return OTelExtensionVerdict( + status="ABSENT", + reasons=(), + placement=placement, + independent_corroboration=False, + ) + if not isinstance(block, dict): + return OTelExtensionVerdict( + status="INFORMATIONAL_ONLY", + reasons=("block-not-object",), + placement=placement, + independent_corroboration=False, + ) + + reasons: list[str] = [] + + for required in ("trace_id", "span_id"): + if required not in block: + reasons.append(f"missing-required:{required}") + + for field, pattern in _HEX.items(): + if field in block: + value = block[field] + if not isinstance(value, str) or not pattern.fullmatch(value): + reasons.append(f"malformed-hex:{field}") + + if "tracestate" in block: + reasons.append("clear-tracestate-forbidden") + if "baggage" in block: + reasons.append("baggage-forbidden") + + for field in block: + if field in _KNOWN_BLOCK_FIELDS: + continue + if field in {"tracestate", "baggage"}: + continue + if not _is_namespaced(field): + reasons.append(f"unnamespaced-unknown-member:{field}") + + _validate_resource(block.get("resource"), reasons) + _validate_semconv(block.get("semconv"), reasons) + + status = "ACCEPTED" if not reasons else "INFORMATIONAL_ONLY" + return OTelExtensionVerdict( + status=status, + reasons=tuple(sorted(set(reasons))), + placement=placement, + independent_corroboration=False, + ) diff --git a/tests/unit/test_aac_otel_extension.py b/tests/unit/test_aac_otel_extension.py new file mode 100644 index 00000000..461ac67a --- /dev/null +++ b/tests/unit/test_aac_otel_extension.py @@ -0,0 +1,159 @@ +from skill_factory.evolution.aac_otel_extension import ( + validate_aac_otel_extension, +) + + +POSITIVE_BLOCK={ + "org.agentactioncapsule.otel":{ + "trace_id":"4bf92f3577b34da6a3ce929d0e0e4736", + "span_id":"00f067aa0ba902b7", + "trace_flags":"01", + "resource":{"service.name":"support-agent"}, + "semconv":{ + "source":"open-telemetry/semantic-conventions-genai@8c1b98a", + "gen_ai.provider.name":"local", + "gen_ai.operation.name":"execute_tool", + }, + } +} + + +def test_published_positive_vector_is_accepted(): + verdict=validate_aac_otel_extension(POSITIVE_BLOCK) + + assert verdict.status=="ACCEPTED" + assert verdict.reasons==() + assert verdict.independent_corroboration is False + + +def test_published_malformed_trace_id_vector_is_informational_only(): + capsule={ + "org.agentactioncapsule.otel":{ + "trace_id":"4BF92F3577b34da6a3ce929d0e0e473", + "span_id":"00f067aa0ba902b7", + } + } + + verdict=validate_aac_otel_extension(capsule) + + assert verdict.status=="INFORMATIONAL_ONLY" + assert "malformed-hex:trace_id" in verdict.reasons + + +def test_published_clear_tracestate_vector_is_informational_only(): + capsule={ + "org.agentactioncapsule.otel":{ + "trace_id":"4bf92f3577b34da6a3ce929d0e0e4736", + "span_id":"00f067aa0ba902b7", + "tracestate":"congo=t61rcWkgMzE", + } + } + + verdict=validate_aac_otel_extension(capsule) + + assert verdict.status=="INFORMATIONAL_ONLY" + assert "clear-tracestate-forbidden" in verdict.reasons + + +def test_published_non_allowlisted_content_vector_is_informational_only(): + capsule={ + "org.agentactioncapsule.otel":{ + "trace_id":"4bf92f3577b34da6a3ce929d0e0e4736", + "span_id":"00f067aa0ba902b7", + "semconv":{ + "source":"open-telemetry/semantic-conventions-genai@8c1b98a", + "gen_ai.input.messages":[{"role":"user","content":"secret"}], + }, + } + } + + verdict=validate_aac_otel_extension(capsule) + + assert verdict.status=="INFORMATIONAL_ONLY" + assert ( + "semconv-never-enters:gen_ai.input.messages" + in verdict.reasons + ) + + +def test_nested_placement_is_treated_identically(): + block=POSITIVE_BLOCK["org.agentactioncapsule.otel"] + capsule={ + "model_attestation":{ + "compute_attestation":{ + "org.agentactioncapsule.otel":block, + } + } + } + + verdict=validate_aac_otel_extension(capsule) + + assert verdict.status=="ACCEPTED" + assert verdict.placement=="model_attestation.compute_attestation" + + +def test_same_producer_telemetry_is_never_independent_corroboration(): + verdict=validate_aac_otel_extension(POSITIVE_BLOCK) + + assert verdict.independent_corroboration is False + + +def test_unpinned_semconv_source_is_not_promoted(): + block=dict(POSITIVE_BLOCK["org.agentactioncapsule.otel"]) + block["semconv"]={ + "gen_ai.provider.name":"local", + "gen_ai.operation.name":"execute_tool", + } + + verdict=validate_aac_otel_extension( + {"org.agentactioncapsule.otel":block} + ) + + assert verdict.status=="INFORMATIONAL_ONLY" + assert "semconv-source-missing-or-unpinned" in verdict.reasons + + +def test_device_identifying_resource_is_not_clear_safe(): + block=dict(POSITIVE_BLOCK["org.agentactioncapsule.otel"]) + block["resource"]={ + "service.name":"support-agent", + "host.id":"host-123", + } + + verdict=validate_aac_otel_extension( + {"org.agentactioncapsule.otel":block} + ) + + assert verdict.status=="INFORMATIONAL_ONLY" + assert "resource-not-clear-safe:host.id" in verdict.reasons + + + +def test_empty_semconv_source_suffix_is_not_pinned(): + block=dict(POSITIVE_BLOCK["org.agentactioncapsule.otel"]) + block["semconv"]={ + "source":"open-telemetry/semantic-conventions-genai@", + "gen_ai.operation.name":"execute_tool", + } + + verdict=validate_aac_otel_extension( + {"org.agentactioncapsule.otel":block} + ) + + assert verdict.status=="INFORMATIONAL_ONLY" + assert "semconv-source-missing-or-unpinned" in verdict.reasons + + +def test_cache_input_token_family_is_admitted_by_draft_pattern(): + block=dict(POSITIVE_BLOCK["org.agentactioncapsule.otel"]) + block["semconv"]={ + "source":"open-telemetry/semantic-conventions-genai@8c1b98a", + "gen_ai.usage.cache_read.input_tokens":17, + } + + verdict=validate_aac_otel_extension( + {"org.agentactioncapsule.otel":block} + ) + + assert verdict.status=="ACCEPTED" + assert verdict.reasons==()