From 749f8a8a6b329afa08b0cfc88e00b22681dc3e9c Mon Sep 17 00:00:00 2001 From: jariy17 Date: Thu, 27 Aug 2026 20:27:28 +0000 Subject: [PATCH 01/15] feat(eval): ondemand simulate handler (batch pattern: ingestion-wait-ms, failures/sessions) --- src/handlers/eval/ondemand/index.tsx | 4 +- src/handlers/eval/ondemand/simulate/index.tsx | 135 ++++++++++++++++++ 2 files changed, 138 insertions(+), 1 deletion(-) create mode 100644 src/handlers/eval/ondemand/simulate/index.tsx diff --git a/src/handlers/eval/ondemand/index.tsx b/src/handlers/eval/ondemand/index.tsx index d46cc38ea..e731df263 100644 --- a/src/handlers/eval/ondemand/index.tsx +++ b/src/handlers/eval/ondemand/index.tsx @@ -3,11 +3,13 @@ import type { AppIO } from "../../../io"; import type { Core } from "../../types"; import { createHelpDefault } from "../../help"; import { createEvaluateOnDemandHandler } from "./evaluate"; +import { createSimulateOnDemandHandler } from "./simulate"; // ondemand groups the synchronous, client-side evaluation commands. It has no TUI // screen (unlike evaluator/online-eval), so a bare invocation prints help. export function createOnDemandHandler(core: Core, io: AppIO): Router { return new Router("ondemand", "evaluate existing sessions synchronously, client-side") .default(createHelpDefault(io)) - .handler(createEvaluateOnDemandHandler(core, io)); + .handler(createEvaluateOnDemandHandler(core, io)) + .handler(createSimulateOnDemandHandler(core, io)); } diff --git a/src/handlers/eval/ondemand/simulate/index.tsx b/src/handlers/eval/ondemand/simulate/index.tsx new file mode 100644 index 000000000..eec914937 --- /dev/null +++ b/src/handlers/eval/ondemand/simulate/index.tsx @@ -0,0 +1,135 @@ +import z from "zod"; +import type { EvaluationReferenceInput } from "@aws-sdk/client-bedrock-agentcore"; +import { createHandler, flag } from "../../../../router"; +import { InputValidationError } from "../../../../errors"; +import { JsonRendererKey } from "../../../../tui"; +import type { AppIO } from "../../../../io"; +import type { Core } from "../../../types"; +import type { InvokedSession } from "../../types"; +import { coreOptsFromCtx } from "../../../utils"; +import { parseRuntimeInvokeHeaders } from "../../../runtime/invoke/request"; + +// Composes invokeDataset (replay) → getTracesForAgent (gather) → evaluate (grade, +// synchronous). The on-demand twin of batch-evaluation simulate: no async job, scores +// print inline. Invoke flags mirror `runtime invoke`. +export const createSimulateOnDemandHandler = (core: Core, _io: AppIO) => + createHandler({ + name: "simulate", + description: "replay a dataset against a runtime, then evaluate the sessions client-side", + flags: [ + flag("runtime-id", "runtime id to invoke per scenario", z.string().optional()), + flag("qualifier", "runtime endpoint qualifier (default DEFAULT)", z.string().optional()), + flag( + "payload-template", + 'JSON payload template; {input} is the scenario input, e.g. {"prompt":"{input}"}', + z.string().optional(), + ), + flag("header", "an ordered application header (repeatable)", z.array(z.string()).optional()), + flag( + "bearer-token", + "CUSTOM_JWT bearer token (for JWT-auth runtimes)", + z.string().optional(), + ), + flag("user-id", "runtime user id", z.string().optional()), + flag("dataset", "dataset source: local JSONL path or a dataset id", z.string().optional()), + flag("dataset-version", "dataset version (with a dataset id)", z.string().optional()), + flag("evaluator", "evaluator id(s) to apply", z.array(z.string()).optional()), + flag( + "ingestion-wait-ms", + "ms to wait for span ingestion before grading (default 180000; 0 to skip)", + z.coerce.number().int().nonnegative().optional(), + ), + ], + handle: async (ctx, flags) => { + if (!flags["runtime-id"]) + throw new InputValidationError("required option '--runtime-id' not specified"); + if (!flags["payload-template"]) { + throw new InputValidationError("required option '--payload-template' not specified"); + } + if (!flags["dataset"]) + throw new InputValidationError("required option '--dataset' not specified"); + if (!flags["evaluator"]?.length) { + throw new InputValidationError( + "required option '--evaluator ' not specified", + ); + } + + // Ctrl-C aborts the run (invokes, the ingestion wait, the dataset download). + const controller = new AbortController(); + const interrupt = () => controller.abort(); + process.once("SIGINT", interrupt); + try { + const opts = coreOptsFromCtx(ctx); + + // 1. Replay the dataset — reuse invokeDataset verbatim (grader-agnostic). + const replay = await core.eval.invokeDataset( + { + runtimeId: flags["runtime-id"], + qualifier: flags["qualifier"], + payloadTemplate: flags["payload-template"], + headers: parseRuntimeInvokeHeaders(flags["header"]), + bearerToken: flags["bearer-token"], + userId: flags["user-id"], + dataset: flags["dataset"], + datasetVersion: flags["dataset-version"], + waitIngestionMs: flags["ingestion-wait-ms"], + }, + opts, + controller.signal, + ); + if (replay.invoked === 0) { + const first = replay.failures[0]; + const detail = first ? `; first error: ${first.exampleId} — ${first.error}` : ""; + throw new InputValidationError( + `no examples could be invoked (${replay.failed} failed) — nothing to evaluate${detail}`, + ); + } + + // 2. Gather the just-created sessions' traces (client-side CloudWatch read). + const traces = await core.eval.getTracesForAgent( + { + agent: flags["runtime-id"], + endpoint: flags["qualifier"], + sessionIds: replay.sessions.map((s) => s.sessionId), + }, + opts, + ); + + // 3. Adapt neutral ground truth → EvaluationReferenceInput[] and grade synchronously. + const groundTruth = replay.sessions.flatMap(toReferenceInputs); + const result = await core.eval.evaluate( + { traces, evaluatorIds: flags["evaluator"], groundTruth }, + opts, + ); + + ctx.require(JsonRendererKey).renderJson({ + ...result, + examplesInvoked: replay.invoked, + examplesFailed: replay.failed, + sessions: replay.sessions.map((s) => ({ + exampleId: s.exampleId, + sessionId: s.sessionId, + })), + failures: replay.failures, + }); + } finally { + process.off("SIGINT", interrupt); + } + }, + }); + +// Adapt one invoked session's neutral InlineGroundTruth to the Evaluate API's +// EvaluationReferenceInput, correlated by sessionId. assertions ({text}[]) and +// expectedTrajectory ({toolNames}) map 1:1. Per-turn expectedResponse is trace-level and +// needs a turn→trace id we don't have here, so it is omitted (batch simulate covers it). +function toReferenceInputs(s: InvokedSession): EvaluationReferenceInput[] { + const gt = s.groundTruth; + if (!gt?.assertions?.length && !gt?.expectedTrajectory) return []; + return [ + { + context: { spanContext: { sessionId: s.sessionId } }, + ...(gt.assertions?.length && { assertions: gt.assertions }), + ...(gt.expectedTrajectory && { expectedTrajectory: gt.expectedTrajectory }), + }, + ]; +} From 452d430a16646b298773f5e188e8be797e2de179 Mon Sep 17 00:00:00 2001 From: jariy17 Date: Thu, 27 Aug 2026 20:42:51 +0000 Subject: [PATCH 02/15] test(eval): ondemand simulate edge tests + register under ondemand --- src/handlers/eval/ondemand/ondemand.test.tsx | 69 +++++++++++++++++++- 1 file changed, 68 insertions(+), 1 deletion(-) diff --git a/src/handlers/eval/ondemand/ondemand.test.tsx b/src/handlers/eval/ondemand/ondemand.test.tsx index 903a9140e..64782c855 100644 --- a/src/handlers/eval/ondemand/ondemand.test.tsx +++ b/src/handlers/eval/ondemand/ondemand.test.tsx @@ -180,7 +180,74 @@ describe("eval ondemand command hierarchy", () => { .find((c) => c.name() === "eval") ?.children() .find((c) => c.name() === "ondemand"); - expect(group?.children().map((c) => c.name())).toEqual(["evaluate"]); + expect(group?.children().map((c) => c.name())).toEqual(["evaluate", "simulate"]); + }); +}); + +describe("eval ondemand simulate", () => { + const BASE = [ + "eval", + "ondemand", + "simulate", + "--runtime-id", + "r-1", + "--payload-template", + '{"prompt":"{input}"}', + "--dataset", + "/tmp/ds.jsonl", + "--evaluator", + "Builtin.Helpfulness", + ]; + + test.each<[RegExp, string[]]>([ + [ + /--runtime-id/, + ["--payload-template", "{}", "--dataset", "/tmp/ds.jsonl", "--evaluator", "E"], + ], + [ + /--payload-template/, + ["--runtime-id", "r-1", "--dataset", "/tmp/ds.jsonl", "--evaluator", "E"], + ], + [/--dataset/, ["--runtime-id", "r-1", "--payload-template", "{}", "--evaluator", "E"]], + [ + /--evaluator/, + ["--runtime-id", "r-1", "--payload-template", "{}", "--dataset", "/tmp/ds.jsonl"], + ], + ])("rejects when a required flag is missing (%s)", async (expected, args) => { + await expect(run(["eval", "ondemand", "simulate", ...args])).rejects.toThrow(expected); + }); + + test("refuses to grade when nothing was invoked, naming the first failure", async () => { + await expect( + run(BASE, (c) => + c.eval.setInvokeDatasetResponse({ + sessions: [], + invoked: 0, + failed: 2, + failures: [ + { exampleId: "e1", error: "HTTP 500" }, + { exampleId: "e2", error: "HTTP 500" }, + ], + }), + ), + ).rejects.toThrow(/no examples could be invoked \(2 failed\).*first error: e1 — HTTP 500/); + }); + + test("passes --ingestion-wait-ms through to invokeDataset and renders failures", async () => { + const { core, stdout } = await run([...BASE, "--ingestion-wait-ms", "0"], (c) => + c.eval.setInvokeDatasetResponse({ + sessions: [{ exampleId: "ok1", sessionId: "s1" }], + invoked: 1, + failed: 1, + failures: [{ exampleId: "bad", error: "HTTP 500" }], + }), + ); + const invoke = core.eval.calls.find((c) => c.method === "invokeDataset"); + expect(invoke).toBeDefined(); + expect((invoke!.args[0] as { waitIngestionMs?: number }).waitIngestionMs).toBe(0); + const out = JSON.parse(stdout); + expect(out.failures).toEqual([{ exampleId: "bad", error: "HTTP 500" }]); + expect(out.sessions).toEqual([{ exampleId: "ok1", sessionId: "s1" }]); }); }); From c29ad3160f6d67852c1616fbf3591a90507402cf Mon Sep 17 00:00:00 2001 From: jariy17 Date: Thu, 27 Aug 2026 21:14:47 +0000 Subject: [PATCH 03/15] test(eval): ondemand simulate fixture golden + clock seam; strip handler comments --- src/core/eval.tsx | 3 +- src/core/index.tsx | 2 + .../EvaluateCommand.6867137a17d29ce5.json | 114 +++ ...tQueryResultsCommand.2e6a36ed61668bc0.json | 344 +++++++ ...tQueryResultsCommand.70382c1aa9a94e3a.json | 960 ++++++++++++++++++ ...eAgentRuntimeCommand.d7f8ec055ea3add0.json | 8 + .../StartQueryCommand.3c11aa80c0c5122e.json | 3 + .../StartQueryCommand.a6424982fd346c1.json | 3 + .../ondemand/__fixtures__/simulate-ds.jsonl | 1 + .../__fixtures__/simulate.golden.json | 125 +++ .../eval/ondemand/ondemand.fixture.test.tsx | 45 + src/handlers/eval/ondemand/simulate/index.tsx | 11 - 12 files changed, 1607 insertions(+), 12 deletions(-) create mode 100644 src/handlers/eval/ondemand/__fixtures__/EvaluateCommand.6867137a17d29ce5.json create mode 100644 src/handlers/eval/ondemand/__fixtures__/GetQueryResultsCommand.2e6a36ed61668bc0.json create mode 100644 src/handlers/eval/ondemand/__fixtures__/GetQueryResultsCommand.70382c1aa9a94e3a.json create mode 100644 src/handlers/eval/ondemand/__fixtures__/InvokeAgentRuntimeCommand.d7f8ec055ea3add0.json create mode 100644 src/handlers/eval/ondemand/__fixtures__/StartQueryCommand.3c11aa80c0c5122e.json create mode 100644 src/handlers/eval/ondemand/__fixtures__/StartQueryCommand.a6424982fd346c1.json create mode 100644 src/handlers/eval/ondemand/__fixtures__/simulate-ds.jsonl create mode 100644 src/handlers/eval/ondemand/__fixtures__/simulate.golden.json diff --git a/src/core/eval.tsx b/src/core/eval.tsx index fd51e04ee..9e4f0f4d2 100644 --- a/src/core/eval.tsx +++ b/src/core/eval.tsx @@ -216,6 +216,7 @@ export class EvalClient implements CoreEvalClient { // logger for batch-evaluation result-log diagnostics private readonly logger: Logger = noopLogger, private readonly newSessionId: () => string = randomUUID, + private readonly now: () => number = () => Date.now(), ) {} async createEvaluator( @@ -582,7 +583,7 @@ export class EvalClient implements CoreEvalClient { const logGroupName = runtimeLogGroup(runtimeId, qualifier); const serviceName = runtimeServiceName(runtimeName, qualifier); - const endMs = input.window ? +input.window.endTime : Date.now(); + const endMs = input.window ? +input.window.endTime : this.now(); const startMs = input.window ? +input.window.startTime : endMs - SEVEN_DAYS_MS; const startSec = Math.floor(startMs / 1000); const endSec = Math.floor(endMs / 1000); diff --git a/src/core/index.tsx b/src/core/index.tsx index d7275b912..82deabb44 100644 --- a/src/core/index.tsx +++ b/src/core/index.tsx @@ -39,6 +39,7 @@ type CoreClientConfig = { logger: Logger; fetch?: CoreFetch; newSessionId?: () => string; + now?: () => number; }; // CoreClient is the single entry point to the Bedrock AgentCore APIs. It owns the @@ -84,6 +85,7 @@ export class CoreClient implements AwsClients { fetch, this.logger.child({ module: "eval" }), config.newSessionId, + config.now, ); this.projectManager = new FsProjectManager({ diff --git a/src/handlers/eval/ondemand/__fixtures__/EvaluateCommand.6867137a17d29ce5.json b/src/handlers/eval/ondemand/__fixtures__/EvaluateCommand.6867137a17d29ce5.json new file mode 100644 index 000000000..b8c9304a2 --- /dev/null +++ b/src/handlers/eval/ondemand/__fixtures__/EvaluateCommand.6867137a17d29ce5.json @@ -0,0 +1,114 @@ +{ + "evaluationResults": [ + { + "evaluatorArn": "arn:aws:bedrock-agentcore:::evaluator/Builtin.Helpfulness", + "evaluatorId": "Builtin.Helpfulness", + "evaluatorName": "Builtin.Helpfulness", + "context": { + "spanContext": { + "sessionId": "00000000-0000-4000-8000-000000000001", + "traceId": "6a8f5b461983df572e2aca3264d32d4d" + } + }, + "explanation": "The user simply said 'hi', which is a greeting with no specific goal expressed. The assistant responded with a polite greeting and an open invitation to help. This is appropriate conversational behavior that maintains the flow of interaction and invites the user to share their actual needs. Since the user hasn't expressed a specific goal yet, the assistant's response is a standard, appropriate reply that keeps the conversation open. It doesn't advance any specific goal (since none exists yet), but it doesn't hinder progress either. This falls into the 'Neutral/Mixed' category as it's appropriate chit-chat for conversation flow with no specific goal to advance.", + "value": 0.5, + "label": "Neutral/Mixed", + "tokenUsage": { + "inputTokens": 815, + "outputTokens": 155, + "totalTokens": 970 + }, + "ignoredReferenceInputFields": [ + "assertions" + ] + }, + { + "evaluatorArn": "arn:aws:bedrock-agentcore:::evaluator/Builtin.Helpfulness", + "evaluatorId": "Builtin.Helpfulness", + "evaluatorName": "Builtin.Helpfulness", + "context": { + "spanContext": { + "sessionId": "00000000-0000-4000-8000-000000000001", + "traceId": "6a8f5bbc1882c05547b64a9246f17ca4" + } + }, + "explanation": "The user has said 'hi' twice without providing any specific goal or request. The assistant's response 'Hi there! How can I assist you today?' is a standard greeting that keeps the conversation open and invites the user to share their needs. Since the user hasn't expressed a specific goal yet, the assistant can't do much more than respond to the greeting and prompt the user to share what they need. This response is appropriate for the conversational context - it's a polite acknowledgment that maintains the conversation flow and opens the door for the user to state their actual needs. It doesn't advance any specific goal (since none has been stated), but it doesn't hinder progress either. This falls into the 'Neutral/Mixed' category as it's appropriate chit-chat for conversation flow with no specific goal to advance.", + "value": 0.5, + "label": "Neutral/Mixed", + "tokenUsage": { + "inputTokens": 887, + "outputTokens": 194, + "totalTokens": 1081 + }, + "ignoredReferenceInputFields": [ + "assertions" + ] + }, + { + "evaluatorArn": "arn:aws:bedrock-agentcore:::evaluator/Builtin.Helpfulness", + "evaluatorId": "Builtin.Helpfulness", + "evaluatorName": "Builtin.Helpfulness", + "context": { + "spanContext": { + "sessionId": "00000000-0000-4000-8000-000000000001", + "traceId": "6a8f639540ebe6cb6f30d5d80762c2e2" + } + }, + "explanation": "The user has sent 'hi' three times in a row without providing any specific request or goal. The assistant's response 'Hello! How can I help you today?' is a standard greeting that keeps the conversation open and invites the user to share their needs. This is the third identical exchange, and the assistant is simply repeating the same greeting. While the response is appropriate and doesn't obstruct any goal, it also doesn't advance any specific goal since the user hasn't expressed one yet. The response is essentially neutral - it's appropriate chit-chat that maintains conversation flow without moving toward any particular goal (since no goal has been stated). The assistant could potentially note that the user has greeted multiple times and ask if they need help with something specific, which would be slightly more proactive, but the current response is still a reasonable reply to a simple greeting.", + "value": 0.5, + "label": "Neutral/Mixed", + "tokenUsage": { + "inputTokens": 958, + "outputTokens": 207, + "totalTokens": 1165 + }, + "ignoredReferenceInputFields": [ + "assertions" + ] + }, + { + "evaluatorArn": "arn:aws:bedrock-agentcore:::evaluator/Builtin.Helpfulness", + "evaluatorId": "Builtin.Helpfulness", + "evaluatorName": "Builtin.Helpfulness", + "context": { + "spanContext": { + "sessionId": "00000000-0000-4000-8000-000000000001", + "traceId": "6a90a5987755c6e457ada5616a16dddb" + } + }, + "explanation": "The user's final request was 'Say hello and offer help.' The assistant's response directly fulfills this request by:\n1. Saying hello with a greeting and wave emoji\n2. Offering help by listing specific capabilities it can assist with\n\nThe response is friendly, clear, and directly addresses what the user asked for. It goes slightly beyond the minimal requirement by providing specific examples of what it can help with (web search, fetching webpages, calculations), which gives the user actionable information about how to proceed.\n\nHowever, the listed capabilities (web search, fetch webpages, perform calculations) seem oddly specific and somewhat limiting - a general AI assistant can help with many more things like writing, analysis, coding, answering questions, etc. This specificity might actually mislead the user about the assistant's full range of capabilities.\n\nDespite this minor issue, the response does exactly what was requested - says hello and offers help - and does so in a clear, organized manner. The user's goal was simple and the assistant met it directly.", + "value": 0.83, + "label": "Very Helpful", + "tokenUsage": { + "inputTokens": 1112, + "outputTokens": 258, + "totalTokens": 1370 + }, + "ignoredReferenceInputFields": [ + "assertions" + ] + }, + { + "evaluatorArn": "arn:aws:bedrock-agentcore:::evaluator/Builtin.Helpfulness", + "evaluatorId": "Builtin.Helpfulness", + "evaluatorName": "Builtin.Helpfulness", + "context": { + "spanContext": { + "sessionId": "00000000-0000-4000-8000-000000000001", + "traceId": "6a90a74750f33f567076943f7bce7966" + } + }, + "explanation": "The user's request is simple and explicit: 'Say hello and offer help.' The assistant's response directly fulfills this request by greeting the user with 'Hello! 👋 Welcome!' and then offering help with a clear, organized list of capabilities (web searches, reading webpages, and calculations). The response is well-formatted, friendly, and ends with an open invitation for the user to specify what they need. This is essentially the same response as the previous turn (with minor variations like adding 'Welcome!'), which is appropriate since the user repeated the same request. The response fully satisfies the user's stated goal of having the assistant say hello and offer help. It's comprehensive and actionable, clearly communicating what the assistant can do. There's nothing missing or problematic about this response given the user's simple, direct request.", + "value": 0.83, + "label": "Very Helpful", + "tokenUsage": { + "inputTokens": 1267, + "outputTokens": 203, + "totalTokens": 1470 + }, + "ignoredReferenceInputFields": [ + "assertions" + ] + } + ] +} \ No newline at end of file diff --git a/src/handlers/eval/ondemand/__fixtures__/GetQueryResultsCommand.2e6a36ed61668bc0.json b/src/handlers/eval/ondemand/__fixtures__/GetQueryResultsCommand.2e6a36ed61668bc0.json new file mode 100644 index 000000000..12a4f7f3f --- /dev/null +++ b/src/handlers/eval/ondemand/__fixtures__/GetQueryResultsCommand.2e6a36ed61668bc0.json @@ -0,0 +1,344 @@ +{ + "queryLanguage": "CWLI", + "results": [ + [ + { + "field": "@message", + "value": "{\"resource\":{\"attributes\":{\"telemetry.sdk.language\":\"python\",\"telemetry.sdk.name\":\"opentelemetry\",\"telemetry.sdk.version\":\"1.40.0\",\"service.name\":\"asdf_MyAgent.DEFAULT\",\"aws.log.group.names\":\"/aws/bedrock-agentcore/runtimes/asdf_MyAgent-3s5axvBC6Q-DEFAULT\",\"aws.log.stream.names\":\"otel-rt-logs\",\"deployment.environment.name\":\"bedrock-agentcore:default\",\"cloud.resource_id\":\"arn:aws:bedrock-agentcore:us-west-2:685197708687:runtime/asdf_MyAgent-3s5axvBC6Q/runtime-endpoint/DEFAULT:DEFAULT\",\"cloud.platform\":\"aws_bedrock_agentcore\",\"cloud.provider\":\"aws\",\"cloud.region\":\"us-west-2\",\"telemetry.auto.version\":\"0.17.0-aws\",\"aws.local.service\":\"asdf_MyAgent.DEFAULT\",\"aws.service.type\":\"gen_ai_agent\"}},\"scope\":{\"name\":\"strands.telemetry.tracer\"},\"timeUnixNano\":1787779917882748907,\"observedTimeUnixNano\":1787779918378293959,\"severityNumber\":9,\"severityText\":\"\",\"body\":{\"input\":{\"messages\":[{\"role\":\"system\",\"content\":{\"content\":\"[{\\\"text\\\": \\\"\\\\n You are a helpful assistant. Use tools when appropriate.\\\\n \\\"}]\"}},{\"role\":\"user\",\"content\":{\"content\":\"[{\\\"text\\\": \\\"hi\\\"}]\"}}]},\"output\":{\"messages\":[{\"role\":\"assistant\",\"content\":{\"finish_reason\":\"end_turn\",\"message\":\"[{\\\"text\\\": \\\"Hello! How can I help you today?\\\"}]\"}}]}},\"attributes\":{\"session.id\":\"00000000-0000-4000-8000-000000000001\",\"event.name\":\"strands.telemetry.tracer\"},\"flags\":1,\"traceId\":\"6a8f5b461983df572e2aca3264d32d4d\",\"spanId\":\"ac9653f70866483d\"}" + }, + { + "field": "sessionId", + "value": "00000000-0000-4000-8000-000000000001" + }, + { + "field": "traceId", + "value": "6a8f5b461983df572e2aca3264d32d4d" + }, + { + "field": "spanId", + "value": "ac9653f70866483d" + }, + { + "field": "@ptr", + "value": "CswBCo8BCkw2ODUxOTc3MDg2ODc6L2F3cy9iZWRyb2NrLWFnZW50Y29yZS9ydW50aW1lcy9hc2RmX015QWdlbnQtM3M1YXh2QkM2US1ERUZBVUxUEAAaJGY2ZGY3NDIyLWVlZTQtNDhkNS05NDg3LTE0NTZkYjdkZGE1YiIOCICQ+IyCNBDnn6qthDRA0cHR4dszSAASNhoYAgakLY/QAAAAAD0Mr1AABqj1tJAAAAQCIAEo/63y/4M0MK/U8v+DNDgfQKbVAUikaFCmRyACEBoYAQ==" + } + ], + [ + { + "field": "@message", + "value": "{\"resource\":{\"attributes\":{\"telemetry.sdk.language\":\"python\",\"telemetry.sdk.name\":\"opentelemetry\",\"telemetry.sdk.version\":\"1.40.0\",\"service.name\":\"asdf_MyAgent.DEFAULT\",\"aws.log.group.names\":\"/aws/bedrock-agentcore/runtimes/asdf_MyAgent-3s5axvBC6Q-DEFAULT\",\"aws.log.stream.names\":\"otel-rt-logs\",\"deployment.environment.name\":\"bedrock-agentcore:default\",\"cloud.resource_id\":\"arn:aws:bedrock-agentcore:us-west-2:685197708687:runtime/asdf_MyAgent-3s5axvBC6Q/runtime-endpoint/DEFAULT:DEFAULT\",\"cloud.platform\":\"aws_bedrock_agentcore\",\"cloud.provider\":\"aws\",\"cloud.region\":\"us-west-2\",\"telemetry.auto.version\":\"0.17.0-aws\",\"aws.local.service\":\"asdf_MyAgent.DEFAULT\",\"aws.service.type\":\"gen_ai_agent\"}},\"scope\":{\"name\":\"strands.telemetry.tracer\"},\"timeUnixNano\":1787779917883212081,\"observedTimeUnixNano\":1787779918378532682,\"severityNumber\":9,\"severityText\":\"\",\"body\":{\"input\":{\"messages\":[{\"role\":\"system\",\"content\":\"\\n You are a helpful assistant. Use tools when appropriate.\\n \"},{\"role\":\"user\",\"content\":{\"content\":\"[{\\\"text\\\": \\\"hi\\\"}]\"}}]},\"output\":{\"messages\":[{\"role\":\"assistant\",\"content\":{\"message\":\"Hello! How can I help you today?\\n\",\"finish_reason\":\"end_turn\"}}]}},\"attributes\":{\"session.id\":\"00000000-0000-4000-8000-000000000001\",\"event.name\":\"strands.telemetry.tracer\"},\"flags\":1,\"traceId\":\"6a8f5b461983df572e2aca3264d32d4d\",\"spanId\":\"82fe397271c56f75\"}" + }, + { + "field": "sessionId", + "value": "00000000-0000-4000-8000-000000000001" + }, + { + "field": "traceId", + "value": "6a8f5b461983df572e2aca3264d32d4d" + }, + { + "field": "spanId", + "value": "82fe397271c56f75" + }, + { + "field": "@ptr", + "value": "CswBCo8BCkw2ODUxOTc3MDg2ODc6L2F3cy9iZWRyb2NrLWFnZW50Y29yZS9ydW50aW1lcy9hc2RmX015QWdlbnQtM3M1YXh2QkM2US1ERUZBVUxUEAAaJGY2ZGY3NDIyLWVlZTQtNDhkNS05NDg3LTE0NTZkYjdkZGE1YiIOCICQ+IyCNBDnn6qthDRA0cHR4dszSAASNhoYAgakLY/QAAAAAD0Mr1AABqj1tJAAAAQCIAEo/63y/4M0MK/U8v+DNDgfQKbVAUikaFCmRyACEBwYAQ==" + } + ], + [ + { + "field": "@message", + "value": "{\"resource\":{\"attributes\":{\"telemetry.sdk.language\":\"python\",\"telemetry.sdk.name\":\"opentelemetry\",\"telemetry.sdk.version\":\"1.40.0\",\"service.name\":\"asdf_MyAgent.DEFAULT\",\"aws.log.group.names\":\"/aws/bedrock-agentcore/runtimes/asdf_MyAgent-3s5axvBC6Q-DEFAULT\",\"aws.log.stream.names\":\"otel-rt-logs\",\"deployment.environment.name\":\"bedrock-agentcore:default\",\"cloud.resource_id\":\"arn:aws:bedrock-agentcore:us-west-2:685197708687:runtime/asdf_MyAgent-3s5axvBC6Q/runtime-endpoint/DEFAULT:DEFAULT\",\"cloud.platform\":\"aws_bedrock_agentcore\",\"cloud.provider\":\"aws\",\"cloud.region\":\"us-west-2\",\"telemetry.auto.version\":\"0.17.0-aws\",\"aws.local.service\":\"asdf_MyAgent.DEFAULT\",\"aws.service.type\":\"gen_ai_agent\"}},\"scope\":{\"name\":\"strands.telemetry.tracer\"},\"timeUnixNano\":1787779917883063841,\"observedTimeUnixNano\":1787779918378428749,\"severityNumber\":9,\"severityText\":\"\",\"body\":{\"input\":{\"messages\":[{\"role\":\"user\",\"content\":{\"content\":\"[{\\\"text\\\": \\\"hi\\\"}]\"}}]}},\"attributes\":{\"session.id\":\"00000000-0000-4000-8000-000000000001\",\"event.name\":\"strands.telemetry.tracer\"},\"flags\":1,\"traceId\":\"6a8f5b461983df572e2aca3264d32d4d\",\"spanId\":\"c9722c3dbb4e781e\"}" + }, + { + "field": "sessionId", + "value": "00000000-0000-4000-8000-000000000001" + }, + { + "field": "traceId", + "value": "6a8f5b461983df572e2aca3264d32d4d" + }, + { + "field": "spanId", + "value": "c9722c3dbb4e781e" + }, + { + "field": "@ptr", + "value": "CswBCo8BCkw2ODUxOTc3MDg2ODc6L2F3cy9iZWRyb2NrLWFnZW50Y29yZS9ydW50aW1lcy9hc2RmX015QWdlbnQtM3M1YXh2QkM2US1ERUZBVUxUEAAaJGY2ZGY3NDIyLWVlZTQtNDhkNS05NDg3LTE0NTZkYjdkZGE1YiIOCICQ+IyCNBDnn6qthDRA0cHR4dszSAASNhoYAgakLY/QAAAAAD0Mr1AABqj1tJAAAAQCIAEo/63y/4M0MK/U8v+DNDgfQKbVAUikaFCmRyACEBsYAQ==" + } + ], + [ + { + "field": "@message", + "value": "{\"resource\":{\"attributes\":{\"telemetry.sdk.language\":\"python\",\"telemetry.sdk.name\":\"opentelemetry\",\"telemetry.sdk.version\":\"1.40.0\",\"service.name\":\"asdf_MyAgent.DEFAULT\",\"aws.log.group.names\":\"/aws/bedrock-agentcore/runtimes/asdf_MyAgent-3s5axvBC6Q-DEFAULT\",\"aws.log.stream.names\":\"otel-rt-logs\",\"deployment.environment.name\":\"bedrock-agentcore:default\",\"cloud.resource_id\":\"arn:aws:bedrock-agentcore:us-west-2:685197708687:runtime/asdf_MyAgent-3s5axvBC6Q/runtime-endpoint/DEFAULT:DEFAULT\",\"cloud.platform\":\"aws_bedrock_agentcore\",\"cloud.provider\":\"aws\",\"cloud.region\":\"us-west-2\",\"telemetry.auto.version\":\"0.17.0-aws\",\"aws.local.service\":\"asdf_MyAgent.DEFAULT\",\"aws.service.type\":\"gen_ai_agent\"}},\"scope\":{\"name\":\"strands.telemetry.tracer\"},\"timeUnixNano\":1787780030855720915,\"observedTimeUnixNano\":1787780033471228591,\"severityNumber\":9,\"severityText\":\"\",\"body\":{\"input\":{\"messages\":[{\"role\":\"system\",\"content\":{\"content\":\"[{\\\"text\\\": \\\"\\\\n You are a helpful assistant. Use tools when appropriate.\\\\n \\\"}]\"}},{\"role\":\"user\",\"content\":{\"content\":\"[{\\\"text\\\": \\\"hi\\\"}]\"}}]},\"output\":{\"messages\":[{\"role\":\"assistant\",\"content\":{\"content\":\"[{\\\"text\\\": \\\"Hello! How can I help you today?\\\"}]\"}},{\"role\":\"assistant\",\"content\":{\"finish_reason\":\"end_turn\",\"message\":\"[{\\\"text\\\": \\\"Hi there! How can I assist you today?\\\"}]\"}}]}},\"attributes\":{\"session.id\":\"00000000-0000-4000-8000-000000000001\",\"event.name\":\"strands.telemetry.tracer\"},\"flags\":1,\"traceId\":\"6a8f5bbc1882c05547b64a9246f17ca4\",\"spanId\":\"2b53da16274ce9f5\"}" + }, + { + "field": "sessionId", + "value": "00000000-0000-4000-8000-000000000001" + }, + { + "field": "traceId", + "value": "6a8f5bbc1882c05547b64a9246f17ca4" + }, + { + "field": "spanId", + "value": "2b53da16274ce9f5" + }, + { + "field": "@ptr", + "value": "Cs4BCo8BCkw2ODUxOTc3MDg2ODc6L2F3cy9iZWRyb2NrLWFnZW50Y29yZS9ydW50aW1lcy9hc2RmX015QWdlbnQtM3M1YXh2QkM2US1ERUZBVUxUEAAaJGY2ZGY3NDIyLWVlZTQtNDhkNS05NDg3LTE0NTZkYjdkZGE1YiIOCICQ+IyCNBDnn6qthDRA0cHR4dszSAASOBoYAgakLY/QAAAAAD0NEwIABqj1vAAAAAQCIAEo1IL2/4M0MIjD+f+DNDgYQKiOAkj11wFQ9oABIAIQFRgB" + } + ], + [ + { + "field": "@message", + "value": "{\"resource\":{\"attributes\":{\"telemetry.sdk.language\":\"python\",\"telemetry.sdk.name\":\"opentelemetry\",\"telemetry.sdk.version\":\"1.40.0\",\"service.name\":\"asdf_MyAgent.DEFAULT\",\"aws.log.group.names\":\"/aws/bedrock-agentcore/runtimes/asdf_MyAgent-3s5axvBC6Q-DEFAULT\",\"aws.log.stream.names\":\"otel-rt-logs\",\"deployment.environment.name\":\"bedrock-agentcore:default\",\"cloud.resource_id\":\"arn:aws:bedrock-agentcore:us-west-2:685197708687:runtime/asdf_MyAgent-3s5axvBC6Q/runtime-endpoint/DEFAULT:DEFAULT\",\"cloud.platform\":\"aws_bedrock_agentcore\",\"cloud.provider\":\"aws\",\"cloud.region\":\"us-west-2\",\"telemetry.auto.version\":\"0.17.0-aws\",\"aws.local.service\":\"asdf_MyAgent.DEFAULT\",\"aws.service.type\":\"gen_ai_agent\"}},\"scope\":{\"name\":\"strands.telemetry.tracer\"},\"timeUnixNano\":1787780030855979475,\"observedTimeUnixNano\":1787780033471374339,\"severityNumber\":9,\"severityText\":\"\",\"body\":{\"input\":{\"messages\":[{\"role\":\"user\",\"content\":{\"content\":\"[{\\\"text\\\": \\\"hi\\\"}]\"}}]},\"output\":{\"messages\":[{\"role\":\"assistant\",\"content\":{\"content\":\"[{\\\"text\\\": \\\"Hello! How can I help you today?\\\"}]\"}}]}},\"attributes\":{\"session.id\":\"00000000-0000-4000-8000-000000000001\",\"event.name\":\"strands.telemetry.tracer\"},\"flags\":1,\"traceId\":\"6a8f5bbc1882c05547b64a9246f17ca4\",\"spanId\":\"43f809568e52d1ca\"}" + }, + { + "field": "sessionId", + "value": "00000000-0000-4000-8000-000000000001" + }, + { + "field": "traceId", + "value": "6a8f5bbc1882c05547b64a9246f17ca4" + }, + { + "field": "spanId", + "value": "43f809568e52d1ca" + }, + { + "field": "@ptr", + "value": "Cs4BCo8BCkw2ODUxOTc3MDg2ODc6L2F3cy9iZWRyb2NrLWFnZW50Y29yZS9ydW50aW1lcy9hc2RmX015QWdlbnQtM3M1YXh2QkM2US1ERUZBVUxUEAAaJGY2ZGY3NDIyLWVlZTQtNDhkNS05NDg3LTE0NTZkYjdkZGE1YiIOCICQ+IyCNBDnn6qthDRA0cHR4dszSAASOBoYAgakLY/QAAAAAD0NEwIABqj1vAAAAAQCIAEo1IL2/4M0MIjD+f+DNDgYQKiOAkj11wFQ9oABIAIQFhgB" + } + ], + [ + { + "field": "@message", + "value": "{\"resource\":{\"attributes\":{\"telemetry.sdk.language\":\"python\",\"telemetry.sdk.name\":\"opentelemetry\",\"telemetry.sdk.version\":\"1.40.0\",\"service.name\":\"asdf_MyAgent.DEFAULT\",\"aws.log.group.names\":\"/aws/bedrock-agentcore/runtimes/asdf_MyAgent-3s5axvBC6Q-DEFAULT\",\"aws.log.stream.names\":\"otel-rt-logs\",\"deployment.environment.name\":\"bedrock-agentcore:default\",\"cloud.resource_id\":\"arn:aws:bedrock-agentcore:us-west-2:685197708687:runtime/asdf_MyAgent-3s5axvBC6Q/runtime-endpoint/DEFAULT:DEFAULT\",\"cloud.platform\":\"aws_bedrock_agentcore\",\"cloud.provider\":\"aws\",\"cloud.region\":\"us-west-2\",\"telemetry.auto.version\":\"0.17.0-aws\",\"aws.local.service\":\"asdf_MyAgent.DEFAULT\",\"aws.service.type\":\"gen_ai_agent\"}},\"scope\":{\"name\":\"strands.telemetry.tracer\"},\"timeUnixNano\":1787780030856119726,\"observedTimeUnixNano\":1787780033471475805,\"severityNumber\":9,\"severityText\":\"\",\"body\":{\"input\":{\"messages\":[{\"role\":\"system\",\"content\":\"\\n You are a helpful assistant. Use tools when appropriate.\\n \"},{\"role\":\"user\",\"content\":{\"content\":\"[{\\\"text\\\": \\\"hi\\\"}]\"}}]},\"output\":{\"messages\":[{\"role\":\"assistant\",\"content\":{\"message\":\"Hi there! How can I assist you today?\\n\",\"finish_reason\":\"end_turn\"}}]}},\"attributes\":{\"session.id\":\"00000000-0000-4000-8000-000000000001\",\"event.name\":\"strands.telemetry.tracer\"},\"flags\":1,\"traceId\":\"6a8f5bbc1882c05547b64a9246f17ca4\",\"spanId\":\"324e3a835d1af5c4\"}" + }, + { + "field": "sessionId", + "value": "00000000-0000-4000-8000-000000000001" + }, + { + "field": "traceId", + "value": "6a8f5bbc1882c05547b64a9246f17ca4" + }, + { + "field": "spanId", + "value": "324e3a835d1af5c4" + }, + { + "field": "@ptr", + "value": "Cs4BCo8BCkw2ODUxOTc3MDg2ODc6L2F3cy9iZWRyb2NrLWFnZW50Y29yZS9ydW50aW1lcy9hc2RmX015QWdlbnQtM3M1YXh2QkM2US1ERUZBVUxUEAAaJGY2ZGY3NDIyLWVlZTQtNDhkNS05NDg3LTE0NTZkYjdkZGE1YiIOCICQ+IyCNBDnn6qthDRA0cHR4dszSAASOBoYAgakLY/QAAAAAD0NEwIABqj1vAAAAAQCIAEo1IL2/4M0MIjD+f+DNDgYQKiOAkj11wFQ9oABIAIQFxgB" + } + ], + [ + { + "field": "@message", + "value": "{\"resource\":{\"attributes\":{\"telemetry.sdk.language\":\"python\",\"telemetry.sdk.name\":\"opentelemetry\",\"telemetry.sdk.version\":\"1.40.0\",\"service.name\":\"asdf_MyAgent.DEFAULT\",\"aws.log.group.names\":\"/aws/bedrock-agentcore/runtimes/asdf_MyAgent-3s5axvBC6Q-DEFAULT\",\"aws.log.stream.names\":\"otel-rt-logs\",\"deployment.environment.name\":\"bedrock-agentcore:default\",\"cloud.resource_id\":\"arn:aws:bedrock-agentcore:us-west-2:685197708687:runtime/asdf_MyAgent-3s5axvBC6Q/runtime-endpoint/DEFAULT:DEFAULT\",\"cloud.platform\":\"aws_bedrock_agentcore\",\"cloud.provider\":\"aws\",\"cloud.region\":\"us-west-2\",\"telemetry.auto.version\":\"0.17.0-aws\",\"aws.local.service\":\"asdf_MyAgent.DEFAULT\",\"aws.service.type\":\"gen_ai_agent\"}},\"scope\":{\"name\":\"strands.telemetry.tracer\"},\"timeUnixNano\":1787782045239528397,\"observedTimeUnixNano\":1787782046488355037,\"severityNumber\":9,\"severityText\":\"\",\"body\":{\"input\":{\"messages\":[{\"role\":\"user\",\"content\":{\"content\":\"[{\\\"text\\\": \\\"hi\\\"}]\"}}]}},\"attributes\":{\"session.id\":\"00000000-0000-4000-8000-000000000001\",\"event.name\":\"strands.telemetry.tracer\"},\"flags\":1,\"traceId\":\"6a8f639540ebe6cb6f30d5d80762c2e2\",\"spanId\":\"fc189f3b80a0c1f4\"}" + }, + { + "field": "sessionId", + "value": "00000000-0000-4000-8000-000000000001" + }, + { + "field": "traceId", + "value": "6a8f639540ebe6cb6f30d5d80762c2e2" + }, + { + "field": "spanId", + "value": "fc189f3b80a0c1f4" + }, + { + "field": "@ptr", + "value": "CswBCo8BCkw2ODUxOTc3MDg2ODc6L2F3cy9iZWRyb2NrLWFnZW50Y29yZS9ydW50aW1lcy9hc2RmX015QWdlbnQtM3M1YXh2QkM2US1ERUZBVUxUEAAaJGY2ZGY3NDIyLWVlZTQtNDhkNS05NDg3LTE0NTZkYjdkZGE1YiIOCICQ+IyCNBDnn6qthDRA0cHR4dszSAASNhoYAgakLY/QAAAAAD0VMD0ABqj2OYAAAAQCIAEo7Z/0gIQ0MJ3G9ICENDggQLfZAUisaFCuRyACEB0YAQ==" + } + ], + [ + { + "field": "@message", + "value": "{\"resource\":{\"attributes\":{\"telemetry.sdk.language\":\"python\",\"telemetry.sdk.name\":\"opentelemetry\",\"telemetry.sdk.version\":\"1.40.0\",\"service.name\":\"asdf_MyAgent.DEFAULT\",\"aws.log.group.names\":\"/aws/bedrock-agentcore/runtimes/asdf_MyAgent-3s5axvBC6Q-DEFAULT\",\"aws.log.stream.names\":\"otel-rt-logs\",\"deployment.environment.name\":\"bedrock-agentcore:default\",\"cloud.resource_id\":\"arn:aws:bedrock-agentcore:us-west-2:685197708687:runtime/asdf_MyAgent-3s5axvBC6Q/runtime-endpoint/DEFAULT:DEFAULT\",\"cloud.platform\":\"aws_bedrock_agentcore\",\"cloud.provider\":\"aws\",\"cloud.region\":\"us-west-2\",\"telemetry.auto.version\":\"0.17.0-aws\",\"aws.local.service\":\"asdf_MyAgent.DEFAULT\",\"aws.service.type\":\"gen_ai_agent\"}},\"scope\":{\"name\":\"strands.telemetry.tracer\"},\"timeUnixNano\":1787782045239678560,\"observedTimeUnixNano\":1787782046488465849,\"severityNumber\":9,\"severityText\":\"\",\"body\":{\"input\":{\"messages\":[{\"role\":\"system\",\"content\":\"\\n You are a helpful assistant. Use tools when appropriate.\\n \"},{\"role\":\"user\",\"content\":{\"content\":\"[{\\\"text\\\": \\\"hi\\\"}]\"}}]},\"output\":{\"messages\":[{\"role\":\"assistant\",\"content\":{\"message\":\"Hello! How can I help you today?\\n\",\"finish_reason\":\"end_turn\"}}]}},\"attributes\":{\"session.id\":\"00000000-0000-4000-8000-000000000001\",\"event.name\":\"strands.telemetry.tracer\"},\"flags\":1,\"traceId\":\"6a8f639540ebe6cb6f30d5d80762c2e2\",\"spanId\":\"6f925a9a78a2fe62\"}" + }, + { + "field": "sessionId", + "value": "00000000-0000-4000-8000-000000000001" + }, + { + "field": "traceId", + "value": "6a8f639540ebe6cb6f30d5d80762c2e2" + }, + { + "field": "spanId", + "value": "6f925a9a78a2fe62" + }, + { + "field": "@ptr", + "value": "CswBCo8BCkw2ODUxOTc3MDg2ODc6L2F3cy9iZWRyb2NrLWFnZW50Y29yZS9ydW50aW1lcy9hc2RmX015QWdlbnQtM3M1YXh2QkM2US1ERUZBVUxUEAAaJGY2ZGY3NDIyLWVlZTQtNDhkNS05NDg3LTE0NTZkYjdkZGE1YiIOCICQ+IyCNBDnn6qthDRA0cHR4dszSAASNhoYAgakLY/QAAAAAD0VMD0ABqj2OYAAAAQCIAEo7Z/0gIQ0MJ3G9ICENDggQLfZAUisaFCuRyACEB4YAQ==" + } + ], + [ + { + "field": "@message", + "value": "{\"resource\":{\"attributes\":{\"telemetry.sdk.language\":\"python\",\"telemetry.sdk.name\":\"opentelemetry\",\"telemetry.sdk.version\":\"1.40.0\",\"service.name\":\"asdf_MyAgent.DEFAULT\",\"aws.log.group.names\":\"/aws/bedrock-agentcore/runtimes/asdf_MyAgent-3s5axvBC6Q-DEFAULT\",\"aws.log.stream.names\":\"otel-rt-logs\",\"deployment.environment.name\":\"bedrock-agentcore:default\",\"cloud.resource_id\":\"arn:aws:bedrock-agentcore:us-west-2:685197708687:runtime/asdf_MyAgent-3s5axvBC6Q/runtime-endpoint/DEFAULT:DEFAULT\",\"cloud.platform\":\"aws_bedrock_agentcore\",\"cloud.provider\":\"aws\",\"cloud.region\":\"us-west-2\",\"telemetry.auto.version\":\"0.17.0-aws\",\"aws.local.service\":\"asdf_MyAgent.DEFAULT\",\"aws.service.type\":\"gen_ai_agent\"}},\"scope\":{\"name\":\"strands.telemetry.tracer\"},\"timeUnixNano\":1787782045239215762,\"observedTimeUnixNano\":1787782046488222333,\"severityNumber\":9,\"severityText\":\"\",\"body\":{\"input\":{\"messages\":[{\"role\":\"system\",\"content\":{\"content\":\"[{\\\"text\\\": \\\"\\\\n You are a helpful assistant. Use tools when appropriate.\\\\n \\\"}]\"}},{\"role\":\"user\",\"content\":{\"content\":\"[{\\\"text\\\": \\\"hi\\\"}]\"}}]},\"output\":{\"messages\":[{\"role\":\"assistant\",\"content\":{\"finish_reason\":\"end_turn\",\"message\":\"[{\\\"text\\\": \\\"Hello! How can I help you today?\\\"}]\"}}]}},\"attributes\":{\"session.id\":\"00000000-0000-4000-8000-000000000001\",\"event.name\":\"strands.telemetry.tracer\"},\"flags\":1,\"traceId\":\"6a8f639540ebe6cb6f30d5d80762c2e2\",\"spanId\":\"0925d4cc59be00d9\"}" + }, + { + "field": "sessionId", + "value": "00000000-0000-4000-8000-000000000001" + }, + { + "field": "traceId", + "value": "6a8f639540ebe6cb6f30d5d80762c2e2" + }, + { + "field": "spanId", + "value": "0925d4cc59be00d9" + }, + { + "field": "@ptr", + "value": "CswBCo8BCkw2ODUxOTc3MDg2ODc6L2F3cy9iZWRyb2NrLWFnZW50Y29yZS9ydW50aW1lcy9hc2RmX015QWdlbnQtM3M1YXh2QkM2US1ERUZBVUxUEAAaJGY2ZGY3NDIyLWVlZTQtNDhkNS05NDg3LTE0NTZkYjdkZGE1YiIOCICQ+IyCNBDnn6qthDRA0cHR4dszSAASNhoYAgakLY/QAAAAAD0VMD0ABqj2OYAAAAQCIAEo7Z/0gIQ0MJ3G9ICENDggQLfZAUisaFCuRyACEBwYAQ==" + } + ], + [ + { + "field": "@message", + "value": "{\"resource\":{\"attributes\":{\"telemetry.sdk.language\":\"python\",\"telemetry.sdk.name\":\"opentelemetry\",\"telemetry.sdk.version\":\"1.40.0\",\"service.name\":\"asdf_MyAgent.DEFAULT\",\"aws.log.group.names\":\"/aws/bedrock-agentcore/runtimes/asdf_MyAgent-3s5axvBC6Q-DEFAULT\",\"aws.log.stream.names\":\"otel-rt-logs\",\"deployment.environment.name\":\"bedrock-agentcore:default\",\"cloud.resource_id\":\"arn:aws:bedrock-agentcore:us-west-2:685197708687:runtime/asdf_MyAgent-3s5axvBC6Q/runtime-endpoint/DEFAULT:DEFAULT\",\"cloud.platform\":\"aws_bedrock_agentcore\",\"cloud.provider\":\"aws\",\"cloud.region\":\"us-west-2\",\"telemetry.auto.version\":\"0.17.0-aws\",\"aws.local.service\":\"asdf_MyAgent.DEFAULT\",\"aws.service.type\":\"gen_ai_agent\"}},\"scope\":{\"name\":\"strands.telemetry.tracer\"},\"timeUnixNano\":1787864481907136303,\"observedTimeUnixNano\":1787864486789333679,\"severityNumber\":9,\"severityText\":\"\",\"body\":{\"input\":{\"messages\":[{\"role\":\"system\",\"content\":{\"content\":\"[{\\\"text\\\": \\\"\\\\n You are a helpful assistant. Use tools when appropriate.\\\\n \\\"}]\"}},{\"role\":\"user\",\"content\":{\"content\":\"[{\\\"text\\\": \\\"Say hello and offer help\\\"}]\"}}]},\"output\":{\"messages\":[{\"role\":\"assistant\",\"content\":{\"finish_reason\":\"end_turn\",\"message\":\"[{\\\"text\\\": \\\"Hello! 👋 \\\\n\\\\nI'm here to help you with a variety of tasks. I can:\\\\n\\\\n- **Search the web** for current information, news, facts, or answers to questions\\\\n- **Fetch and read webpages** to get detailed content from specific URLs\\\\n- **Perform calculations** like adding numbers together\\\\n\\\\nWhat can I help you with today?\\\"}]\"}}]}},\"attributes\":{\"session.id\":\"00000000-0000-4000-8000-000000000001\",\"event.name\":\"strands.telemetry.tracer\"},\"flags\":1,\"traceId\":\"6a90a5987755c6e457ada5616a16dddb\",\"spanId\":\"d473e096a2db85a8\"}" + }, + { + "field": "sessionId", + "value": "00000000-0000-4000-8000-000000000001" + }, + { + "field": "traceId", + "value": "6a90a5987755c6e457ada5616a16dddb" + }, + { + "field": "spanId", + "value": "d473e096a2db85a8" + }, + { + "field": "@ptr", + "value": "CswBCo8BCkw2ODUxOTc3MDg2ODc6L2F3cy9iZWRyb2NrLWFnZW50Y29yZS9ydW50aW1lcy9hc2RmX015QWdlbnQtM3M1YXh2QkM2US1ERUZBVUxUEAAaJGY2ZGY3NDIyLWVlZTQtNDhkNS05NDg3LTE0NTZkYjdkZGE1YiIOCICQ+IyCNBDnn6qthDRA0cHR4dszSAASNhoYAgahOL2DAAAAAJJ5XU0ABqkKWdAAAAYyIAEojNybqIQ0MPOAnKiENDgkQJLgAUj9blD9TSACECEYAQ==" + } + ], + [ + { + "field": "@message", + "value": "{\"resource\":{\"attributes\":{\"telemetry.sdk.language\":\"python\",\"telemetry.sdk.name\":\"opentelemetry\",\"telemetry.sdk.version\":\"1.40.0\",\"service.name\":\"asdf_MyAgent.DEFAULT\",\"aws.log.group.names\":\"/aws/bedrock-agentcore/runtimes/asdf_MyAgent-3s5axvBC6Q-DEFAULT\",\"aws.log.stream.names\":\"otel-rt-logs\",\"deployment.environment.name\":\"bedrock-agentcore:default\",\"cloud.resource_id\":\"arn:aws:bedrock-agentcore:us-west-2:685197708687:runtime/asdf_MyAgent-3s5axvBC6Q/runtime-endpoint/DEFAULT:DEFAULT\",\"cloud.platform\":\"aws_bedrock_agentcore\",\"cloud.provider\":\"aws\",\"cloud.region\":\"us-west-2\",\"telemetry.auto.version\":\"0.17.0-aws\",\"aws.local.service\":\"asdf_MyAgent.DEFAULT\",\"aws.service.type\":\"gen_ai_agent\"}},\"scope\":{\"name\":\"strands.telemetry.tracer\"},\"timeUnixNano\":1787864481907743241,\"observedTimeUnixNano\":1787864486789635040,\"severityNumber\":9,\"severityText\":\"\",\"body\":{\"input\":{\"messages\":[{\"role\":\"system\",\"content\":\"\\n You are a helpful assistant. Use tools when appropriate.\\n \"},{\"role\":\"user\",\"content\":{\"content\":\"[{\\\"text\\\": \\\"Say hello and offer help\\\"}]\"}}]},\"output\":{\"messages\":[{\"role\":\"assistant\",\"content\":{\"message\":\"Hello! 👋 \\n\\nI'm here to help you with a variety of tasks. I can:\\n\\n- **Search the web** for current information, news, facts, or answers to questions\\n- **Fetch and read webpages** to get detailed content from specific URLs\\n- **Perform calculations** like adding numbers together\\n\\nWhat can I help you with today?\\n\",\"finish_reason\":\"end_turn\"}}]}},\"attributes\":{\"session.id\":\"00000000-0000-4000-8000-000000000001\",\"event.name\":\"strands.telemetry.tracer\"},\"flags\":1,\"traceId\":\"6a90a5987755c6e457ada5616a16dddb\",\"spanId\":\"2619914761138b83\"}" + }, + { + "field": "sessionId", + "value": "00000000-0000-4000-8000-000000000001" + }, + { + "field": "traceId", + "value": "6a90a5987755c6e457ada5616a16dddb" + }, + { + "field": "spanId", + "value": "2619914761138b83" + }, + { + "field": "@ptr", + "value": "CswBCo8BCkw2ODUxOTc3MDg2ODc6L2F3cy9iZWRyb2NrLWFnZW50Y29yZS9ydW50aW1lcy9hc2RmX015QWdlbnQtM3M1YXh2QkM2US1ERUZBVUxUEAAaJGY2ZGY3NDIyLWVlZTQtNDhkNS05NDg3LTE0NTZkYjdkZGE1YiIOCICQ+IyCNBDnn6qthDRA0cHR4dszSAASNhoYAgahOL2DAAAAAJJ5XU0ABqkKWdAAAAYyIAEojNybqIQ0MPOAnKiENDgkQJLgAUj9blD9TSACECMYAQ==" + } + ], + [ + { + "field": "@message", + "value": "{\"resource\":{\"attributes\":{\"telemetry.sdk.language\":\"python\",\"telemetry.sdk.name\":\"opentelemetry\",\"telemetry.sdk.version\":\"1.40.0\",\"service.name\":\"asdf_MyAgent.DEFAULT\",\"aws.log.group.names\":\"/aws/bedrock-agentcore/runtimes/asdf_MyAgent-3s5axvBC6Q-DEFAULT\",\"aws.log.stream.names\":\"otel-rt-logs\",\"deployment.environment.name\":\"bedrock-agentcore:default\",\"cloud.resource_id\":\"arn:aws:bedrock-agentcore:us-west-2:685197708687:runtime/asdf_MyAgent-3s5axvBC6Q/runtime-endpoint/DEFAULT:DEFAULT\",\"cloud.platform\":\"aws_bedrock_agentcore\",\"cloud.provider\":\"aws\",\"cloud.region\":\"us-west-2\",\"telemetry.auto.version\":\"0.17.0-aws\",\"aws.local.service\":\"asdf_MyAgent.DEFAULT\",\"aws.service.type\":\"gen_ai_agent\"}},\"scope\":{\"name\":\"strands.telemetry.tracer\"},\"timeUnixNano\":1787864481907561373,\"observedTimeUnixNano\":1787864486789498361,\"severityNumber\":9,\"severityText\":\"\",\"body\":{\"input\":{\"messages\":[{\"role\":\"user\",\"content\":{\"content\":\"[{\\\"text\\\": \\\"Say hello and offer help\\\"}]\"}}]}},\"attributes\":{\"session.id\":\"00000000-0000-4000-8000-000000000001\",\"event.name\":\"strands.telemetry.tracer\"},\"flags\":1,\"traceId\":\"6a90a5987755c6e457ada5616a16dddb\",\"spanId\":\"17fb5fcf047ef303\"}" + }, + { + "field": "sessionId", + "value": "00000000-0000-4000-8000-000000000001" + }, + { + "field": "traceId", + "value": "6a90a5987755c6e457ada5616a16dddb" + }, + { + "field": "spanId", + "value": "17fb5fcf047ef303" + }, + { + "field": "@ptr", + "value": "CswBCo8BCkw2ODUxOTc3MDg2ODc6L2F3cy9iZWRyb2NrLWFnZW50Y29yZS9ydW50aW1lcy9hc2RmX015QWdlbnQtM3M1YXh2QkM2US1ERUZBVUxUEAAaJGY2ZGY3NDIyLWVlZTQtNDhkNS05NDg3LTE0NTZkYjdkZGE1YiIOCICQ+IyCNBDnn6qthDRA0cHR4dszSAASNhoYAgahOL2DAAAAAJJ5XU0ABqkKWdAAAAYyIAEojNybqIQ0MPOAnKiENDgkQJLgAUj9blD9TSACECIYAQ==" + } + ], + [ + { + "field": "@message", + "value": "{\"resource\":{\"attributes\":{\"telemetry.sdk.language\":\"python\",\"telemetry.sdk.name\":\"opentelemetry\",\"telemetry.sdk.version\":\"1.40.0\",\"service.name\":\"asdf_MyAgent.DEFAULT\",\"aws.log.group.names\":\"/aws/bedrock-agentcore/runtimes/asdf_MyAgent-3s5axvBC6Q-DEFAULT\",\"aws.log.stream.names\":\"otel-rt-logs\",\"deployment.environment.name\":\"bedrock-agentcore:default\",\"cloud.resource_id\":\"arn:aws:bedrock-agentcore:us-west-2:685197708687:runtime/asdf_MyAgent-3s5axvBC6Q/runtime-endpoint/DEFAULT:DEFAULT\",\"cloud.platform\":\"aws_bedrock_agentcore\",\"cloud.provider\":\"aws\",\"cloud.region\":\"us-west-2\",\"telemetry.auto.version\":\"0.17.0-aws\",\"aws.local.service\":\"asdf_MyAgent.DEFAULT\",\"aws.service.type\":\"gen_ai_agent\"}},\"scope\":{\"name\":\"strands.telemetry.tracer\"},\"timeUnixNano\":1787864908109701143,\"observedTimeUnixNano\":1787864911841960802,\"severityNumber\":9,\"severityText\":\"\",\"body\":{\"input\":{\"messages\":[{\"role\":\"system\",\"content\":{\"content\":\"[{\\\"text\\\": \\\"\\\\n You are a helpful assistant. Use tools when appropriate.\\\\n \\\"}]\"}},{\"role\":\"user\",\"content\":{\"content\":\"[{\\\"text\\\": \\\"Say hello and offer help\\\"}]\"}}]},\"output\":{\"messages\":[{\"role\":\"assistant\",\"content\":{\"content\":\"[{\\\"text\\\": \\\"Hello! 👋 \\\\n\\\\nI'm here to help you with a variety of tasks. I can:\\\\n\\\\n- **Search the web** for current information, news, facts, or answers to questions\\\\n- **Fetch and read webpages** to get detailed content from specific URLs\\\\n- **Perform calculations** like adding numbers together\\\\n\\\\nWhat can I help you with today?\\\"}]\"}},{\"role\":\"assistant\",\"content\":{\"finish_reason\":\"end_turn\",\"message\":\"[{\\\"text\\\": \\\"Hello! 👋 Welcome!\\\\n\\\\nI'm here to assist you today. I can help you with:\\\\n\\\\n- **Web searches** - Find current information, news, research, or answers to questions on any topic\\\\n- **Reading webpages** - Extract and read content from specific URLs or articles\\\\n- **Simple calculations** - Add numbers together\\\\n\\\\nWhat would you like help with?\\\"}]\"}}]}},\"attributes\":{\"session.id\":\"00000000-0000-4000-8000-000000000001\",\"event.name\":\"strands.telemetry.tracer\"},\"flags\":1,\"traceId\":\"6a90a74750f33f567076943f7bce7966\",\"spanId\":\"3bf0732214c6e32c\"}" + }, + { + "field": "sessionId", + "value": "00000000-0000-4000-8000-000000000001" + }, + { + "field": "traceId", + "value": "6a90a74750f33f567076943f7bce7966" + }, + { + "field": "spanId", + "value": "3bf0732214c6e32c" + }, + { + "field": "@ptr", + "value": "CssBCo8BCkw2ODUxOTc3MDg2ODc6L2F3cy9iZWRyb2NrLWFnZW50Y29yZS9ydW50aW1lcy9hc2RmX015QWdlbnQtM3M1YXh2QkM2US1ERUZBVUxUEAAaJGY2ZGY3NDIyLWVlZTQtNDhkNS05NDg3LTE0NTZkYjdkZGE1YiIOCICQ+IyCNBDnn6qthDRA0cHR4dszSAASNRoYAgahOL2DAAAAAJJ6V08ABqkKdOAAAAYyIAEon/i1qIQ0MM6CtqiENDgGQIk5SIhKUNMvIAIQARgB" + } + ], + [ + { + "field": "@message", + "value": "{\"resource\":{\"attributes\":{\"telemetry.sdk.language\":\"python\",\"telemetry.sdk.name\":\"opentelemetry\",\"telemetry.sdk.version\":\"1.40.0\",\"service.name\":\"asdf_MyAgent.DEFAULT\",\"aws.log.group.names\":\"/aws/bedrock-agentcore/runtimes/asdf_MyAgent-3s5axvBC6Q-DEFAULT\",\"aws.log.stream.names\":\"otel-rt-logs\",\"deployment.environment.name\":\"bedrock-agentcore:default\",\"cloud.resource_id\":\"arn:aws:bedrock-agentcore:us-west-2:685197708687:runtime/asdf_MyAgent-3s5axvBC6Q/runtime-endpoint/DEFAULT:DEFAULT\",\"cloud.platform\":\"aws_bedrock_agentcore\",\"cloud.provider\":\"aws\",\"cloud.region\":\"us-west-2\",\"telemetry.auto.version\":\"0.17.0-aws\",\"aws.local.service\":\"asdf_MyAgent.DEFAULT\",\"aws.service.type\":\"gen_ai_agent\"}},\"scope\":{\"name\":\"strands.telemetry.tracer\"},\"timeUnixNano\":1787864908110047680,\"observedTimeUnixNano\":1787864911842132631,\"severityNumber\":9,\"severityText\":\"\",\"body\":{\"input\":{\"messages\":[{\"role\":\"user\",\"content\":{\"content\":\"[{\\\"text\\\": \\\"Say hello and offer help\\\"}]\"}}]},\"output\":{\"messages\":[{\"role\":\"assistant\",\"content\":{\"content\":\"[{\\\"text\\\": \\\"Hello! 👋 \\\\n\\\\nI'm here to help you with a variety of tasks. I can:\\\\n\\\\n- **Search the web** for current information, news, facts, or answers to questions\\\\n- **Fetch and read webpages** to get detailed content from specific URLs\\\\n- **Perform calculations** like adding numbers together\\\\n\\\\nWhat can I help you with today?\\\"}]\"}}]}},\"attributes\":{\"session.id\":\"00000000-0000-4000-8000-000000000001\",\"event.name\":\"strands.telemetry.tracer\"},\"flags\":1,\"traceId\":\"6a90a74750f33f567076943f7bce7966\",\"spanId\":\"5e744ea5e19f1053\"}" + }, + { + "field": "sessionId", + "value": "00000000-0000-4000-8000-000000000001" + }, + { + "field": "traceId", + "value": "6a90a74750f33f567076943f7bce7966" + }, + { + "field": "spanId", + "value": "5e744ea5e19f1053" + }, + { + "field": "@ptr", + "value": "CssBCo8BCkw2ODUxOTc3MDg2ODc6L2F3cy9iZWRyb2NrLWFnZW50Y29yZS9ydW50aW1lcy9hc2RmX015QWdlbnQtM3M1YXh2QkM2US1ERUZBVUxUEAAaJGY2ZGY3NDIyLWVlZTQtNDhkNS05NDg3LTE0NTZkYjdkZGE1YiIOCICQ+IyCNBDnn6qthDRA0cHR4dszSAASNRoYAgahOL2DAAAAAJJ6V08ABqkKdOAAAAYyIAEon/i1qIQ0MM6CtqiENDgGQIk5SIhKUNMvIAIQAhgB" + } + ], + [ + { + "field": "@message", + "value": "{\"resource\":{\"attributes\":{\"telemetry.sdk.language\":\"python\",\"telemetry.sdk.name\":\"opentelemetry\",\"telemetry.sdk.version\":\"1.40.0\",\"service.name\":\"asdf_MyAgent.DEFAULT\",\"aws.log.group.names\":\"/aws/bedrock-agentcore/runtimes/asdf_MyAgent-3s5axvBC6Q-DEFAULT\",\"aws.log.stream.names\":\"otel-rt-logs\",\"deployment.environment.name\":\"bedrock-agentcore:default\",\"cloud.resource_id\":\"arn:aws:bedrock-agentcore:us-west-2:685197708687:runtime/asdf_MyAgent-3s5axvBC6Q/runtime-endpoint/DEFAULT:DEFAULT\",\"cloud.platform\":\"aws_bedrock_agentcore\",\"cloud.provider\":\"aws\",\"cloud.region\":\"us-west-2\",\"telemetry.auto.version\":\"0.17.0-aws\",\"aws.local.service\":\"asdf_MyAgent.DEFAULT\",\"aws.service.type\":\"gen_ai_agent\"}},\"scope\":{\"name\":\"strands.telemetry.tracer\"},\"timeUnixNano\":1787864908110229781,\"observedTimeUnixNano\":1787864911842264993,\"severityNumber\":9,\"severityText\":\"\",\"body\":{\"input\":{\"messages\":[{\"role\":\"system\",\"content\":\"\\n You are a helpful assistant. Use tools when appropriate.\\n \"},{\"role\":\"user\",\"content\":{\"content\":\"[{\\\"text\\\": \\\"Say hello and offer help\\\"}]\"}}]},\"output\":{\"messages\":[{\"role\":\"assistant\",\"content\":{\"message\":\"Hello! 👋 Welcome!\\n\\nI'm here to assist you today. I can help you with:\\n\\n- **Web searches** - Find current information, news, research, or answers to questions on any topic\\n- **Reading webpages** - Extract and read content from specific URLs or articles\\n- **Simple calculations** - Add numbers together\\n\\nWhat would you like help with?\\n\",\"finish_reason\":\"end_turn\"}}]}},\"attributes\":{\"session.id\":\"00000000-0000-4000-8000-000000000001\",\"event.name\":\"strands.telemetry.tracer\"},\"flags\":1,\"traceId\":\"6a90a74750f33f567076943f7bce7966\",\"spanId\":\"b970895264868438\"}" + }, + { + "field": "sessionId", + "value": "00000000-0000-4000-8000-000000000001" + }, + { + "field": "traceId", + "value": "6a90a74750f33f567076943f7bce7966" + }, + { + "field": "spanId", + "value": "b970895264868438" + }, + { + "field": "@ptr", + "value": "CssBCo8BCkw2ODUxOTc3MDg2ODc6L2F3cy9iZWRyb2NrLWFnZW50Y29yZS9ydW50aW1lcy9hc2RmX015QWdlbnQtM3M1YXh2QkM2US1ERUZBVUxUEAAaJGY2ZGY3NDIyLWVlZTQtNDhkNS05NDg3LTE0NTZkYjdkZGE1YiIOCICQ+IyCNBDnn6qthDRA0cHR4dszSAASNRoYAgahOL2DAAAAAJJ6V08ABqkKdOAAAAYyIAEon/i1qIQ0MM6CtqiENDgGQIk5SIhKUNMvIAIQAxgB" + } + ] + ], + "statistics": { + "recordsMatched": 15, + "recordsScanned": 142, + "estimatedRecordsSkipped": 20, + "bytesScanned": 134730, + "estimatedBytesSkipped": 23963, + "logGroupsScanned": 1 + }, + "status": "Complete" +} \ No newline at end of file diff --git a/src/handlers/eval/ondemand/__fixtures__/GetQueryResultsCommand.70382c1aa9a94e3a.json b/src/handlers/eval/ondemand/__fixtures__/GetQueryResultsCommand.70382c1aa9a94e3a.json new file mode 100644 index 000000000..9e456cd44 --- /dev/null +++ b/src/handlers/eval/ondemand/__fixtures__/GetQueryResultsCommand.70382c1aa9a94e3a.json @@ -0,0 +1,960 @@ +{ + "queryLanguage": "CWLI", + "results": [ + [ + { + "field": "@message", + "value": "{\"resource\":{\"attributes\":{\"deployment.environment.name\":\"bedrock-agentcore:default\",\"aws.local.service\":\"asdf_MyAgent.DEFAULT\",\"service.name\":\"asdf_MyAgent.DEFAULT\",\"cloud.region\":\"us-west-2\",\"aws.log.stream.names\":\"otel-rt-logs\",\"telemetry.sdk.name\":\"opentelemetry\",\"aws.service.type\":\"gen_ai_agent\",\"telemetry.sdk.language\":\"python\",\"cloud.provider\":\"aws\",\"cloud.resource_id\":\"arn:aws:bedrock-agentcore:us-west-2:685197708687:runtime/asdf_MyAgent-3s5axvBC6Q/runtime-endpoint/DEFAULT:DEFAULT\",\"aws.log.group.names\":\"/aws/bedrock-agentcore/runtimes/asdf_MyAgent-3s5axvBC6Q-DEFAULT\",\"telemetry.sdk.version\":\"1.40.0\",\"cloud.platform\":\"aws_bedrock_agentcore\",\"telemetry.auto.version\":\"0.17.0-aws\"}},\"scope\":{\"name\":\"opentelemetry.instrumentation.urllib3\",\"version\":\"0.61b0\"},\"traceId\":\"6a8f5b461983df572e2aca3264d32d4d\",\"spanId\":\"e4198cd154238815\",\"parentSpanId\":\"3497dea65b2e2550\",\"flags\":256,\"name\":\"PUT\",\"kind\":\"CLIENT\",\"startTimeUnixNano\":1787779915187794321,\"endTimeUnixNano\":1787779915188812325,\"durationNano\":1018004,\"attributes\":{\"aws.local.service\":\"asdf_MyAgent.DEFAULT\",\"telemetry.extended\":\"true\",\"http.url\":\"http://169.254.169.254/latest/api/token\",\"aws.remote.service\":\"169.254.169.254\",\"aws.local.environment\":\"bedrock-agentcore:default\",\"aws.remote.operation\":\"PUT /latest\",\"http.status_code\":200,\"aws.local.operation\":\"UnmappedOperation\",\"aws.span.kind\":\"CLIENT\",\"PlatformType\":\"AWS::BedrockAgentCore\",\"http.method\":\"PUT\",\"http.response.status_code\":200,\"session.id\":\"00000000-0000-4000-8000-000000000001\"},\"status\":{\"code\":\"UNSET\"}}" + }, + { + "field": "sessionId", + "value": "00000000-0000-4000-8000-000000000001" + }, + { + "field": "traceId", + "value": "6a8f5b461983df572e2aca3264d32d4d" + }, + { + "field": "spanId", + "value": "e4198cd154238815" + }, + { + "field": "@ptr", + "value": "CpwBCmAKFjY4NTE5NzcwODY4Nzphd3Mvc3BhbnMQABokZTE0NTI5MzgtZjVjZi00OTc1LWE0N2QtYzk3YzMwZmFlY2ZlIg4IgJD4jII0EOefqq2ENDjv37W+yTNAyO+wktMzSAASNhoYAganxnVsAAAAACC2BcQABqj1tNAAAAXCIAEotLvy/4M0MLvQ8v+DNDgLQNCZAUijZ1COPCACEAAYAQ==" + } + ], + [ + { + "field": "@message", + "value": "{\"resource\":{\"attributes\":{\"deployment.environment.name\":\"bedrock-agentcore:default\",\"aws.local.service\":\"asdf_MyAgent.DEFAULT\",\"service.name\":\"asdf_MyAgent.DEFAULT\",\"cloud.region\":\"us-west-2\",\"aws.log.stream.names\":\"otel-rt-logs\",\"telemetry.sdk.name\":\"opentelemetry\",\"aws.service.type\":\"gen_ai_agent\",\"telemetry.sdk.language\":\"python\",\"cloud.provider\":\"aws\",\"cloud.resource_id\":\"arn:aws:bedrock-agentcore:us-west-2:685197708687:runtime/asdf_MyAgent-3s5axvBC6Q/runtime-endpoint/DEFAULT:DEFAULT\",\"aws.log.group.names\":\"/aws/bedrock-agentcore/runtimes/asdf_MyAgent-3s5axvBC6Q-DEFAULT\",\"telemetry.sdk.version\":\"1.40.0\",\"cloud.platform\":\"aws_bedrock_agentcore\",\"telemetry.auto.version\":\"0.17.0-aws\"}},\"scope\":{\"name\":\"opentelemetry.instrumentation.urllib3\",\"version\":\"0.61b0\"},\"traceId\":\"6a8f5b461983df572e2aca3264d32d4d\",\"spanId\":\"664e729f0f3e33fe\",\"parentSpanId\":\"3497dea65b2e2550\",\"flags\":256,\"name\":\"GET\",\"kind\":\"CLIENT\",\"startTimeUnixNano\":1787779915189208828,\"endTimeUnixNano\":1787779915189765731,\"durationNano\":556903,\"attributes\":{\"aws.local.service\":\"asdf_MyAgent.DEFAULT\",\"telemetry.extended\":\"true\",\"http.url\":\"http://169.254.169.254/latest/meta-data/iam/security-credentials/\",\"aws.remote.service\":\"169.254.169.254\",\"aws.local.environment\":\"bedrock-agentcore:default\",\"aws.remote.operation\":\"GET /latest\",\"http.status_code\":200,\"aws.local.operation\":\"UnmappedOperation\",\"aws.span.kind\":\"CLIENT\",\"PlatformType\":\"AWS::BedrockAgentCore\",\"http.method\":\"GET\",\"http.response.status_code\":200,\"session.id\":\"00000000-0000-4000-8000-000000000001\"},\"status\":{\"code\":\"UNSET\"}}" + }, + { + "field": "sessionId", + "value": "00000000-0000-4000-8000-000000000001" + }, + { + "field": "traceId", + "value": "6a8f5b461983df572e2aca3264d32d4d" + }, + { + "field": "spanId", + "value": "664e729f0f3e33fe" + }, + { + "field": "@ptr", + "value": "CpwBCmAKFjY4NTE5NzcwODY4Nzphd3Mvc3BhbnMQABokZTE0NTI5MzgtZjVjZi00OTc1LWE0N2QtYzk3YzMwZmFlY2ZlIg4IgJD4jII0EOefqq2ENDjv37W+yTNAyO+wktMzSAASNhoYAganxnVsAAAAACC2BcQABqj1tNAAAAXCIAEotLvy/4M0MLvQ8v+DNDgLQNCZAUijZ1COPCACEAEYAQ==" + } + ], + [ + { + "field": "@message", + "value": "{\"resource\":{\"attributes\":{\"deployment.environment.name\":\"bedrock-agentcore:default\",\"aws.local.service\":\"asdf_MyAgent.DEFAULT\",\"service.name\":\"asdf_MyAgent.DEFAULT\",\"cloud.region\":\"us-west-2\",\"aws.log.stream.names\":\"otel-rt-logs\",\"telemetry.sdk.name\":\"opentelemetry\",\"aws.service.type\":\"gen_ai_agent\",\"telemetry.sdk.language\":\"python\",\"cloud.provider\":\"aws\",\"cloud.resource_id\":\"arn:aws:bedrock-agentcore:us-west-2:685197708687:runtime/asdf_MyAgent-3s5axvBC6Q/runtime-endpoint/DEFAULT:DEFAULT\",\"aws.log.group.names\":\"/aws/bedrock-agentcore/runtimes/asdf_MyAgent-3s5axvBC6Q-DEFAULT\",\"telemetry.sdk.version\":\"1.40.0\",\"cloud.platform\":\"aws_bedrock_agentcore\",\"telemetry.auto.version\":\"0.17.0-aws\"}},\"scope\":{\"name\":\"opentelemetry.instrumentation.urllib3\",\"version\":\"0.61b0\"},\"traceId\":\"6a8f5b461983df572e2aca3264d32d4d\",\"spanId\":\"8a9482865c52e8fc\",\"parentSpanId\":\"3497dea65b2e2550\",\"flags\":256,\"name\":\"GET\",\"kind\":\"CLIENT\",\"startTimeUnixNano\":1787779915190080195,\"endTimeUnixNano\":1787779915190585314,\"durationNano\":505119,\"attributes\":{\"aws.local.service\":\"asdf_MyAgent.DEFAULT\",\"telemetry.extended\":\"true\",\"http.url\":\"http://169.254.169.254/latest/meta-data/iam/security-credentials/execution_role\",\"aws.remote.service\":\"169.254.169.254\",\"aws.local.environment\":\"bedrock-agentcore:default\",\"aws.remote.operation\":\"GET /latest\",\"http.status_code\":200,\"aws.local.operation\":\"UnmappedOperation\",\"aws.span.kind\":\"CLIENT\",\"PlatformType\":\"AWS::BedrockAgentCore\",\"http.method\":\"GET\",\"http.response.status_code\":200,\"session.id\":\"00000000-0000-4000-8000-000000000001\"},\"status\":{\"code\":\"UNSET\"}}" + }, + { + "field": "sessionId", + "value": "00000000-0000-4000-8000-000000000001" + }, + { + "field": "traceId", + "value": "6a8f5b461983df572e2aca3264d32d4d" + }, + { + "field": "spanId", + "value": "8a9482865c52e8fc" + }, + { + "field": "@ptr", + "value": "CpwBCmAKFjY4NTE5NzcwODY4Nzphd3Mvc3BhbnMQABokZTE0NTI5MzgtZjVjZi00OTc1LWE0N2QtYzk3YzMwZmFlY2ZlIg4IgJD4jII0EOefqq2ENDjv37W+yTNAyO+wktMzSAASNhoYAganxnVsAAAAACC2BcQABqj1tNAAAAXCIAEotLvy/4M0MLvQ8v+DNDgLQNCZAUijZ1COPCACEAIYAQ==" + } + ], + [ + { + "field": "@message", + "value": "{\"resource\":{\"attributes\":{\"deployment.environment.name\":\"bedrock-agentcore:default\",\"aws.local.service\":\"asdf_MyAgent.DEFAULT\",\"service.name\":\"asdf_MyAgent.DEFAULT\",\"cloud.region\":\"us-west-2\",\"aws.log.stream.names\":\"otel-rt-logs\",\"telemetry.sdk.name\":\"opentelemetry\",\"aws.service.type\":\"gen_ai_agent\",\"telemetry.sdk.language\":\"python\",\"cloud.provider\":\"aws\",\"cloud.resource_id\":\"arn:aws:bedrock-agentcore:us-west-2:685197708687:runtime/asdf_MyAgent-3s5axvBC6Q/runtime-endpoint/DEFAULT:DEFAULT\",\"aws.log.group.names\":\"/aws/bedrock-agentcore/runtimes/asdf_MyAgent-3s5axvBC6Q-DEFAULT\",\"telemetry.sdk.version\":\"1.40.0\",\"cloud.platform\":\"aws_bedrock_agentcore\",\"telemetry.auto.version\":\"0.17.0-aws\"}},\"scope\":{\"name\":\"opentelemetry.instrumentation.httpx\",\"version\":\"0.61b0\"},\"traceId\":\"6a8f5b461983df572e2aca3264d32d4d\",\"spanId\":\"8aa39d663ee028ef\",\"parentSpanId\":\"3497dea65b2e2550\",\"flags\":256,\"name\":\"POST\",\"kind\":\"CLIENT\",\"startTimeUnixNano\":1787779915296590217,\"endTimeUnixNano\":1787779915340065584,\"durationNano\":43475367,\"attributes\":{\"aws.local.service\":\"asdf_MyAgent.DEFAULT\",\"telemetry.extended\":\"true\",\"http.url\":\"https://bugbashgw1776978672-zsy8cbqwts.gateway.bedrock-agentcore.us-west-2.amazonaws.com/mcp\",\"aws.remote.service\":\"bugbashgw1776978672-zsy8cbqwts.gateway.bedrock-agentcore.us-west-2.amazonaws.com\",\"aws.local.environment\":\"bedrock-agentcore:default\",\"aws.remote.operation\":\"POST /mcp\",\"http.status_code\":200,\"aws.local.operation\":\"UnmappedOperation\",\"aws.span.kind\":\"CLIENT\",\"PlatformType\":\"AWS::BedrockAgentCore\",\"http.method\":\"POST\",\"http.response.status_code\":200,\"session.id\":\"00000000-0000-4000-8000-000000000001\"},\"status\":{\"code\":\"UNSET\"}}" + }, + { + "field": "sessionId", + "value": "00000000-0000-4000-8000-000000000001" + }, + { + "field": "traceId", + "value": "6a8f5b461983df572e2aca3264d32d4d" + }, + { + "field": "spanId", + "value": "8aa39d663ee028ef" + }, + { + "field": "@ptr", + "value": "CpwBCmAKFjY4NTE5NzcwODY4Nzphd3Mvc3BhbnMQABokZTE0NTI5MzgtZjVjZi00OTc1LWE0N2QtYzk3YzMwZmFlY2ZlIg4IgJD4jII0EOefqq2ENDjv37W+yTNAyO+wktMzSAASNhoYAganxnVsAAAAACC2BcQABqj1tNAAAAXCIAEotLvy/4M0MLvQ8v+DNDgLQNCZAUijZ1COPCACEAMYAQ==" + } + ], + [ + { + "field": "@message", + "value": "{\"resource\":{\"attributes\":{\"deployment.environment.name\":\"bedrock-agentcore:default\",\"aws.local.service\":\"asdf_MyAgent.DEFAULT\",\"service.name\":\"asdf_MyAgent.DEFAULT\",\"cloud.region\":\"us-west-2\",\"aws.log.stream.names\":\"otel-rt-logs\",\"telemetry.sdk.name\":\"opentelemetry\",\"aws.service.type\":\"gen_ai_agent\",\"telemetry.sdk.language\":\"python\",\"cloud.provider\":\"aws\",\"cloud.resource_id\":\"arn:aws:bedrock-agentcore:us-west-2:685197708687:runtime/asdf_MyAgent-3s5axvBC6Q/runtime-endpoint/DEFAULT:DEFAULT\",\"aws.log.group.names\":\"/aws/bedrock-agentcore/runtimes/asdf_MyAgent-3s5axvBC6Q-DEFAULT\",\"telemetry.sdk.version\":\"1.40.0\",\"cloud.platform\":\"aws_bedrock_agentcore\",\"telemetry.auto.version\":\"0.17.0-aws\"}},\"scope\":{\"name\":\"opentelemetry.instrumentation.httpx\",\"version\":\"0.61b0\"},\"traceId\":\"6a8f5b461983df572e2aca3264d32d4d\",\"spanId\":\"c3411a35a4406d31\",\"parentSpanId\":\"3497dea65b2e2550\",\"flags\":256,\"name\":\"POST\",\"kind\":\"CLIENT\",\"startTimeUnixNano\":1787779915342535411,\"endTimeUnixNano\":1787779915372977571,\"durationNano\":30442160,\"attributes\":{\"aws.local.service\":\"asdf_MyAgent.DEFAULT\",\"telemetry.extended\":\"true\",\"http.url\":\"https://bugbashgw1776978672-zsy8cbqwts.gateway.bedrock-agentcore.us-west-2.amazonaws.com/mcp\",\"aws.remote.service\":\"bugbashgw1776978672-zsy8cbqwts.gateway.bedrock-agentcore.us-west-2.amazonaws.com\",\"aws.local.environment\":\"bedrock-agentcore:default\",\"aws.remote.operation\":\"POST /mcp\",\"http.status_code\":202,\"aws.local.operation\":\"UnmappedOperation\",\"aws.span.kind\":\"CLIENT\",\"PlatformType\":\"AWS::BedrockAgentCore\",\"http.method\":\"POST\",\"http.response.status_code\":202,\"session.id\":\"00000000-0000-4000-8000-000000000001\"},\"status\":{\"code\":\"UNSET\"}}" + }, + { + "field": "sessionId", + "value": "00000000-0000-4000-8000-000000000001" + }, + { + "field": "traceId", + "value": "6a8f5b461983df572e2aca3264d32d4d" + }, + { + "field": "spanId", + "value": "c3411a35a4406d31" + }, + { + "field": "@ptr", + "value": "CpwBCmAKFjY4NTE5NzcwODY4Nzphd3Mvc3BhbnMQABokZTE0NTI5MzgtZjVjZi00OTc1LWE0N2QtYzk3YzMwZmFlY2ZlIg4IgJD4jII0EOefqq2ENDjv37W+yTNAyO+wktMzSAASNhoYAganxnVsAAAAACC2BcQABqj1tNAAAAXCIAEotLvy/4M0MLvQ8v+DNDgLQNCZAUijZ1COPCACEAQYAQ==" + } + ], + [ + { + "field": "@message", + "value": "{\"resource\":{\"attributes\":{\"deployment.environment.name\":\"bedrock-agentcore:default\",\"aws.local.service\":\"asdf_MyAgent.DEFAULT\",\"service.name\":\"asdf_MyAgent.DEFAULT\",\"cloud.region\":\"us-west-2\",\"aws.log.stream.names\":\"otel-rt-logs\",\"telemetry.sdk.name\":\"opentelemetry\",\"aws.service.type\":\"gen_ai_agent\",\"telemetry.sdk.language\":\"python\",\"cloud.provider\":\"aws\",\"cloud.resource_id\":\"arn:aws:bedrock-agentcore:us-west-2:685197708687:runtime/asdf_MyAgent-3s5axvBC6Q/runtime-endpoint/DEFAULT:DEFAULT\",\"aws.log.group.names\":\"/aws/bedrock-agentcore/runtimes/asdf_MyAgent-3s5axvBC6Q-DEFAULT\",\"telemetry.sdk.version\":\"1.40.0\",\"cloud.platform\":\"aws_bedrock_agentcore\",\"telemetry.auto.version\":\"0.17.0-aws\"}},\"scope\":{\"name\":\"opentelemetry.instrumentation.httpx\",\"version\":\"0.61b0\"},\"traceId\":\"6a8f5b461983df572e2aca3264d32d4d\",\"spanId\":\"92319b090266de53\",\"parentSpanId\":\"3497dea65b2e2550\",\"flags\":256,\"name\":\"POST\",\"kind\":\"CLIENT\",\"startTimeUnixNano\":1787779915374228814,\"endTimeUnixNano\":1787779915441287083,\"durationNano\":67058269,\"attributes\":{\"aws.local.service\":\"asdf_MyAgent.DEFAULT\",\"telemetry.extended\":\"true\",\"http.url\":\"https://bugbashgw1776978672-zsy8cbqwts.gateway.bedrock-agentcore.us-west-2.amazonaws.com/mcp\",\"aws.remote.service\":\"bugbashgw1776978672-zsy8cbqwts.gateway.bedrock-agentcore.us-west-2.amazonaws.com\",\"aws.local.environment\":\"bedrock-agentcore:default\",\"aws.remote.operation\":\"POST /mcp\",\"http.status_code\":200,\"aws.local.operation\":\"UnmappedOperation\",\"aws.span.kind\":\"CLIENT\",\"PlatformType\":\"AWS::BedrockAgentCore\",\"http.method\":\"POST\",\"http.response.status_code\":200,\"session.id\":\"00000000-0000-4000-8000-000000000001\"},\"status\":{\"code\":\"UNSET\"}}" + }, + { + "field": "sessionId", + "value": "00000000-0000-4000-8000-000000000001" + }, + { + "field": "traceId", + "value": "6a8f5b461983df572e2aca3264d32d4d" + }, + { + "field": "spanId", + "value": "92319b090266de53" + }, + { + "field": "@ptr", + "value": "CpwBCmAKFjY4NTE5NzcwODY4Nzphd3Mvc3BhbnMQABokZTE0NTI5MzgtZjVjZi00OTc1LWE0N2QtYzk3YzMwZmFlY2ZlIg4IgJD4jII0EOefqq2ENDjv37W+yTNAyO+wktMzSAASNhoYAganxnVsAAAAACC2BcQABqj1tNAAAAXCIAEotLvy/4M0MLvQ8v+DNDgLQNCZAUijZ1COPCACEAUYAQ==" + } + ], + [ + { + "field": "@message", + "value": "{\"resource\":{\"attributes\":{\"deployment.environment.name\":\"bedrock-agentcore:default\",\"aws.local.service\":\"asdf_MyAgent.DEFAULT\",\"service.name\":\"asdf_MyAgent.DEFAULT\",\"cloud.region\":\"us-west-2\",\"aws.log.stream.names\":\"otel-rt-logs\",\"telemetry.sdk.name\":\"opentelemetry\",\"aws.service.type\":\"gen_ai_agent\",\"telemetry.sdk.language\":\"python\",\"cloud.provider\":\"aws\",\"cloud.resource_id\":\"arn:aws:bedrock-agentcore:us-west-2:685197708687:runtime/asdf_MyAgent-3s5axvBC6Q/runtime-endpoint/DEFAULT:DEFAULT\",\"aws.log.group.names\":\"/aws/bedrock-agentcore/runtimes/asdf_MyAgent-3s5axvBC6Q-DEFAULT\",\"telemetry.sdk.version\":\"1.40.0\",\"cloud.platform\":\"aws_bedrock_agentcore\",\"telemetry.auto.version\":\"0.17.0-aws\"}},\"scope\":{\"name\":\"strands.telemetry.tracer\",\"version\":\"\"},\"traceId\":\"6a8f5b461983df572e2aca3264d32d4d\",\"spanId\":\"ac9653f70866483d\",\"parentSpanId\":\"c9722c3dbb4e781e\",\"flags\":256,\"name\":\"chat\",\"kind\":\"INTERNAL\",\"startTimeUnixNano\":1787779915444785352,\"endTimeUnixNano\":1787779917882748907,\"durationNano\":2437963555,\"attributes\":{\"aws.local.service\":\"asdf_MyAgent.DEFAULT\",\"gen_ai.usage.prompt_tokens\":1199,\"gen_ai.usage.output_tokens\":12,\"gen_ai.server.request.duration\":2404,\"gen_ai.usage.total_tokens\":1211,\"gen_ai.usage.completion_tokens\":12,\"aws.genai.span_kind\":\"LLM\",\"gen_ai.event.start_time\":\"2026-08-26T21:31:55.444793+00:00\",\"gen_ai.server.time_to_first_token\":2292,\"aws.local.environment\":\"bedrock-agentcore:default\",\"gen_ai.provider.name\":\"strands-agents\",\"gen_ai.operation.name\":\"chat\",\"gen_ai.event.end_time\":\"2026-08-26T21:31:57.882714+00:00\",\"gen_ai.usage.input_tokens\":1199,\"aws.genai.token_count_total\":1211,\"gen_ai.request.model\":\"global.anthropic.claude-sonnet-4-5-20250929-v1:0\",\"PlatformType\":\"AWS::BedrockAgentCore\",\"session.id\":\"00000000-0000-4000-8000-000000000001\",\"gen_ai.system\":\"strands-agents\"},\"status\":{\"code\":\"OK\"}}" + }, + { + "field": "sessionId", + "value": "00000000-0000-4000-8000-000000000001" + }, + { + "field": "traceId", + "value": "6a8f5b461983df572e2aca3264d32d4d" + }, + { + "field": "spanId", + "value": "ac9653f70866483d" + }, + { + "field": "@ptr", + "value": "CpwBCmAKFjY4NTE5NzcwODY4Nzphd3Mvc3BhbnMQABokZTE0NTI5MzgtZjVjZi00OTc1LWE0N2QtYzk3YzMwZmFlY2ZlIg4IgJD4jII0EOefqq2ENDjv37W+yTNAyO+wktMzSAASNhoYAganxnVsAAAAACC2BcQABqj1tNAAAAXCIAEotLvy/4M0MLvQ8v+DNDgLQNCZAUijZ1COPCACEAcYAQ==" + } + ], + [ + { + "field": "@message", + "value": "{\"resource\":{\"attributes\":{\"deployment.environment.name\":\"bedrock-agentcore:default\",\"aws.local.service\":\"asdf_MyAgent.DEFAULT\",\"service.name\":\"asdf_MyAgent.DEFAULT\",\"cloud.region\":\"us-west-2\",\"aws.log.stream.names\":\"otel-rt-logs\",\"telemetry.sdk.name\":\"opentelemetry\",\"aws.service.type\":\"gen_ai_agent\",\"telemetry.sdk.language\":\"python\",\"cloud.provider\":\"aws\",\"cloud.resource_id\":\"arn:aws:bedrock-agentcore:us-west-2:685197708687:runtime/asdf_MyAgent-3s5axvBC6Q/runtime-endpoint/DEFAULT:DEFAULT\",\"aws.log.group.names\":\"/aws/bedrock-agentcore/runtimes/asdf_MyAgent-3s5axvBC6Q-DEFAULT\",\"telemetry.sdk.version\":\"1.40.0\",\"cloud.platform\":\"aws_bedrock_agentcore\",\"telemetry.auto.version\":\"0.17.0-aws\"}},\"scope\":{\"name\":\"opentelemetry.instrumentation.botocore.bedrock-runtime\",\"version\":\"0.61b0\"},\"traceId\":\"6a8f5b461983df572e2aca3264d32d4d\",\"spanId\":\"ac5f4cb63375480b\",\"parentSpanId\":\"ac9653f70866483d\",\"flags\":256,\"name\":\"chat global.anthropic.claude-sonnet-4-5-20250929-v1:0\",\"kind\":\"CLIENT\",\"startTimeUnixNano\":1787779915457699487,\"endTimeUnixNano\":1787779917882096447,\"durationNano\":2424396960,\"attributes\":{\"aws.local.service\":\"asdf_MyAgent.DEFAULT\",\"rpc.service\":\"Bedrock Runtime\",\"aws.remote.resource.identifier\":\"global.anthropic.claude-sonnet-4-5-20250929-v1:0\",\"aws.local.environment\":\"bedrock-agentcore:default\",\"aws.remote.operation\":\"ConverseStream\",\"gen_ai.provider.name\":\"aws.bedrock\",\"server.address\":\"bedrock-runtime.us-west-2.amazonaws.com\",\"aws.request_id\":\"abfa0ec8-9efc-4f12-aa53-1a4e19dbfb25\",\"aws.local.operation\":\"UnmappedOperation\",\"aws.span.kind\":\"CLIENT\",\"aws.auth.region\":\"us-west-2\",\"rpc.method\":\"ConverseStream\",\"gen_ai.response.finish_reasons\":[\"end_turn\"],\"server.port\":443,\"gen_ai.request.model\":\"global.anthropic.claude-sonnet-4-5-20250929-v1:0\",\"http.response.status_code\":200,\"gen_ai.system\":\"aws.bedrock\",\"telemetry.extended\":\"true\",\"gen_ai.usage.output_tokens\":12,\"aws.genai.span_kind\":\"LLM\",\"rpc.system\":\"aws-api\",\"aws.remote.service\":\"AWS::BedrockRuntime\",\"http.status_code\":200,\"aws.region\":\"us-west-2\",\"aws.remote.resource.type\":\"AWS::Bedrock::Model\",\"gen_ai.operation.name\":\"chat\",\"gen_ai.usage.input_tokens\":1199,\"aws.genai.token_count_total\":1211,\"retry_attempts\":0,\"PlatformType\":\"AWS::BedrockAgentCore\",\"aws.auth.account.access_key\":\"ASIAZ7CHXJWHTC5VPKVH\",\"session.id\":\"00000000-0000-4000-8000-000000000001\"},\"status\":{\"code\":\"UNSET\"}}" + }, + { + "field": "sessionId", + "value": "00000000-0000-4000-8000-000000000001" + }, + { + "field": "traceId", + "value": "6a8f5b461983df572e2aca3264d32d4d" + }, + { + "field": "spanId", + "value": "ac5f4cb63375480b" + }, + { + "field": "@ptr", + "value": "CpwBCmAKFjY4NTE5NzcwODY4Nzphd3Mvc3BhbnMQABokZTE0NTI5MzgtZjVjZi00OTc1LWE0N2QtYzk3YzMwZmFlY2ZlIg4IgJD4jII0EOefqq2ENDjv37W+yTNAyO+wktMzSAASNhoYAganxnVsAAAAACC2BcQABqj1tNAAAAXCIAEotLvy/4M0MLvQ8v+DNDgLQNCZAUijZ1COPCACEAYYAQ==" + } + ], + [ + { + "field": "@message", + "value": "{\"resource\":{\"attributes\":{\"deployment.environment.name\":\"bedrock-agentcore:default\",\"aws.local.service\":\"asdf_MyAgent.DEFAULT\",\"service.name\":\"asdf_MyAgent.DEFAULT\",\"cloud.region\":\"us-west-2\",\"aws.log.stream.names\":\"otel-rt-logs\",\"telemetry.sdk.name\":\"opentelemetry\",\"aws.service.type\":\"gen_ai_agent\",\"telemetry.sdk.language\":\"python\",\"cloud.provider\":\"aws\",\"cloud.resource_id\":\"arn:aws:bedrock-agentcore:us-west-2:685197708687:runtime/asdf_MyAgent-3s5axvBC6Q/runtime-endpoint/DEFAULT:DEFAULT\",\"aws.log.group.names\":\"/aws/bedrock-agentcore/runtimes/asdf_MyAgent-3s5axvBC6Q-DEFAULT\",\"telemetry.sdk.version\":\"1.40.0\",\"cloud.platform\":\"aws_bedrock_agentcore\",\"telemetry.auto.version\":\"0.17.0-aws\"}},\"scope\":{\"name\":\"opentelemetry.instrumentation.starlette\",\"version\":\"0.61b0\"},\"traceId\":\"6a8f5b461983df572e2aca3264d32d4d\",\"spanId\":\"3497dea65b2e2550\",\"parentSpanId\":\"6f895a24f81f3726\",\"flags\":768,\"name\":\"POST /invocations\",\"kind\":\"SERVER\",\"startTimeUnixNano\":1787779915147134575,\"endTimeUnixNano\":1787779917883512213,\"durationNano\":2736377638,\"attributes\":{\"aws.local.service\":\"asdf_MyAgent.DEFAULT\",\"net.peer.port\":48806,\"telemetry.extended\":\"true\",\"http.target\":\"/invocations\",\"http.flavor\":\"1.1\",\"http.url\":\"http://cell01.us-west-2.prod.arp.kepler-analytics.aws.dev/invocations\",\"net.peer.ip\":\"127.0.0.1\",\"http.host\":\"127.0.0.1:8080\",\"aws.local.environment\":\"bedrock-agentcore:default\",\"http.status_code\":200,\"aws.local.operation\":\"POST /invocations\",\"aws.span.kind\":\"SERVER\",\"http.server_name\":\"cell01.us-west-2.prod.arp.kepler-analytics.aws.dev\",\"net.host.port\":8080,\"http.route\":\"/invocations\",\"PlatformType\":\"AWS::BedrockAgentCore\",\"http.method\":\"POST\",\"http.response.status_code\":200,\"session.id\":\"00000000-0000-4000-8000-000000000001\",\"http.scheme\":\"http\"},\"status\":{\"code\":\"UNSET\"}}" + }, + { + "field": "sessionId", + "value": "00000000-0000-4000-8000-000000000001" + }, + { + "field": "traceId", + "value": "6a8f5b461983df572e2aca3264d32d4d" + }, + { + "field": "spanId", + "value": "3497dea65b2e2550" + }, + { + "field": "@ptr", + "value": "CpwBCmAKFjY4NTE5NzcwODY4Nzphd3Mvc3BhbnMQABokZTE0NTI5MzgtZjVjZi00OTc1LWE0N2QtYzk3YzMwZmFlY2ZlIg4IgJD4jII0EOefqq2ENDjv37W+yTNAyO+wktMzSAASNhoYAganxnVsAAAAACC2BcQABqj1tNAAAAXCIAEotLvy/4M0MLvQ8v+DNDgLQNCZAUijZ1COPCACEAoYAQ==" + } + ], + [ + { + "field": "@message", + "value": "{\"resource\":{\"attributes\":{\"deployment.environment.name\":\"bedrock-agentcore:default\",\"aws.local.service\":\"asdf_MyAgent.DEFAULT\",\"service.name\":\"asdf_MyAgent.DEFAULT\",\"cloud.region\":\"us-west-2\",\"aws.log.stream.names\":\"otel-rt-logs\",\"telemetry.sdk.name\":\"opentelemetry\",\"aws.service.type\":\"gen_ai_agent\",\"telemetry.sdk.language\":\"python\",\"cloud.provider\":\"aws\",\"cloud.resource_id\":\"arn:aws:bedrock-agentcore:us-west-2:685197708687:runtime/asdf_MyAgent-3s5axvBC6Q/runtime-endpoint/DEFAULT:DEFAULT\",\"aws.log.group.names\":\"/aws/bedrock-agentcore/runtimes/asdf_MyAgent-3s5axvBC6Q-DEFAULT\",\"telemetry.sdk.version\":\"1.40.0\",\"cloud.platform\":\"aws_bedrock_agentcore\",\"telemetry.auto.version\":\"0.17.0-aws\"}},\"scope\":{\"name\":\"strands.telemetry.tracer\",\"version\":\"\"},\"traceId\":\"6a8f5b461983df572e2aca3264d32d4d\",\"spanId\":\"c9722c3dbb4e781e\",\"parentSpanId\":\"82fe397271c56f75\",\"flags\":256,\"name\":\"execute_event_loop_cycle\",\"kind\":\"INTERNAL\",\"startTimeUnixNano\":1787779915444636525,\"endTimeUnixNano\":1787779917883063841,\"durationNano\":2438427316,\"attributes\":{\"aws.local.service\":\"asdf_MyAgent.DEFAULT\",\"gen_ai.operation.name\":\"execute_event_loop_cycle\",\"gen_ai.event.end_time\":\"2026-08-26T21:31:57.883048+00:00\",\"event_loop.cycle_id\":\"3eac5258-e59c-44d1-afa9-02a1a42a10c3\",\"gen_ai.event.start_time\":\"2026-08-26T21:31:55.444648+00:00\",\"PlatformType\":\"AWS::BedrockAgentCore\",\"session.id\":\"00000000-0000-4000-8000-000000000001\",\"gen_ai.system\":\"strands-agents\",\"aws.local.environment\":\"bedrock-agentcore:default\",\"gen_ai.provider.name\":\"strands-agents\"},\"status\":{\"code\":\"OK\"}}" + }, + { + "field": "sessionId", + "value": "00000000-0000-4000-8000-000000000001" + }, + { + "field": "traceId", + "value": "6a8f5b461983df572e2aca3264d32d4d" + }, + { + "field": "spanId", + "value": "c9722c3dbb4e781e" + }, + { + "field": "@ptr", + "value": "CpwBCmAKFjY4NTE5NzcwODY4Nzphd3Mvc3BhbnMQABokZTE0NTI5MzgtZjVjZi00OTc1LWE0N2QtYzk3YzMwZmFlY2ZlIg4IgJD4jII0EOefqq2ENDjv37W+yTNAyO+wktMzSAASNhoYAganxnVsAAAAACC2BcQABqj1tNAAAAXCIAEotLvy/4M0MLvQ8v+DNDgLQNCZAUijZ1COPCACEAgYAQ==" + } + ], + [ + { + "field": "@message", + "value": "{\"resource\":{\"attributes\":{\"deployment.environment.name\":\"bedrock-agentcore:default\",\"aws.local.service\":\"asdf_MyAgent.DEFAULT\",\"service.name\":\"asdf_MyAgent.DEFAULT\",\"cloud.region\":\"us-west-2\",\"aws.log.stream.names\":\"otel-rt-logs\",\"telemetry.sdk.name\":\"opentelemetry\",\"aws.service.type\":\"gen_ai_agent\",\"telemetry.sdk.language\":\"python\",\"cloud.provider\":\"aws\",\"cloud.resource_id\":\"arn:aws:bedrock-agentcore:us-west-2:685197708687:runtime/asdf_MyAgent-3s5axvBC6Q/runtime-endpoint/DEFAULT:DEFAULT\",\"aws.log.group.names\":\"/aws/bedrock-agentcore/runtimes/asdf_MyAgent-3s5axvBC6Q-DEFAULT\",\"telemetry.sdk.version\":\"1.40.0\",\"cloud.platform\":\"aws_bedrock_agentcore\",\"telemetry.auto.version\":\"0.17.0-aws\"}},\"scope\":{\"name\":\"strands.telemetry.tracer\",\"version\":\"\"},\"traceId\":\"6a8f5b461983df572e2aca3264d32d4d\",\"spanId\":\"82fe397271c56f75\",\"parentSpanId\":\"3497dea65b2e2550\",\"flags\":256,\"name\":\"invoke_agent Strands Agents\",\"kind\":\"INTERNAL\",\"startTimeUnixNano\":1787779915443995518,\"endTimeUnixNano\":1787779917883212081,\"durationNano\":2439216563,\"attributes\":{\"aws.local.service\":\"asdf_MyAgent.DEFAULT\",\"gen_ai.usage.prompt_tokens\":1199,\"gen_ai.usage.output_tokens\":12,\"gen_ai.usage.cache_write_input_tokens\":0,\"gen_ai.agent.name\":\"Strands Agents\",\"gen_ai.usage.total_tokens\":1211,\"gen_ai.usage.completion_tokens\":12,\"aws.genai.span_kind\":\"AGENT\",\"gen_ai.event.start_time\":\"2026-08-26T21:31:55.444014+00:00\",\"aws.local.environment\":\"bedrock-agentcore:default\",\"gen_ai.provider.name\":\"strands-agents\",\"gen_ai.operation.name\":\"invoke_agent\",\"gen_ai.event.end_time\":\"2026-08-26T21:31:57.883192+00:00\",\"gen_ai.usage.input_tokens\":1199,\"aws.genai.token_count_total\":1211,\"gen_ai.request.model\":\"global.anthropic.claude-sonnet-4-5-20250929-v1:0\",\"gen_ai.usage.cache_read_input_tokens\":0,\"gen_ai.agent.tools\":\"[\\\"add_numbers\\\", \\\"x_amz_bedrock_agentcore_search\\\", \\\"mcpTarget___web_fetch_exa\\\", \\\"mcpTarget___web_search_exa\\\"]\",\"PlatformType\":\"AWS::BedrockAgentCore\",\"session.id\":\"00000000-0000-4000-8000-000000000001\",\"gen_ai.system\":\"strands-agents\"},\"status\":{\"code\":\"OK\"}}" + }, + { + "field": "sessionId", + "value": "00000000-0000-4000-8000-000000000001" + }, + { + "field": "traceId", + "value": "6a8f5b461983df572e2aca3264d32d4d" + }, + { + "field": "spanId", + "value": "82fe397271c56f75" + }, + { + "field": "@ptr", + "value": "CpwBCmAKFjY4NTE5NzcwODY4Nzphd3Mvc3BhbnMQABokZTE0NTI5MzgtZjVjZi00OTc1LWE0N2QtYzk3YzMwZmFlY2ZlIg4IgJD4jII0EOefqq2ENDjv37W+yTNAyO+wktMzSAASNhoYAganxnVsAAAAACC2BcQABqj1tNAAAAXCIAEotLvy/4M0MLvQ8v+DNDgLQNCZAUijZ1COPCACEAkYAQ==" + } + ], + [ + { + "field": "@message", + "value": "{\"resource\":{\"attributes\":{\"deployment.environment.name\":\"bedrock-agentcore:default\",\"aws.local.service\":\"asdf_MyAgent.DEFAULT\",\"service.name\":\"asdf_MyAgent.DEFAULT\",\"cloud.region\":\"us-west-2\",\"aws.log.stream.names\":\"otel-rt-logs\",\"telemetry.sdk.name\":\"opentelemetry\",\"aws.service.type\":\"gen_ai_agent\",\"telemetry.sdk.language\":\"python\",\"cloud.provider\":\"aws\",\"cloud.resource_id\":\"arn:aws:bedrock-agentcore:us-west-2:685197708687:runtime/asdf_MyAgent-3s5axvBC6Q/runtime-endpoint/DEFAULT:DEFAULT\",\"aws.log.group.names\":\"/aws/bedrock-agentcore/runtimes/asdf_MyAgent-3s5axvBC6Q-DEFAULT\",\"telemetry.sdk.version\":\"1.40.0\",\"cloud.platform\":\"aws_bedrock_agentcore\",\"telemetry.auto.version\":\"0.17.0-aws\"}},\"scope\":{\"name\":\"opentelemetry.instrumentation.botocore.bedrock-runtime\",\"version\":\"0.61b0\"},\"traceId\":\"6a8f5bbc1882c05547b64a9246f17ca4\",\"spanId\":\"c6fce016050b427c\",\"parentSpanId\":\"2b53da16274ce9f5\",\"flags\":256,\"name\":\"chat global.anthropic.claude-sonnet-4-5-20250929-v1:0\",\"kind\":\"CLIENT\",\"startTimeUnixNano\":1787780028788804860,\"endTimeUnixNano\":1787780030855083391,\"durationNano\":2066278531,\"attributes\":{\"aws.local.service\":\"asdf_MyAgent.DEFAULT\",\"rpc.service\":\"Bedrock Runtime\",\"aws.remote.resource.identifier\":\"global.anthropic.claude-sonnet-4-5-20250929-v1:0\",\"aws.local.environment\":\"bedrock-agentcore:default\",\"aws.remote.operation\":\"ConverseStream\",\"gen_ai.provider.name\":\"aws.bedrock\",\"server.address\":\"bedrock-runtime.us-west-2.amazonaws.com\",\"aws.request_id\":\"ff6866c3-28d5-400d-8947-249c9a7576fb\",\"aws.local.operation\":\"UnmappedOperation\",\"aws.span.kind\":\"CLIENT\",\"aws.auth.region\":\"us-west-2\",\"rpc.method\":\"ConverseStream\",\"gen_ai.response.finish_reasons\":[\"end_turn\"],\"server.port\":443,\"gen_ai.request.model\":\"global.anthropic.claude-sonnet-4-5-20250929-v1:0\",\"http.response.status_code\":200,\"gen_ai.system\":\"aws.bedrock\",\"telemetry.extended\":\"true\",\"gen_ai.usage.output_tokens\":13,\"aws.genai.span_kind\":\"LLM\",\"rpc.system\":\"aws-api\",\"aws.remote.service\":\"AWS::BedrockRuntime\",\"http.status_code\":200,\"aws.region\":\"us-west-2\",\"aws.remote.resource.type\":\"AWS::Bedrock::Model\",\"gen_ai.operation.name\":\"chat\",\"gen_ai.usage.input_tokens\":1215,\"aws.genai.token_count_total\":1228,\"retry_attempts\":0,\"PlatformType\":\"AWS::BedrockAgentCore\",\"aws.auth.account.access_key\":\"ASIAZ7CHXJWHTC5VPKVH\",\"session.id\":\"00000000-0000-4000-8000-000000000001\"},\"status\":{\"code\":\"UNSET\"}}" + }, + { + "field": "sessionId", + "value": "00000000-0000-4000-8000-000000000001" + }, + { + "field": "traceId", + "value": "6a8f5bbc1882c05547b64a9246f17ca4" + }, + { + "field": "spanId", + "value": "c6fce016050b427c" + }, + { + "field": "@ptr", + "value": "CpsBCmAKFjY4NTE5NzcwODY4Nzphd3Mvc3BhbnMQABokZTE0NTI5MzgtZjVjZi00OTc1LWE0N2QtYzk3YzMwZmFlY2ZlIg4IgJD4jII0EOefqq2ENDjv37W+yTNAyO+wktMzSAASNRoYAganxnVsAAAAACC2y4gABqj1vAAAAAXCIAEoh8P5/4M0MIjD+f+DNDgFQK9MSKleUJYzIAIQABgB" + } + ], + [ + { + "field": "@message", + "value": "{\"resource\":{\"attributes\":{\"deployment.environment.name\":\"bedrock-agentcore:default\",\"aws.local.service\":\"asdf_MyAgent.DEFAULT\",\"service.name\":\"asdf_MyAgent.DEFAULT\",\"cloud.region\":\"us-west-2\",\"aws.log.stream.names\":\"otel-rt-logs\",\"telemetry.sdk.name\":\"opentelemetry\",\"aws.service.type\":\"gen_ai_agent\",\"telemetry.sdk.language\":\"python\",\"cloud.provider\":\"aws\",\"cloud.resource_id\":\"arn:aws:bedrock-agentcore:us-west-2:685197708687:runtime/asdf_MyAgent-3s5axvBC6Q/runtime-endpoint/DEFAULT:DEFAULT\",\"aws.log.group.names\":\"/aws/bedrock-agentcore/runtimes/asdf_MyAgent-3s5axvBC6Q-DEFAULT\",\"telemetry.sdk.version\":\"1.40.0\",\"cloud.platform\":\"aws_bedrock_agentcore\",\"telemetry.auto.version\":\"0.17.0-aws\"}},\"scope\":{\"name\":\"strands.telemetry.tracer\",\"version\":\"\"},\"traceId\":\"6a8f5bbc1882c05547b64a9246f17ca4\",\"spanId\":\"2b53da16274ce9f5\",\"parentSpanId\":\"43f809568e52d1ca\",\"flags\":256,\"name\":\"chat\",\"kind\":\"INTERNAL\",\"startTimeUnixNano\":1787780028788254696,\"endTimeUnixNano\":1787780030855720915,\"durationNano\":2067466219,\"attributes\":{\"aws.local.service\":\"asdf_MyAgent.DEFAULT\",\"gen_ai.usage.prompt_tokens\":1215,\"gen_ai.usage.output_tokens\":13,\"gen_ai.server.request.duration\":2060,\"gen_ai.usage.total_tokens\":1228,\"gen_ai.usage.completion_tokens\":13,\"aws.genai.span_kind\":\"LLM\",\"gen_ai.event.start_time\":\"2026-08-26T21:33:48.788261+00:00\",\"gen_ai.server.time_to_first_token\":1710,\"aws.local.environment\":\"bedrock-agentcore:default\",\"gen_ai.provider.name\":\"strands-agents\",\"gen_ai.operation.name\":\"chat\",\"gen_ai.event.end_time\":\"2026-08-26T21:33:50.855686+00:00\",\"gen_ai.usage.input_tokens\":1215,\"aws.genai.token_count_total\":1228,\"gen_ai.request.model\":\"global.anthropic.claude-sonnet-4-5-20250929-v1:0\",\"PlatformType\":\"AWS::BedrockAgentCore\",\"session.id\":\"00000000-0000-4000-8000-000000000001\",\"gen_ai.system\":\"strands-agents\"},\"status\":{\"code\":\"OK\"}}" + }, + { + "field": "sessionId", + "value": "00000000-0000-4000-8000-000000000001" + }, + { + "field": "traceId", + "value": "6a8f5bbc1882c05547b64a9246f17ca4" + }, + { + "field": "spanId", + "value": "2b53da16274ce9f5" + }, + { + "field": "@ptr", + "value": "CpsBCmAKFjY4NTE5NzcwODY4Nzphd3Mvc3BhbnMQABokZTE0NTI5MzgtZjVjZi00OTc1LWE0N2QtYzk3YzMwZmFlY2ZlIg4IgJD4jII0EOefqq2ENDjv37W+yTNAyO+wktMzSAASNRoYAganxnVsAAAAACC2y4gABqj1vAAAAAXCIAEoh8P5/4M0MIjD+f+DNDgFQK9MSKleUJYzIAIQARgB" + } + ], + [ + { + "field": "@message", + "value": "{\"resource\":{\"attributes\":{\"deployment.environment.name\":\"bedrock-agentcore:default\",\"aws.local.service\":\"asdf_MyAgent.DEFAULT\",\"service.name\":\"asdf_MyAgent.DEFAULT\",\"cloud.region\":\"us-west-2\",\"aws.log.stream.names\":\"otel-rt-logs\",\"telemetry.sdk.name\":\"opentelemetry\",\"aws.service.type\":\"gen_ai_agent\",\"telemetry.sdk.language\":\"python\",\"cloud.provider\":\"aws\",\"cloud.resource_id\":\"arn:aws:bedrock-agentcore:us-west-2:685197708687:runtime/asdf_MyAgent-3s5axvBC6Q/runtime-endpoint/DEFAULT:DEFAULT\",\"aws.log.group.names\":\"/aws/bedrock-agentcore/runtimes/asdf_MyAgent-3s5axvBC6Q-DEFAULT\",\"telemetry.sdk.version\":\"1.40.0\",\"cloud.platform\":\"aws_bedrock_agentcore\",\"telemetry.auto.version\":\"0.17.0-aws\"}},\"scope\":{\"name\":\"strands.telemetry.tracer\",\"version\":\"\"},\"traceId\":\"6a8f5bbc1882c05547b64a9246f17ca4\",\"spanId\":\"43f809568e52d1ca\",\"parentSpanId\":\"324e3a835d1af5c4\",\"flags\":256,\"name\":\"execute_event_loop_cycle\",\"kind\":\"INTERNAL\",\"startTimeUnixNano\":1787780028788110333,\"endTimeUnixNano\":1787780030855979475,\"durationNano\":2067869142,\"attributes\":{\"aws.local.service\":\"asdf_MyAgent.DEFAULT\",\"gen_ai.operation.name\":\"execute_event_loop_cycle\",\"gen_ai.event.end_time\":\"2026-08-26T21:33:50.855965+00:00\",\"event_loop.cycle_id\":\"6e6ce2c7-3386-4efc-a0d4-f65e3832a565\",\"gen_ai.event.start_time\":\"2026-08-26T21:33:48.788119+00:00\",\"PlatformType\":\"AWS::BedrockAgentCore\",\"session.id\":\"00000000-0000-4000-8000-000000000001\",\"gen_ai.system\":\"strands-agents\",\"aws.local.environment\":\"bedrock-agentcore:default\",\"gen_ai.provider.name\":\"strands-agents\"},\"status\":{\"code\":\"OK\"}}" + }, + { + "field": "sessionId", + "value": "00000000-0000-4000-8000-000000000001" + }, + { + "field": "traceId", + "value": "6a8f5bbc1882c05547b64a9246f17ca4" + }, + { + "field": "spanId", + "value": "43f809568e52d1ca" + }, + { + "field": "@ptr", + "value": "CpsBCmAKFjY4NTE5NzcwODY4Nzphd3Mvc3BhbnMQABokZTE0NTI5MzgtZjVjZi00OTc1LWE0N2QtYzk3YzMwZmFlY2ZlIg4IgJD4jII0EOefqq2ENDjv37W+yTNAyO+wktMzSAASNRoYAganxnVsAAAAACC2y4gABqj1vAAAAAXCIAEoh8P5/4M0MIjD+f+DNDgFQK9MSKleUJYzIAIQAhgB" + } + ], + [ + { + "field": "@message", + "value": "{\"resource\":{\"attributes\":{\"deployment.environment.name\":\"bedrock-agentcore:default\",\"aws.local.service\":\"asdf_MyAgent.DEFAULT\",\"service.name\":\"asdf_MyAgent.DEFAULT\",\"cloud.region\":\"us-west-2\",\"aws.log.stream.names\":\"otel-rt-logs\",\"telemetry.sdk.name\":\"opentelemetry\",\"aws.service.type\":\"gen_ai_agent\",\"telemetry.sdk.language\":\"python\",\"cloud.provider\":\"aws\",\"cloud.resource_id\":\"arn:aws:bedrock-agentcore:us-west-2:685197708687:runtime/asdf_MyAgent-3s5axvBC6Q/runtime-endpoint/DEFAULT:DEFAULT\",\"aws.log.group.names\":\"/aws/bedrock-agentcore/runtimes/asdf_MyAgent-3s5axvBC6Q-DEFAULT\",\"telemetry.sdk.version\":\"1.40.0\",\"cloud.platform\":\"aws_bedrock_agentcore\",\"telemetry.auto.version\":\"0.17.0-aws\"}},\"scope\":{\"name\":\"strands.telemetry.tracer\",\"version\":\"\"},\"traceId\":\"6a8f5bbc1882c05547b64a9246f17ca4\",\"spanId\":\"324e3a835d1af5c4\",\"parentSpanId\":\"eba2750a08d48390\",\"flags\":256,\"name\":\"invoke_agent Strands Agents\",\"kind\":\"INTERNAL\",\"startTimeUnixNano\":1787780028787818615,\"endTimeUnixNano\":1787780030856119726,\"durationNano\":2068301111,\"attributes\":{\"aws.local.service\":\"asdf_MyAgent.DEFAULT\",\"gen_ai.usage.prompt_tokens\":2414,\"gen_ai.usage.output_tokens\":25,\"gen_ai.usage.cache_write_input_tokens\":0,\"gen_ai.agent.name\":\"Strands Agents\",\"gen_ai.usage.total_tokens\":2439,\"gen_ai.usage.completion_tokens\":25,\"aws.genai.span_kind\":\"AGENT\",\"gen_ai.event.start_time\":\"2026-08-26T21:33:48.787832+00:00\",\"aws.local.environment\":\"bedrock-agentcore:default\",\"gen_ai.provider.name\":\"strands-agents\",\"gen_ai.operation.name\":\"invoke_agent\",\"gen_ai.event.end_time\":\"2026-08-26T21:33:50.856099+00:00\",\"gen_ai.usage.input_tokens\":2414,\"aws.genai.token_count_total\":2439,\"gen_ai.request.model\":\"global.anthropic.claude-sonnet-4-5-20250929-v1:0\",\"gen_ai.usage.cache_read_input_tokens\":0,\"gen_ai.agent.tools\":\"[\\\"add_numbers\\\", \\\"x_amz_bedrock_agentcore_search\\\", \\\"mcpTarget___web_fetch_exa\\\", \\\"mcpTarget___web_search_exa\\\"]\",\"PlatformType\":\"AWS::BedrockAgentCore\",\"session.id\":\"00000000-0000-4000-8000-000000000001\",\"gen_ai.system\":\"strands-agents\"},\"status\":{\"code\":\"OK\"}}" + }, + { + "field": "sessionId", + "value": "00000000-0000-4000-8000-000000000001" + }, + { + "field": "traceId", + "value": "6a8f5bbc1882c05547b64a9246f17ca4" + }, + { + "field": "spanId", + "value": "324e3a835d1af5c4" + }, + { + "field": "@ptr", + "value": "CpsBCmAKFjY4NTE5NzcwODY4Nzphd3Mvc3BhbnMQABokZTE0NTI5MzgtZjVjZi00OTc1LWE0N2QtYzk3YzMwZmFlY2ZlIg4IgJD4jII0EOefqq2ENDjv37W+yTNAyO+wktMzSAASNRoYAganxnVsAAAAACC2y4gABqj1vAAAAAXCIAEoh8P5/4M0MIjD+f+DNDgFQK9MSKleUJYzIAIQAxgB" + } + ], + [ + { + "field": "@message", + "value": "{\"resource\":{\"attributes\":{\"deployment.environment.name\":\"bedrock-agentcore:default\",\"aws.local.service\":\"asdf_MyAgent.DEFAULT\",\"service.name\":\"asdf_MyAgent.DEFAULT\",\"cloud.region\":\"us-west-2\",\"aws.log.stream.names\":\"otel-rt-logs\",\"telemetry.sdk.name\":\"opentelemetry\",\"aws.service.type\":\"gen_ai_agent\",\"telemetry.sdk.language\":\"python\",\"cloud.provider\":\"aws\",\"cloud.resource_id\":\"arn:aws:bedrock-agentcore:us-west-2:685197708687:runtime/asdf_MyAgent-3s5axvBC6Q/runtime-endpoint/DEFAULT:DEFAULT\",\"aws.log.group.names\":\"/aws/bedrock-agentcore/runtimes/asdf_MyAgent-3s5axvBC6Q-DEFAULT\",\"telemetry.sdk.version\":\"1.40.0\",\"cloud.platform\":\"aws_bedrock_agentcore\",\"telemetry.auto.version\":\"0.17.0-aws\"}},\"scope\":{\"name\":\"opentelemetry.instrumentation.starlette\",\"version\":\"0.61b0\"},\"traceId\":\"6a8f5bbc1882c05547b64a9246f17ca4\",\"spanId\":\"eba2750a08d48390\",\"parentSpanId\":\"4c77c79e708e39db\",\"flags\":768,\"name\":\"POST /invocations\",\"kind\":\"SERVER\",\"startTimeUnixNano\":1787780028786050715,\"endTimeUnixNano\":1787780030856458693,\"durationNano\":2070407978,\"attributes\":{\"aws.local.service\":\"asdf_MyAgent.DEFAULT\",\"net.peer.port\":37368,\"telemetry.extended\":\"true\",\"http.target\":\"/invocations\",\"http.flavor\":\"1.1\",\"http.url\":\"http://cell01.us-west-2.prod.arp.kepler-analytics.aws.dev/invocations\",\"net.peer.ip\":\"127.0.0.1\",\"http.host\":\"127.0.0.1:8080\",\"aws.local.environment\":\"bedrock-agentcore:default\",\"http.status_code\":200,\"aws.local.operation\":\"POST /invocations\",\"aws.span.kind\":\"SERVER\",\"http.server_name\":\"cell01.us-west-2.prod.arp.kepler-analytics.aws.dev\",\"net.host.port\":8080,\"http.route\":\"/invocations\",\"PlatformType\":\"AWS::BedrockAgentCore\",\"http.method\":\"POST\",\"http.response.status_code\":200,\"session.id\":\"00000000-0000-4000-8000-000000000001\",\"http.scheme\":\"http\"},\"status\":{\"code\":\"UNSET\"}}" + }, + { + "field": "sessionId", + "value": "00000000-0000-4000-8000-000000000001" + }, + { + "field": "traceId", + "value": "6a8f5bbc1882c05547b64a9246f17ca4" + }, + { + "field": "spanId", + "value": "eba2750a08d48390" + }, + { + "field": "@ptr", + "value": "CpsBCmAKFjY4NTE5NzcwODY4Nzphd3Mvc3BhbnMQABokZTE0NTI5MzgtZjVjZi00OTc1LWE0N2QtYzk3YzMwZmFlY2ZlIg4IgJD4jII0EOefqq2ENDjv37W+yTNAyO+wktMzSAASNRoYAganxnVsAAAAACC2y4gABqj1vAAAAAXCIAEoh8P5/4M0MIjD+f+DNDgFQK9MSKleUJYzIAIQBBgB" + } + ], + [ + { + "field": "@message", + "value": "{\"resource\":{\"attributes\":{\"deployment.environment.name\":\"bedrock-agentcore:default\",\"aws.local.service\":\"asdf_MyAgent.DEFAULT\",\"service.name\":\"asdf_MyAgent.DEFAULT\",\"cloud.region\":\"us-west-2\",\"aws.log.stream.names\":\"otel-rt-logs\",\"telemetry.sdk.name\":\"opentelemetry\",\"aws.service.type\":\"gen_ai_agent\",\"telemetry.sdk.language\":\"python\",\"cloud.provider\":\"aws\",\"cloud.resource_id\":\"arn:aws:bedrock-agentcore:us-west-2:685197708687:runtime/asdf_MyAgent-3s5axvBC6Q/runtime-endpoint/DEFAULT:DEFAULT\",\"aws.log.group.names\":\"/aws/bedrock-agentcore/runtimes/asdf_MyAgent-3s5axvBC6Q-DEFAULT\",\"telemetry.sdk.version\":\"1.40.0\",\"cloud.platform\":\"aws_bedrock_agentcore\",\"telemetry.auto.version\":\"0.17.0-aws\"}},\"scope\":{\"name\":\"opentelemetry.instrumentation.urllib3\",\"version\":\"0.61b0\"},\"traceId\":\"6a8f639540ebe6cb6f30d5d80762c2e2\",\"spanId\":\"09f7e5c0e6870466\",\"parentSpanId\":\"9a98ae2cca8efc3d\",\"flags\":256,\"name\":\"PUT\",\"kind\":\"CLIENT\",\"startTimeUnixNano\":1787782043375063062,\"endTimeUnixNano\":1787782043376082417,\"durationNano\":1019355,\"attributes\":{\"aws.local.service\":\"asdf_MyAgent.DEFAULT\",\"telemetry.extended\":\"true\",\"http.url\":\"http://169.254.169.254/latest/api/token\",\"aws.remote.service\":\"169.254.169.254\",\"aws.local.environment\":\"bedrock-agentcore:default\",\"aws.remote.operation\":\"PUT /latest\",\"http.status_code\":200,\"aws.local.operation\":\"UnmappedOperation\",\"aws.span.kind\":\"CLIENT\",\"PlatformType\":\"AWS::BedrockAgentCore\",\"http.method\":\"PUT\",\"http.response.status_code\":200,\"session.id\":\"00000000-0000-4000-8000-000000000001\"},\"status\":{\"code\":\"UNSET\"}}" + }, + { + "field": "sessionId", + "value": "00000000-0000-4000-8000-000000000001" + }, + { + "field": "traceId", + "value": "6a8f639540ebe6cb6f30d5d80762c2e2" + }, + { + "field": "spanId", + "value": "09f7e5c0e6870466" + }, + { + "field": "@ptr", + "value": "CpwBCmAKFjY4NTE5NzcwODY4Nzphd3Mvc3BhbnMQABokZTE0NTI5MzgtZjVjZi00OTc1LWE0N2QtYzk3YzMwZmFlY2ZlIg4IgJD4jII0EOefqq2ENDjv37W+yTNAyO+wktMzSAASNhoYAganxnVsAAAAACDEC4EABqj2OeAAAAXCIAEo8K30gIQ0MLe89ICENDgLQNCZAUidZ1CIPCACEAAYAQ==" + } + ], + [ + { + "field": "@message", + "value": "{\"resource\":{\"attributes\":{\"deployment.environment.name\":\"bedrock-agentcore:default\",\"aws.local.service\":\"asdf_MyAgent.DEFAULT\",\"service.name\":\"asdf_MyAgent.DEFAULT\",\"cloud.region\":\"us-west-2\",\"aws.log.stream.names\":\"otel-rt-logs\",\"telemetry.sdk.name\":\"opentelemetry\",\"aws.service.type\":\"gen_ai_agent\",\"telemetry.sdk.language\":\"python\",\"cloud.provider\":\"aws\",\"cloud.resource_id\":\"arn:aws:bedrock-agentcore:us-west-2:685197708687:runtime/asdf_MyAgent-3s5axvBC6Q/runtime-endpoint/DEFAULT:DEFAULT\",\"aws.log.group.names\":\"/aws/bedrock-agentcore/runtimes/asdf_MyAgent-3s5axvBC6Q-DEFAULT\",\"telemetry.sdk.version\":\"1.40.0\",\"cloud.platform\":\"aws_bedrock_agentcore\",\"telemetry.auto.version\":\"0.17.0-aws\"}},\"scope\":{\"name\":\"opentelemetry.instrumentation.urllib3\",\"version\":\"0.61b0\"},\"traceId\":\"6a8f639540ebe6cb6f30d5d80762c2e2\",\"spanId\":\"f7718c78b1f483d8\",\"parentSpanId\":\"9a98ae2cca8efc3d\",\"flags\":256,\"name\":\"GET\",\"kind\":\"CLIENT\",\"startTimeUnixNano\":1787782043376507637,\"endTimeUnixNano\":1787782043377035089,\"durationNano\":527452,\"attributes\":{\"aws.local.service\":\"asdf_MyAgent.DEFAULT\",\"telemetry.extended\":\"true\",\"http.url\":\"http://169.254.169.254/latest/meta-data/iam/security-credentials/\",\"aws.remote.service\":\"169.254.169.254\",\"aws.local.environment\":\"bedrock-agentcore:default\",\"aws.remote.operation\":\"GET /latest\",\"http.status_code\":200,\"aws.local.operation\":\"UnmappedOperation\",\"aws.span.kind\":\"CLIENT\",\"PlatformType\":\"AWS::BedrockAgentCore\",\"http.method\":\"GET\",\"http.response.status_code\":200,\"session.id\":\"00000000-0000-4000-8000-000000000001\"},\"status\":{\"code\":\"UNSET\"}}" + }, + { + "field": "sessionId", + "value": "00000000-0000-4000-8000-000000000001" + }, + { + "field": "traceId", + "value": "6a8f639540ebe6cb6f30d5d80762c2e2" + }, + { + "field": "spanId", + "value": "f7718c78b1f483d8" + }, + { + "field": "@ptr", + "value": "CpwBCmAKFjY4NTE5NzcwODY4Nzphd3Mvc3BhbnMQABokZTE0NTI5MzgtZjVjZi00OTc1LWE0N2QtYzk3YzMwZmFlY2ZlIg4IgJD4jII0EOefqq2ENDjv37W+yTNAyO+wktMzSAASNhoYAganxnVsAAAAACDEC4EABqj2OeAAAAXCIAEo8K30gIQ0MLe89ICENDgLQNCZAUidZ1CIPCACEAEYAQ==" + } + ], + [ + { + "field": "@message", + "value": "{\"resource\":{\"attributes\":{\"deployment.environment.name\":\"bedrock-agentcore:default\",\"aws.local.service\":\"asdf_MyAgent.DEFAULT\",\"service.name\":\"asdf_MyAgent.DEFAULT\",\"cloud.region\":\"us-west-2\",\"aws.log.stream.names\":\"otel-rt-logs\",\"telemetry.sdk.name\":\"opentelemetry\",\"aws.service.type\":\"gen_ai_agent\",\"telemetry.sdk.language\":\"python\",\"cloud.provider\":\"aws\",\"cloud.resource_id\":\"arn:aws:bedrock-agentcore:us-west-2:685197708687:runtime/asdf_MyAgent-3s5axvBC6Q/runtime-endpoint/DEFAULT:DEFAULT\",\"aws.log.group.names\":\"/aws/bedrock-agentcore/runtimes/asdf_MyAgent-3s5axvBC6Q-DEFAULT\",\"telemetry.sdk.version\":\"1.40.0\",\"cloud.platform\":\"aws_bedrock_agentcore\",\"telemetry.auto.version\":\"0.17.0-aws\"}},\"scope\":{\"name\":\"opentelemetry.instrumentation.urllib3\",\"version\":\"0.61b0\"},\"traceId\":\"6a8f639540ebe6cb6f30d5d80762c2e2\",\"spanId\":\"4f5b2a32c8aa0586\",\"parentSpanId\":\"9a98ae2cca8efc3d\",\"flags\":256,\"name\":\"GET\",\"kind\":\"CLIENT\",\"startTimeUnixNano\":1787782043377350818,\"endTimeUnixNano\":1787782043377867116,\"durationNano\":516298,\"attributes\":{\"aws.local.service\":\"asdf_MyAgent.DEFAULT\",\"telemetry.extended\":\"true\",\"http.url\":\"http://169.254.169.254/latest/meta-data/iam/security-credentials/execution_role\",\"aws.remote.service\":\"169.254.169.254\",\"aws.local.environment\":\"bedrock-agentcore:default\",\"aws.remote.operation\":\"GET /latest\",\"http.status_code\":200,\"aws.local.operation\":\"UnmappedOperation\",\"aws.span.kind\":\"CLIENT\",\"PlatformType\":\"AWS::BedrockAgentCore\",\"http.method\":\"GET\",\"http.response.status_code\":200,\"session.id\":\"00000000-0000-4000-8000-000000000001\"},\"status\":{\"code\":\"UNSET\"}}" + }, + { + "field": "sessionId", + "value": "00000000-0000-4000-8000-000000000001" + }, + { + "field": "traceId", + "value": "6a8f639540ebe6cb6f30d5d80762c2e2" + }, + { + "field": "spanId", + "value": "4f5b2a32c8aa0586" + }, + { + "field": "@ptr", + "value": "CpwBCmAKFjY4NTE5NzcwODY4Nzphd3Mvc3BhbnMQABokZTE0NTI5MzgtZjVjZi00OTc1LWE0N2QtYzk3YzMwZmFlY2ZlIg4IgJD4jII0EOefqq2ENDjv37W+yTNAyO+wktMzSAASNhoYAganxnVsAAAAACDEC4EABqj2OeAAAAXCIAEo8K30gIQ0MLe89ICENDgLQNCZAUidZ1CIPCACEAIYAQ==" + } + ], + [ + { + "field": "@message", + "value": "{\"resource\":{\"attributes\":{\"deployment.environment.name\":\"bedrock-agentcore:default\",\"aws.local.service\":\"asdf_MyAgent.DEFAULT\",\"service.name\":\"asdf_MyAgent.DEFAULT\",\"cloud.region\":\"us-west-2\",\"aws.log.stream.names\":\"otel-rt-logs\",\"telemetry.sdk.name\":\"opentelemetry\",\"aws.service.type\":\"gen_ai_agent\",\"telemetry.sdk.language\":\"python\",\"cloud.provider\":\"aws\",\"cloud.resource_id\":\"arn:aws:bedrock-agentcore:us-west-2:685197708687:runtime/asdf_MyAgent-3s5axvBC6Q/runtime-endpoint/DEFAULT:DEFAULT\",\"aws.log.group.names\":\"/aws/bedrock-agentcore/runtimes/asdf_MyAgent-3s5axvBC6Q-DEFAULT\",\"telemetry.sdk.version\":\"1.40.0\",\"cloud.platform\":\"aws_bedrock_agentcore\",\"telemetry.auto.version\":\"0.17.0-aws\"}},\"scope\":{\"name\":\"opentelemetry.instrumentation.httpx\",\"version\":\"0.61b0\"},\"traceId\":\"6a8f639540ebe6cb6f30d5d80762c2e2\",\"spanId\":\"7bad4d2a822b9d46\",\"parentSpanId\":\"9a98ae2cca8efc3d\",\"flags\":256,\"name\":\"POST\",\"kind\":\"CLIENT\",\"startTimeUnixNano\":1787782043483277501,\"endTimeUnixNano\":1787782043527189690,\"durationNano\":43912189,\"attributes\":{\"aws.local.service\":\"asdf_MyAgent.DEFAULT\",\"telemetry.extended\":\"true\",\"http.url\":\"https://bugbashgw1776978672-zsy8cbqwts.gateway.bedrock-agentcore.us-west-2.amazonaws.com/mcp\",\"aws.remote.service\":\"bugbashgw1776978672-zsy8cbqwts.gateway.bedrock-agentcore.us-west-2.amazonaws.com\",\"aws.local.environment\":\"bedrock-agentcore:default\",\"aws.remote.operation\":\"POST /mcp\",\"http.status_code\":200,\"aws.local.operation\":\"UnmappedOperation\",\"aws.span.kind\":\"CLIENT\",\"PlatformType\":\"AWS::BedrockAgentCore\",\"http.method\":\"POST\",\"http.response.status_code\":200,\"session.id\":\"00000000-0000-4000-8000-000000000001\"},\"status\":{\"code\":\"UNSET\"}}" + }, + { + "field": "sessionId", + "value": "00000000-0000-4000-8000-000000000001" + }, + { + "field": "traceId", + "value": "6a8f639540ebe6cb6f30d5d80762c2e2" + }, + { + "field": "spanId", + "value": "7bad4d2a822b9d46" + }, + { + "field": "@ptr", + "value": "CpwBCmAKFjY4NTE5NzcwODY4Nzphd3Mvc3BhbnMQABokZTE0NTI5MzgtZjVjZi00OTc1LWE0N2QtYzk3YzMwZmFlY2ZlIg4IgJD4jII0EOefqq2ENDjv37W+yTNAyO+wktMzSAASNhoYAganxnVsAAAAACDEC4EABqj2OeAAAAXCIAEo8K30gIQ0MLe89ICENDgLQNCZAUidZ1CIPCACEAMYAQ==" + } + ], + [ + { + "field": "@message", + "value": "{\"resource\":{\"attributes\":{\"deployment.environment.name\":\"bedrock-agentcore:default\",\"aws.local.service\":\"asdf_MyAgent.DEFAULT\",\"service.name\":\"asdf_MyAgent.DEFAULT\",\"cloud.region\":\"us-west-2\",\"aws.log.stream.names\":\"otel-rt-logs\",\"telemetry.sdk.name\":\"opentelemetry\",\"aws.service.type\":\"gen_ai_agent\",\"telemetry.sdk.language\":\"python\",\"cloud.provider\":\"aws\",\"cloud.resource_id\":\"arn:aws:bedrock-agentcore:us-west-2:685197708687:runtime/asdf_MyAgent-3s5axvBC6Q/runtime-endpoint/DEFAULT:DEFAULT\",\"aws.log.group.names\":\"/aws/bedrock-agentcore/runtimes/asdf_MyAgent-3s5axvBC6Q-DEFAULT\",\"telemetry.sdk.version\":\"1.40.0\",\"cloud.platform\":\"aws_bedrock_agentcore\",\"telemetry.auto.version\":\"0.17.0-aws\"}},\"scope\":{\"name\":\"opentelemetry.instrumentation.httpx\",\"version\":\"0.61b0\"},\"traceId\":\"6a8f639540ebe6cb6f30d5d80762c2e2\",\"spanId\":\"d14fb48aa6858b3d\",\"parentSpanId\":\"9a98ae2cca8efc3d\",\"flags\":256,\"name\":\"POST\",\"kind\":\"CLIENT\",\"startTimeUnixNano\":1787782043529850664,\"endTimeUnixNano\":1787782043562515163,\"durationNano\":32664499,\"attributes\":{\"aws.local.service\":\"asdf_MyAgent.DEFAULT\",\"telemetry.extended\":\"true\",\"http.url\":\"https://bugbashgw1776978672-zsy8cbqwts.gateway.bedrock-agentcore.us-west-2.amazonaws.com/mcp\",\"aws.remote.service\":\"bugbashgw1776978672-zsy8cbqwts.gateway.bedrock-agentcore.us-west-2.amazonaws.com\",\"aws.local.environment\":\"bedrock-agentcore:default\",\"aws.remote.operation\":\"POST /mcp\",\"http.status_code\":202,\"aws.local.operation\":\"UnmappedOperation\",\"aws.span.kind\":\"CLIENT\",\"PlatformType\":\"AWS::BedrockAgentCore\",\"http.method\":\"POST\",\"http.response.status_code\":202,\"session.id\":\"00000000-0000-4000-8000-000000000001\"},\"status\":{\"code\":\"UNSET\"}}" + }, + { + "field": "sessionId", + "value": "00000000-0000-4000-8000-000000000001" + }, + { + "field": "traceId", + "value": "6a8f639540ebe6cb6f30d5d80762c2e2" + }, + { + "field": "spanId", + "value": "d14fb48aa6858b3d" + }, + { + "field": "@ptr", + "value": "CpwBCmAKFjY4NTE5NzcwODY4Nzphd3Mvc3BhbnMQABokZTE0NTI5MzgtZjVjZi00OTc1LWE0N2QtYzk3YzMwZmFlY2ZlIg4IgJD4jII0EOefqq2ENDjv37W+yTNAyO+wktMzSAASNhoYAganxnVsAAAAACDEC4EABqj2OeAAAAXCIAEo8K30gIQ0MLe89ICENDgLQNCZAUidZ1CIPCACEAQYAQ==" + } + ], + [ + { + "field": "@message", + "value": "{\"resource\":{\"attributes\":{\"deployment.environment.name\":\"bedrock-agentcore:default\",\"aws.local.service\":\"asdf_MyAgent.DEFAULT\",\"service.name\":\"asdf_MyAgent.DEFAULT\",\"cloud.region\":\"us-west-2\",\"aws.log.stream.names\":\"otel-rt-logs\",\"telemetry.sdk.name\":\"opentelemetry\",\"aws.service.type\":\"gen_ai_agent\",\"telemetry.sdk.language\":\"python\",\"cloud.provider\":\"aws\",\"cloud.resource_id\":\"arn:aws:bedrock-agentcore:us-west-2:685197708687:runtime/asdf_MyAgent-3s5axvBC6Q/runtime-endpoint/DEFAULT:DEFAULT\",\"aws.log.group.names\":\"/aws/bedrock-agentcore/runtimes/asdf_MyAgent-3s5axvBC6Q-DEFAULT\",\"telemetry.sdk.version\":\"1.40.0\",\"cloud.platform\":\"aws_bedrock_agentcore\",\"telemetry.auto.version\":\"0.17.0-aws\"}},\"scope\":{\"name\":\"opentelemetry.instrumentation.httpx\",\"version\":\"0.61b0\"},\"traceId\":\"6a8f639540ebe6cb6f30d5d80762c2e2\",\"spanId\":\"d779e95172d4d1ba\",\"parentSpanId\":\"9a98ae2cca8efc3d\",\"flags\":256,\"name\":\"POST\",\"kind\":\"CLIENT\",\"startTimeUnixNano\":1787782043564209768,\"endTimeUnixNano\":1787782043624521554,\"durationNano\":60311786,\"attributes\":{\"aws.local.service\":\"asdf_MyAgent.DEFAULT\",\"telemetry.extended\":\"true\",\"http.url\":\"https://bugbashgw1776978672-zsy8cbqwts.gateway.bedrock-agentcore.us-west-2.amazonaws.com/mcp\",\"aws.remote.service\":\"bugbashgw1776978672-zsy8cbqwts.gateway.bedrock-agentcore.us-west-2.amazonaws.com\",\"aws.local.environment\":\"bedrock-agentcore:default\",\"aws.remote.operation\":\"POST /mcp\",\"http.status_code\":200,\"aws.local.operation\":\"UnmappedOperation\",\"aws.span.kind\":\"CLIENT\",\"PlatformType\":\"AWS::BedrockAgentCore\",\"http.method\":\"POST\",\"http.response.status_code\":200,\"session.id\":\"00000000-0000-4000-8000-000000000001\"},\"status\":{\"code\":\"UNSET\"}}" + }, + { + "field": "sessionId", + "value": "00000000-0000-4000-8000-000000000001" + }, + { + "field": "traceId", + "value": "6a8f639540ebe6cb6f30d5d80762c2e2" + }, + { + "field": "spanId", + "value": "d779e95172d4d1ba" + }, + { + "field": "@ptr", + "value": "CpwBCmAKFjY4NTE5NzcwODY4Nzphd3Mvc3BhbnMQABokZTE0NTI5MzgtZjVjZi00OTc1LWE0N2QtYzk3YzMwZmFlY2ZlIg4IgJD4jII0EOefqq2ENDjv37W+yTNAyO+wktMzSAASNhoYAganxnVsAAAAACDEC4EABqj2OeAAAAXCIAEo8K30gIQ0MLe89ICENDgLQNCZAUidZ1CIPCACEAUYAQ==" + } + ], + [ + { + "field": "@message", + "value": "{\"resource\":{\"attributes\":{\"deployment.environment.name\":\"bedrock-agentcore:default\",\"aws.local.service\":\"asdf_MyAgent.DEFAULT\",\"service.name\":\"asdf_MyAgent.DEFAULT\",\"cloud.region\":\"us-west-2\",\"aws.log.stream.names\":\"otel-rt-logs\",\"telemetry.sdk.name\":\"opentelemetry\",\"aws.service.type\":\"gen_ai_agent\",\"telemetry.sdk.language\":\"python\",\"cloud.provider\":\"aws\",\"cloud.resource_id\":\"arn:aws:bedrock-agentcore:us-west-2:685197708687:runtime/asdf_MyAgent-3s5axvBC6Q/runtime-endpoint/DEFAULT:DEFAULT\",\"aws.log.group.names\":\"/aws/bedrock-agentcore/runtimes/asdf_MyAgent-3s5axvBC6Q-DEFAULT\",\"telemetry.sdk.version\":\"1.40.0\",\"cloud.platform\":\"aws_bedrock_agentcore\",\"telemetry.auto.version\":\"0.17.0-aws\"}},\"scope\":{\"name\":\"opentelemetry.instrumentation.botocore.bedrock-runtime\",\"version\":\"0.61b0\"},\"traceId\":\"6a8f639540ebe6cb6f30d5d80762c2e2\",\"spanId\":\"0e31289cb4c82723\",\"parentSpanId\":\"0925d4cc59be00d9\",\"flags\":256,\"name\":\"chat global.anthropic.claude-sonnet-4-5-20250929-v1:0\",\"kind\":\"CLIENT\",\"startTimeUnixNano\":1787782043640469865,\"endTimeUnixNano\":1787782045238681022,\"durationNano\":1598211157,\"attributes\":{\"aws.local.service\":\"asdf_MyAgent.DEFAULT\",\"rpc.service\":\"Bedrock Runtime\",\"aws.remote.resource.identifier\":\"global.anthropic.claude-sonnet-4-5-20250929-v1:0\",\"aws.local.environment\":\"bedrock-agentcore:default\",\"aws.remote.operation\":\"ConverseStream\",\"gen_ai.provider.name\":\"aws.bedrock\",\"server.address\":\"bedrock-runtime.us-west-2.amazonaws.com\",\"aws.request_id\":\"45f112bf-ae3f-4cd2-8110-8f6199d04114\",\"aws.local.operation\":\"UnmappedOperation\",\"aws.span.kind\":\"CLIENT\",\"aws.auth.region\":\"us-west-2\",\"rpc.method\":\"ConverseStream\",\"gen_ai.response.finish_reasons\":[\"end_turn\"],\"server.port\":443,\"gen_ai.request.model\":\"global.anthropic.claude-sonnet-4-5-20250929-v1:0\",\"http.response.status_code\":200,\"gen_ai.system\":\"aws.bedrock\",\"telemetry.extended\":\"true\",\"gen_ai.usage.output_tokens\":12,\"aws.genai.span_kind\":\"LLM\",\"rpc.system\":\"aws-api\",\"aws.remote.service\":\"AWS::BedrockRuntime\",\"http.status_code\":200,\"aws.region\":\"us-west-2\",\"aws.remote.resource.type\":\"AWS::Bedrock::Model\",\"gen_ai.operation.name\":\"chat\",\"gen_ai.usage.input_tokens\":1199,\"aws.genai.token_count_total\":1211,\"retry_attempts\":0,\"PlatformType\":\"AWS::BedrockAgentCore\",\"aws.auth.account.access_key\":\"ASIAZ7CHXJWHWD7B2RCF\",\"session.id\":\"00000000-0000-4000-8000-000000000001\"},\"status\":{\"code\":\"UNSET\"}}" + }, + { + "field": "sessionId", + "value": "00000000-0000-4000-8000-000000000001" + }, + { + "field": "traceId", + "value": "6a8f639540ebe6cb6f30d5d80762c2e2" + }, + { + "field": "spanId", + "value": "0e31289cb4c82723" + }, + { + "field": "@ptr", + "value": "CpwBCmAKFjY4NTE5NzcwODY4Nzphd3Mvc3BhbnMQABokZTE0NTI5MzgtZjVjZi00OTc1LWE0N2QtYzk3YzMwZmFlY2ZlIg4IgJD4jII0EOefqq2ENDjv37W+yTNAyO+wktMzSAASNhoYAganxnVsAAAAACDEC4EABqj2OeAAAAXCIAEo8K30gIQ0MLe89ICENDgLQNCZAUidZ1CIPCACEAYYAQ==" + } + ], + [ + { + "field": "@message", + "value": "{\"resource\":{\"attributes\":{\"deployment.environment.name\":\"bedrock-agentcore:default\",\"aws.local.service\":\"asdf_MyAgent.DEFAULT\",\"service.name\":\"asdf_MyAgent.DEFAULT\",\"cloud.region\":\"us-west-2\",\"aws.log.stream.names\":\"otel-rt-logs\",\"telemetry.sdk.name\":\"opentelemetry\",\"aws.service.type\":\"gen_ai_agent\",\"telemetry.sdk.language\":\"python\",\"cloud.provider\":\"aws\",\"cloud.resource_id\":\"arn:aws:bedrock-agentcore:us-west-2:685197708687:runtime/asdf_MyAgent-3s5axvBC6Q/runtime-endpoint/DEFAULT:DEFAULT\",\"aws.log.group.names\":\"/aws/bedrock-agentcore/runtimes/asdf_MyAgent-3s5axvBC6Q-DEFAULT\",\"telemetry.sdk.version\":\"1.40.0\",\"cloud.platform\":\"aws_bedrock_agentcore\",\"telemetry.auto.version\":\"0.17.0-aws\"}},\"scope\":{\"name\":\"strands.telemetry.tracer\",\"version\":\"\"},\"traceId\":\"6a8f639540ebe6cb6f30d5d80762c2e2\",\"spanId\":\"6f925a9a78a2fe62\",\"parentSpanId\":\"9a98ae2cca8efc3d\",\"flags\":256,\"name\":\"invoke_agent Strands Agents\",\"kind\":\"INTERNAL\",\"startTimeUnixNano\":1787782043627084668,\"endTimeUnixNano\":1787782045239678560,\"durationNano\":1612593892,\"attributes\":{\"aws.local.service\":\"asdf_MyAgent.DEFAULT\",\"gen_ai.usage.prompt_tokens\":1199,\"gen_ai.usage.output_tokens\":12,\"gen_ai.usage.cache_write_input_tokens\":0,\"gen_ai.agent.name\":\"Strands Agents\",\"gen_ai.usage.total_tokens\":1211,\"gen_ai.usage.completion_tokens\":12,\"aws.genai.span_kind\":\"AGENT\",\"gen_ai.event.start_time\":\"2026-08-26T22:07:23.627100+00:00\",\"aws.local.environment\":\"bedrock-agentcore:default\",\"gen_ai.provider.name\":\"strands-agents\",\"gen_ai.operation.name\":\"invoke_agent\",\"gen_ai.event.end_time\":\"2026-08-26T22:07:25.239658+00:00\",\"gen_ai.usage.input_tokens\":1199,\"aws.genai.token_count_total\":1211,\"gen_ai.request.model\":\"global.anthropic.claude-sonnet-4-5-20250929-v1:0\",\"gen_ai.usage.cache_read_input_tokens\":0,\"gen_ai.agent.tools\":\"[\\\"add_numbers\\\", \\\"x_amz_bedrock_agentcore_search\\\", \\\"mcpTarget___web_fetch_exa\\\", \\\"mcpTarget___web_search_exa\\\"]\",\"PlatformType\":\"AWS::BedrockAgentCore\",\"session.id\":\"00000000-0000-4000-8000-000000000001\",\"gen_ai.system\":\"strands-agents\"},\"status\":{\"code\":\"OK\"}}" + }, + { + "field": "sessionId", + "value": "00000000-0000-4000-8000-000000000001" + }, + { + "field": "traceId", + "value": "6a8f639540ebe6cb6f30d5d80762c2e2" + }, + { + "field": "spanId", + "value": "6f925a9a78a2fe62" + }, + { + "field": "@ptr", + "value": "CpwBCmAKFjY4NTE5NzcwODY4Nzphd3Mvc3BhbnMQABokZTE0NTI5MzgtZjVjZi00OTc1LWE0N2QtYzk3YzMwZmFlY2ZlIg4IgJD4jII0EOefqq2ENDjv37W+yTNAyO+wktMzSAASNhoYAganxnVsAAAAACDEC4EABqj2OeAAAAXCIAEo8K30gIQ0MLe89ICENDgLQNCZAUidZ1CIPCACEAkYAQ==" + } + ], + [ + { + "field": "@message", + "value": "{\"resource\":{\"attributes\":{\"deployment.environment.name\":\"bedrock-agentcore:default\",\"aws.local.service\":\"asdf_MyAgent.DEFAULT\",\"service.name\":\"asdf_MyAgent.DEFAULT\",\"cloud.region\":\"us-west-2\",\"aws.log.stream.names\":\"otel-rt-logs\",\"telemetry.sdk.name\":\"opentelemetry\",\"aws.service.type\":\"gen_ai_agent\",\"telemetry.sdk.language\":\"python\",\"cloud.provider\":\"aws\",\"cloud.resource_id\":\"arn:aws:bedrock-agentcore:us-west-2:685197708687:runtime/asdf_MyAgent-3s5axvBC6Q/runtime-endpoint/DEFAULT:DEFAULT\",\"aws.log.group.names\":\"/aws/bedrock-agentcore/runtimes/asdf_MyAgent-3s5axvBC6Q-DEFAULT\",\"telemetry.sdk.version\":\"1.40.0\",\"cloud.platform\":\"aws_bedrock_agentcore\",\"telemetry.auto.version\":\"0.17.0-aws\"}},\"scope\":{\"name\":\"strands.telemetry.tracer\",\"version\":\"\"},\"traceId\":\"6a8f639540ebe6cb6f30d5d80762c2e2\",\"spanId\":\"fc189f3b80a0c1f4\",\"parentSpanId\":\"6f925a9a78a2fe62\",\"flags\":256,\"name\":\"execute_event_loop_cycle\",\"kind\":\"INTERNAL\",\"startTimeUnixNano\":1787782043627671178,\"endTimeUnixNano\":1787782045239528397,\"durationNano\":1611857219,\"attributes\":{\"aws.local.service\":\"asdf_MyAgent.DEFAULT\",\"gen_ai.operation.name\":\"execute_event_loop_cycle\",\"gen_ai.event.end_time\":\"2026-08-26T22:07:25.239513+00:00\",\"event_loop.cycle_id\":\"cc82280b-018e-4cea-ac5a-526dae34b998\",\"gen_ai.event.start_time\":\"2026-08-26T22:07:23.627681+00:00\",\"PlatformType\":\"AWS::BedrockAgentCore\",\"session.id\":\"00000000-0000-4000-8000-000000000001\",\"gen_ai.system\":\"strands-agents\",\"aws.local.environment\":\"bedrock-agentcore:default\",\"gen_ai.provider.name\":\"strands-agents\"},\"status\":{\"code\":\"OK\"}}" + }, + { + "field": "sessionId", + "value": "00000000-0000-4000-8000-000000000001" + }, + { + "field": "traceId", + "value": "6a8f639540ebe6cb6f30d5d80762c2e2" + }, + { + "field": "spanId", + "value": "fc189f3b80a0c1f4" + }, + { + "field": "@ptr", + "value": "CpwBCmAKFjY4NTE5NzcwODY4Nzphd3Mvc3BhbnMQABokZTE0NTI5MzgtZjVjZi00OTc1LWE0N2QtYzk3YzMwZmFlY2ZlIg4IgJD4jII0EOefqq2ENDjv37W+yTNAyO+wktMzSAASNhoYAganxnVsAAAAACDEC4EABqj2OeAAAAXCIAEo8K30gIQ0MLe89ICENDgLQNCZAUidZ1CIPCACEAgYAQ==" + } + ], + [ + { + "field": "@message", + "value": "{\"resource\":{\"attributes\":{\"deployment.environment.name\":\"bedrock-agentcore:default\",\"aws.local.service\":\"asdf_MyAgent.DEFAULT\",\"service.name\":\"asdf_MyAgent.DEFAULT\",\"cloud.region\":\"us-west-2\",\"aws.log.stream.names\":\"otel-rt-logs\",\"telemetry.sdk.name\":\"opentelemetry\",\"aws.service.type\":\"gen_ai_agent\",\"telemetry.sdk.language\":\"python\",\"cloud.provider\":\"aws\",\"cloud.resource_id\":\"arn:aws:bedrock-agentcore:us-west-2:685197708687:runtime/asdf_MyAgent-3s5axvBC6Q/runtime-endpoint/DEFAULT:DEFAULT\",\"aws.log.group.names\":\"/aws/bedrock-agentcore/runtimes/asdf_MyAgent-3s5axvBC6Q-DEFAULT\",\"telemetry.sdk.version\":\"1.40.0\",\"cloud.platform\":\"aws_bedrock_agentcore\",\"telemetry.auto.version\":\"0.17.0-aws\"}},\"scope\":{\"name\":\"opentelemetry.instrumentation.starlette\",\"version\":\"0.61b0\"},\"traceId\":\"6a8f639540ebe6cb6f30d5d80762c2e2\",\"spanId\":\"9a98ae2cca8efc3d\",\"parentSpanId\":\"6efc239e6bd4139f\",\"flags\":768,\"name\":\"POST /invocations\",\"kind\":\"SERVER\",\"startTimeUnixNano\":1787782043334606170,\"endTimeUnixNano\":1787782045239993499,\"durationNano\":1905387329,\"attributes\":{\"aws.local.service\":\"asdf_MyAgent.DEFAULT\",\"net.peer.port\":34872,\"telemetry.extended\":\"true\",\"http.target\":\"/invocations\",\"http.flavor\":\"1.1\",\"http.url\":\"http://cell01.us-west-2.prod.arp.kepler-analytics.aws.dev/invocations\",\"net.peer.ip\":\"127.0.0.1\",\"http.host\":\"127.0.0.1:8080\",\"aws.local.environment\":\"bedrock-agentcore:default\",\"http.status_code\":200,\"aws.local.operation\":\"POST /invocations\",\"aws.span.kind\":\"SERVER\",\"http.server_name\":\"cell01.us-west-2.prod.arp.kepler-analytics.aws.dev\",\"net.host.port\":8080,\"http.route\":\"/invocations\",\"PlatformType\":\"AWS::BedrockAgentCore\",\"http.method\":\"POST\",\"http.response.status_code\":200,\"session.id\":\"00000000-0000-4000-8000-000000000001\",\"http.scheme\":\"http\"},\"status\":{\"code\":\"UNSET\"}}" + }, + { + "field": "sessionId", + "value": "00000000-0000-4000-8000-000000000001" + }, + { + "field": "traceId", + "value": "6a8f639540ebe6cb6f30d5d80762c2e2" + }, + { + "field": "spanId", + "value": "9a98ae2cca8efc3d" + }, + { + "field": "@ptr", + "value": "CpwBCmAKFjY4NTE5NzcwODY4Nzphd3Mvc3BhbnMQABokZTE0NTI5MzgtZjVjZi00OTc1LWE0N2QtYzk3YzMwZmFlY2ZlIg4IgJD4jII0EOefqq2ENDjv37W+yTNAyO+wktMzSAASNhoYAganxnVsAAAAACDEC4EABqj2OeAAAAXCIAEo8K30gIQ0MLe89ICENDgLQNCZAUidZ1CIPCACEAoYAQ==" + } + ], + [ + { + "field": "@message", + "value": "{\"resource\":{\"attributes\":{\"deployment.environment.name\":\"bedrock-agentcore:default\",\"aws.local.service\":\"asdf_MyAgent.DEFAULT\",\"service.name\":\"asdf_MyAgent.DEFAULT\",\"cloud.region\":\"us-west-2\",\"aws.log.stream.names\":\"otel-rt-logs\",\"telemetry.sdk.name\":\"opentelemetry\",\"aws.service.type\":\"gen_ai_agent\",\"telemetry.sdk.language\":\"python\",\"cloud.provider\":\"aws\",\"cloud.resource_id\":\"arn:aws:bedrock-agentcore:us-west-2:685197708687:runtime/asdf_MyAgent-3s5axvBC6Q/runtime-endpoint/DEFAULT:DEFAULT\",\"aws.log.group.names\":\"/aws/bedrock-agentcore/runtimes/asdf_MyAgent-3s5axvBC6Q-DEFAULT\",\"telemetry.sdk.version\":\"1.40.0\",\"cloud.platform\":\"aws_bedrock_agentcore\",\"telemetry.auto.version\":\"0.17.0-aws\"}},\"scope\":{\"name\":\"strands.telemetry.tracer\",\"version\":\"\"},\"traceId\":\"6a8f639540ebe6cb6f30d5d80762c2e2\",\"spanId\":\"0925d4cc59be00d9\",\"parentSpanId\":\"fc189f3b80a0c1f4\",\"flags\":256,\"name\":\"chat\",\"kind\":\"INTERNAL\",\"startTimeUnixNano\":1787782043627802055,\"endTimeUnixNano\":1787782045239215762,\"durationNano\":1611413707,\"attributes\":{\"aws.local.service\":\"asdf_MyAgent.DEFAULT\",\"gen_ai.usage.prompt_tokens\":1199,\"gen_ai.usage.output_tokens\":12,\"gen_ai.server.request.duration\":1579,\"gen_ai.usage.total_tokens\":1211,\"gen_ai.usage.completion_tokens\":12,\"aws.genai.span_kind\":\"LLM\",\"gen_ai.event.start_time\":\"2026-08-26T22:07:23.627809+00:00\",\"gen_ai.server.time_to_first_token\":1504,\"aws.local.environment\":\"bedrock-agentcore:default\",\"gen_ai.provider.name\":\"strands-agents\",\"gen_ai.operation.name\":\"chat\",\"gen_ai.event.end_time\":\"2026-08-26T22:07:25.239178+00:00\",\"gen_ai.usage.input_tokens\":1199,\"aws.genai.token_count_total\":1211,\"gen_ai.request.model\":\"global.anthropic.claude-sonnet-4-5-20250929-v1:0\",\"PlatformType\":\"AWS::BedrockAgentCore\",\"session.id\":\"00000000-0000-4000-8000-000000000001\",\"gen_ai.system\":\"strands-agents\"},\"status\":{\"code\":\"OK\"}}" + }, + { + "field": "sessionId", + "value": "00000000-0000-4000-8000-000000000001" + }, + { + "field": "traceId", + "value": "6a8f639540ebe6cb6f30d5d80762c2e2" + }, + { + "field": "spanId", + "value": "0925d4cc59be00d9" + }, + { + "field": "@ptr", + "value": "CpwBCmAKFjY4NTE5NzcwODY4Nzphd3Mvc3BhbnMQABokZTE0NTI5MzgtZjVjZi00OTc1LWE0N2QtYzk3YzMwZmFlY2ZlIg4IgJD4jII0EOefqq2ENDjv37W+yTNAyO+wktMzSAASNhoYAganxnVsAAAAACDEC4EABqj2OeAAAAXCIAEo8K30gIQ0MLe89ICENDgLQNCZAUidZ1CIPCACEAcYAQ==" + } + ], + [ + { + "field": "@message", + "value": "{\"resource\":{\"attributes\":{\"deployment.environment.name\":\"bedrock-agentcore:default\",\"aws.local.service\":\"asdf_MyAgent.DEFAULT\",\"service.name\":\"asdf_MyAgent.DEFAULT\",\"cloud.region\":\"us-west-2\",\"aws.log.stream.names\":\"otel-rt-logs\",\"telemetry.sdk.name\":\"opentelemetry\",\"aws.service.type\":\"gen_ai_agent\",\"telemetry.sdk.language\":\"python\",\"cloud.provider\":\"aws\",\"cloud.resource_id\":\"arn:aws:bedrock-agentcore:us-west-2:685197708687:runtime/asdf_MyAgent-3s5axvBC6Q/runtime-endpoint/DEFAULT:DEFAULT\",\"aws.log.group.names\":\"/aws/bedrock-agentcore/runtimes/asdf_MyAgent-3s5axvBC6Q-DEFAULT\",\"telemetry.sdk.version\":\"1.40.0\",\"cloud.platform\":\"aws_bedrock_agentcore\",\"telemetry.auto.version\":\"0.17.0-aws\"}},\"scope\":{\"name\":\"opentelemetry.instrumentation.urllib3\",\"version\":\"0.61b0\"},\"traceId\":\"6a90a5987755c6e457ada5616a16dddb\",\"spanId\":\"dc14ad43d77bfb36\",\"parentSpanId\":\"98dc1b886c48967e\",\"flags\":256,\"name\":\"PUT\",\"kind\":\"CLIENT\",\"startTimeUnixNano\":1787864478709074216,\"endTimeUnixNano\":1787864478710322708,\"durationNano\":1248492,\"attributes\":{\"aws.local.service\":\"asdf_MyAgent.DEFAULT\",\"telemetry.extended\":\"true\",\"http.url\":\"http://169.254.169.254/latest/api/token\",\"aws.remote.service\":\"169.254.169.254\",\"aws.local.environment\":\"bedrock-agentcore:default\",\"aws.remote.operation\":\"PUT /latest\",\"http.status_code\":200,\"aws.local.operation\":\"UnmappedOperation\",\"aws.span.kind\":\"CLIENT\",\"PlatformType\":\"AWS::BedrockAgentCore\",\"http.method\":\"PUT\",\"http.response.status_code\":200,\"session.id\":\"00000000-0000-4000-8000-000000000001\"},\"status\":{\"code\":\"UNSET\"}}" + }, + { + "field": "sessionId", + "value": "00000000-0000-4000-8000-000000000001" + }, + { + "field": "traceId", + "value": "6a90a5987755c6e457ada5616a16dddb" + }, + { + "field": "spanId", + "value": "dc14ad43d77bfb36" + }, + { + "field": "@ptr", + "value": "CpsBCmAKFjY4NTE5NzcwODY4Nzphd3Mvc3BhbnMQABokZTE0NTI5MzgtZjVjZi00OTc1LWE0N2QtYzk3YzMwZmFlY2ZlIg4IgJD4jII0EOefqq2ENDjv37W+yTNAyO+wktMzSAASNRoYAgakSqFXAAAAAiX3oQcABqkKWgAAAANyIAEo9uebqIQ0MLrqm6iENDgGQKFNSLk1UOgfIAIQABgB" + } + ], + [ + { + "field": "@message", + "value": "{\"resource\":{\"attributes\":{\"deployment.environment.name\":\"bedrock-agentcore:default\",\"aws.local.service\":\"asdf_MyAgent.DEFAULT\",\"service.name\":\"asdf_MyAgent.DEFAULT\",\"cloud.region\":\"us-west-2\",\"aws.log.stream.names\":\"otel-rt-logs\",\"telemetry.sdk.name\":\"opentelemetry\",\"aws.service.type\":\"gen_ai_agent\",\"telemetry.sdk.language\":\"python\",\"cloud.provider\":\"aws\",\"cloud.resource_id\":\"arn:aws:bedrock-agentcore:us-west-2:685197708687:runtime/asdf_MyAgent-3s5axvBC6Q/runtime-endpoint/DEFAULT:DEFAULT\",\"aws.log.group.names\":\"/aws/bedrock-agentcore/runtimes/asdf_MyAgent-3s5axvBC6Q-DEFAULT\",\"telemetry.sdk.version\":\"1.40.0\",\"cloud.platform\":\"aws_bedrock_agentcore\",\"telemetry.auto.version\":\"0.17.0-aws\"}},\"scope\":{\"name\":\"opentelemetry.instrumentation.urllib3\",\"version\":\"0.61b0\"},\"traceId\":\"6a90a5987755c6e457ada5616a16dddb\",\"spanId\":\"cc7a992ec6c59a21\",\"parentSpanId\":\"98dc1b886c48967e\",\"flags\":256,\"name\":\"GET\",\"kind\":\"CLIENT\",\"startTimeUnixNano\":1787864478710808895,\"endTimeUnixNano\":1787864478711561018,\"durationNano\":752123,\"attributes\":{\"aws.local.service\":\"asdf_MyAgent.DEFAULT\",\"telemetry.extended\":\"true\",\"http.url\":\"http://169.254.169.254/latest/meta-data/iam/security-credentials/\",\"aws.remote.service\":\"169.254.169.254\",\"aws.local.environment\":\"bedrock-agentcore:default\",\"aws.remote.operation\":\"GET /latest\",\"http.status_code\":200,\"aws.local.operation\":\"UnmappedOperation\",\"aws.span.kind\":\"CLIENT\",\"PlatformType\":\"AWS::BedrockAgentCore\",\"http.method\":\"GET\",\"http.response.status_code\":200,\"session.id\":\"00000000-0000-4000-8000-000000000001\"},\"status\":{\"code\":\"UNSET\"}}" + }, + { + "field": "sessionId", + "value": "00000000-0000-4000-8000-000000000001" + }, + { + "field": "traceId", + "value": "6a90a5987755c6e457ada5616a16dddb" + }, + { + "field": "spanId", + "value": "cc7a992ec6c59a21" + }, + { + "field": "@ptr", + "value": "CpsBCmAKFjY4NTE5NzcwODY4Nzphd3Mvc3BhbnMQABokZTE0NTI5MzgtZjVjZi00OTc1LWE0N2QtYzk3YzMwZmFlY2ZlIg4IgJD4jII0EOefqq2ENDjv37W+yTNAyO+wktMzSAASNRoYAgakSqFXAAAAAiX3oQcABqkKWgAAAANyIAEo9uebqIQ0MLrqm6iENDgGQKFNSLk1UOgfIAIQARgB" + } + ], + [ + { + "field": "@message", + "value": "{\"resource\":{\"attributes\":{\"deployment.environment.name\":\"bedrock-agentcore:default\",\"aws.local.service\":\"asdf_MyAgent.DEFAULT\",\"service.name\":\"asdf_MyAgent.DEFAULT\",\"cloud.region\":\"us-west-2\",\"aws.log.stream.names\":\"otel-rt-logs\",\"telemetry.sdk.name\":\"opentelemetry\",\"aws.service.type\":\"gen_ai_agent\",\"telemetry.sdk.language\":\"python\",\"cloud.provider\":\"aws\",\"cloud.resource_id\":\"arn:aws:bedrock-agentcore:us-west-2:685197708687:runtime/asdf_MyAgent-3s5axvBC6Q/runtime-endpoint/DEFAULT:DEFAULT\",\"aws.log.group.names\":\"/aws/bedrock-agentcore/runtimes/asdf_MyAgent-3s5axvBC6Q-DEFAULT\",\"telemetry.sdk.version\":\"1.40.0\",\"cloud.platform\":\"aws_bedrock_agentcore\",\"telemetry.auto.version\":\"0.17.0-aws\"}},\"scope\":{\"name\":\"opentelemetry.instrumentation.urllib3\",\"version\":\"0.61b0\"},\"traceId\":\"6a90a5987755c6e457ada5616a16dddb\",\"spanId\":\"f155589448edb57d\",\"parentSpanId\":\"98dc1b886c48967e\",\"flags\":256,\"name\":\"GET\",\"kind\":\"CLIENT\",\"startTimeUnixNano\":1787864478711972677,\"endTimeUnixNano\":1787864478712747135,\"durationNano\":774458,\"attributes\":{\"aws.local.service\":\"asdf_MyAgent.DEFAULT\",\"telemetry.extended\":\"true\",\"http.url\":\"http://169.254.169.254/latest/meta-data/iam/security-credentials/execution_role\",\"aws.remote.service\":\"169.254.169.254\",\"aws.local.environment\":\"bedrock-agentcore:default\",\"aws.remote.operation\":\"GET /latest\",\"http.status_code\":200,\"aws.local.operation\":\"UnmappedOperation\",\"aws.span.kind\":\"CLIENT\",\"PlatformType\":\"AWS::BedrockAgentCore\",\"http.method\":\"GET\",\"http.response.status_code\":200,\"session.id\":\"00000000-0000-4000-8000-000000000001\"},\"status\":{\"code\":\"UNSET\"}}" + }, + { + "field": "sessionId", + "value": "00000000-0000-4000-8000-000000000001" + }, + { + "field": "traceId", + "value": "6a90a5987755c6e457ada5616a16dddb" + }, + { + "field": "spanId", + "value": "f155589448edb57d" + }, + { + "field": "@ptr", + "value": "CpsBCmAKFjY4NTE5NzcwODY4Nzphd3Mvc3BhbnMQABokZTE0NTI5MzgtZjVjZi00OTc1LWE0N2QtYzk3YzMwZmFlY2ZlIg4IgJD4jII0EOefqq2ENDjv37W+yTNAyO+wktMzSAASNRoYAgakSqFXAAAAAiX3oQcABqkKWgAAAANyIAEo9uebqIQ0MLrqm6iENDgGQKFNSLk1UOgfIAIQAhgB" + } + ], + [ + { + "field": "@message", + "value": "{\"resource\":{\"attributes\":{\"deployment.environment.name\":\"bedrock-agentcore:default\",\"aws.local.service\":\"asdf_MyAgent.DEFAULT\",\"service.name\":\"asdf_MyAgent.DEFAULT\",\"cloud.region\":\"us-west-2\",\"aws.log.stream.names\":\"otel-rt-logs\",\"telemetry.sdk.name\":\"opentelemetry\",\"aws.service.type\":\"gen_ai_agent\",\"telemetry.sdk.language\":\"python\",\"cloud.provider\":\"aws\",\"cloud.resource_id\":\"arn:aws:bedrock-agentcore:us-west-2:685197708687:runtime/asdf_MyAgent-3s5axvBC6Q/runtime-endpoint/DEFAULT:DEFAULT\",\"aws.log.group.names\":\"/aws/bedrock-agentcore/runtimes/asdf_MyAgent-3s5axvBC6Q-DEFAULT\",\"telemetry.sdk.version\":\"1.40.0\",\"cloud.platform\":\"aws_bedrock_agentcore\",\"telemetry.auto.version\":\"0.17.0-aws\"}},\"scope\":{\"name\":\"opentelemetry.instrumentation.httpx\",\"version\":\"0.61b0\"},\"traceId\":\"6a90a5987755c6e457ada5616a16dddb\",\"spanId\":\"e350f049b9ee9a83\",\"parentSpanId\":\"98dc1b886c48967e\",\"flags\":256,\"name\":\"POST\",\"kind\":\"CLIENT\",\"startTimeUnixNano\":1787864478853179249,\"endTimeUnixNano\":1787864478920600443,\"durationNano\":67421194,\"attributes\":{\"aws.local.service\":\"asdf_MyAgent.DEFAULT\",\"telemetry.extended\":\"true\",\"http.url\":\"https://bugbashgw1776978672-zsy8cbqwts.gateway.bedrock-agentcore.us-west-2.amazonaws.com/mcp\",\"aws.remote.service\":\"bugbashgw1776978672-zsy8cbqwts.gateway.bedrock-agentcore.us-west-2.amazonaws.com\",\"aws.local.environment\":\"bedrock-agentcore:default\",\"aws.remote.operation\":\"POST /mcp\",\"http.status_code\":200,\"aws.local.operation\":\"UnmappedOperation\",\"aws.span.kind\":\"CLIENT\",\"PlatformType\":\"AWS::BedrockAgentCore\",\"http.method\":\"POST\",\"http.response.status_code\":200,\"session.id\":\"00000000-0000-4000-8000-000000000001\"},\"status\":{\"code\":\"UNSET\"}}" + }, + { + "field": "sessionId", + "value": "00000000-0000-4000-8000-000000000001" + }, + { + "field": "traceId", + "value": "6a90a5987755c6e457ada5616a16dddb" + }, + { + "field": "spanId", + "value": "e350f049b9ee9a83" + }, + { + "field": "@ptr", + "value": "CpsBCmAKFjY4NTE5NzcwODY4Nzphd3Mvc3BhbnMQABokZTE0NTI5MzgtZjVjZi00OTc1LWE0N2QtYzk3YzMwZmFlY2ZlIg4IgJD4jII0EOefqq2ENDjv37W+yTNAyO+wktMzSAASNRoYAgakSqFXAAAAAiX3oQcABqkKWgAAAANyIAEo9uebqIQ0MLrqm6iENDgGQKFNSLk1UOgfIAIQAxgB" + } + ], + [ + { + "field": "@message", + "value": "{\"resource\":{\"attributes\":{\"deployment.environment.name\":\"bedrock-agentcore:default\",\"aws.local.service\":\"asdf_MyAgent.DEFAULT\",\"service.name\":\"asdf_MyAgent.DEFAULT\",\"cloud.region\":\"us-west-2\",\"aws.log.stream.names\":\"otel-rt-logs\",\"telemetry.sdk.name\":\"opentelemetry\",\"aws.service.type\":\"gen_ai_agent\",\"telemetry.sdk.language\":\"python\",\"cloud.provider\":\"aws\",\"cloud.resource_id\":\"arn:aws:bedrock-agentcore:us-west-2:685197708687:runtime/asdf_MyAgent-3s5axvBC6Q/runtime-endpoint/DEFAULT:DEFAULT\",\"aws.log.group.names\":\"/aws/bedrock-agentcore/runtimes/asdf_MyAgent-3s5axvBC6Q-DEFAULT\",\"telemetry.sdk.version\":\"1.40.0\",\"cloud.platform\":\"aws_bedrock_agentcore\",\"telemetry.auto.version\":\"0.17.0-aws\"}},\"scope\":{\"name\":\"opentelemetry.instrumentation.httpx\",\"version\":\"0.61b0\"},\"traceId\":\"6a90a5987755c6e457ada5616a16dddb\",\"spanId\":\"c8a7a6467d7a7d7c\",\"parentSpanId\":\"98dc1b886c48967e\",\"flags\":256,\"name\":\"POST\",\"kind\":\"CLIENT\",\"startTimeUnixNano\":1787864478923846540,\"endTimeUnixNano\":1787864478960458576,\"durationNano\":36612036,\"attributes\":{\"aws.local.service\":\"asdf_MyAgent.DEFAULT\",\"telemetry.extended\":\"true\",\"http.url\":\"https://bugbashgw1776978672-zsy8cbqwts.gateway.bedrock-agentcore.us-west-2.amazonaws.com/mcp\",\"aws.remote.service\":\"bugbashgw1776978672-zsy8cbqwts.gateway.bedrock-agentcore.us-west-2.amazonaws.com\",\"aws.local.environment\":\"bedrock-agentcore:default\",\"aws.remote.operation\":\"POST /mcp\",\"http.status_code\":202,\"aws.local.operation\":\"UnmappedOperation\",\"aws.span.kind\":\"CLIENT\",\"PlatformType\":\"AWS::BedrockAgentCore\",\"http.method\":\"POST\",\"http.response.status_code\":202,\"session.id\":\"00000000-0000-4000-8000-000000000001\"},\"status\":{\"code\":\"UNSET\"}}" + }, + { + "field": "sessionId", + "value": "00000000-0000-4000-8000-000000000001" + }, + { + "field": "traceId", + "value": "6a90a5987755c6e457ada5616a16dddb" + }, + { + "field": "spanId", + "value": "c8a7a6467d7a7d7c" + }, + { + "field": "@ptr", + "value": "CpsBCmAKFjY4NTE5NzcwODY4Nzphd3Mvc3BhbnMQABokZTE0NTI5MzgtZjVjZi00OTc1LWE0N2QtYzk3YzMwZmFlY2ZlIg4IgJD4jII0EOefqq2ENDjv37W+yTNAyO+wktMzSAASNRoYAgakSqFXAAAAAiX3oQcABqkKWgAAAANyIAEo9uebqIQ0MLrqm6iENDgGQKFNSLk1UOgfIAIQBBgB" + } + ], + [ + { + "field": "@message", + "value": "{\"resource\":{\"attributes\":{\"deployment.environment.name\":\"bedrock-agentcore:default\",\"aws.local.service\":\"asdf_MyAgent.DEFAULT\",\"service.name\":\"asdf_MyAgent.DEFAULT\",\"cloud.region\":\"us-west-2\",\"aws.log.stream.names\":\"otel-rt-logs\",\"telemetry.sdk.name\":\"opentelemetry\",\"aws.service.type\":\"gen_ai_agent\",\"telemetry.sdk.language\":\"python\",\"cloud.provider\":\"aws\",\"cloud.resource_id\":\"arn:aws:bedrock-agentcore:us-west-2:685197708687:runtime/asdf_MyAgent-3s5axvBC6Q/runtime-endpoint/DEFAULT:DEFAULT\",\"aws.log.group.names\":\"/aws/bedrock-agentcore/runtimes/asdf_MyAgent-3s5axvBC6Q-DEFAULT\",\"telemetry.sdk.version\":\"1.40.0\",\"cloud.platform\":\"aws_bedrock_agentcore\",\"telemetry.auto.version\":\"0.17.0-aws\"}},\"scope\":{\"name\":\"opentelemetry.instrumentation.httpx\",\"version\":\"0.61b0\"},\"traceId\":\"6a90a5987755c6e457ada5616a16dddb\",\"spanId\":\"29da3bc6bf4e4025\",\"parentSpanId\":\"98dc1b886c48967e\",\"flags\":256,\"name\":\"POST\",\"kind\":\"CLIENT\",\"startTimeUnixNano\":1787864478962011733,\"endTimeUnixNano\":1787864479034750244,\"durationNano\":72738511,\"attributes\":{\"aws.local.service\":\"asdf_MyAgent.DEFAULT\",\"telemetry.extended\":\"true\",\"http.url\":\"https://bugbashgw1776978672-zsy8cbqwts.gateway.bedrock-agentcore.us-west-2.amazonaws.com/mcp\",\"aws.remote.service\":\"bugbashgw1776978672-zsy8cbqwts.gateway.bedrock-agentcore.us-west-2.amazonaws.com\",\"aws.local.environment\":\"bedrock-agentcore:default\",\"aws.remote.operation\":\"POST /mcp\",\"http.status_code\":200,\"aws.local.operation\":\"UnmappedOperation\",\"aws.span.kind\":\"CLIENT\",\"PlatformType\":\"AWS::BedrockAgentCore\",\"http.method\":\"POST\",\"http.response.status_code\":200,\"session.id\":\"00000000-0000-4000-8000-000000000001\"},\"status\":{\"code\":\"UNSET\"}}" + }, + { + "field": "sessionId", + "value": "00000000-0000-4000-8000-000000000001" + }, + { + "field": "traceId", + "value": "6a90a5987755c6e457ada5616a16dddb" + }, + { + "field": "spanId", + "value": "29da3bc6bf4e4025" + }, + { + "field": "@ptr", + "value": "CpsBCmAKFjY4NTE5NzcwODY4Nzphd3Mvc3BhbnMQABokZTE0NTI5MzgtZjVjZi00OTc1LWE0N2QtYzk3YzMwZmFlY2ZlIg4IgJD4jII0EOefqq2ENDjv37W+yTNAyO+wktMzSAASNRoYAgakSqFXAAAAAiX3oQcABqkKWgAAAANyIAEo9uebqIQ0MLrqm6iENDgGQKFNSLk1UOgfIAIQBRgB" + } + ], + [ + { + "field": "@message", + "value": "{\"resource\":{\"attributes\":{\"deployment.environment.name\":\"bedrock-agentcore:default\",\"aws.local.service\":\"asdf_MyAgent.DEFAULT\",\"service.name\":\"asdf_MyAgent.DEFAULT\",\"cloud.region\":\"us-west-2\",\"aws.log.stream.names\":\"otel-rt-logs\",\"telemetry.sdk.name\":\"opentelemetry\",\"aws.service.type\":\"gen_ai_agent\",\"telemetry.sdk.language\":\"python\",\"cloud.provider\":\"aws\",\"cloud.resource_id\":\"arn:aws:bedrock-agentcore:us-west-2:685197708687:runtime/asdf_MyAgent-3s5axvBC6Q/runtime-endpoint/DEFAULT:DEFAULT\",\"aws.log.group.names\":\"/aws/bedrock-agentcore/runtimes/asdf_MyAgent-3s5axvBC6Q-DEFAULT\",\"telemetry.sdk.version\":\"1.40.0\",\"cloud.platform\":\"aws_bedrock_agentcore\",\"telemetry.auto.version\":\"0.17.0-aws\"}},\"scope\":{\"name\":\"opentelemetry.instrumentation.botocore.bedrock-runtime\",\"version\":\"0.61b0\"},\"traceId\":\"6a90a5987755c6e457ada5616a16dddb\",\"spanId\":\"12835ac323e13dde\",\"parentSpanId\":\"d473e096a2db85a8\",\"flags\":256,\"name\":\"chat global.anthropic.claude-sonnet-4-5-20250929-v1:0\",\"kind\":\"CLIENT\",\"startTimeUnixNano\":1787864479055516061,\"endTimeUnixNano\":1787864481906556152,\"durationNano\":2851040091,\"attributes\":{\"aws.local.service\":\"asdf_MyAgent.DEFAULT\",\"rpc.service\":\"Bedrock Runtime\",\"aws.remote.resource.identifier\":\"global.anthropic.claude-sonnet-4-5-20250929-v1:0\",\"aws.local.environment\":\"bedrock-agentcore:default\",\"aws.remote.operation\":\"ConverseStream\",\"gen_ai.provider.name\":\"aws.bedrock\",\"server.address\":\"bedrock-runtime.us-west-2.amazonaws.com\",\"aws.request_id\":\"8ce89304-4900-46ba-8429-2815f2f4922c\",\"aws.local.operation\":\"UnmappedOperation\",\"aws.span.kind\":\"CLIENT\",\"aws.auth.region\":\"us-west-2\",\"rpc.method\":\"ConverseStream\",\"gen_ai.response.finish_reasons\":[\"end_turn\"],\"server.port\":443,\"gen_ai.request.model\":\"global.anthropic.claude-sonnet-4-5-20250929-v1:0\",\"http.response.status_code\":200,\"gen_ai.system\":\"aws.bedrock\",\"telemetry.extended\":\"true\",\"gen_ai.usage.output_tokens\":82,\"aws.genai.span_kind\":\"LLM\",\"rpc.system\":\"aws-api\",\"aws.remote.service\":\"AWS::BedrockRuntime\",\"http.status_code\":200,\"aws.region\":\"us-west-2\",\"aws.remote.resource.type\":\"AWS::Bedrock::Model\",\"gen_ai.operation.name\":\"chat\",\"gen_ai.usage.input_tokens\":1203,\"aws.genai.token_count_total\":1285,\"retry_attempts\":0,\"PlatformType\":\"AWS::BedrockAgentCore\",\"aws.auth.account.access_key\":\"ASIAZ7CHXJWH4OBTTP6T\",\"session.id\":\"00000000-0000-4000-8000-000000000001\"},\"status\":{\"code\":\"UNSET\"}}" + }, + { + "field": "sessionId", + "value": "00000000-0000-4000-8000-000000000001" + }, + { + "field": "traceId", + "value": "6a90a5987755c6e457ada5616a16dddb" + }, + { + "field": "spanId", + "value": "12835ac323e13dde" + }, + { + "field": "@ptr", + "value": "CpsBCmAKFjY4NTE5NzcwODY4Nzphd3Mvc3BhbnMQABokZTE0NTI5MzgtZjVjZi00OTc1LWE0N2QtYzk3YzMwZmFlY2ZlIg4IgJD4jII0EOefqq2ENDjv37W+yTNAyO+wktMzSAASNRoYAgakSqFXAAAAAiX3ojcABqkKWmAAAANyIAEo8oCcqIQ0MPSAnKiENDgFQK9MSKdkUJQ5IAIQABgB" + } + ], + [ + { + "field": "@message", + "value": "{\"resource\":{\"attributes\":{\"deployment.environment.name\":\"bedrock-agentcore:default\",\"aws.local.service\":\"asdf_MyAgent.DEFAULT\",\"service.name\":\"asdf_MyAgent.DEFAULT\",\"cloud.region\":\"us-west-2\",\"aws.log.stream.names\":\"otel-rt-logs\",\"telemetry.sdk.name\":\"opentelemetry\",\"aws.service.type\":\"gen_ai_agent\",\"telemetry.sdk.language\":\"python\",\"cloud.provider\":\"aws\",\"cloud.resource_id\":\"arn:aws:bedrock-agentcore:us-west-2:685197708687:runtime/asdf_MyAgent-3s5axvBC6Q/runtime-endpoint/DEFAULT:DEFAULT\",\"aws.log.group.names\":\"/aws/bedrock-agentcore/runtimes/asdf_MyAgent-3s5axvBC6Q-DEFAULT\",\"telemetry.sdk.version\":\"1.40.0\",\"cloud.platform\":\"aws_bedrock_agentcore\",\"telemetry.auto.version\":\"0.17.0-aws\"}},\"scope\":{\"name\":\"strands.telemetry.tracer\",\"version\":\"\"},\"traceId\":\"6a90a5987755c6e457ada5616a16dddb\",\"spanId\":\"17fb5fcf047ef303\",\"parentSpanId\":\"2619914761138b83\",\"flags\":256,\"name\":\"execute_event_loop_cycle\",\"kind\":\"INTERNAL\",\"startTimeUnixNano\":1787864479038684800,\"endTimeUnixNano\":1787864481907561373,\"durationNano\":2868876573,\"attributes\":{\"aws.local.service\":\"asdf_MyAgent.DEFAULT\",\"gen_ai.operation.name\":\"execute_event_loop_cycle\",\"gen_ai.event.end_time\":\"2026-08-27T21:01:21.907543+00:00\",\"event_loop.cycle_id\":\"7b2c4a18-cab6-4ea0-931d-07a005ba1f88\",\"gen_ai.event.start_time\":\"2026-08-27T21:01:19.038699+00:00\",\"PlatformType\":\"AWS::BedrockAgentCore\",\"session.id\":\"00000000-0000-4000-8000-000000000001\",\"gen_ai.system\":\"strands-agents\",\"aws.local.environment\":\"bedrock-agentcore:default\",\"gen_ai.provider.name\":\"strands-agents\"},\"status\":{\"code\":\"OK\"}}" + }, + { + "field": "sessionId", + "value": "00000000-0000-4000-8000-000000000001" + }, + { + "field": "traceId", + "value": "6a90a5987755c6e457ada5616a16dddb" + }, + { + "field": "spanId", + "value": "17fb5fcf047ef303" + }, + { + "field": "@ptr", + "value": "CpsBCmAKFjY4NTE5NzcwODY4Nzphd3Mvc3BhbnMQABokZTE0NTI5MzgtZjVjZi00OTc1LWE0N2QtYzk3YzMwZmFlY2ZlIg4IgJD4jII0EOefqq2ENDjv37W+yTNAyO+wktMzSAASNRoYAgakSqFXAAAAAiX3ojcABqkKWmAAAANyIAEo8oCcqIQ0MPSAnKiENDgFQK9MSKdkUJQ5IAIQAhgB" + } + ], + [ + { + "field": "@message", + "value": "{\"resource\":{\"attributes\":{\"deployment.environment.name\":\"bedrock-agentcore:default\",\"aws.local.service\":\"asdf_MyAgent.DEFAULT\",\"service.name\":\"asdf_MyAgent.DEFAULT\",\"cloud.region\":\"us-west-2\",\"aws.log.stream.names\":\"otel-rt-logs\",\"telemetry.sdk.name\":\"opentelemetry\",\"aws.service.type\":\"gen_ai_agent\",\"telemetry.sdk.language\":\"python\",\"cloud.provider\":\"aws\",\"cloud.resource_id\":\"arn:aws:bedrock-agentcore:us-west-2:685197708687:runtime/asdf_MyAgent-3s5axvBC6Q/runtime-endpoint/DEFAULT:DEFAULT\",\"aws.log.group.names\":\"/aws/bedrock-agentcore/runtimes/asdf_MyAgent-3s5axvBC6Q-DEFAULT\",\"telemetry.sdk.version\":\"1.40.0\",\"cloud.platform\":\"aws_bedrock_agentcore\",\"telemetry.auto.version\":\"0.17.0-aws\"}},\"scope\":{\"name\":\"strands.telemetry.tracer\",\"version\":\"\"},\"traceId\":\"6a90a5987755c6e457ada5616a16dddb\",\"spanId\":\"2619914761138b83\",\"parentSpanId\":\"98dc1b886c48967e\",\"flags\":256,\"name\":\"invoke_agent Strands Agents\",\"kind\":\"INTERNAL\",\"startTimeUnixNano\":1787864479037932324,\"endTimeUnixNano\":1787864481907743241,\"durationNano\":2869810917,\"attributes\":{\"aws.local.service\":\"asdf_MyAgent.DEFAULT\",\"gen_ai.usage.prompt_tokens\":1203,\"gen_ai.usage.output_tokens\":82,\"gen_ai.usage.cache_write_input_tokens\":0,\"gen_ai.agent.name\":\"Strands Agents\",\"gen_ai.usage.total_tokens\":1285,\"gen_ai.usage.completion_tokens\":82,\"aws.genai.span_kind\":\"AGENT\",\"gen_ai.event.start_time\":\"2026-08-27T21:01:19.037951+00:00\",\"aws.local.environment\":\"bedrock-agentcore:default\",\"gen_ai.provider.name\":\"strands-agents\",\"gen_ai.operation.name\":\"invoke_agent\",\"gen_ai.event.end_time\":\"2026-08-27T21:01:21.907716+00:00\",\"gen_ai.usage.input_tokens\":1203,\"aws.genai.token_count_total\":1285,\"gen_ai.request.model\":\"global.anthropic.claude-sonnet-4-5-20250929-v1:0\",\"gen_ai.usage.cache_read_input_tokens\":0,\"gen_ai.agent.tools\":\"[\\\"add_numbers\\\", \\\"x_amz_bedrock_agentcore_search\\\", \\\"mcpTarget___web_fetch_exa\\\", \\\"mcpTarget___web_search_exa\\\"]\",\"PlatformType\":\"AWS::BedrockAgentCore\",\"session.id\":\"00000000-0000-4000-8000-000000000001\",\"gen_ai.system\":\"strands-agents\"},\"status\":{\"code\":\"OK\"}}" + }, + { + "field": "sessionId", + "value": "00000000-0000-4000-8000-000000000001" + }, + { + "field": "traceId", + "value": "6a90a5987755c6e457ada5616a16dddb" + }, + { + "field": "spanId", + "value": "2619914761138b83" + }, + { + "field": "@ptr", + "value": "CpsBCmAKFjY4NTE5NzcwODY4Nzphd3Mvc3BhbnMQABokZTE0NTI5MzgtZjVjZi00OTc1LWE0N2QtYzk3YzMwZmFlY2ZlIg4IgJD4jII0EOefqq2ENDjv37W+yTNAyO+wktMzSAASNRoYAgakSqFXAAAAAiX3ojcABqkKWmAAAANyIAEo8oCcqIQ0MPSAnKiENDgFQK9MSKdkUJQ5IAIQAxgB" + } + ], + [ + { + "field": "@message", + "value": "{\"resource\":{\"attributes\":{\"deployment.environment.name\":\"bedrock-agentcore:default\",\"aws.local.service\":\"asdf_MyAgent.DEFAULT\",\"service.name\":\"asdf_MyAgent.DEFAULT\",\"cloud.region\":\"us-west-2\",\"aws.log.stream.names\":\"otel-rt-logs\",\"telemetry.sdk.name\":\"opentelemetry\",\"aws.service.type\":\"gen_ai_agent\",\"telemetry.sdk.language\":\"python\",\"cloud.provider\":\"aws\",\"cloud.resource_id\":\"arn:aws:bedrock-agentcore:us-west-2:685197708687:runtime/asdf_MyAgent-3s5axvBC6Q/runtime-endpoint/DEFAULT:DEFAULT\",\"aws.log.group.names\":\"/aws/bedrock-agentcore/runtimes/asdf_MyAgent-3s5axvBC6Q-DEFAULT\",\"telemetry.sdk.version\":\"1.40.0\",\"cloud.platform\":\"aws_bedrock_agentcore\",\"telemetry.auto.version\":\"0.17.0-aws\"}},\"scope\":{\"name\":\"strands.telemetry.tracer\",\"version\":\"\"},\"traceId\":\"6a90a5987755c6e457ada5616a16dddb\",\"spanId\":\"d473e096a2db85a8\",\"parentSpanId\":\"17fb5fcf047ef303\",\"flags\":256,\"name\":\"chat\",\"kind\":\"INTERNAL\",\"startTimeUnixNano\":1787864479038852619,\"endTimeUnixNano\":1787864481907136303,\"durationNano\":2868283684,\"attributes\":{\"aws.local.service\":\"asdf_MyAgent.DEFAULT\",\"gen_ai.usage.prompt_tokens\":1203,\"gen_ai.usage.output_tokens\":82,\"gen_ai.server.request.duration\":2813,\"gen_ai.usage.total_tokens\":1285,\"gen_ai.usage.completion_tokens\":82,\"aws.genai.span_kind\":\"LLM\",\"gen_ai.event.start_time\":\"2026-08-27T21:01:19.038861+00:00\",\"gen_ai.server.time_to_first_token\":1508,\"aws.local.environment\":\"bedrock-agentcore:default\",\"gen_ai.provider.name\":\"strands-agents\",\"gen_ai.operation.name\":\"chat\",\"gen_ai.event.end_time\":\"2026-08-27T21:01:21.907090+00:00\",\"gen_ai.usage.input_tokens\":1203,\"aws.genai.token_count_total\":1285,\"gen_ai.request.model\":\"global.anthropic.claude-sonnet-4-5-20250929-v1:0\",\"PlatformType\":\"AWS::BedrockAgentCore\",\"session.id\":\"00000000-0000-4000-8000-000000000001\",\"gen_ai.system\":\"strands-agents\"},\"status\":{\"code\":\"OK\"}}" + }, + { + "field": "sessionId", + "value": "00000000-0000-4000-8000-000000000001" + }, + { + "field": "traceId", + "value": "6a90a5987755c6e457ada5616a16dddb" + }, + { + "field": "spanId", + "value": "d473e096a2db85a8" + }, + { + "field": "@ptr", + "value": "CpsBCmAKFjY4NTE5NzcwODY4Nzphd3Mvc3BhbnMQABokZTE0NTI5MzgtZjVjZi00OTc1LWE0N2QtYzk3YzMwZmFlY2ZlIg4IgJD4jII0EOefqq2ENDjv37W+yTNAyO+wktMzSAASNRoYAgakSqFXAAAAAiX3ojcABqkKWmAAAANyIAEo8oCcqIQ0MPSAnKiENDgFQK9MSKdkUJQ5IAIQARgB" + } + ], + [ + { + "field": "@message", + "value": "{\"resource\":{\"attributes\":{\"deployment.environment.name\":\"bedrock-agentcore:default\",\"aws.local.service\":\"asdf_MyAgent.DEFAULT\",\"service.name\":\"asdf_MyAgent.DEFAULT\",\"cloud.region\":\"us-west-2\",\"aws.log.stream.names\":\"otel-rt-logs\",\"telemetry.sdk.name\":\"opentelemetry\",\"aws.service.type\":\"gen_ai_agent\",\"telemetry.sdk.language\":\"python\",\"cloud.provider\":\"aws\",\"cloud.resource_id\":\"arn:aws:bedrock-agentcore:us-west-2:685197708687:runtime/asdf_MyAgent-3s5axvBC6Q/runtime-endpoint/DEFAULT:DEFAULT\",\"aws.log.group.names\":\"/aws/bedrock-agentcore/runtimes/asdf_MyAgent-3s5axvBC6Q-DEFAULT\",\"telemetry.sdk.version\":\"1.40.0\",\"cloud.platform\":\"aws_bedrock_agentcore\",\"telemetry.auto.version\":\"0.17.0-aws\"}},\"scope\":{\"name\":\"opentelemetry.instrumentation.starlette\",\"version\":\"0.61b0\"},\"traceId\":\"6a90a5987755c6e457ada5616a16dddb\",\"spanId\":\"98dc1b886c48967e\",\"parentSpanId\":\"1305dbef52b28faa\",\"flags\":768,\"name\":\"POST /invocations\",\"kind\":\"SERVER\",\"startTimeUnixNano\":1787864478654367295,\"endTimeUnixNano\":1787864481908108495,\"durationNano\":3253741200,\"attributes\":{\"aws.local.service\":\"asdf_MyAgent.DEFAULT\",\"net.peer.port\":50084,\"telemetry.extended\":\"true\",\"http.target\":\"/invocations\",\"http.flavor\":\"1.1\",\"http.url\":\"http://cell01.us-west-2.prod.arp.kepler-analytics.aws.dev/invocations\",\"net.peer.ip\":\"127.0.0.1\",\"http.host\":\"127.0.0.1:8080\",\"aws.local.environment\":\"bedrock-agentcore:default\",\"http.status_code\":200,\"aws.local.operation\":\"POST /invocations\",\"aws.span.kind\":\"SERVER\",\"http.server_name\":\"cell01.us-west-2.prod.arp.kepler-analytics.aws.dev\",\"net.host.port\":8080,\"http.route\":\"/invocations\",\"PlatformType\":\"AWS::BedrockAgentCore\",\"http.method\":\"POST\",\"http.response.status_code\":200,\"session.id\":\"00000000-0000-4000-8000-000000000001\",\"http.scheme\":\"http\"},\"status\":{\"code\":\"UNSET\"}}" + }, + { + "field": "sessionId", + "value": "00000000-0000-4000-8000-000000000001" + }, + { + "field": "traceId", + "value": "6a90a5987755c6e457ada5616a16dddb" + }, + { + "field": "spanId", + "value": "98dc1b886c48967e" + }, + { + "field": "@ptr", + "value": "CpsBCmAKFjY4NTE5NzcwODY4Nzphd3Mvc3BhbnMQABokZTE0NTI5MzgtZjVjZi00OTc1LWE0N2QtYzk3YzMwZmFlY2ZlIg4IgJD4jII0EOefqq2ENDjv37W+yTNAyO+wktMzSAASNRoYAgakSqFXAAAAAiX3ojcABqkKWmAAAANyIAEo8oCcqIQ0MPSAnKiENDgFQK9MSKdkUJQ5IAIQBBgB" + } + ], + [ + { + "field": "@message", + "value": "{\"resource\":{\"attributes\":{\"deployment.environment.name\":\"bedrock-agentcore:default\",\"aws.local.service\":\"asdf_MyAgent.DEFAULT\",\"service.name\":\"asdf_MyAgent.DEFAULT\",\"cloud.region\":\"us-west-2\",\"aws.log.stream.names\":\"otel-rt-logs\",\"telemetry.sdk.name\":\"opentelemetry\",\"aws.service.type\":\"gen_ai_agent\",\"telemetry.sdk.language\":\"python\",\"cloud.provider\":\"aws\",\"cloud.resource_id\":\"arn:aws:bedrock-agentcore:us-west-2:685197708687:runtime/asdf_MyAgent-3s5axvBC6Q/runtime-endpoint/DEFAULT:DEFAULT\",\"aws.log.group.names\":\"/aws/bedrock-agentcore/runtimes/asdf_MyAgent-3s5axvBC6Q-DEFAULT\",\"telemetry.sdk.version\":\"1.40.0\",\"cloud.platform\":\"aws_bedrock_agentcore\",\"telemetry.auto.version\":\"0.17.0-aws\"}},\"scope\":{\"name\":\"opentelemetry.instrumentation.botocore.bedrock-runtime\",\"version\":\"0.61b0\"},\"traceId\":\"6a90a74750f33f567076943f7bce7966\",\"spanId\":\"8214bd2dfefe43fc\",\"parentSpanId\":\"3bf0732214c6e32c\",\"flags\":256,\"name\":\"chat global.anthropic.claude-sonnet-4-5-20250929-v1:0\",\"kind\":\"CLIENT\",\"startTimeUnixNano\":1787864903784381474,\"endTimeUnixNano\":1787864908108981863,\"durationNano\":4324600389,\"attributes\":{\"aws.local.service\":\"asdf_MyAgent.DEFAULT\",\"rpc.service\":\"Bedrock Runtime\",\"aws.remote.resource.identifier\":\"global.anthropic.claude-sonnet-4-5-20250929-v1:0\",\"aws.local.environment\":\"bedrock-agentcore:default\",\"aws.remote.operation\":\"ConverseStream\",\"gen_ai.provider.name\":\"aws.bedrock\",\"server.address\":\"bedrock-runtime.us-west-2.amazonaws.com\",\"aws.request_id\":\"26c3b497-0da3-49ff-bc10-3eee9841d247\",\"aws.local.operation\":\"UnmappedOperation\",\"aws.span.kind\":\"CLIENT\",\"aws.auth.region\":\"us-west-2\",\"rpc.method\":\"ConverseStream\",\"gen_ai.response.finish_reasons\":[\"end_turn\"],\"server.port\":443,\"gen_ai.request.model\":\"global.anthropic.claude-sonnet-4-5-20250929-v1:0\",\"http.response.status_code\":200,\"gen_ai.system\":\"aws.bedrock\",\"telemetry.extended\":\"true\",\"gen_ai.usage.output_tokens\":84,\"aws.genai.span_kind\":\"LLM\",\"rpc.system\":\"aws-api\",\"aws.remote.service\":\"AWS::BedrockRuntime\",\"http.status_code\":200,\"aws.region\":\"us-west-2\",\"aws.remote.resource.type\":\"AWS::Bedrock::Model\",\"gen_ai.operation.name\":\"chat\",\"gen_ai.usage.input_tokens\":1293,\"aws.genai.token_count_total\":1377,\"retry_attempts\":1,\"PlatformType\":\"AWS::BedrockAgentCore\",\"aws.auth.account.access_key\":\"ASIAZ7CHXJWH4OBTTP6T\",\"session.id\":\"00000000-0000-4000-8000-000000000001\"},\"status\":{\"code\":\"UNSET\"}}" + }, + { + "field": "sessionId", + "value": "00000000-0000-4000-8000-000000000001" + }, + { + "field": "traceId", + "value": "6a90a74750f33f567076943f7bce7966" + }, + { + "field": "spanId", + "value": "8214bd2dfefe43fc" + }, + { + "field": "@ptr", + "value": "CpsBCmAKFjY4NTE5NzcwODY4Nzphd3Mvc3BhbnMQABokZTE0NTI5MzgtZjVjZi00OTc1LWE0N2QtYzk3YzMwZmFlY2ZlIg4IgJD4jII0EOefqq2ENDjv37W+yTNAyO+wktMzSAASNRoYAgakSqFXAAAAAiX4KqgABqkKdPAAAANyIAEozIK2qIQ0MM6CtqiENDgFQLFMSLpkUKc5IAIQABgB" + } + ], + [ + { + "field": "@message", + "value": "{\"resource\":{\"attributes\":{\"deployment.environment.name\":\"bedrock-agentcore:default\",\"aws.local.service\":\"asdf_MyAgent.DEFAULT\",\"service.name\":\"asdf_MyAgent.DEFAULT\",\"cloud.region\":\"us-west-2\",\"aws.log.stream.names\":\"otel-rt-logs\",\"telemetry.sdk.name\":\"opentelemetry\",\"aws.service.type\":\"gen_ai_agent\",\"telemetry.sdk.language\":\"python\",\"cloud.provider\":\"aws\",\"cloud.resource_id\":\"arn:aws:bedrock-agentcore:us-west-2:685197708687:runtime/asdf_MyAgent-3s5axvBC6Q/runtime-endpoint/DEFAULT:DEFAULT\",\"aws.log.group.names\":\"/aws/bedrock-agentcore/runtimes/asdf_MyAgent-3s5axvBC6Q-DEFAULT\",\"telemetry.sdk.version\":\"1.40.0\",\"cloud.platform\":\"aws_bedrock_agentcore\",\"telemetry.auto.version\":\"0.17.0-aws\"}},\"scope\":{\"name\":\"strands.telemetry.tracer\",\"version\":\"\"},\"traceId\":\"6a90a74750f33f567076943f7bce7966\",\"spanId\":\"3bf0732214c6e32c\",\"parentSpanId\":\"5e744ea5e19f1053\",\"flags\":256,\"name\":\"chat\",\"kind\":\"INTERNAL\",\"startTimeUnixNano\":1787864903783684675,\"endTimeUnixNano\":1787864908109701143,\"durationNano\":4326016468,\"attributes\":{\"aws.local.service\":\"asdf_MyAgent.DEFAULT\",\"gen_ai.usage.prompt_tokens\":1293,\"gen_ai.usage.output_tokens\":84,\"gen_ai.server.request.duration\":3357,\"gen_ai.usage.total_tokens\":1377,\"gen_ai.usage.completion_tokens\":84,\"aws.genai.span_kind\":\"LLM\",\"gen_ai.event.start_time\":\"2026-08-27T21:08:23.783693+00:00\",\"gen_ai.server.time_to_first_token\":2101,\"aws.local.environment\":\"bedrock-agentcore:default\",\"gen_ai.provider.name\":\"strands-agents\",\"gen_ai.operation.name\":\"chat\",\"gen_ai.event.end_time\":\"2026-08-27T21:08:28.109656+00:00\",\"gen_ai.usage.input_tokens\":1293,\"aws.genai.token_count_total\":1377,\"gen_ai.request.model\":\"global.anthropic.claude-sonnet-4-5-20250929-v1:0\",\"PlatformType\":\"AWS::BedrockAgentCore\",\"session.id\":\"00000000-0000-4000-8000-000000000001\",\"gen_ai.system\":\"strands-agents\"},\"status\":{\"code\":\"OK\"}}" + }, + { + "field": "sessionId", + "value": "00000000-0000-4000-8000-000000000001" + }, + { + "field": "traceId", + "value": "6a90a74750f33f567076943f7bce7966" + }, + { + "field": "spanId", + "value": "3bf0732214c6e32c" + }, + { + "field": "@ptr", + "value": "CpsBCmAKFjY4NTE5NzcwODY4Nzphd3Mvc3BhbnMQABokZTE0NTI5MzgtZjVjZi00OTc1LWE0N2QtYzk3YzMwZmFlY2ZlIg4IgJD4jII0EOefqq2ENDjv37W+yTNAyO+wktMzSAASNRoYAgakSqFXAAAAAiX4KqgABqkKdPAAAANyIAEozIK2qIQ0MM6CtqiENDgFQLFMSLpkUKc5IAIQARgB" + } + ], + [ + { + "field": "@message", + "value": "{\"resource\":{\"attributes\":{\"deployment.environment.name\":\"bedrock-agentcore:default\",\"aws.local.service\":\"asdf_MyAgent.DEFAULT\",\"service.name\":\"asdf_MyAgent.DEFAULT\",\"cloud.region\":\"us-west-2\",\"aws.log.stream.names\":\"otel-rt-logs\",\"telemetry.sdk.name\":\"opentelemetry\",\"aws.service.type\":\"gen_ai_agent\",\"telemetry.sdk.language\":\"python\",\"cloud.provider\":\"aws\",\"cloud.resource_id\":\"arn:aws:bedrock-agentcore:us-west-2:685197708687:runtime/asdf_MyAgent-3s5axvBC6Q/runtime-endpoint/DEFAULT:DEFAULT\",\"aws.log.group.names\":\"/aws/bedrock-agentcore/runtimes/asdf_MyAgent-3s5axvBC6Q-DEFAULT\",\"telemetry.sdk.version\":\"1.40.0\",\"cloud.platform\":\"aws_bedrock_agentcore\",\"telemetry.auto.version\":\"0.17.0-aws\"}},\"scope\":{\"name\":\"strands.telemetry.tracer\",\"version\":\"\"},\"traceId\":\"6a90a74750f33f567076943f7bce7966\",\"spanId\":\"b970895264868438\",\"parentSpanId\":\"e98a035e27f5b117\",\"flags\":256,\"name\":\"invoke_agent Strands Agents\",\"kind\":\"INTERNAL\",\"startTimeUnixNano\":1787864903783116063,\"endTimeUnixNano\":1787864908110229781,\"durationNano\":4327113718,\"attributes\":{\"aws.local.service\":\"asdf_MyAgent.DEFAULT\",\"gen_ai.usage.prompt_tokens\":2496,\"gen_ai.usage.output_tokens\":166,\"gen_ai.usage.cache_write_input_tokens\":0,\"gen_ai.agent.name\":\"Strands Agents\",\"gen_ai.usage.total_tokens\":2662,\"gen_ai.usage.completion_tokens\":166,\"aws.genai.span_kind\":\"AGENT\",\"gen_ai.event.start_time\":\"2026-08-27T21:08:23.783133+00:00\",\"aws.local.environment\":\"bedrock-agentcore:default\",\"gen_ai.provider.name\":\"strands-agents\",\"gen_ai.operation.name\":\"invoke_agent\",\"gen_ai.event.end_time\":\"2026-08-27T21:08:28.110203+00:00\",\"gen_ai.usage.input_tokens\":2496,\"aws.genai.token_count_total\":2662,\"gen_ai.request.model\":\"global.anthropic.claude-sonnet-4-5-20250929-v1:0\",\"gen_ai.usage.cache_read_input_tokens\":0,\"gen_ai.agent.tools\":\"[\\\"add_numbers\\\", \\\"x_amz_bedrock_agentcore_search\\\", \\\"mcpTarget___web_fetch_exa\\\", \\\"mcpTarget___web_search_exa\\\"]\",\"PlatformType\":\"AWS::BedrockAgentCore\",\"session.id\":\"00000000-0000-4000-8000-000000000001\",\"gen_ai.system\":\"strands-agents\"},\"status\":{\"code\":\"OK\"}}" + }, + { + "field": "sessionId", + "value": "00000000-0000-4000-8000-000000000001" + }, + { + "field": "traceId", + "value": "6a90a74750f33f567076943f7bce7966" + }, + { + "field": "spanId", + "value": "b970895264868438" + }, + { + "field": "@ptr", + "value": "CpsBCmAKFjY4NTE5NzcwODY4Nzphd3Mvc3BhbnMQABokZTE0NTI5MzgtZjVjZi00OTc1LWE0N2QtYzk3YzMwZmFlY2ZlIg4IgJD4jII0EOefqq2ENDjv37W+yTNAyO+wktMzSAASNRoYAgakSqFXAAAAAiX4KqgABqkKdPAAAANyIAEozIK2qIQ0MM6CtqiENDgFQLFMSLpkUKc5IAIQAxgB" + } + ], + [ + { + "field": "@message", + "value": "{\"resource\":{\"attributes\":{\"deployment.environment.name\":\"bedrock-agentcore:default\",\"aws.local.service\":\"asdf_MyAgent.DEFAULT\",\"service.name\":\"asdf_MyAgent.DEFAULT\",\"cloud.region\":\"us-west-2\",\"aws.log.stream.names\":\"otel-rt-logs\",\"telemetry.sdk.name\":\"opentelemetry\",\"aws.service.type\":\"gen_ai_agent\",\"telemetry.sdk.language\":\"python\",\"cloud.provider\":\"aws\",\"cloud.resource_id\":\"arn:aws:bedrock-agentcore:us-west-2:685197708687:runtime/asdf_MyAgent-3s5axvBC6Q/runtime-endpoint/DEFAULT:DEFAULT\",\"aws.log.group.names\":\"/aws/bedrock-agentcore/runtimes/asdf_MyAgent-3s5axvBC6Q-DEFAULT\",\"telemetry.sdk.version\":\"1.40.0\",\"cloud.platform\":\"aws_bedrock_agentcore\",\"telemetry.auto.version\":\"0.17.0-aws\"}},\"scope\":{\"name\":\"opentelemetry.instrumentation.starlette\",\"version\":\"0.61b0\"},\"traceId\":\"6a90a74750f33f567076943f7bce7966\",\"spanId\":\"e98a035e27f5b117\",\"parentSpanId\":\"f9316ed3e092f921\",\"flags\":768,\"name\":\"POST /invocations\",\"kind\":\"SERVER\",\"startTimeUnixNano\":1787864903781172232,\"endTimeUnixNano\":1787864908110619156,\"durationNano\":4329446924,\"attributes\":{\"aws.local.service\":\"asdf_MyAgent.DEFAULT\",\"net.peer.port\":56230,\"telemetry.extended\":\"true\",\"http.target\":\"/invocations\",\"http.flavor\":\"1.1\",\"http.url\":\"http://cell01.us-west-2.prod.arp.kepler-analytics.aws.dev/invocations\",\"net.peer.ip\":\"127.0.0.1\",\"http.host\":\"127.0.0.1:8080\",\"aws.local.environment\":\"bedrock-agentcore:default\",\"http.status_code\":200,\"aws.local.operation\":\"POST /invocations\",\"aws.span.kind\":\"SERVER\",\"http.server_name\":\"cell01.us-west-2.prod.arp.kepler-analytics.aws.dev\",\"net.host.port\":8080,\"http.route\":\"/invocations\",\"PlatformType\":\"AWS::BedrockAgentCore\",\"http.method\":\"POST\",\"http.response.status_code\":200,\"session.id\":\"00000000-0000-4000-8000-000000000001\",\"http.scheme\":\"http\"},\"status\":{\"code\":\"UNSET\"}}" + }, + { + "field": "sessionId", + "value": "00000000-0000-4000-8000-000000000001" + }, + { + "field": "traceId", + "value": "6a90a74750f33f567076943f7bce7966" + }, + { + "field": "spanId", + "value": "e98a035e27f5b117" + }, + { + "field": "@ptr", + "value": "CpsBCmAKFjY4NTE5NzcwODY4Nzphd3Mvc3BhbnMQABokZTE0NTI5MzgtZjVjZi00OTc1LWE0N2QtYzk3YzMwZmFlY2ZlIg4IgJD4jII0EOefqq2ENDjv37W+yTNAyO+wktMzSAASNRoYAgakSqFXAAAAAiX4KqgABqkKdPAAAANyIAEozIK2qIQ0MM6CtqiENDgFQLFMSLpkUKc5IAIQBBgB" + } + ], + [ + { + "field": "@message", + "value": "{\"resource\":{\"attributes\":{\"deployment.environment.name\":\"bedrock-agentcore:default\",\"aws.local.service\":\"asdf_MyAgent.DEFAULT\",\"service.name\":\"asdf_MyAgent.DEFAULT\",\"cloud.region\":\"us-west-2\",\"aws.log.stream.names\":\"otel-rt-logs\",\"telemetry.sdk.name\":\"opentelemetry\",\"aws.service.type\":\"gen_ai_agent\",\"telemetry.sdk.language\":\"python\",\"cloud.provider\":\"aws\",\"cloud.resource_id\":\"arn:aws:bedrock-agentcore:us-west-2:685197708687:runtime/asdf_MyAgent-3s5axvBC6Q/runtime-endpoint/DEFAULT:DEFAULT\",\"aws.log.group.names\":\"/aws/bedrock-agentcore/runtimes/asdf_MyAgent-3s5axvBC6Q-DEFAULT\",\"telemetry.sdk.version\":\"1.40.0\",\"cloud.platform\":\"aws_bedrock_agentcore\",\"telemetry.auto.version\":\"0.17.0-aws\"}},\"scope\":{\"name\":\"strands.telemetry.tracer\",\"version\":\"\"},\"traceId\":\"6a90a74750f33f567076943f7bce7966\",\"spanId\":\"5e744ea5e19f1053\",\"parentSpanId\":\"b970895264868438\",\"flags\":256,\"name\":\"execute_event_loop_cycle\",\"kind\":\"INTERNAL\",\"startTimeUnixNano\":1787864903783489151,\"endTimeUnixNano\":1787864908110047680,\"durationNano\":4326558529,\"attributes\":{\"aws.local.service\":\"asdf_MyAgent.DEFAULT\",\"gen_ai.operation.name\":\"execute_event_loop_cycle\",\"gen_ai.event.end_time\":\"2026-08-27T21:08:28.110029+00:00\",\"event_loop.cycle_id\":\"8071f25d-d51e-465d-a206-3c93648afe25\",\"gen_ai.event.start_time\":\"2026-08-27T21:08:23.783500+00:00\",\"PlatformType\":\"AWS::BedrockAgentCore\",\"session.id\":\"00000000-0000-4000-8000-000000000001\",\"gen_ai.system\":\"strands-agents\",\"aws.local.environment\":\"bedrock-agentcore:default\",\"gen_ai.provider.name\":\"strands-agents\"},\"status\":{\"code\":\"OK\"}}" + }, + { + "field": "sessionId", + "value": "00000000-0000-4000-8000-000000000001" + }, + { + "field": "traceId", + "value": "6a90a74750f33f567076943f7bce7966" + }, + { + "field": "spanId", + "value": "5e744ea5e19f1053" + }, + { + "field": "@ptr", + "value": "CpsBCmAKFjY4NTE5NzcwODY4Nzphd3Mvc3BhbnMQABokZTE0NTI5MzgtZjVjZi00OTc1LWE0N2QtYzk3YzMwZmFlY2ZlIg4IgJD4jII0EOefqq2ENDjv37W+yTNAyO+wktMzSAASNRoYAgakSqFXAAAAAiX4KqgABqkKdPAAAANyIAEozIK2qIQ0MM6CtqiENDgFQLFMSLpkUKc5IAIQAhgB" + } + ] + ], + "statistics": { + "recordsMatched": 43, + "recordsScanned": 43, + "estimatedRecordsSkipped": 0, + "bytesScanned": 78544, + "estimatedBytesSkipped": 0, + "logGroupsScanned": 1 + }, + "status": "Complete" +} \ No newline at end of file diff --git a/src/handlers/eval/ondemand/__fixtures__/InvokeAgentRuntimeCommand.d7f8ec055ea3add0.json b/src/handlers/eval/ondemand/__fixtures__/InvokeAgentRuntimeCommand.d7f8ec055ea3add0.json new file mode 100644 index 000000000..9f288bec2 --- /dev/null +++ b/src/handlers/eval/ondemand/__fixtures__/InvokeAgentRuntimeCommand.d7f8ec055ea3add0.json @@ -0,0 +1,8 @@ +{ + "contentType": "text/event-stream; charset=utf-8", + "runtimeSessionId": "00000000-0000-4000-8000-000000000001", + "response": { + "$stream": "data: \"Hello! 👋 Welcome\"\n\ndata: \"!\"\n\ndata: \"\\n\\nI'm here to assist\"\n\ndata: \" you today\"\n\ndata: \". I can help you\"\n\ndata: \" with:\"\n\ndata: \"\\n\\n- **Web\"\n\ndata: \" searches** -\"\n\ndata: \" Find\"\n\ndata: \" current information, news, research\"\n\ndata: \", or answers\"\n\ndata: \" to questions on\"\n\ndata: \" any\"\n\ndata: \" topic\\n- **Reading\"\n\ndata: \" webpages** - Extract and\"\n\ndata: \" read content\"\n\ndata: \" from specific URLs or\"\n\ndata: \" articles\"\n\ndata: \"\\n- **Simple\"\n\ndata: \" calculations** - Add numbers together\"\n\ndata: \"\\n\\nWhat would you like help\"\n\ndata: \" with?\"\n\n" + }, + "statusCode": 200 +} \ No newline at end of file diff --git a/src/handlers/eval/ondemand/__fixtures__/StartQueryCommand.3c11aa80c0c5122e.json b/src/handlers/eval/ondemand/__fixtures__/StartQueryCommand.3c11aa80c0c5122e.json new file mode 100644 index 000000000..f2dc06c01 --- /dev/null +++ b/src/handlers/eval/ondemand/__fixtures__/StartQueryCommand.3c11aa80c0c5122e.json @@ -0,0 +1,3 @@ +{ + "queryId": "143d23b0-0eb9-4542-940b-ffd637ebb0a8" +} \ No newline at end of file diff --git a/src/handlers/eval/ondemand/__fixtures__/StartQueryCommand.a6424982fd346c1.json b/src/handlers/eval/ondemand/__fixtures__/StartQueryCommand.a6424982fd346c1.json new file mode 100644 index 000000000..31d074708 --- /dev/null +++ b/src/handlers/eval/ondemand/__fixtures__/StartQueryCommand.a6424982fd346c1.json @@ -0,0 +1,3 @@ +{ + "queryId": "9a682dcb-330e-449b-b7b4-9897021c47bc" +} \ No newline at end of file diff --git a/src/handlers/eval/ondemand/__fixtures__/simulate-ds.jsonl b/src/handlers/eval/ondemand/__fixtures__/simulate-ds.jsonl new file mode 100644 index 000000000..34738ea89 --- /dev/null +++ b/src/handlers/eval/ondemand/__fixtures__/simulate-ds.jsonl @@ -0,0 +1 @@ +{"example_id":"greet","turns":[{"input":"Say hello and offer help"}],"assertions":["is polite"]} diff --git a/src/handlers/eval/ondemand/__fixtures__/simulate.golden.json b/src/handlers/eval/ondemand/__fixtures__/simulate.golden.json new file mode 100644 index 000000000..38b65e80b --- /dev/null +++ b/src/handlers/eval/ondemand/__fixtures__/simulate.golden.json @@ -0,0 +1,125 @@ +{ + "sessionsRequested": 1, + "sessionsEvaluated": 1, + "results": [ + { + "evaluatorArn": "arn:aws:bedrock-agentcore:::evaluator/Builtin.Helpfulness", + "evaluatorId": "Builtin.Helpfulness", + "evaluatorName": "Builtin.Helpfulness", + "context": { + "spanContext": { + "sessionId": "00000000-0000-4000-8000-000000000001", + "traceId": "6a8f5b461983df572e2aca3264d32d4d" + } + }, + "explanation": "The user simply said 'hi', which is a greeting with no specific goal expressed. The assistant responded with a polite greeting and an open invitation to help. This is appropriate conversational behavior that maintains the flow of interaction and invites the user to share their actual needs. Since the user hasn't expressed a specific goal yet, the assistant's response is a standard, appropriate reply that keeps the conversation open. It doesn't advance any specific goal (since none exists yet), but it doesn't hinder progress either. This falls into the 'Neutral/Mixed' category as it's appropriate chit-chat for conversation flow with no specific goal to advance.", + "value": 0.5, + "label": "Neutral/Mixed", + "tokenUsage": { + "inputTokens": 815, + "outputTokens": 155, + "totalTokens": 970 + }, + "ignoredReferenceInputFields": [ + "assertions" + ] + }, + { + "evaluatorArn": "arn:aws:bedrock-agentcore:::evaluator/Builtin.Helpfulness", + "evaluatorId": "Builtin.Helpfulness", + "evaluatorName": "Builtin.Helpfulness", + "context": { + "spanContext": { + "sessionId": "00000000-0000-4000-8000-000000000001", + "traceId": "6a8f5bbc1882c05547b64a9246f17ca4" + } + }, + "explanation": "The user has said 'hi' twice without providing any specific goal or request. The assistant's response 'Hi there! How can I assist you today?' is a standard greeting that keeps the conversation open and invites the user to share their needs. Since the user hasn't expressed a specific goal yet, the assistant can't do much more than respond to the greeting and prompt the user to share what they need. This response is appropriate for the conversational context - it's a polite acknowledgment that maintains the conversation flow and opens the door for the user to state their actual needs. It doesn't advance any specific goal (since none has been stated), but it doesn't hinder progress either. This falls into the 'Neutral/Mixed' category as it's appropriate chit-chat for conversation flow with no specific goal to advance.", + "value": 0.5, + "label": "Neutral/Mixed", + "tokenUsage": { + "inputTokens": 887, + "outputTokens": 194, + "totalTokens": 1081 + }, + "ignoredReferenceInputFields": [ + "assertions" + ] + }, + { + "evaluatorArn": "arn:aws:bedrock-agentcore:::evaluator/Builtin.Helpfulness", + "evaluatorId": "Builtin.Helpfulness", + "evaluatorName": "Builtin.Helpfulness", + "context": { + "spanContext": { + "sessionId": "00000000-0000-4000-8000-000000000001", + "traceId": "6a8f639540ebe6cb6f30d5d80762c2e2" + } + }, + "explanation": "The user has sent 'hi' three times in a row without providing any specific request or goal. The assistant's response 'Hello! How can I help you today?' is a standard greeting that keeps the conversation open and invites the user to share their needs. This is the third identical exchange, and the assistant is simply repeating the same greeting. While the response is appropriate and doesn't obstruct any goal, it also doesn't advance any specific goal since the user hasn't expressed one yet. The response is essentially neutral - it's appropriate chit-chat that maintains conversation flow without moving toward any particular goal (since no goal has been stated). The assistant could potentially note that the user has greeted multiple times and ask if they need help with something specific, which would be slightly more proactive, but the current response is still a reasonable reply to a simple greeting.", + "value": 0.5, + "label": "Neutral/Mixed", + "tokenUsage": { + "inputTokens": 958, + "outputTokens": 207, + "totalTokens": 1165 + }, + "ignoredReferenceInputFields": [ + "assertions" + ] + }, + { + "evaluatorArn": "arn:aws:bedrock-agentcore:::evaluator/Builtin.Helpfulness", + "evaluatorId": "Builtin.Helpfulness", + "evaluatorName": "Builtin.Helpfulness", + "context": { + "spanContext": { + "sessionId": "00000000-0000-4000-8000-000000000001", + "traceId": "6a90a5987755c6e457ada5616a16dddb" + } + }, + "explanation": "The user's final request was 'Say hello and offer help.' The assistant's response directly fulfills this request by:\n1. Saying hello with a greeting and wave emoji\n2. Offering help by listing specific capabilities it can assist with\n\nThe response is friendly, clear, and directly addresses what the user asked for. It goes slightly beyond the minimal requirement by providing specific examples of what it can help with (web search, fetching webpages, calculations), which gives the user actionable information about how to proceed.\n\nHowever, the listed capabilities (web search, fetch webpages, perform calculations) seem oddly specific and somewhat limiting - a general AI assistant can help with many more things like writing, analysis, coding, answering questions, etc. This specificity might actually mislead the user about the assistant's full range of capabilities.\n\nDespite this minor issue, the response does exactly what was requested - says hello and offers help - and does so in a clear, organized manner. The user's goal was simple and the assistant met it directly.", + "value": 0.83, + "label": "Very Helpful", + "tokenUsage": { + "inputTokens": 1112, + "outputTokens": 258, + "totalTokens": 1370 + }, + "ignoredReferenceInputFields": [ + "assertions" + ] + }, + { + "evaluatorArn": "arn:aws:bedrock-agentcore:::evaluator/Builtin.Helpfulness", + "evaluatorId": "Builtin.Helpfulness", + "evaluatorName": "Builtin.Helpfulness", + "context": { + "spanContext": { + "sessionId": "00000000-0000-4000-8000-000000000001", + "traceId": "6a90a74750f33f567076943f7bce7966" + } + }, + "explanation": "The user's request is simple and explicit: 'Say hello and offer help.' The assistant's response directly fulfills this request by greeting the user with 'Hello! 👋 Welcome!' and then offering help with a clear, organized list of capabilities (web searches, reading webpages, and calculations). The response is well-formatted, friendly, and ends with an open invitation for the user to specify what they need. This is essentially the same response as the previous turn (with minor variations like adding 'Welcome!'), which is appropriate since the user repeated the same request. The response fully satisfies the user's stated goal of having the assistant say hello and offer help. It's comprehensive and actionable, clearly communicating what the assistant can do. There's nothing missing or problematic about this response given the user's simple, direct request.", + "value": 0.83, + "label": "Very Helpful", + "tokenUsage": { + "inputTokens": 1267, + "outputTokens": 203, + "totalTokens": 1470 + }, + "ignoredReferenceInputFields": [ + "assertions" + ] + } + ], + "examplesInvoked": 1, + "examplesFailed": 0, + "sessions": [ + { + "exampleId": "greet", + "sessionId": "00000000-0000-4000-8000-000000000001" + } + ], + "failures": [] +} \ No newline at end of file diff --git a/src/handlers/eval/ondemand/ondemand.fixture.test.tsx b/src/handlers/eval/ondemand/ondemand.fixture.test.tsx index 39bf5767f..c83263707 100644 --- a/src/handlers/eval/ondemand/ondemand.fixture.test.tsx +++ b/src/handlers/eval/ondemand/ondemand.fixture.test.tsx @@ -4,6 +4,7 @@ import { CoreClient } from "../../../core"; import { createSilentLogger, fixtureFactories, + isRecording, matchGolden, TestGlobalConfigAccessor, testIO, @@ -14,6 +15,7 @@ const REGION = "us-west-2"; const FIXTURES = join(import.meta.dir, "__fixtures__"); const FIXTURE_AGENT = "asdf_MyAgent-3s5axvBC6Q"; +const SIMULATE_DATASET = join(FIXTURES, "simulate-ds.jsonl"); const FIXTURE_SESSION_IDS = [ "67ebf93b-65e3-4127-9e13-483b239f256a", "7f983b9f-9569-4a4d-bdc2-5c997ff346dd", @@ -70,4 +72,47 @@ describe("eval ondemand evaluate (fixture-backed)", () => { // Recording polls live CloudWatch Insights (1s between polls) and calls Evaluate // per session, so it needs well over bun's 5s default; replay is instant. }, 180_000); + + test("simulate replays a dataset, then evaluates the created sessions client-side", async () => { + let n = 0; + const { createControlClient, createDataClient, createIamClient, createLogsClient } = + fixtureFactories(FIXTURES); + const core = new CoreClient({ + createControlClient, + createDataClient, + createIamClient, + createLogsClient, + logger: createSilentLogger(), + newSessionId: () => `00000000-0000-4000-8000-${String(++n).padStart(12, "0")}`, + now: () => Date.parse("2026-08-28T00:00:00Z"), + }); + const io = testIO(); + const root = createRootHandler(core, { + io: io.io, + logger: createSilentLogger(), + globalConfigAccessor: new TestGlobalConfigAccessor(), + }); + + await root.route([ + "node", + "agentcore", + "eval", + "ondemand", + "simulate", + "--runtime-id", + FIXTURE_AGENT, + "--payload-template", + '{"prompt":"{input}"}', + "--dataset", + SIMULATE_DATASET, + "--evaluator", + "Builtin.Helpfulness", + "--ingestion-wait-ms", + isRecording() ? "150000" : "0", + "--region", + REGION, + ]); + + matchGolden(FIXTURES, "simulate.golden.json", io.stdout()); + }, 200_000); }); diff --git a/src/handlers/eval/ondemand/simulate/index.tsx b/src/handlers/eval/ondemand/simulate/index.tsx index eec914937..1b13ac7d9 100644 --- a/src/handlers/eval/ondemand/simulate/index.tsx +++ b/src/handlers/eval/ondemand/simulate/index.tsx @@ -9,9 +9,6 @@ import type { InvokedSession } from "../../types"; import { coreOptsFromCtx } from "../../../utils"; import { parseRuntimeInvokeHeaders } from "../../../runtime/invoke/request"; -// Composes invokeDataset (replay) → getTracesForAgent (gather) → evaluate (grade, -// synchronous). The on-demand twin of batch-evaluation simulate: no async job, scores -// print inline. Invoke flags mirror `runtime invoke`. export const createSimulateOnDemandHandler = (core: Core, _io: AppIO) => createHandler({ name: "simulate", @@ -54,14 +51,12 @@ export const createSimulateOnDemandHandler = (core: Core, _io: AppIO) => ); } - // Ctrl-C aborts the run (invokes, the ingestion wait, the dataset download). const controller = new AbortController(); const interrupt = () => controller.abort(); process.once("SIGINT", interrupt); try { const opts = coreOptsFromCtx(ctx); - // 1. Replay the dataset — reuse invokeDataset verbatim (grader-agnostic). const replay = await core.eval.invokeDataset( { runtimeId: flags["runtime-id"], @@ -85,7 +80,6 @@ export const createSimulateOnDemandHandler = (core: Core, _io: AppIO) => ); } - // 2. Gather the just-created sessions' traces (client-side CloudWatch read). const traces = await core.eval.getTracesForAgent( { agent: flags["runtime-id"], @@ -95,7 +89,6 @@ export const createSimulateOnDemandHandler = (core: Core, _io: AppIO) => opts, ); - // 3. Adapt neutral ground truth → EvaluationReferenceInput[] and grade synchronously. const groundTruth = replay.sessions.flatMap(toReferenceInputs); const result = await core.eval.evaluate( { traces, evaluatorIds: flags["evaluator"], groundTruth }, @@ -118,10 +111,6 @@ export const createSimulateOnDemandHandler = (core: Core, _io: AppIO) => }, }); -// Adapt one invoked session's neutral InlineGroundTruth to the Evaluate API's -// EvaluationReferenceInput, correlated by sessionId. assertions ({text}[]) and -// expectedTrajectory ({toolNames}) map 1:1. Per-turn expectedResponse is trace-level and -// needs a turn→trace id we don't have here, so it is omitted (batch simulate covers it). function toReferenceInputs(s: InvokedSession): EvaluationReferenceInput[] { const gt = s.groundTruth; if (!gt?.assertions?.length && !gt?.expectedTrajectory) return []; From 95a1b67c45f6ce4c8988f414bc196636df893cf1 Mon Sep 17 00:00:00 2001 From: jariy17 Date: Wed, 26 Aug 2026 17:37:44 +0000 Subject: [PATCH 04/15] feat(eval): add ab-test config-bundle run (create) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Adds `agentcore eval ab-test config-bundle run` — the first create in the ab-test family. Runs an A/B test between two config-bundle versions on one gateway. --control / --treatment / --gateway-filter each accept inline JSON, file://, or - (stdin) via SourceResolver (same shape as online-eval create's --filters). Core provisions an IAM execution role when --role-arn is omitted (mirrors online-eval create + retryWhileRolePropagates), and rolls the role back if CreateABTest fails. Validation is server-side (gateway READY, bundles, online-eval enabled iff enableOnCreate) — the CLI resolves ids to ARNs and surfaces the service's 4xx cleanly. Note: --runtime dropped (not a CreateABTest field) and deviates from doc. --- src/core/abTestExecutionRole.tsx | 149 +++++++++++++++ src/core/eval.tsx | 80 +++++++++ .../eval/ab-test/ab-test.create.test.tsx | 169 ++++++++++++++++++ .../eval/ab-test/ab-test.write.test.tsx | 1 + .../eval/ab-test/config-bundle/index.tsx | 10 ++ .../eval/ab-test/config-bundle/run/index.tsx | 115 ++++++++++++ src/handlers/eval/ab-test/index.tsx | 4 +- src/handlers/eval/types.tsx | 20 +++ src/testing/TestCoreClient.tsx | 18 ++ 9 files changed, 565 insertions(+), 1 deletion(-) create mode 100644 src/core/abTestExecutionRole.tsx create mode 100644 src/handlers/eval/ab-test/ab-test.create.test.tsx create mode 100644 src/handlers/eval/ab-test/config-bundle/index.tsx create mode 100644 src/handlers/eval/ab-test/config-bundle/run/index.tsx diff --git a/src/core/abTestExecutionRole.tsx b/src/core/abTestExecutionRole.tsx new file mode 100644 index 000000000..a4463f5f8 --- /dev/null +++ b/src/core/abTestExecutionRole.tsx @@ -0,0 +1,149 @@ +import { + CreateRoleCommand, + GetRoleCommand, + PutRolePolicyCommand, + DeleteRoleCommand, + DeleteRolePolicyCommand, + type IAMClient, +} from "@aws-sdk/client-iam"; +import { createHash } from "node:crypto"; + +const AB_TEST_POLICY_NAME = "ABTestExecutionPolicy"; + +export function abTestExecutionRoleName(testName: string): string { + const hash = createHash("sha256").update(`ab-test:${testName}`).digest("hex").slice(0, 8); + const base = `AgentCoreABTest-${testName}`; + return `${base.slice(0, 55)}-${hash}`; +} + +export function roleNameFromArn(roleArn: string): string { + const parts = roleArn.split("/"); + return parts[parts.length - 1] ?? roleArn; +} + +function trustPolicy(accountId: string, region: string): string { + return JSON.stringify({ + Version: "2012-10-17", + Statement: [ + { + Effect: "Allow", + Principal: { Service: "bedrock-agentcore.amazonaws.com" }, + Action: "sts:AssumeRole", + Condition: { + StringEquals: { "aws:SourceAccount": accountId }, + ArnLike: { + "aws:SourceArn": `arn:aws:bedrock-agentcore:${region}:${accountId}:ab-test/*`, + }, + }, + }, + ], + }); +} + +function executionPolicy(accountId: string, region: string): string { + return JSON.stringify({ + Version: "2012-10-17", + Statement: [ + { + Sid: "AgentCoreResources", + Effect: "Allow", + Action: [ + "bedrock-agentcore:GetGateway", + "bedrock-agentcore:GetGatewayTarget", + "bedrock-agentcore:ListGatewayTargets", + "bedrock-agentcore:CreateGatewayRule", + "bedrock-agentcore:UpdateGatewayRule", + "bedrock-agentcore:GetGatewayRule", + "bedrock-agentcore:DeleteGatewayRule", + "bedrock-agentcore:ListGatewayRules", + "bedrock-agentcore:GetOnlineEvaluationConfig", + "bedrock-agentcore:GetEvaluator", + "bedrock-agentcore:GetConfigurationBundle", + "bedrock-agentcore:GetConfigurationBundleVersion", + "bedrock-agentcore:ListConfigurationBundleVersions", + ], + Resource: `arn:aws:bedrock-agentcore:${region}:${accountId}:*`, + Condition: { StringEquals: { "aws:ResourceAccount": accountId } }, + }, + { + Sid: "CloudWatchLogsDescribe", + Effect: "Allow", + Action: ["logs:DescribeLogGroups"], + Resource: "*", + }, + { + Sid: "CloudWatchLogs", + Effect: "Allow", + Action: [ + "logs:DescribeIndexPolicies", + "logs:PutIndexPolicy", + "logs:StartQuery", + "logs:GetQueryResults", + "logs:StopQuery", + "logs:FilterLogEvents", + "logs:GetLogEvents", + ], + Resource: [ + `arn:aws:logs:${region}:${accountId}:log-group:/aws/bedrock-agentcore/evaluations/*`, + `arn:aws:logs:${region}:${accountId}:log-group:/aws/bedrock-agentcore/runtimes/*`, + `arn:aws:logs:${region}:${accountId}:log-group:aws/spans`, + `arn:aws:logs:${region}:${accountId}:log-group:aws/spans:*`, + ], + }, + ], + }); +} + +export async function provisionAbTestRole( + iam: IAMClient, + testName: string, + gatewayArn: string, + region: string, +): Promise<{ roleArn: string; created: boolean }> { + const accountId = gatewayArn.split(":")[4] ?? "*"; + const roleName = abTestExecutionRoleName(testName); + + let roleArn: string; + let created = false; + try { + const existing = await iam.send(new GetRoleCommand({ RoleName: roleName })); + roleArn = existing.Role!.Arn!; + } catch (error) { + if ((error as Error).name !== "NoSuchEntityException") throw error; + const result = await iam.send( + new CreateRoleCommand({ + RoleName: roleName, + AssumeRolePolicyDocument: trustPolicy(accountId, region), + Description: `Execution role for AgentCore A/B test "${testName}" (created by agentcore CLI)`, + }), + ); + roleArn = result.Role!.Arn!; + created = true; + } + + await iam.send( + new PutRolePolicyCommand({ + RoleName: roleName, + PolicyName: AB_TEST_POLICY_NAME, + PolicyDocument: executionPolicy(accountId, region), + }), + ); + + return { roleArn, created }; +} + +export async function deleteAbTestRole(iam: IAMClient, roleArn: string): Promise { + const roleName = roleNameFromArn(roleArn); + try { + await iam.send( + new DeleteRolePolicyCommand({ RoleName: roleName, PolicyName: AB_TEST_POLICY_NAME }), + ); + } catch { + // best effort + } + try { + await iam.send(new DeleteRoleCommand({ RoleName: roleName })); + } catch { + // best effort + } +} diff --git a/src/core/eval.tsx b/src/core/eval.tsx index fd51e04ee..b06308745 100644 --- a/src/core/eval.tsx +++ b/src/core/eval.tsx @@ -16,6 +16,7 @@ import { GetDatasetCommand, GetEvaluatorCommand, GetHarnessCommand, + GetGatewayCommand, GetOnlineEvaluationConfigCommand, ListConfigurationBundlesCommand, ListConfigurationBundleVersionsCommand, @@ -60,6 +61,7 @@ import { import { DeleteRecommendationCommand, EvaluateCommand, + CreateABTestCommand, GetABTestCommand, ListABTestsCommand, UpdateABTestCommand, @@ -74,6 +76,7 @@ import { type EvaluationReferenceInput, type EvaluationResultContent, type EvaluationTarget, + type CreateABTestResponse, type GetABTestResponse, type ListABTestsResponse, type ABTestExecutionStatus, @@ -119,6 +122,7 @@ import type { RoleScopeWarning, CoreEvalClient, CreateConfigurationBundleInput, + CreateConfigBundleABTestInput, CreateDatasetInput, CreateOnlineEvalInput, CreateOnlineInsightInput, @@ -165,6 +169,7 @@ import { revokeOnlineEvalScope, scopePolicyName, } from "./onlineEvalExecutionRole"; +import { deleteAbTestRole, provisionAbTestRole } from "./abTestExecutionRole"; const DEFAULT_ENDPOINT_QUALIFIER = "DEFAULT"; const DEFAULT_INGESTION_WAIT_MS = 180_000; @@ -476,6 +481,81 @@ export class EvalClient implements CoreEvalClient { .send(new DeleteABTestCommand({ abTestId: id })); } + async createConfigBundleABTest( + input: CreateConfigBundleABTestInput, + options: CoreOptions, + ): Promise { + const control = this.clients.control(toClientConfig(options)); + const gateway = await control.send(new GetGatewayCommand({ gatewayIdentifier: input.gateway })); + const gatewayArn = gateway.gatewayArn!; + const accountId = gatewayArn.split(":")[4] ?? "*"; + + const controlBundleArn = `arn:aws:bedrock-agentcore:${options.region}:${accountId}:configuration-bundle/${input.control.configBundle}`; + const treatmentBundleArn = `arn:aws:bedrock-agentcore:${options.region}:${accountId}:configuration-bundle/${input.treatment.configBundle}`; + const onlineEvaluationConfigArn = `arn:aws:bedrock-agentcore:${options.region}:${accountId}:online-evaluation-config/${input.onlineEval}`; + + const treatmentWeight = input.treatmentWeight ?? 50; + const variants = [ + { + name: "C", + weight: 100 - treatmentWeight, + variantConfiguration: { + configurationBundle: { + bundleArn: controlBundleArn, + bundleVersion: input.control.bundleVersion, + }, + }, + }, + { + name: "T1", + weight: treatmentWeight, + variantConfiguration: { + configurationBundle: { + bundleArn: treatmentBundleArn, + bundleVersion: input.treatment.bundleVersion, + }, + }, + }, + ]; + + let roleArn = input.roleArn; + let provisionedRoleArn: string | undefined; + if (!roleArn) { + const iam = this.clients.iam({ region: options.region }); + const provisioned = await provisionAbTestRole(iam, input.name, gatewayArn, options.region); + roleArn = provisioned.roleArn; + if (provisioned.created) provisionedRoleArn = provisioned.roleArn; + } + + const command = new CreateABTestCommand({ + name: input.name, + gatewayArn, + variants, + evaluationConfig: { onlineEvaluationConfigArn }, + roleArn, + gatewayFilter: input.gatewayFilter, + enableOnCreate: !input.disableOnCreate, + clientToken: randomUUID(), + }); + + try { + return input.roleArn + ? await this.clients.data(toClientConfig(options)).send(command) + : await retryWhileRolePropagates(() => + this.clients.data(toClientConfig(options)).send(command), + ); + } catch (error) { + if (provisionedRoleArn) { + try { + await deleteAbTestRole(this.clients.iam({ region: options.region }), provisionedRoleArn); + } catch { + // best effort + } + } + throw error; + } + } + async listBatchInsights( nextToken: string | undefined, maxResults: number | undefined, diff --git a/src/handlers/eval/ab-test/ab-test.create.test.tsx b/src/handlers/eval/ab-test/ab-test.create.test.tsx new file mode 100644 index 000000000..c96a31dd3 --- /dev/null +++ b/src/handlers/eval/ab-test/ab-test.create.test.tsx @@ -0,0 +1,169 @@ +import { test, expect, describe } from "bun:test"; +import type { CreateABTestResponse } from "@aws-sdk/client-bedrock-agentcore"; +import { createRootHandler } from "../../index"; +import { createSilentLogger, TestCoreClient, testIO } from "../../../testing"; +import { TestGlobalConfigAccessor } from "../../../testing/"; + +const OK: CreateABTestResponse = { + abTestId: "orders-v2-abc123", + abTestArn: "arn:aws:bedrock-agentcore:us-west-2:123456789012:ab-test/orders-v2-abc123", + name: "orders-v2", + status: "CREATING", + executionStatus: "NOT_STARTED", + createdAt: new Date("2026-08-26T10:00:00.000Z"), +} satisfies CreateABTestResponse; + +async function run(args: string[], configure?: (core: TestCoreClient) => void) { + const core = new TestCoreClient(); + core.eval.setAbTestCreateResponse(OK); + configure?.(core); + const io = testIO(); + const root = createRootHandler(core, { + io: io.io, + logger: createSilentLogger(), + globalConfigAccessor: new TestGlobalConfigAccessor(), + }); + await root.route(["node", "agentcore", ...args, "--region", "us-west-2"]); + return { core, stdout: io.stdout() }; +} + +const BASE = [ + "eval", + "ab-test", + "config-bundle", + "run", + "--name", + "orders-v2", + "--gateway", + "orders-gateway-abc123", + "--control", + '{"config-bundle":"orders-prompt-abc","bundle-version":"1111"}', + "--treatment", + '{"config-bundle":"orders-prompt-abc","bundle-version":"2222"}', + "--online-eval", + "online-eval-abc123", + "--json", +]; + +describe("eval ab-test config-bundle run", () => { + test("registers under ab-test → config-bundle", () => { + const io = testIO(); + const root = createRootHandler(new TestCoreClient(), { + io: io.io, + logger: createSilentLogger(), + globalConfigAccessor: new TestGlobalConfigAccessor(), + }); + const abTest = root + .children() + .find((c) => c.name() === "eval") + ?.children() + .find((c) => c.name() === "ab-test"); + expect(abTest?.children().map((c) => c.name())).toContain("config-bundle"); + const cb = abTest?.children().find((c) => c.name() === "config-bundle"); + expect(cb?.children().map((c) => c.name())).toEqual(["run"]); + }); + + test("maps flags to a createConfigBundleABTest call", async () => { + const { core, stdout } = await run([...BASE, "--treatment-weight", "20"]); + expect(JSON.parse(stdout).abTestId).toBe("orders-v2-abc123"); + const call = core.eval.calls.find((c) => c.method === "createConfigBundleABTest"); + expect(call?.args[0]).toEqual({ + name: "orders-v2", + gateway: "orders-gateway-abc123", + control: { configBundle: "orders-prompt-abc", bundleVersion: "1111" }, + treatment: { configBundle: "orders-prompt-abc", bundleVersion: "2222" }, + onlineEval: "online-eval-abc123", + treatmentWeight: 20, + gatewayFilter: undefined, + roleArn: undefined, + disableOnCreate: false, + }); + expect(call?.args[1]).toEqual({ region: "us-west-2" }); + }); + + test("passes --gateway-filter through as a GatewayFilter", async () => { + const { core } = await run([ + ...BASE, + "--gateway-filter", + '{"targetPaths":["/orders/checkout"]}', + ]); + const call = core.eval.calls.find((c) => c.method === "createConfigBundleABTest"); + expect(call).toBeDefined(); + expect((call!.args[0] as { gatewayFilter?: unknown }).gatewayFilter).toEqual({ + targetPaths: ["/orders/checkout"], + }); + }); + + test("passes --disable-on-create and --role-arn through", async () => { + const { core } = await run([ + ...BASE, + "--disable-on-create", + "--role-arn", + "arn:aws:iam::123456789012:role/customer-owned", + ]); + const call = core.eval.calls.find((c) => c.method === "createConfigBundleABTest"); + const input = call?.args[0] as { disableOnCreate?: boolean; roleArn?: string }; + expect(input.disableOnCreate).toBe(true); + expect(input.roleArn).toBe("arn:aws:iam::123456789012:role/customer-owned"); + }); + + test("rejects equal control/treatment bundle-versions", async () => { + await expect( + run([ + "eval", + "ab-test", + "config-bundle", + "run", + "--name", + "x", + "--gateway", + "g", + "--control", + '{"config-bundle":"b","bundle-version":"same"}', + "--treatment", + '{"config-bundle":"b","bundle-version":"same"}', + "--online-eval", + "o", + "--json", + ]), + ).rejects.toThrow(/must differ/); + }); + + test("rejects --treatment-weight outside 1-99", async () => { + await expect(run([...BASE, "--treatment-weight", "0"])).rejects.toThrow(/1 and 99/); + await expect(run([...BASE, "--treatment-weight", "100"])).rejects.toThrow(/1 and 99/); + }); + + test.each(["name", "gateway", "control", "treatment", "online-eval"] as const)( + "requires --%s", + async (missing) => { + const args = BASE.filter((_, i, arr) => { + const prev = arr[i - 1]; + return prev !== `--${missing}` && arr[i] !== `--${missing}`; + }); + await expect(run(args)).rejects.toThrow(new RegExp(`--${missing}`)); + }, + ); + + test("rejects a malformed control JSON shape", async () => { + await expect( + run([ + "eval", + "ab-test", + "config-bundle", + "run", + "--name", + "x", + "--gateway", + "g", + "--control", + '{"wrong":"shape"}', + "--treatment", + '{"config-bundle":"b","bundle-version":"2"}', + "--online-eval", + "o", + "--json", + ]), + ).rejects.toThrow(/--control must be/); + }); +}); diff --git a/src/handlers/eval/ab-test/ab-test.write.test.tsx b/src/handlers/eval/ab-test/ab-test.write.test.tsx index 31116a1a5..3a7bd3d9d 100644 --- a/src/handlers/eval/ab-test/ab-test.write.test.tsx +++ b/src/handlers/eval/ab-test/ab-test.write.test.tsx @@ -36,6 +36,7 @@ describe("eval ab-test command hierarchy", () => { "resume", "stop", "delete", + "config-bundle", ]); }); }); diff --git a/src/handlers/eval/ab-test/config-bundle/index.tsx b/src/handlers/eval/ab-test/config-bundle/index.tsx new file mode 100644 index 000000000..1c04c571c --- /dev/null +++ b/src/handlers/eval/ab-test/config-bundle/index.tsx @@ -0,0 +1,10 @@ +import { Router } from "../../../../router"; +import type { AppIO } from "../../../../io"; +import type { Core } from "../../../types"; +import { createConfigBundleRunHandler } from "./run"; + +export function createConfigBundleAbTestHandler(core: Core, io: AppIO): Router { + return new Router("config-bundle", "config-bundle A/B tests").handler( + createConfigBundleRunHandler(core, io), + ); +} diff --git a/src/handlers/eval/ab-test/config-bundle/run/index.tsx b/src/handlers/eval/ab-test/config-bundle/run/index.tsx new file mode 100644 index 000000000..c8016fbdc --- /dev/null +++ b/src/handlers/eval/ab-test/config-bundle/run/index.tsx @@ -0,0 +1,115 @@ +import type { GatewayFilter } from "@aws-sdk/client-bedrock-agentcore"; +import z from "zod"; +import { createHandler, flag } from "../../../../../router"; +import { InputValidationError } from "../../../../../errors"; +import { JsonRendererKey } from "../../../../../tui"; +import { SourceResolver, type AppIO } from "../../../../../io"; +import type { Core } from "../../../../types"; +import type { BundleRef } from "../../../types"; +import { coreOptsFromCtx } from "../../../../utils"; +import { parseJsonFlag } from "../../../../utils"; + +const bundleRefSchema = z + .object({ + "config-bundle": z.string().min(1), + "bundle-version": z.string().min(1), + }) + .strict(); + +function toBundleRef(name: string, raw: unknown): BundleRef { + const parsed = bundleRefSchema.safeParse(raw); + if (!parsed.success) { + throw new InputValidationError( + `--${name} must be {"config-bundle": "", "bundle-version": ""}`, + ); + } + return { + configBundle: parsed.data["config-bundle"], + bundleVersion: parsed.data["bundle-version"], + }; +} + +export const createConfigBundleRunHandler = (core: Core, io: AppIO) => + createHandler({ + name: "run", + description: "run an A/B test between two config-bundle versions on one gateway", + flags: [ + flag("name", "the A/B test name", z.string().optional()), + flag("gateway", "deployed gateway id", z.string().optional()), + flag( + "control", + 'control JSON {"config-bundle","bundle-version"} (inline, file://, or -)', + z.string().optional(), + ), + flag( + "treatment", + 'treatment JSON {"config-bundle","bundle-version"} (inline, file://, or -)', + z.string().optional(), + ), + flag("online-eval", "online-evaluation config id", z.string().optional()), + flag( + "treatment-weight", + "1-99; control weight = 100 - this (default 50)", + z.number().optional(), + ), + flag( + "gateway-filter", + 'GatewayFilter JSON, e.g. {"targetPaths":["/orders"]} (inline, file://, or -)', + z.string().optional(), + ), + flag( + "role-arn", + "execution-role override (default: auto-provisioned)", + z.string().optional(), + ), + flag("disable-on-create", "create without starting", z.boolean().optional()), + ], + handle: async (ctx, flags) => { + const required = ["name", "gateway", "control", "treatment", "online-eval"] as const; + for (const f of required) { + if (!flags[f]) throw new InputValidationError(`required option '--${f}' not specified`); + } + + const source = new SourceResolver({ stdin: io.stdin }); + const controlRaw = parseJsonFlag( + "control", + await source.resolveText("control", flags["control"]), + ); + const treatmentRaw = parseJsonFlag( + "treatment", + await source.resolveText("treatment", flags["treatment"]), + ); + const gatewayFilter = parseJsonFlag( + "gateway-filter", + await source.resolveText("gateway-filter", flags["gateway-filter"]), + ); + + const control = toBundleRef("control", controlRaw); + const treatment = toBundleRef("treatment", treatmentRaw); + if (control.bundleVersion === treatment.bundleVersion) { + throw new InputValidationError("treatment bundle-version must differ from control"); + } + + const treatmentWeight = flags["treatment-weight"]; + if (treatmentWeight !== undefined && (treatmentWeight < 1 || treatmentWeight > 99)) { + throw new InputValidationError("--treatment-weight must be between 1 and 99"); + } + + const result = await core.eval.createConfigBundleABTest( + { + name: flags["name"]!, + gateway: flags["gateway"]!, + control, + treatment, + onlineEval: flags["online-eval"]!, + treatmentWeight, + gatewayFilter, + roleArn: flags["role-arn"], + disableOnCreate: flags["disable-on-create"], + }, + coreOptsFromCtx(ctx), + ); + + ctx.require(JsonRendererKey).renderJson(result); + }, + }); diff --git a/src/handlers/eval/ab-test/index.tsx b/src/handlers/eval/ab-test/index.tsx index 6bccf91fc..4e42ada3a 100644 --- a/src/handlers/eval/ab-test/index.tsx +++ b/src/handlers/eval/ab-test/index.tsx @@ -9,6 +9,7 @@ import { createPauseAbTestHandler } from "./pause"; import { createResumeAbTestHandler } from "./resume"; import { createStopAbTestHandler } from "./stop"; import { createDeleteAbTestHandler } from "./delete"; +import { createConfigBundleAbTestHandler } from "./config-bundle"; export function createAbTestHandler(core: Core, io: AppIO): Router { return new Router("ab-test", "inspect AgentCore A/B tests") @@ -20,7 +21,8 @@ export function createAbTestHandler(core: Core, io: AppIO): Router { .handler(createPauseAbTestHandler(core)) .handler(createResumeAbTestHandler(core)) .handler(createStopAbTestHandler(core)) - .handler(createDeleteAbTestHandler(core)); + .handler(createDeleteAbTestHandler(core)) + .handler(createConfigBundleAbTestHandler(core, io)); } export { AbTestScreen } from "./screen.tsx"; diff --git a/src/handlers/eval/types.tsx b/src/handlers/eval/types.tsx index b1be77f4b..2ce7cd2fd 100644 --- a/src/handlers/eval/types.tsx +++ b/src/handlers/eval/types.tsx @@ -30,6 +30,8 @@ import type { UpdateOnlineEvaluationConfigResponse, } from "@aws-sdk/client-bedrock-agentcore-control"; import type { + CreateABTestResponse, + GatewayFilter, GetABTestResponse, ListABTestsResponse, ABTestExecutionStatus, @@ -231,6 +233,20 @@ export type RoleScopeWarning = { logGroupNames: string[]; }; +export type BundleRef = { configBundle: string; bundleVersion: string }; + +export type CreateConfigBundleABTestInput = { + name: string; + gateway: string; + control: BundleRef; + treatment: BundleRef; + onlineEval: string; + treatmentWeight?: number; + gatewayFilter?: GatewayFilter; + roleArn?: string; + disableOnCreate?: boolean; +}; + export type CreateDatasetInput = CreateDatasetRequest; export type StartRecommendationInput = { name: string; @@ -427,6 +443,10 @@ export interface CoreEvalClient { options: CoreOptions, ): Promise; deleteABTest(id: string, options: CoreOptions): Promise; + createConfigBundleABTest( + input: CreateConfigBundleABTestInput, + options: CoreOptions, + ): Promise; // startBatchEvaluation submits an async, service-side evaluation over sessions // the service gathers from the resolved data source. Returns the durable job id // + RUNNING status; poll with getBatchEvaluation. diff --git a/src/testing/TestCoreClient.tsx b/src/testing/TestCoreClient.tsx index 8fcce7a46..94a888c47 100644 --- a/src/testing/TestCoreClient.tsx +++ b/src/testing/TestCoreClient.tsx @@ -76,6 +76,7 @@ import type { GetABTestResponse, ListABTestsResponse, ABTestExecutionStatus, + CreateABTestResponse, UpdateABTestResponse, DeleteABTestResponse, DeleteRecommendationResponse, @@ -137,6 +138,7 @@ import type { CodeBasedUpdate, CoreEvalClient, CreateConfigurationBundleInput, + CreateConfigBundleABTestInput, CreateDatasetInput, CreateOnlineEvalInput, CreateOnlineInsightInput, @@ -281,6 +283,7 @@ const DEFAULT_GET_ABTEST_RESPONSE = {} as GetABTestResponse; const DEFAULT_LIST_ABTESTS_RESPONSE: ListABTestsResponse = { abTests: [] }; const DEFAULT_UPDATE_ABTEST_RESPONSE = {} as UpdateABTestResponse; const DEFAULT_DELETE_ABTEST_RESPONSE = {} as DeleteABTestResponse; +const DEFAULT_CREATE_ABTEST_RESPONSE = {} as CreateABTestResponse; const DEFAULT_START_BATCH_EVAL_RESPONSE = { batchEvaluationId: "batch-eval-test", status: "RUNNING", @@ -1438,6 +1441,7 @@ export class TestEvalClient implements CoreEvalClient { private abTestListResponses = new Map(); private abTestUpdateResponse: UpdateABTestResponse = DEFAULT_UPDATE_ABTEST_RESPONSE; private abTestDeleteResponse: DeleteABTestResponse = DEFAULT_DELETE_ABTEST_RESPONSE; + private abTestCreateResponse: CreateABTestResponse = DEFAULT_CREATE_ABTEST_RESPONSE; private batchEvalResults: BatchEvaluationResultEntry[] = []; private batchEvalResultsError?: unknown; private startBatchEvalResponse: StartBatchEvaluationResponse = DEFAULT_START_BATCH_EVAL_RESPONSE; @@ -1685,6 +1689,11 @@ export class TestEvalClient implements CoreEvalClient { return this; } + setAbTestCreateResponse(response: CreateABTestResponse): this { + this.abTestCreateResponse = response; + return this; + } + // setUpdateDatasetResult sets what updateDatasetExamples resolves to (when not // erroring). setUpdateDatasetResult(result: DatasetUpdateResult): this { @@ -1894,6 +1903,15 @@ export class TestEvalClient implements CoreEvalClient { return this.abTestDeleteResponse; } + async createConfigBundleABTest( + input: CreateConfigBundleABTestInput, + options: CoreOptions, + ): Promise { + this.calls.push({ method: "createConfigBundleABTest", args: [input, options] }); + if (this.error) throw this.error; + return this.abTestCreateResponse; + } + async startBatchEvaluation( input: StartBatchEvaluationInput, options: CoreOptions, From ac398b838bd2c340488df479fd6150fbcf0cfcfc Mon Sep 17 00:00:00 2001 From: jariy17 Date: Wed, 26 Aug 2026 17:45:09 +0000 Subject: [PATCH 05/15] fix(eval): address ab-test config-bundle run review - Reject control/treatment only when the (config-bundle, bundle-version) pair is identical, not on version-string collision across different bundles. - Retry CreateABTest on data-plane AccessDenied (403), not just the control-plane role-not-propagated phrasing, so a freshly provisioned role that is mid-propagation is retried. - Extract accountId via a throwing helper instead of a silent '*' fallback. - --treatment-weight must be an integer. - Add unit tests for the execution-role module (name cap, trust + inline policy, create vs reuse). --- src/core/abTestExecutionRole.test.ts | 100 ++++++++++++++++++ src/core/abTestExecutionRole.tsx | 12 ++- src/core/eval.tsx | 33 +++++- .../eval/ab-test/ab-test.create.test.tsx | 2 +- .../eval/ab-test/config-bundle/run/index.tsx | 11 +- 5 files changed, 147 insertions(+), 11 deletions(-) create mode 100644 src/core/abTestExecutionRole.test.ts diff --git a/src/core/abTestExecutionRole.test.ts b/src/core/abTestExecutionRole.test.ts new file mode 100644 index 000000000..7409d61c7 --- /dev/null +++ b/src/core/abTestExecutionRole.test.ts @@ -0,0 +1,100 @@ +import { test, expect, describe } from "bun:test"; +import { CreateRoleCommand, GetRoleCommand, type IAMClient } from "@aws-sdk/client-iam"; +import { + abTestExecutionRoleName, + accountIdFromArn, + provisionAbTestRole, +} from "./abTestExecutionRole"; + +const GATEWAY_ARN = "arn:aws:bedrock-agentcore:us-west-2:123456789012:gateway/orders-gw"; + +type Sent = { name: string; input: unknown }; + +function fakeIam(onGet: "found" | "missing"): { iam: IAMClient; sent: Sent[] } { + const sent: Sent[] = []; + const iam = { + send: async (command: { constructor: { name: string }; input: unknown }) => { + sent.push({ name: command.constructor.name, input: command.input }); + if (command instanceof GetRoleCommand) { + if (onGet === "missing") { + throw Object.assign(new Error("no such entity"), { name: "NoSuchEntityException" }); + } + return { + Role: { + Arn: `arn:aws:iam::123456789012:role/${(command.input as { RoleName: string }).RoleName}`, + }, + }; + } + if (command instanceof CreateRoleCommand) { + return { + Role: { + Arn: `arn:aws:iam::123456789012:role/${(command.input as { RoleName: string }).RoleName}`, + }, + }; + } + return {}; + }, + } as unknown as IAMClient; + return { iam, sent }; +} + +describe("abTestExecutionRoleName", () => { + test("stays within IAM's 64-char limit and is deterministic", () => { + const long = abTestExecutionRoleName("x".repeat(120)); + expect(long.length).toBeLessThanOrEqual(64); + expect(abTestExecutionRoleName("orders")).toBe(abTestExecutionRoleName("orders")); + }); + + test("distinct names for distinct tests", () => { + expect(abTestExecutionRoleName("a")).not.toBe(abTestExecutionRoleName("b")); + }); +}); + +describe("accountIdFromArn", () => { + test("extracts the account segment", () => { + expect(accountIdFromArn(GATEWAY_ARN)).toBe("123456789012"); + }); + test("throws on a malformed ARN", () => { + expect(() => accountIdFromArn("not-an-arn")).toThrow(/account id/); + }); +}); + +describe("provisionAbTestRole", () => { + test("creates the role + inline policy and reports created=true", async () => { + const { iam, sent } = fakeIam("missing"); + const result = await provisionAbTestRole(iam, "orders-v2", GATEWAY_ARN, "us-west-2"); + + expect(result.created).toBe(true); + expect(result.roleArn).toContain(":role/"); + expect(sent.map((s) => s.name)).toEqual([ + "GetRoleCommand", + "CreateRoleCommand", + "PutRolePolicyCommand", + ]); + + const create = sent.find((s) => s.name === "CreateRoleCommand")!.input as { + AssumeRolePolicyDocument: string; + }; + const trust = JSON.parse(create.AssumeRolePolicyDocument); + expect(trust.Statement[0].Principal.Service).toBe("bedrock-agentcore.amazonaws.com"); + expect(trust.Statement[0].Condition.StringEquals["aws:SourceAccount"]).toBe("123456789012"); + expect(trust.Statement[0].Condition.ArnLike["aws:SourceArn"]).toContain(":ab-test/*"); + + const policy = sent.find((s) => s.name === "PutRolePolicyCommand")!.input as { + PolicyDocument: string; + }; + const doc = JSON.parse(policy.PolicyDocument); + const actions = doc.Statement.flatMap((s: { Action: string[] }) => s.Action); + expect(actions).toContain("bedrock-agentcore:GetGateway"); + expect(actions).toContain("bedrock-agentcore:GetConfigurationBundleVersion"); + expect(actions).toContain("bedrock-agentcore:GetOnlineEvaluationConfig"); + }); + + test("reuses an existing role and reports created=false", async () => { + const { iam, sent } = fakeIam("found"); + const result = await provisionAbTestRole(iam, "orders-v2", GATEWAY_ARN, "us-west-2"); + + expect(result.created).toBe(false); + expect(sent.map((s) => s.name)).toEqual(["GetRoleCommand", "PutRolePolicyCommand"]); + }); +}); diff --git a/src/core/abTestExecutionRole.tsx b/src/core/abTestExecutionRole.tsx index a4463f5f8..721f4d026 100644 --- a/src/core/abTestExecutionRole.tsx +++ b/src/core/abTestExecutionRole.tsx @@ -21,6 +21,12 @@ export function roleNameFromArn(roleArn: string): string { return parts[parts.length - 1] ?? roleArn; } +export function accountIdFromArn(arn: string): string { + const accountId = arn.split(":")[4]; + if (!accountId) throw new Error(`could not extract account id from ARN: ${arn}`); + return accountId; +} + function trustPolicy(accountId: string, region: string): string { return JSON.stringify({ Version: "2012-10-17", @@ -100,7 +106,7 @@ export async function provisionAbTestRole( gatewayArn: string, region: string, ): Promise<{ roleArn: string; created: boolean }> { - const accountId = gatewayArn.split(":")[4] ?? "*"; + const accountId = accountIdFromArn(gatewayArn); const roleName = abTestExecutionRoleName(testName); let roleArn: string; @@ -139,11 +145,11 @@ export async function deleteAbTestRole(iam: IAMClient, roleArn: string): Promise new DeleteRolePolicyCommand({ RoleName: roleName, PolicyName: AB_TEST_POLICY_NAME }), ); } catch { - // best effort + void 0; } try { await iam.send(new DeleteRoleCommand({ RoleName: roleName })); } catch { - // best effort + void 0; } } diff --git a/src/core/eval.tsx b/src/core/eval.tsx index b06308745..6e282633b 100644 --- a/src/core/eval.tsx +++ b/src/core/eval.tsx @@ -169,7 +169,7 @@ import { revokeOnlineEvalScope, scopePolicyName, } from "./onlineEvalExecutionRole"; -import { deleteAbTestRole, provisionAbTestRole } from "./abTestExecutionRole"; +import { accountIdFromArn, deleteAbTestRole, provisionAbTestRole } from "./abTestExecutionRole"; const DEFAULT_ENDPOINT_QUALIFIER = "DEFAULT"; const DEFAULT_INGESTION_WAIT_MS = 180_000; @@ -488,7 +488,7 @@ export class EvalClient implements CoreEvalClient { const control = this.clients.control(toClientConfig(options)); const gateway = await control.send(new GetGatewayCommand({ gatewayIdentifier: input.gateway })); const gatewayArn = gateway.gatewayArn!; - const accountId = gatewayArn.split(":")[4] ?? "*"; + const accountId = accountIdFromArn(gatewayArn); const controlBundleArn = `arn:aws:bedrock-agentcore:${options.region}:${accountId}:configuration-bundle/${input.control.configBundle}`; const treatmentBundleArn = `arn:aws:bedrock-agentcore:${options.region}:${accountId}:configuration-bundle/${input.treatment.configBundle}`; @@ -541,7 +541,7 @@ export class EvalClient implements CoreEvalClient { try { return input.roleArn ? await this.clients.data(toClientConfig(options)).send(command) - : await retryWhileRolePropagates(() => + : await retryWhileRoleUnassumable(() => this.clients.data(toClientConfig(options)).send(command), ); } catch (error) { @@ -549,7 +549,7 @@ export class EvalClient implements CoreEvalClient { try { await deleteAbTestRole(this.clients.iam({ region: options.region }), provisionedRoleArn); } catch { - // best effort + void 0; } } throw error; @@ -2106,6 +2106,31 @@ function chunk(items: T[], size: number): T[][] { const ROLE_NOT_PROPAGATED = /role cannot be assumed|does not have permissions to (create log group|access the specified log groups)/i; +// CreateABTest is on the data plane and surfaces a freshly-provisioned role that +// has not propagated as a plain AccessDenied rather than the control-plane phrasing +// ROLE_NOT_PROPAGATED matches, so the ab-test create path retries on that too. +async function retryWhileRoleUnassumable(send: () => Promise): Promise { + const delaysMs = [1_000, 2_000, 4_000, 8_000]; + for (const delay of delaysMs) { + try { + return await send(); + } catch (error) { + const err = error as { + name?: string; + message?: string; + $metadata?: { httpStatusCode?: number }; + }; + const retryable = + err.name === "AccessDeniedException" || + err.$metadata?.httpStatusCode === 403 || + ROLE_NOT_PROPAGATED.test(err.message ?? ""); + if (!retryable) throw error; + await new Promise((resolve) => setTimeout(resolve, delay)); + } + } + return send(); +} + // retryWhileRolePropagates retries `send` while the service reports the execution // role as unusable, which is how a not-yet-propagated role or policy surfaces. // Bounded and short: propagation is normally a few seconds, and a role that is diff --git a/src/handlers/eval/ab-test/ab-test.create.test.tsx b/src/handlers/eval/ab-test/ab-test.create.test.tsx index c96a31dd3..38ebd8fa6 100644 --- a/src/handlers/eval/ab-test/ab-test.create.test.tsx +++ b/src/handlers/eval/ab-test/ab-test.create.test.tsx @@ -126,7 +126,7 @@ describe("eval ab-test config-bundle run", () => { "o", "--json", ]), - ).rejects.toThrow(/must differ/); + ).rejects.toThrow(/must reference a different/); }); test("rejects --treatment-weight outside 1-99", async () => { diff --git a/src/handlers/eval/ab-test/config-bundle/run/index.tsx b/src/handlers/eval/ab-test/config-bundle/run/index.tsx index c8016fbdc..1c852f421 100644 --- a/src/handlers/eval/ab-test/config-bundle/run/index.tsx +++ b/src/handlers/eval/ab-test/config-bundle/run/index.tsx @@ -50,7 +50,7 @@ export const createConfigBundleRunHandler = (core: Core, io: AppIO) => flag( "treatment-weight", "1-99; control weight = 100 - this (default 50)", - z.number().optional(), + z.number().int().optional(), ), flag( "gateway-filter", @@ -86,8 +86,13 @@ export const createConfigBundleRunHandler = (core: Core, io: AppIO) => const control = toBundleRef("control", controlRaw); const treatment = toBundleRef("treatment", treatmentRaw); - if (control.bundleVersion === treatment.bundleVersion) { - throw new InputValidationError("treatment bundle-version must differ from control"); + if ( + control.configBundle === treatment.configBundle && + control.bundleVersion === treatment.bundleVersion + ) { + throw new InputValidationError( + "control and treatment must reference a different config-bundle or bundle-version", + ); } const treatmentWeight = flags["treatment-weight"]; From 653dc9fd710bda6f3163faf152eb7f7a05107312 Mon Sep 17 00:00:00 2001 From: jariy17 Date: Wed, 26 Aug 2026 17:54:25 +0000 Subject: [PATCH 06/15] feat(eval): use --enable-on-create for ab-test config-bundle run Match online-eval create's flag ergonomics: replace the boolean opt-out --disable-on-create with a value flag --enable-on-create (default true). Input carries enableOnCreate?: boolean; core sends enableOnCreate ?? true. --- src/core/eval.tsx | 2 +- .../eval/ab-test/ab-test.create.test.tsx | 18 +++++++++++++----- .../eval/ab-test/config-bundle/run/index.tsx | 11 +++++++++-- src/handlers/eval/types.tsx | 2 +- 4 files changed, 24 insertions(+), 9 deletions(-) diff --git a/src/core/eval.tsx b/src/core/eval.tsx index 6e282633b..825a4c2a7 100644 --- a/src/core/eval.tsx +++ b/src/core/eval.tsx @@ -534,7 +534,7 @@ export class EvalClient implements CoreEvalClient { evaluationConfig: { onlineEvaluationConfigArn }, roleArn, gatewayFilter: input.gatewayFilter, - enableOnCreate: !input.disableOnCreate, + enableOnCreate: input.enableOnCreate ?? true, clientToken: randomUUID(), }); diff --git a/src/handlers/eval/ab-test/ab-test.create.test.tsx b/src/handlers/eval/ab-test/ab-test.create.test.tsx index 38ebd8fa6..a20ae75c2 100644 --- a/src/handlers/eval/ab-test/ab-test.create.test.tsx +++ b/src/handlers/eval/ab-test/ab-test.create.test.tsx @@ -76,7 +76,7 @@ describe("eval ab-test config-bundle run", () => { treatmentWeight: 20, gatewayFilter: undefined, roleArn: undefined, - disableOnCreate: false, + enableOnCreate: undefined, }); expect(call?.args[1]).toEqual({ region: "us-west-2" }); }); @@ -94,19 +94,27 @@ describe("eval ab-test config-bundle run", () => { }); }); - test("passes --disable-on-create and --role-arn through", async () => { + test("passes --enable-on-create false and --role-arn through", async () => { const { core } = await run([ ...BASE, - "--disable-on-create", + "--enable-on-create", + "false", "--role-arn", "arn:aws:iam::123456789012:role/customer-owned", ]); const call = core.eval.calls.find((c) => c.method === "createConfigBundleABTest"); - const input = call?.args[0] as { disableOnCreate?: boolean; roleArn?: string }; - expect(input.disableOnCreate).toBe(true); + const input = call?.args[0] as { enableOnCreate?: boolean; roleArn?: string }; + expect(input.enableOnCreate).toBe(false); expect(input.roleArn).toBe("arn:aws:iam::123456789012:role/customer-owned"); }); + test("--enable-on-create true is passed through", async () => { + const { core } = await run([...BASE, "--enable-on-create", "true"]); + const call = core.eval.calls.find((c) => c.method === "createConfigBundleABTest"); + expect(call).toBeDefined(); + expect((call!.args[0] as { enableOnCreate?: boolean }).enableOnCreate).toBe(true); + }); + test("rejects equal control/treatment bundle-versions", async () => { await expect( run([ diff --git a/src/handlers/eval/ab-test/config-bundle/run/index.tsx b/src/handlers/eval/ab-test/config-bundle/run/index.tsx index 1c852f421..a7abd1e5d 100644 --- a/src/handlers/eval/ab-test/config-bundle/run/index.tsx +++ b/src/handlers/eval/ab-test/config-bundle/run/index.tsx @@ -62,7 +62,11 @@ export const createConfigBundleRunHandler = (core: Core, io: AppIO) => "execution-role override (default: auto-provisioned)", z.string().optional(), ), - flag("disable-on-create", "create without starting", z.boolean().optional()), + flag( + "enable-on-create", + "whether to start the test immediately (default true; pass false to create it paused)", + z.enum(["true", "false"]).optional(), + ), ], handle: async (ctx, flags) => { const required = ["name", "gateway", "control", "treatment", "online-eval"] as const; @@ -110,7 +114,10 @@ export const createConfigBundleRunHandler = (core: Core, io: AppIO) => treatmentWeight, gatewayFilter, roleArn: flags["role-arn"], - disableOnCreate: flags["disable-on-create"], + enableOnCreate: + flags["enable-on-create"] === undefined + ? undefined + : flags["enable-on-create"] === "true", }, coreOptsFromCtx(ctx), ); diff --git a/src/handlers/eval/types.tsx b/src/handlers/eval/types.tsx index 2ce7cd2fd..b13c6f9cd 100644 --- a/src/handlers/eval/types.tsx +++ b/src/handlers/eval/types.tsx @@ -244,7 +244,7 @@ export type CreateConfigBundleABTestInput = { treatmentWeight?: number; gatewayFilter?: GatewayFilter; roleArn?: string; - disableOnCreate?: boolean; + enableOnCreate?: boolean; }; export type CreateDatasetInput = CreateDatasetRequest; From 0ac67548d84481c0b6673aa4d292e7c5d707f7cd Mon Sep 17 00:00:00 2001 From: jariy17 Date: Wed, 26 Aug 2026 22:49:09 +0000 Subject: [PATCH 07/15] refactor(eval): reuse retryWhileRolePropagates for ab-test create Drop the duplicate retryWhileRoleUnassumable I added; broaden the existing retryWhileRolePropagates to also retry on data-plane AccessDenied/403 (how a freshly-provisioned role surfaces on CreateABTest) and reuse it. Removes the cross-file name collision with harness's helper. --- src/core/eval.tsx | 24 ++---------------------- 1 file changed, 2 insertions(+), 22 deletions(-) diff --git a/src/core/eval.tsx b/src/core/eval.tsx index 825a4c2a7..859bfd215 100644 --- a/src/core/eval.tsx +++ b/src/core/eval.tsx @@ -541,7 +541,7 @@ export class EvalClient implements CoreEvalClient { try { return input.roleArn ? await this.clients.data(toClientConfig(options)).send(command) - : await retryWhileRoleUnassumable(() => + : await retryWhileRolePropagates(() => this.clients.data(toClientConfig(options)).send(command), ); } catch (error) { @@ -2106,10 +2106,7 @@ function chunk(items: T[], size: number): T[][] { const ROLE_NOT_PROPAGATED = /role cannot be assumed|does not have permissions to (create log group|access the specified log groups)/i; -// CreateABTest is on the data plane and surfaces a freshly-provisioned role that -// has not propagated as a plain AccessDenied rather than the control-plane phrasing -// ROLE_NOT_PROPAGATED matches, so the ab-test create path retries on that too. -async function retryWhileRoleUnassumable(send: () => Promise): Promise { +async function retryWhileRolePropagates(send: () => Promise): Promise { const delaysMs = [1_000, 2_000, 4_000, 8_000]; for (const delay of delaysMs) { try { @@ -2131,23 +2128,6 @@ async function retryWhileRoleUnassumable(send: () => Promise): Promise return send(); } -// retryWhileRolePropagates retries `send` while the service reports the execution -// role as unusable, which is how a not-yet-propagated role or policy surfaces. -// Bounded and short: propagation is normally a few seconds, and a role that is -// genuinely misconfigured should fail fast rather than hang. -async function retryWhileRolePropagates(send: () => Promise): Promise { - const delaysMs = [1_000, 2_000, 4_000, 8_000]; - for (const delay of delaysMs) { - try { - return await send(); - } catch (error) { - if (!ROLE_NOT_PROPAGATED.test((error as Error).message)) throw error; - await new Promise((resolve) => setTimeout(resolve, delay)); - } - } - return send(); -} - // evaluatorKmsKeys collects the customer managed KMS keys of the referenced // evaluators. The service validates that the execution role can decrypt them when // the config is created, so a provisioned role has to grant kms:Decrypt on exactly From f325a1a9d43d5c90daf3e2e4c8cf648ec6667f1f Mon Sep 17 00:00:00 2001 From: jariy17 Date: Thu, 27 Aug 2026 21:06:41 +0000 Subject: [PATCH 08/15] test(eval): golden fixture for ab-test config-bundle run Record a self-contained config-bundle run golden in account 685197708687 (matches the config-bundle fixtures): create a bundle (v1 -> v2), a paused online-eval on a real runtime, then run the paused A/B test; afterAll tears down the ab-test, online-eval, provisioned role, and bundle. Broaden retryWhileRolePropagates to also retry ValidationException 'unable to assume the provided IAM role' -- how CreateABTest surfaces a freshly provisioned role mid-propagation. Drop the TestCoreClient happy-path mapping tests the golden now covers; keep the local validation/error cases. --- src/core/eval.tsx | 5 +- .../eval/ab-test/ab-test.create.test.tsx | 39 ---- .../CreateABTestCommand.506cd57a7653b22c.json | 6 + .../CreateABTestCommand.a4666d7f80bc7cb0.json | 10 + ...urationBundleCommand.e8ee73bc166ad5e4.json | 8 + ...luationConfigCommand.ae1ee3532d19f571.json | 14 ++ .../CreateRoleCommand.1fa2ac2a7f0f7fc0.json | 12 ++ .../CreateRoleCommand.7b030b47662eee32.json | 12 ++ .../DeleteRoleCommand.c6a8dc12fb95054d.json | 1 + ...eteRolePolicyCommand.3826bd85235b40f0.json | 1 + ...tAgentRuntimeCommand.9f77333d1b9dcf5d.json | 47 +++++ ...urationBundleCommand.cf3c23a25f6bf298.json | 23 ++ ...urationBundleCommand.ed835ef9d614b3e6.json | 23 ++ .../GetEvaluatorCommand.716589b0884f35c0.json | 72 +++++++ .../GetGatewayCommand.4216a59651bb046a.json | 20 ++ .../GetRoleCommand.2ca20231e1472584.json | 15 ++ .../GetRoleCommand.c6a8dc12fb95054d.json | 6 + ...PutRolePolicyCommand.8bf1be5c5e52a2ec.json | 1 + ...PutRolePolicyCommand.bc0b72d6bad3afad.json | 1 + ...urationBundleCommand.aca9ba06670aa395.json | 8 + ...urationBundleCommand.de17ce40709b64eb.json | 8 + .../run-bundle-create.golden.json | 6 + .../run-bundle-update.golden.json | 6 + .../__fixtures__/run-online-eval.golden.json | 12 ++ .../__fixtures__/run.golden.json | 8 + .../config-bundle-run.fixture.test.tsx | 198 ++++++++++++++++++ 26 files changed, 521 insertions(+), 41 deletions(-) create mode 100644 src/handlers/eval/ab-test/config-bundle/__fixtures__/CreateABTestCommand.506cd57a7653b22c.json create mode 100644 src/handlers/eval/ab-test/config-bundle/__fixtures__/CreateABTestCommand.a4666d7f80bc7cb0.json create mode 100644 src/handlers/eval/ab-test/config-bundle/__fixtures__/CreateConfigurationBundleCommand.e8ee73bc166ad5e4.json create mode 100644 src/handlers/eval/ab-test/config-bundle/__fixtures__/CreateOnlineEvaluationConfigCommand.ae1ee3532d19f571.json create mode 100644 src/handlers/eval/ab-test/config-bundle/__fixtures__/CreateRoleCommand.1fa2ac2a7f0f7fc0.json create mode 100644 src/handlers/eval/ab-test/config-bundle/__fixtures__/CreateRoleCommand.7b030b47662eee32.json create mode 100644 src/handlers/eval/ab-test/config-bundle/__fixtures__/DeleteRoleCommand.c6a8dc12fb95054d.json create mode 100644 src/handlers/eval/ab-test/config-bundle/__fixtures__/DeleteRolePolicyCommand.3826bd85235b40f0.json create mode 100644 src/handlers/eval/ab-test/config-bundle/__fixtures__/GetAgentRuntimeCommand.9f77333d1b9dcf5d.json create mode 100644 src/handlers/eval/ab-test/config-bundle/__fixtures__/GetConfigurationBundleCommand.cf3c23a25f6bf298.json create mode 100644 src/handlers/eval/ab-test/config-bundle/__fixtures__/GetConfigurationBundleCommand.ed835ef9d614b3e6.json create mode 100644 src/handlers/eval/ab-test/config-bundle/__fixtures__/GetEvaluatorCommand.716589b0884f35c0.json create mode 100644 src/handlers/eval/ab-test/config-bundle/__fixtures__/GetGatewayCommand.4216a59651bb046a.json create mode 100644 src/handlers/eval/ab-test/config-bundle/__fixtures__/GetRoleCommand.2ca20231e1472584.json create mode 100644 src/handlers/eval/ab-test/config-bundle/__fixtures__/GetRoleCommand.c6a8dc12fb95054d.json create mode 100644 src/handlers/eval/ab-test/config-bundle/__fixtures__/PutRolePolicyCommand.8bf1be5c5e52a2ec.json create mode 100644 src/handlers/eval/ab-test/config-bundle/__fixtures__/PutRolePolicyCommand.bc0b72d6bad3afad.json create mode 100644 src/handlers/eval/ab-test/config-bundle/__fixtures__/UpdateConfigurationBundleCommand.aca9ba06670aa395.json create mode 100644 src/handlers/eval/ab-test/config-bundle/__fixtures__/UpdateConfigurationBundleCommand.de17ce40709b64eb.json create mode 100644 src/handlers/eval/ab-test/config-bundle/__fixtures__/run-bundle-create.golden.json create mode 100644 src/handlers/eval/ab-test/config-bundle/__fixtures__/run-bundle-update.golden.json create mode 100644 src/handlers/eval/ab-test/config-bundle/__fixtures__/run-online-eval.golden.json create mode 100644 src/handlers/eval/ab-test/config-bundle/__fixtures__/run.golden.json create mode 100644 src/handlers/eval/ab-test/config-bundle/config-bundle-run.fixture.test.tsx diff --git a/src/core/eval.tsx b/src/core/eval.tsx index 859bfd215..439c67f4f 100644 --- a/src/core/eval.tsx +++ b/src/core/eval.tsx @@ -2104,10 +2104,10 @@ function chunk(items: T[], size: number): T[][] { // is created. It surfaces as one of two messages depending on which part has not // propagated yet. const ROLE_NOT_PROPAGATED = - /role cannot be assumed|does not have permissions to (create log group|access the specified log groups)/i; + /cannot be assumed|unable to assume|does not have permissions to (create log group|access the specified log groups)/i; async function retryWhileRolePropagates(send: () => Promise): Promise { - const delaysMs = [1_000, 2_000, 4_000, 8_000]; + const delaysMs = [2_000, 4_000, 8_000, 15_000]; for (const delay of delaysMs) { try { return await send(); @@ -2120,6 +2120,7 @@ async function retryWhileRolePropagates(send: () => Promise): Promise { const retryable = err.name === "AccessDeniedException" || err.$metadata?.httpStatusCode === 403 || + (err.name === "ValidationException" && /assume|role|trust/i.test(err.message ?? "")) || ROLE_NOT_PROPAGATED.test(err.message ?? ""); if (!retryable) throw error; await new Promise((resolve) => setTimeout(resolve, delay)); diff --git a/src/handlers/eval/ab-test/ab-test.create.test.tsx b/src/handlers/eval/ab-test/ab-test.create.test.tsx index a20ae75c2..ca0d4f519 100644 --- a/src/handlers/eval/ab-test/ab-test.create.test.tsx +++ b/src/handlers/eval/ab-test/ab-test.create.test.tsx @@ -63,24 +63,6 @@ describe("eval ab-test config-bundle run", () => { expect(cb?.children().map((c) => c.name())).toEqual(["run"]); }); - test("maps flags to a createConfigBundleABTest call", async () => { - const { core, stdout } = await run([...BASE, "--treatment-weight", "20"]); - expect(JSON.parse(stdout).abTestId).toBe("orders-v2-abc123"); - const call = core.eval.calls.find((c) => c.method === "createConfigBundleABTest"); - expect(call?.args[0]).toEqual({ - name: "orders-v2", - gateway: "orders-gateway-abc123", - control: { configBundle: "orders-prompt-abc", bundleVersion: "1111" }, - treatment: { configBundle: "orders-prompt-abc", bundleVersion: "2222" }, - onlineEval: "online-eval-abc123", - treatmentWeight: 20, - gatewayFilter: undefined, - roleArn: undefined, - enableOnCreate: undefined, - }); - expect(call?.args[1]).toEqual({ region: "us-west-2" }); - }); - test("passes --gateway-filter through as a GatewayFilter", async () => { const { core } = await run([ ...BASE, @@ -94,27 +76,6 @@ describe("eval ab-test config-bundle run", () => { }); }); - test("passes --enable-on-create false and --role-arn through", async () => { - const { core } = await run([ - ...BASE, - "--enable-on-create", - "false", - "--role-arn", - "arn:aws:iam::123456789012:role/customer-owned", - ]); - const call = core.eval.calls.find((c) => c.method === "createConfigBundleABTest"); - const input = call?.args[0] as { enableOnCreate?: boolean; roleArn?: string }; - expect(input.enableOnCreate).toBe(false); - expect(input.roleArn).toBe("arn:aws:iam::123456789012:role/customer-owned"); - }); - - test("--enable-on-create true is passed through", async () => { - const { core } = await run([...BASE, "--enable-on-create", "true"]); - const call = core.eval.calls.find((c) => c.method === "createConfigBundleABTest"); - expect(call).toBeDefined(); - expect((call!.args[0] as { enableOnCreate?: boolean }).enableOnCreate).toBe(true); - }); - test("rejects equal control/treatment bundle-versions", async () => { await expect( run([ diff --git a/src/handlers/eval/ab-test/config-bundle/__fixtures__/CreateABTestCommand.506cd57a7653b22c.json b/src/handlers/eval/ab-test/config-bundle/__fixtures__/CreateABTestCommand.506cd57a7653b22c.json new file mode 100644 index 000000000..51aff6399 --- /dev/null +++ b/src/handlers/eval/ab-test/config-bundle/__fixtures__/CreateABTestCommand.506cd57a7653b22c.json @@ -0,0 +1,6 @@ +{ + "$error": { + "name": "ValidationException", + "message": "Unable to assume the provided IAM role. Verify the role exists and its trust policy allows bedrock-agentcore.amazonaws.com to assume it." + } +} \ No newline at end of file diff --git a/src/handlers/eval/ab-test/config-bundle/__fixtures__/CreateABTestCommand.a4666d7f80bc7cb0.json b/src/handlers/eval/ab-test/config-bundle/__fixtures__/CreateABTestCommand.a4666d7f80bc7cb0.json new file mode 100644 index 000000000..971f7115e --- /dev/null +++ b/src/handlers/eval/ab-test/config-bundle/__fixtures__/CreateABTestCommand.a4666d7f80bc7cb0.json @@ -0,0 +1,10 @@ +{ + "abTestId": "agentcore_cli_abtest_run-8e10bf2f27", + "abTestArn": "arn:aws:bedrock-agentcore:us-west-2:685197708687:ab-test/agentcore_cli_abtest_run-8e10bf2f27", + "status": "CREATING", + "executionStatus": "NOT_STARTED", + "createdAt": { + "$date": "2026-08-27T21:03:19.535Z" + }, + "name": "agentcore_cli_abtest_run" +} \ No newline at end of file diff --git a/src/handlers/eval/ab-test/config-bundle/__fixtures__/CreateConfigurationBundleCommand.e8ee73bc166ad5e4.json b/src/handlers/eval/ab-test/config-bundle/__fixtures__/CreateConfigurationBundleCommand.e8ee73bc166ad5e4.json new file mode 100644 index 000000000..265aebbc3 --- /dev/null +++ b/src/handlers/eval/ab-test/config-bundle/__fixtures__/CreateConfigurationBundleCommand.e8ee73bc166ad5e4.json @@ -0,0 +1,8 @@ +{ + "bundleArn": "arn:aws:bedrock-agentcore:us-west-2:685197708687:configuration-bundle/agentcore_cli_abtest_run_bundle-QTjmqQ9zT6", + "bundleId": "agentcore_cli_abtest_run_bundle-QTjmqQ9zT6", + "versionId": "e3c144d3-22ce-4a82-9413-cc9be04eb8a5", + "createdAt": { + "$date": "2026-08-27T21:02:58.851Z" + } +} \ No newline at end of file diff --git a/src/handlers/eval/ab-test/config-bundle/__fixtures__/CreateOnlineEvaluationConfigCommand.ae1ee3532d19f571.json b/src/handlers/eval/ab-test/config-bundle/__fixtures__/CreateOnlineEvaluationConfigCommand.ae1ee3532d19f571.json new file mode 100644 index 000000000..1d2638815 --- /dev/null +++ b/src/handlers/eval/ab-test/config-bundle/__fixtures__/CreateOnlineEvaluationConfigCommand.ae1ee3532d19f571.json @@ -0,0 +1,14 @@ +{ + "onlineEvaluationConfigArn": "arn:aws:bedrock-agentcore:us-west-2:685197708687:online-evaluation-config/agentcore_cli_abtest_run_eval-i7s3ryDRdt", + "onlineEvaluationConfigId": "agentcore_cli_abtest_run_eval-i7s3ryDRdt", + "createdAt": { + "$date": "2026-08-27T21:03:03.249Z" + }, + "status": "CREATING", + "executionStatus": "DISABLED", + "outputConfig": { + "cloudWatchConfig": { + "logGroupName": "/aws/bedrock-agentcore/evaluations/results/agentcore_cli_abtest_run_eval-i7s3ryDRdt" + } + } +} \ No newline at end of file diff --git a/src/handlers/eval/ab-test/config-bundle/__fixtures__/CreateRoleCommand.1fa2ac2a7f0f7fc0.json b/src/handlers/eval/ab-test/config-bundle/__fixtures__/CreateRoleCommand.1fa2ac2a7f0f7fc0.json new file mode 100644 index 000000000..40fb878c0 --- /dev/null +++ b/src/handlers/eval/ab-test/config-bundle/__fixtures__/CreateRoleCommand.1fa2ac2a7f0f7fc0.json @@ -0,0 +1,12 @@ +{ + "Role": { + "Path": "/", + "RoleName": "AgentCoreABTest-agentcore_cli_abtest_run-1cc7229b", + "RoleId": "AROAZ7CHXJWHZBY6C35TH", + "Arn": "arn:aws:iam::685197708687:role/AgentCoreABTest-agentcore_cli_abtest_run-1cc7229b", + "CreateDate": { + "$date": "2026-08-27T21:03:03.000Z" + }, + "AssumeRolePolicyDocument": "%7B%22Version%22%3A%222012-10-17%22%2C%22Statement%22%3A%5B%7B%22Effect%22%3A%22Allow%22%2C%22Principal%22%3A%7B%22Service%22%3A%22bedrock-agentcore.amazonaws.com%22%7D%2C%22Action%22%3A%22sts%3AAssumeRole%22%2C%22Condition%22%3A%7B%22StringEquals%22%3A%7B%22aws%3ASourceAccount%22%3A%22685197708687%22%7D%2C%22ArnLike%22%3A%7B%22aws%3ASourceArn%22%3A%22arn%3Aaws%3Abedrock-agentcore%3Aus-west-2%3A685197708687%3Aab-test%2F%2A%22%7D%7D%7D%5D%7D" + } +} \ No newline at end of file diff --git a/src/handlers/eval/ab-test/config-bundle/__fixtures__/CreateRoleCommand.7b030b47662eee32.json b/src/handlers/eval/ab-test/config-bundle/__fixtures__/CreateRoleCommand.7b030b47662eee32.json new file mode 100644 index 000000000..1968c9437 --- /dev/null +++ b/src/handlers/eval/ab-test/config-bundle/__fixtures__/CreateRoleCommand.7b030b47662eee32.json @@ -0,0 +1,12 @@ +{ + "Role": { + "Path": "/", + "RoleName": "AgentCoreOnlineEval-agentcore_cli_abtest_run_eval", + "RoleId": "AROAZ7CHXJWH3HARLZYI3", + "Arn": "arn:aws:iam::685197708687:role/AgentCoreOnlineEval-agentcore_cli_abtest_run_eval", + "CreateDate": { + "$date": "2026-08-27T21:01:46.000Z" + }, + "AssumeRolePolicyDocument": "%7B%22Version%22%3A%222012-10-17%22%2C%22Statement%22%3A%5B%7B%22Effect%22%3A%22Allow%22%2C%22Principal%22%3A%7B%22Service%22%3A%22bedrock-agentcore.amazonaws.com%22%7D%2C%22Action%22%3A%22sts%3AAssumeRole%22%7D%5D%7D" + } +} \ No newline at end of file diff --git a/src/handlers/eval/ab-test/config-bundle/__fixtures__/DeleteRoleCommand.c6a8dc12fb95054d.json b/src/handlers/eval/ab-test/config-bundle/__fixtures__/DeleteRoleCommand.c6a8dc12fb95054d.json new file mode 100644 index 000000000..9e26dfeeb --- /dev/null +++ b/src/handlers/eval/ab-test/config-bundle/__fixtures__/DeleteRoleCommand.c6a8dc12fb95054d.json @@ -0,0 +1 @@ +{} \ No newline at end of file diff --git a/src/handlers/eval/ab-test/config-bundle/__fixtures__/DeleteRolePolicyCommand.3826bd85235b40f0.json b/src/handlers/eval/ab-test/config-bundle/__fixtures__/DeleteRolePolicyCommand.3826bd85235b40f0.json new file mode 100644 index 000000000..9e26dfeeb --- /dev/null +++ b/src/handlers/eval/ab-test/config-bundle/__fixtures__/DeleteRolePolicyCommand.3826bd85235b40f0.json @@ -0,0 +1 @@ +{} \ No newline at end of file diff --git a/src/handlers/eval/ab-test/config-bundle/__fixtures__/GetAgentRuntimeCommand.9f77333d1b9dcf5d.json b/src/handlers/eval/ab-test/config-bundle/__fixtures__/GetAgentRuntimeCommand.9f77333d1b9dcf5d.json new file mode 100644 index 000000000..2d3b5e713 --- /dev/null +++ b/src/handlers/eval/ab-test/config-bundle/__fixtures__/GetAgentRuntimeCommand.9f77333d1b9dcf5d.json @@ -0,0 +1,47 @@ +{ + "agentRuntimeArn": "arn:aws:bedrock-agentcore:us-west-2:685197708687:runtime/asdf_MyAgent-3s5axvBC6Q", + "agentRuntimeName": "asdf_MyAgent", + "agentRuntimeId": "asdf_MyAgent-3s5axvBC6Q", + "agentRuntimeVersion": "1", + "createdAt": { + "$date": "2026-04-23T21:17:21.895Z" + }, + "lastUpdatedAt": { + "$date": "2026-04-23T21:17:35.159Z" + }, + "roleArn": "arn:aws:iam::685197708687:role/AgentCore-asdf-default-ApplicationAgentMyAgentRunti-KdyUbgImzDRK", + "networkConfiguration": { + "networkMode": "PUBLIC" + }, + "status": "READY", + "lifecycleConfiguration": { + "idleRuntimeSessionTimeout": 900, + "maxLifetime": 28800 + }, + "description": "AgentCore Runtime: asdf_MyAgent", + "workloadIdentityDetails": { + "workloadIdentityArn": "arn:aws:bedrock-agentcore:us-west-2:685197708687:workload-identity-directory/default/workload-identity/asdf_MyAgent-3s5axvBC6Q" + }, + "agentRuntimeArtifact": { + "codeConfiguration": { + "code": { + "s3": { + "bucket": "cdk-hnb659fds-assets-685197708687-us-west-2", + "prefix": "a07977786dda1e2e5be304cb7485237a19ed24d5e05b02e73ca91a43fd2e7280.zip" + } + }, + "runtime": "PYTHON_3_13", + "entryPoint": [ + "opentelemetry-instrument", + "main.py" + ] + } + }, + "environmentVariables": { + "AGENTCORE_GATEWAY_BUGBASHGW1776978672_AUTH_TYPE": "NONE", + "AGENTCORE_GATEWAY_BUGBASHGW1776978672_URL": "https://bugbashgw1776978672-zsy8cbqwts.gateway.bedrock-agentcore.us-west-2.amazonaws.com/mcp" + }, + "metadataConfiguration": { + "requireMMDSV2": true + } +} \ No newline at end of file diff --git a/src/handlers/eval/ab-test/config-bundle/__fixtures__/GetConfigurationBundleCommand.cf3c23a25f6bf298.json b/src/handlers/eval/ab-test/config-bundle/__fixtures__/GetConfigurationBundleCommand.cf3c23a25f6bf298.json new file mode 100644 index 000000000..7876da211 --- /dev/null +++ b/src/handlers/eval/ab-test/config-bundle/__fixtures__/GetConfigurationBundleCommand.cf3c23a25f6bf298.json @@ -0,0 +1,23 @@ +{ + "bundleArn": "arn:aws:bedrock-agentcore:us-west-2:685197708687:configuration-bundle/agentcore_cli_abtest_run_bundle-QTjmqQ9zT6", + "bundleId": "agentcore_cli_abtest_run_bundle-QTjmqQ9zT6", + "bundleName": "agentcore_cli_abtest_run_bundle", + "versionId": "e3c144d3-22ce-4a82-9413-cc9be04eb8a5", + "components": { + "arn:aws:bedrock-agentcore:us-west-2:685197708687:runtime/asdf_MyAgent-3s5axvBC6Q": { + "configuration": { + "system_prompt": "A/B run fixture v1." + } + } + }, + "createdAt": { + "$date": "2026-08-27T21:02:58.851Z" + }, + "updatedAt": { + "$date": "2026-08-27T21:02:58.851Z" + }, + "lineageMetadata": { + "parentVersionIds": [], + "branchName": "mainline" + } +} \ No newline at end of file diff --git a/src/handlers/eval/ab-test/config-bundle/__fixtures__/GetConfigurationBundleCommand.ed835ef9d614b3e6.json b/src/handlers/eval/ab-test/config-bundle/__fixtures__/GetConfigurationBundleCommand.ed835ef9d614b3e6.json new file mode 100644 index 000000000..489607a72 --- /dev/null +++ b/src/handlers/eval/ab-test/config-bundle/__fixtures__/GetConfigurationBundleCommand.ed835ef9d614b3e6.json @@ -0,0 +1,23 @@ +{ + "bundleArn": "arn:aws:bedrock-agentcore:us-west-2:685197708687:configuration-bundle/agentcore_cli_abtest_run_bundle-Fcuyfm8hxE", + "bundleId": "agentcore_cli_abtest_run_bundle-Fcuyfm8hxE", + "bundleName": "agentcore_cli_abtest_run_bundle", + "versionId": "ee7a2803-a60c-4b93-ac4d-5c20f32856da", + "components": { + "arn:aws:bedrock-agentcore:us-west-2:685197708687:runtime/asdf_MyAgent-3s5axvBC6Q": { + "configuration": { + "system_prompt": "A/B run fixture v1." + } + } + }, + "createdAt": { + "$date": "2026-08-27T21:01:43.000Z" + }, + "updatedAt": { + "$date": "2026-08-27T21:01:43.000Z" + }, + "lineageMetadata": { + "parentVersionIds": [], + "branchName": "mainline" + } +} \ No newline at end of file diff --git a/src/handlers/eval/ab-test/config-bundle/__fixtures__/GetEvaluatorCommand.716589b0884f35c0.json b/src/handlers/eval/ab-test/config-bundle/__fixtures__/GetEvaluatorCommand.716589b0884f35c0.json new file mode 100644 index 000000000..ebf428698 --- /dev/null +++ b/src/handlers/eval/ab-test/config-bundle/__fixtures__/GetEvaluatorCommand.716589b0884f35c0.json @@ -0,0 +1,72 @@ +{ + "evaluatorArn": "arn:aws:bedrock-agentcore:::evaluator/Builtin.Helpfulness", + "evaluatorId": "Builtin.Helpfulness", + "evaluatorName": "Builtin.Helpfulness", + "evaluatorConfig": { + "llmAsAJudge": { + "ratingScale": { + "numerical": [ + { + "value": { + "string": "0.0", + "type": "bigDecimal" + }, + "label": "Not helpful at all" + }, + { + "value": { + "string": "1.0", + "type": "bigDecimal" + }, + "label": "Very unhelpful" + }, + { + "value": { + "string": "2.0", + "type": "bigDecimal" + }, + "label": "Somewhat unhelpful" + }, + { + "value": { + "string": "3.0", + "type": "bigDecimal" + }, + "label": "Neutral/Mixed" + }, + { + "value": { + "string": "4.0", + "type": "bigDecimal" + }, + "label": "Somewhat helpful" + }, + { + "value": { + "string": "5.0", + "type": "bigDecimal" + }, + "label": "Very helpful" + }, + { + "value": { + "string": "6.0", + "type": "bigDecimal" + }, + "label": "Above and beyond" + } + ] + } + } + }, + "level": "TRACE", + "status": "ACTIVE", + "createdAt": { + "$date": "2024-10-22T00:00:00.000Z" + }, + "updatedAt": { + "$date": "2024-10-22T00:00:00.000Z" + }, + "description": "Response Quality Metric. Evaluates from user's perspective how useful and valuable the agent's response is", + "lockedForModification": true +} \ No newline at end of file diff --git a/src/handlers/eval/ab-test/config-bundle/__fixtures__/GetGatewayCommand.4216a59651bb046a.json b/src/handlers/eval/ab-test/config-bundle/__fixtures__/GetGatewayCommand.4216a59651bb046a.json new file mode 100644 index 000000000..1007a1d2f --- /dev/null +++ b/src/handlers/eval/ab-test/config-bundle/__fixtures__/GetGatewayCommand.4216a59651bb046a.json @@ -0,0 +1,20 @@ +{ + "gatewayArn": "arn:aws:bedrock-agentcore:us-west-2:685197708687:gateway/agentcore-cli-gateway-read-fixture-a-l6opkbe2kd", + "gatewayId": "agentcore-cli-gateway-read-fixture-a-l6opkbe2kd", + "createdAt": { + "$date": "2026-07-29T22:19:37.409Z" + }, + "updatedAt": { + "$date": "2026-07-29T22:19:37.971Z" + }, + "status": "READY", + "name": "agentcore-cli-gateway-read-fixture-a", + "authorizerType": "NONE", + "gatewayUrl": "https://agentcore-cli-gateway-read-fixture-a-l6opkbe2kd.gateway.bedrock-agentcore.us-west-2.amazonaws.com/mcp", + "description": "AgentCore CLI persistent Gateway read fixture", + "roleArn": "arn:aws:iam::685197708687:role/AgentCoreCliGatewayReadFixtureRole", + "protocolType": "MCP", + "workloadIdentityDetails": { + "workloadIdentityArn": "arn:aws:bedrock-agentcore:us-west-2:685197708687:workload-identity-directory/default/workload-identity/agentcore-cli-gateway-read-fixture-a-l6opkbe2kd" + } +} \ No newline at end of file diff --git a/src/handlers/eval/ab-test/config-bundle/__fixtures__/GetRoleCommand.2ca20231e1472584.json b/src/handlers/eval/ab-test/config-bundle/__fixtures__/GetRoleCommand.2ca20231e1472584.json new file mode 100644 index 000000000..afe04004e --- /dev/null +++ b/src/handlers/eval/ab-test/config-bundle/__fixtures__/GetRoleCommand.2ca20231e1472584.json @@ -0,0 +1,15 @@ +{ + "Role": { + "Path": "/", + "RoleName": "AgentCoreOnlineEval-agentcore_cli_abtest_run_eval", + "RoleId": "AROAZ7CHXJWH3HARLZYI3", + "Arn": "arn:aws:iam::685197708687:role/AgentCoreOnlineEval-agentcore_cli_abtest_run_eval", + "CreateDate": { + "$date": "2026-08-27T21:01:46.000Z" + }, + "AssumeRolePolicyDocument": "%7B%22Version%22%3A%222012-10-17%22%2C%22Statement%22%3A%5B%7B%22Effect%22%3A%22Allow%22%2C%22Principal%22%3A%7B%22Service%22%3A%22bedrock-agentcore.amazonaws.com%22%7D%2C%22Action%22%3A%22sts%3AAssumeRole%22%7D%5D%7D", + "Description": "Default execution role for the AgentCore online evaluation config \"agentcore_cli_abtest_run_eval\" (created by the agentcore CLI)", + "MaxSessionDuration": 3600, + "RoleLastUsed": {} + } +} \ No newline at end of file diff --git a/src/handlers/eval/ab-test/config-bundle/__fixtures__/GetRoleCommand.c6a8dc12fb95054d.json b/src/handlers/eval/ab-test/config-bundle/__fixtures__/GetRoleCommand.c6a8dc12fb95054d.json new file mode 100644 index 000000000..8bda1283f --- /dev/null +++ b/src/handlers/eval/ab-test/config-bundle/__fixtures__/GetRoleCommand.c6a8dc12fb95054d.json @@ -0,0 +1,6 @@ +{ + "$error": { + "name": "NoSuchEntityException", + "message": "The role with name AgentCoreABTest-agentcore_cli_abtest_run-1cc7229b cannot be found." + } +} \ No newline at end of file diff --git a/src/handlers/eval/ab-test/config-bundle/__fixtures__/PutRolePolicyCommand.8bf1be5c5e52a2ec.json b/src/handlers/eval/ab-test/config-bundle/__fixtures__/PutRolePolicyCommand.8bf1be5c5e52a2ec.json new file mode 100644 index 000000000..9e26dfeeb --- /dev/null +++ b/src/handlers/eval/ab-test/config-bundle/__fixtures__/PutRolePolicyCommand.8bf1be5c5e52a2ec.json @@ -0,0 +1 @@ +{} \ No newline at end of file diff --git a/src/handlers/eval/ab-test/config-bundle/__fixtures__/PutRolePolicyCommand.bc0b72d6bad3afad.json b/src/handlers/eval/ab-test/config-bundle/__fixtures__/PutRolePolicyCommand.bc0b72d6bad3afad.json new file mode 100644 index 000000000..9e26dfeeb --- /dev/null +++ b/src/handlers/eval/ab-test/config-bundle/__fixtures__/PutRolePolicyCommand.bc0b72d6bad3afad.json @@ -0,0 +1 @@ +{} \ No newline at end of file diff --git a/src/handlers/eval/ab-test/config-bundle/__fixtures__/UpdateConfigurationBundleCommand.aca9ba06670aa395.json b/src/handlers/eval/ab-test/config-bundle/__fixtures__/UpdateConfigurationBundleCommand.aca9ba06670aa395.json new file mode 100644 index 000000000..6284b216a --- /dev/null +++ b/src/handlers/eval/ab-test/config-bundle/__fixtures__/UpdateConfigurationBundleCommand.aca9ba06670aa395.json @@ -0,0 +1,8 @@ +{ + "bundleArn": "arn:aws:bedrock-agentcore:us-west-2:685197708687:configuration-bundle/agentcore_cli_abtest_run_bundle-QTjmqQ9zT6", + "bundleId": "agentcore_cli_abtest_run_bundle-QTjmqQ9zT6", + "versionId": "d9379d05-8c1c-4764-afcb-8b5f852d6830", + "updatedAt": { + "$date": "2026-08-27T21:03:02.235Z" + } +} \ No newline at end of file diff --git a/src/handlers/eval/ab-test/config-bundle/__fixtures__/UpdateConfigurationBundleCommand.de17ce40709b64eb.json b/src/handlers/eval/ab-test/config-bundle/__fixtures__/UpdateConfigurationBundleCommand.de17ce40709b64eb.json new file mode 100644 index 000000000..8772cde4d --- /dev/null +++ b/src/handlers/eval/ab-test/config-bundle/__fixtures__/UpdateConfigurationBundleCommand.de17ce40709b64eb.json @@ -0,0 +1,8 @@ +{ + "bundleArn": "arn:aws:bedrock-agentcore:us-west-2:685197708687:configuration-bundle/agentcore_cli_abtest_run_bundle-Fcuyfm8hxE", + "bundleId": "agentcore_cli_abtest_run_bundle-Fcuyfm8hxE", + "versionId": "e4d7ad4b-764b-41a5-a007-7b7f0ddac0df", + "updatedAt": { + "$date": "2026-08-27T21:01:46.324Z" + } +} \ No newline at end of file diff --git a/src/handlers/eval/ab-test/config-bundle/__fixtures__/run-bundle-create.golden.json b/src/handlers/eval/ab-test/config-bundle/__fixtures__/run-bundle-create.golden.json new file mode 100644 index 000000000..af43b1ef8 --- /dev/null +++ b/src/handlers/eval/ab-test/config-bundle/__fixtures__/run-bundle-create.golden.json @@ -0,0 +1,6 @@ +{ + "bundleArn": "arn:aws:bedrock-agentcore:us-west-2:685197708687:configuration-bundle/agentcore_cli_abtest_run_bundle-QTjmqQ9zT6", + "bundleId": "agentcore_cli_abtest_run_bundle-QTjmqQ9zT6", + "versionId": "e3c144d3-22ce-4a82-9413-cc9be04eb8a5", + "createdAt": "2026-08-27T21:02:58.851Z" +} \ No newline at end of file diff --git a/src/handlers/eval/ab-test/config-bundle/__fixtures__/run-bundle-update.golden.json b/src/handlers/eval/ab-test/config-bundle/__fixtures__/run-bundle-update.golden.json new file mode 100644 index 000000000..f60044fae --- /dev/null +++ b/src/handlers/eval/ab-test/config-bundle/__fixtures__/run-bundle-update.golden.json @@ -0,0 +1,6 @@ +{ + "bundleArn": "arn:aws:bedrock-agentcore:us-west-2:685197708687:configuration-bundle/agentcore_cli_abtest_run_bundle-QTjmqQ9zT6", + "bundleId": "agentcore_cli_abtest_run_bundle-QTjmqQ9zT6", + "versionId": "d9379d05-8c1c-4764-afcb-8b5f852d6830", + "updatedAt": "2026-08-27T21:03:02.235Z" +} \ No newline at end of file diff --git a/src/handlers/eval/ab-test/config-bundle/__fixtures__/run-online-eval.golden.json b/src/handlers/eval/ab-test/config-bundle/__fixtures__/run-online-eval.golden.json new file mode 100644 index 000000000..654a825b1 --- /dev/null +++ b/src/handlers/eval/ab-test/config-bundle/__fixtures__/run-online-eval.golden.json @@ -0,0 +1,12 @@ +{ + "onlineEvaluationConfigArn": "arn:aws:bedrock-agentcore:us-west-2:685197708687:online-evaluation-config/agentcore_cli_abtest_run_eval-i7s3ryDRdt", + "onlineEvaluationConfigId": "agentcore_cli_abtest_run_eval-i7s3ryDRdt", + "createdAt": "2026-08-27T21:03:03.249Z", + "status": "CREATING", + "executionStatus": "DISABLED", + "outputConfig": { + "cloudWatchConfig": { + "logGroupName": "/aws/bedrock-agentcore/evaluations/results/agentcore_cli_abtest_run_eval-i7s3ryDRdt" + } + } +} \ No newline at end of file diff --git a/src/handlers/eval/ab-test/config-bundle/__fixtures__/run.golden.json b/src/handlers/eval/ab-test/config-bundle/__fixtures__/run.golden.json new file mode 100644 index 000000000..66b3dfd1e --- /dev/null +++ b/src/handlers/eval/ab-test/config-bundle/__fixtures__/run.golden.json @@ -0,0 +1,8 @@ +{ + "abTestId": "agentcore_cli_abtest_run-8e10bf2f27", + "abTestArn": "arn:aws:bedrock-agentcore:us-west-2:685197708687:ab-test/agentcore_cli_abtest_run-8e10bf2f27", + "status": "CREATING", + "executionStatus": "NOT_STARTED", + "createdAt": "2026-08-27T21:03:19.535Z", + "name": "agentcore_cli_abtest_run" +} \ No newline at end of file diff --git a/src/handlers/eval/ab-test/config-bundle/config-bundle-run.fixture.test.tsx b/src/handlers/eval/ab-test/config-bundle/config-bundle-run.fixture.test.tsx new file mode 100644 index 000000000..3b54ca580 --- /dev/null +++ b/src/handlers/eval/ab-test/config-bundle/config-bundle-run.fixture.test.tsx @@ -0,0 +1,198 @@ +import { afterAll, describe, expect, test } from "bun:test"; +import { + DeleteConfigurationBundleCommand, + GetConfigurationBundleCommand, + DeleteOnlineEvaluationConfigCommand, +} from "@aws-sdk/client-bedrock-agentcore-control"; +import { DeleteABTestCommand } from "@aws-sdk/client-bedrock-agentcore"; +import { join } from "node:path"; +import { CoreClient } from "../../../../core"; +import { + createSilentLogger, + fixtureFactories, + isRecording, + matchGolden, + TestGlobalConfigAccessor, + testIO, +} from "../../../../testing"; +import { createControlClient, createDataClient, createIamClient } from "../../../../core/factories"; +import { abTestExecutionRoleName, deleteAbTestRole } from "../../../../core/abTestExecutionRole"; +import { createRootHandler } from "../../../index"; + +const REGION = "us-west-2"; +const FIXTURES = join(import.meta.dir, "__fixtures__"); + +const RUNTIME_ARN = + "arn:aws:bedrock-agentcore:us-west-2:685197708687:runtime/asdf_MyAgent-3s5axvBC6Q"; +const AGENT_ID = "asdf_MyAgent-3s5axvBC6Q"; +const EVALUATOR_ID = "Builtin.Helpfulness"; +const GATEWAY_ID = "agentcore-cli-gateway-read-fixture-a-l6opkbe2kd"; +const BUNDLE_NAME = "agentcore_cli_abtest_run_bundle"; +const ONLINE_EVAL_NAME = "agentcore_cli_abtest_run_eval"; + +const COMPONENTS_V1 = { + [RUNTIME_ARN]: { configuration: { system_prompt: "A/B run fixture v1." } }, +}; +const COMPONENTS_V2 = { + [RUNTIME_ARN]: { configuration: { system_prompt: "A/B run fixture v2." } }, +}; + +function createFixtureCore(): CoreClient { + const { createControlClient, createDataClient, createIamClient, createLogsClient } = + fixtureFactories(FIXTURES); + return new CoreClient({ + createControlClient, + createDataClient, + createIamClient, + createLogsClient, + logger: createSilentLogger(), + }); +} + +async function run(args: string[]): Promise { + const io = testIO(); + const root = createRootHandler(createFixtureCore(), { + io: io.io, + logger: createSilentLogger(), + globalConfigAccessor: new TestGlobalConfigAccessor(), + }); + await root.route(["node", "agentcore", ...args, "--region", REGION]); + return io.stdout(); +} + +async function settle(): Promise { + if (isRecording()) await new Promise((resolve) => setTimeout(resolve, 3000)); +} + +const created: { + bundleId?: string; + v1?: string; + v2?: string; + onlineEvalId?: string; + abTestId?: string; +} = {}; + +afterAll(async () => { + if (!isRecording()) return; + const control = createControlClient({ region: REGION }); + const data = createDataClient({ region: REGION }); + if (created.abTestId) { + try { + await data.send(new DeleteABTestCommand({ abTestId: created.abTestId })); + } catch (error) { + console.error("cleanup ab-test:", error); + } + try { + await deleteAbTestRole( + createIamClient({ region: REGION }), + abTestExecutionRoleName("agentcore_cli_abtest_run"), + ); + } catch (error) { + console.error("cleanup ab-test role:", error); + } + } + if (created.onlineEvalId) { + try { + await control.send( + new DeleteOnlineEvaluationConfigCommand({ onlineEvaluationConfigId: created.onlineEvalId }), + ); + } catch (error) { + console.error("cleanup online-eval:", error); + } + } + if (created.bundleId) { + try { + await control.send( + new GetConfigurationBundleCommand({ bundleId: created.bundleId, branchName: "mainline" }), + ); + await control.send(new DeleteConfigurationBundleCommand({ bundleId: created.bundleId })); + } catch (error) { + if ((error as Error).name !== "ResourceNotFoundException") { + console.error("cleanup bundle:", error); + } + } + } +}); + +describe("eval ab-test config-bundle run (fixture-backed)", () => { + test("provisions a bundle with two versions", async () => { + const v1 = await run([ + "eval", + "config-bundle", + "create", + "--name", + BUNDLE_NAME, + "--components", + JSON.stringify(COMPONENTS_V1), + ]); + matchGolden(FIXTURES, "run-bundle-create.golden.json", v1); + const first = JSON.parse(v1); + created.bundleId = first.bundleId; + created.v1 = first.versionId; + + await settle(); + + const v2 = await run([ + "eval", + "config-bundle", + "update", + "--id", + created.bundleId!, + "--components", + JSON.stringify(COMPONENTS_V2), + "--commit-message", + "A/B run fixture v2", + ]); + matchGolden(FIXTURES, "run-bundle-update.golden.json", v2); + created.v2 = JSON.parse(v2).versionId; + expect(created.v2).not.toBe(created.v1); + }, 180_000); + + test("provisions a paused online evaluation config", async () => { + const out = await run([ + "eval", + "online-eval", + "create", + "--name", + ONLINE_EVAL_NAME, + "--agent", + AGENT_ID, + "--evaluator", + EVALUATOR_ID, + "--sampling-rate", + "100", + "--enable-on-create", + "false", + ]); + matchGolden(FIXTURES, "run-online-eval.golden.json", out); + created.onlineEvalId = JSON.parse(out).onlineEvaluationConfigId; + }, 180_000); + + test("runs a paused config-bundle A/B test", async () => { + const out = await run([ + "eval", + "ab-test", + "config-bundle", + "run", + "--name", + "agentcore_cli_abtest_run", + "--gateway", + GATEWAY_ID, + "--control", + JSON.stringify({ "config-bundle": created.bundleId, "bundle-version": created.v1 }), + "--treatment", + JSON.stringify({ "config-bundle": created.bundleId, "bundle-version": created.v2 }), + "--online-eval", + created.onlineEvalId!, + "--treatment-weight", + "20", + "--enable-on-create", + "false", + ]); + matchGolden(FIXTURES, "run.golden.json", out); + const abTest = JSON.parse(out); + created.abTestId = abTest.abTestId; + expect(abTest.abTestId).toBeString(); + expect(abTest.executionStatus).toBe("NOT_STARTED"); + }, 180_000); +}); From f09cea01a1df96d5aa9954c55205559ac09c8623 Mon Sep 17 00:00:00 2001 From: jariy17 Date: Thu, 27 Aug 2026 21:10:23 +0000 Subject: [PATCH 09/15] test(eval): consolidate ab-test command-flow + unhappy paths into one file Merge ab-test.write.test.tsx + ab-test.create.test.tsx into a single ab-test.test.tsx (mirrors batch-evaluation.test.tsx): hierarchy, get/list/ pause/resume/stop/delete happy paths, and every unhappy path in one place -- missing --id (now covers get, which regressed), Core-error surfacing per op (not-found / invalid-transition / not-stopped), and config-bundle run validation (required flags, malformed + mis-shaped JSON, identical variants, weight bounds). Golden fixture files unchanged. --- .../eval/ab-test/ab-test.create.test.tsx | 138 --------- src/handlers/eval/ab-test/ab-test.test.tsx | 266 ++++++++++++++++++ .../eval/ab-test/ab-test.write.test.tsx | 89 ------ 3 files changed, 266 insertions(+), 227 deletions(-) delete mode 100644 src/handlers/eval/ab-test/ab-test.create.test.tsx create mode 100644 src/handlers/eval/ab-test/ab-test.test.tsx delete mode 100644 src/handlers/eval/ab-test/ab-test.write.test.tsx diff --git a/src/handlers/eval/ab-test/ab-test.create.test.tsx b/src/handlers/eval/ab-test/ab-test.create.test.tsx deleted file mode 100644 index ca0d4f519..000000000 --- a/src/handlers/eval/ab-test/ab-test.create.test.tsx +++ /dev/null @@ -1,138 +0,0 @@ -import { test, expect, describe } from "bun:test"; -import type { CreateABTestResponse } from "@aws-sdk/client-bedrock-agentcore"; -import { createRootHandler } from "../../index"; -import { createSilentLogger, TestCoreClient, testIO } from "../../../testing"; -import { TestGlobalConfigAccessor } from "../../../testing/"; - -const OK: CreateABTestResponse = { - abTestId: "orders-v2-abc123", - abTestArn: "arn:aws:bedrock-agentcore:us-west-2:123456789012:ab-test/orders-v2-abc123", - name: "orders-v2", - status: "CREATING", - executionStatus: "NOT_STARTED", - createdAt: new Date("2026-08-26T10:00:00.000Z"), -} satisfies CreateABTestResponse; - -async function run(args: string[], configure?: (core: TestCoreClient) => void) { - const core = new TestCoreClient(); - core.eval.setAbTestCreateResponse(OK); - configure?.(core); - const io = testIO(); - const root = createRootHandler(core, { - io: io.io, - logger: createSilentLogger(), - globalConfigAccessor: new TestGlobalConfigAccessor(), - }); - await root.route(["node", "agentcore", ...args, "--region", "us-west-2"]); - return { core, stdout: io.stdout() }; -} - -const BASE = [ - "eval", - "ab-test", - "config-bundle", - "run", - "--name", - "orders-v2", - "--gateway", - "orders-gateway-abc123", - "--control", - '{"config-bundle":"orders-prompt-abc","bundle-version":"1111"}', - "--treatment", - '{"config-bundle":"orders-prompt-abc","bundle-version":"2222"}', - "--online-eval", - "online-eval-abc123", - "--json", -]; - -describe("eval ab-test config-bundle run", () => { - test("registers under ab-test → config-bundle", () => { - const io = testIO(); - const root = createRootHandler(new TestCoreClient(), { - io: io.io, - logger: createSilentLogger(), - globalConfigAccessor: new TestGlobalConfigAccessor(), - }); - const abTest = root - .children() - .find((c) => c.name() === "eval") - ?.children() - .find((c) => c.name() === "ab-test"); - expect(abTest?.children().map((c) => c.name())).toContain("config-bundle"); - const cb = abTest?.children().find((c) => c.name() === "config-bundle"); - expect(cb?.children().map((c) => c.name())).toEqual(["run"]); - }); - - test("passes --gateway-filter through as a GatewayFilter", async () => { - const { core } = await run([ - ...BASE, - "--gateway-filter", - '{"targetPaths":["/orders/checkout"]}', - ]); - const call = core.eval.calls.find((c) => c.method === "createConfigBundleABTest"); - expect(call).toBeDefined(); - expect((call!.args[0] as { gatewayFilter?: unknown }).gatewayFilter).toEqual({ - targetPaths: ["/orders/checkout"], - }); - }); - - test("rejects equal control/treatment bundle-versions", async () => { - await expect( - run([ - "eval", - "ab-test", - "config-bundle", - "run", - "--name", - "x", - "--gateway", - "g", - "--control", - '{"config-bundle":"b","bundle-version":"same"}', - "--treatment", - '{"config-bundle":"b","bundle-version":"same"}', - "--online-eval", - "o", - "--json", - ]), - ).rejects.toThrow(/must reference a different/); - }); - - test("rejects --treatment-weight outside 1-99", async () => { - await expect(run([...BASE, "--treatment-weight", "0"])).rejects.toThrow(/1 and 99/); - await expect(run([...BASE, "--treatment-weight", "100"])).rejects.toThrow(/1 and 99/); - }); - - test.each(["name", "gateway", "control", "treatment", "online-eval"] as const)( - "requires --%s", - async (missing) => { - const args = BASE.filter((_, i, arr) => { - const prev = arr[i - 1]; - return prev !== `--${missing}` && arr[i] !== `--${missing}`; - }); - await expect(run(args)).rejects.toThrow(new RegExp(`--${missing}`)); - }, - ); - - test("rejects a malformed control JSON shape", async () => { - await expect( - run([ - "eval", - "ab-test", - "config-bundle", - "run", - "--name", - "x", - "--gateway", - "g", - "--control", - '{"wrong":"shape"}', - "--treatment", - '{"config-bundle":"b","bundle-version":"2"}', - "--online-eval", - "o", - "--json", - ]), - ).rejects.toThrow(/--control must be/); - }); -}); diff --git a/src/handlers/eval/ab-test/ab-test.test.tsx b/src/handlers/eval/ab-test/ab-test.test.tsx new file mode 100644 index 000000000..3f75f8647 --- /dev/null +++ b/src/handlers/eval/ab-test/ab-test.test.tsx @@ -0,0 +1,266 @@ +import { test, expect, describe } from "bun:test"; +import type { GetABTestResponse, ListABTestsResponse } from "@aws-sdk/client-bedrock-agentcore"; +import { createRootHandler } from "../../index"; +import { createSilentLogger, TestCoreClient, testIO } from "../../../testing"; +import { TestGlobalConfigAccessor } from "../../../testing/"; + +async function run(args: string[], configure?: (core: TestCoreClient) => void) { + const core = new TestCoreClient(); + configure?.(core); + const io = testIO(); + const root = createRootHandler(core, { + io: io.io, + logger: createSilentLogger(), + globalConfigAccessor: new TestGlobalConfigAccessor(), + }); + await root.route(["node", "agentcore", ...args, "--region", "us-west-2"]); + return { core, stdout: io.stdout() }; +} + +const ARN = "arn:aws:bedrock-agentcore:us-west-2:123456789012:ab-test/ab-test-1"; + +const GET_RESPONSE = { + abTestId: "ab-test-1", + abTestArn: ARN, + name: "orders-v2", + status: "ACTIVE", + executionStatus: "RUNNING", + gatewayArn: "arn:aws:bedrock-agentcore:us-west-2:123456789012:gateway/orders", + variants: [], + evaluationConfig: { + onlineEvaluationConfigArn: + "arn:aws:bedrock-agentcore:us-west-2:123456789012:online-evaluation-config/x", + }, + createdAt: new Date("2026-07-19T01:02:03.000Z"), + updatedAt: new Date("2026-07-20T12:34:56.000Z"), +} satisfies GetABTestResponse; + +const LIST_RESPONSE = { + abTests: [{ abTestId: "ab-test-1", status: "ACTIVE" }], + nextToken: "next", +} as ListABTestsResponse; + +const RUN_BASE = [ + "eval", + "ab-test", + "config-bundle", + "run", + "--name", + "orders-v2", + "--gateway", + "orders-gateway-abc123", + "--control", + '{"config-bundle":"orders-prompt-abc","bundle-version":"1111"}', + "--treatment", + '{"config-bundle":"orders-prompt-abc","bundle-version":"2222"}', + "--online-eval", + "online-eval-abc123", + "--json", +]; + +describe("eval ab-test command hierarchy", () => { + test("registers get, list, pause, resume, stop, delete, config-bundle", () => { + const io = testIO(); + const root = createRootHandler(new TestCoreClient(), { + io: io.io, + logger: createSilentLogger(), + globalConfigAccessor: new TestGlobalConfigAccessor(), + }); + const group = root + .children() + .find((c) => c.name() === "eval") + ?.children() + .find((c) => c.name() === "ab-test"); + expect(group?.children().map((c) => c.name())).toEqual([ + "get", + "list", + "pause", + "resume", + "stop", + "delete", + "config-bundle", + ]); + const cb = group?.children().find((c) => c.name() === "config-bundle"); + expect(cb?.children().map((c) => c.name())).toEqual(["run"]); + }); +}); + +describe("eval ab-test get", () => { + test("returns the test by id", async () => { + const { core, stdout } = await run( + ["eval", "ab-test", "get", "--id", "ab-test-1", "--json"], + (c) => c.eval.setAbTestGetResponse(GET_RESPONSE), + ); + expect(JSON.parse(stdout).abTestId).toBe("ab-test-1"); + expect(core.eval.calls).toEqual([ + { method: "getABTest", args: ["ab-test-1", { region: "us-west-2" }] }, + ]); + }); + + test("requires --id", async () => { + await expect(run(["eval", "ab-test", "get", "--json"])).rejects.toThrow(/--id/); + }); + + test("surfaces a Core error", async () => { + await expect( + run(["eval", "ab-test", "get", "--id", "missing", "--json"], (c) => + c.eval.setError(new Error("ResourceNotFound")), + ), + ).rejects.toThrow(/ResourceNotFound/); + }); +}); + +describe("eval ab-test list", () => { + test("passes pagination through", async () => { + const { core, stdout } = await run( + ["eval", "ab-test", "list", "--max-results", "10", "--json"], + (c) => c.eval.setAbTestListResponse(LIST_RESPONSE), + ); + expect(JSON.parse(stdout).nextToken).toBe("next"); + expect(core.eval.calls[0]?.args).toEqual([undefined, 10, { region: "us-west-2" }]); + }); + + test("surfaces a Core error", async () => { + await expect( + run(["eval", "ab-test", "list", "--json"], (c) => c.eval.setError(new Error("boom"))), + ).rejects.toThrow(/boom/); + }); +}); + +describe("eval ab-test transitions", () => { + test.each([ + ["pause", "PAUSED"], + ["resume", "RUNNING"], + ["stop", "STOPPED"], + ] as const)("%s sets executionStatus %s via Core", async (command, status) => { + const { core } = await run(["eval", "ab-test", command, "--id", "ab-test-1", "--json"], (c) => + c.eval.setAbTestUpdateResponse({ + abTestId: "ab-test-1", + abTestArn: ARN, + status: "ACTIVE", + executionStatus: status, + updatedAt: new Date("2026-07-20T12:34:56.000Z"), + }), + ); + expect(core.eval.calls).toEqual([ + { method: "setABTestExecutionStatus", args: ["ab-test-1", status, { region: "us-west-2" }] }, + ]); + }); + + test.each(["pause", "resume", "stop"] as const)("%s requires --id", async (command) => { + await expect(run(["eval", "ab-test", command, "--json"])).rejects.toThrow(/--id/); + }); + + test.each(["pause", "resume", "stop"] as const)("%s surfaces a Core error", async (command) => { + await expect( + run(["eval", "ab-test", command, "--id", "ab-test-1", "--json"], (c) => + c.eval.setError(new Error("invalid transition")), + ), + ).rejects.toThrow(/invalid transition/); + }); +}); + +describe("eval ab-test delete", () => { + test("deletes by id via Core", async () => { + const { core, stdout } = await run( + ["eval", "ab-test", "delete", "--id", "ab-test-1", "--json"], + (c) => + c.eval.setAbTestDeleteResponse({ + abTestId: "ab-test-1", + abTestArn: ARN, + status: "DELETING", + }), + ); + expect(JSON.parse(stdout).abTestId).toBe("ab-test-1"); + expect(core.eval.calls).toEqual([ + { method: "deleteABTest", args: ["ab-test-1", { region: "us-west-2" }] }, + ]); + }); + + test("requires --id", async () => { + await expect(run(["eval", "ab-test", "delete", "--json"])).rejects.toThrow(/--id/); + }); + + test("surfaces a Core error (e.g. not stopped)", async () => { + await expect( + run(["eval", "ab-test", "delete", "--id", "ab-test-1", "--json"], (c) => + c.eval.setError(new Error("must be stopped")), + ), + ).rejects.toThrow(/must be stopped/); + }); +}); + +describe("eval ab-test config-bundle run validation", () => { + test.each(["name", "gateway", "control", "treatment", "online-eval"] as const)( + "requires --%s", + async (missing) => { + const args = RUN_BASE.filter( + (a, i) => a !== `--${missing}` && RUN_BASE[i - 1] !== `--${missing}`, + ); + await expect(run(args)).rejects.toThrow(new RegExp(`--${missing}`)); + }, + ); + + test("rejects malformed --control JSON", async () => { + const args = RUN_BASE.map((a) => + a === '{"config-bundle":"orders-prompt-abc","bundle-version":"1111"}' ? "notjson" : a, + ); + await expect(run(args)).rejects.toThrow(/Invalid JSON/); + }); + + test("rejects a mis-shaped --control object", async () => { + const args = RUN_BASE.map((a) => + a === '{"config-bundle":"orders-prompt-abc","bundle-version":"1111"}' + ? '{"wrong":"shape"}' + : a, + ); + await expect(run(args)).rejects.toThrow(/--control must be/); + }); + + test("rejects identical control/treatment", async () => { + const same = '{"config-bundle":"b","bundle-version":"same"}'; + await expect( + run([ + "eval", + "ab-test", + "config-bundle", + "run", + "--name", + "x", + "--gateway", + "g", + "--control", + same, + "--treatment", + same, + "--online-eval", + "o", + "--json", + ]), + ).rejects.toThrow(/must reference a different/); + }); + + test.each(["0", "100"])("rejects --treatment-weight %s", async (w) => { + await expect(run([...RUN_BASE, "--treatment-weight", w])).rejects.toThrow(/1 and 99/); + }); + + test("passes --gateway-filter through as a GatewayFilter", async () => { + const { core } = await run( + [...RUN_BASE, "--gateway-filter", '{"targetPaths":["/orders/checkout"]}'], + (c) => + c.eval.setAbTestCreateResponse({ + abTestId: "x", + abTestArn: ARN, + name: "x", + status: "CREATING", + executionStatus: "NOT_STARTED", + createdAt: new Date("2026-08-26T10:00:00.000Z"), + }), + ); + const call = core.eval.calls.find((c) => c.method === "createConfigBundleABTest"); + expect(call).toBeDefined(); + expect((call!.args[0] as { gatewayFilter?: unknown }).gatewayFilter).toEqual({ + targetPaths: ["/orders/checkout"], + }); + }); +}); diff --git a/src/handlers/eval/ab-test/ab-test.write.test.tsx b/src/handlers/eval/ab-test/ab-test.write.test.tsx deleted file mode 100644 index 3a7bd3d9d..000000000 --- a/src/handlers/eval/ab-test/ab-test.write.test.tsx +++ /dev/null @@ -1,89 +0,0 @@ -import { test, expect, describe } from "bun:test"; -import { createRootHandler } from "../../index"; -import { createSilentLogger, TestCoreClient, testIO } from "../../../testing"; -import { TestGlobalConfigAccessor } from "../../../testing/"; - -async function run(args: string[], configure?: (core: TestCoreClient) => void) { - const core = new TestCoreClient(); - configure?.(core); - const io = testIO(); - const root = createRootHandler(core, { - io: io.io, - logger: createSilentLogger(), - globalConfigAccessor: new TestGlobalConfigAccessor(), - }); - await root.route(["node", "agentcore", ...args, "--region", "us-west-2"]); - return { core, stdout: io.stdout() }; -} - -describe("eval ab-test command hierarchy", () => { - test("registers get, list, pause, resume, stop, delete", () => { - const io = testIO(); - const root = createRootHandler(new TestCoreClient(), { - io: io.io, - logger: createSilentLogger(), - globalConfigAccessor: new TestGlobalConfigAccessor(), - }); - const group = root - .children() - .find((c) => c.name() === "eval") - ?.children() - .find((c) => c.name() === "ab-test"); - expect(group?.children().map((c) => c.name())).toEqual([ - "get", - "list", - "pause", - "resume", - "stop", - "delete", - "config-bundle", - ]); - }); -}); - -describe("eval ab-test transitions", () => { - test.each([ - ["pause", "PAUSED"], - ["resume", "RUNNING"], - ["stop", "STOPPED"], - ] as const)("%s sets executionStatus %s via Core", async (command, status) => { - const { core } = await run(["eval", "ab-test", command, "--id", "ab-test-1", "--json"], (c) => - c.eval.setAbTestUpdateResponse({ - abTestId: "ab-test-1", - abTestArn: "arn:aws:bedrock-agentcore:us-west-2:123456789012:ab-test/ab-test-1", - status: "ACTIVE", - executionStatus: status, - updatedAt: new Date("2026-07-20T12:34:56.000Z"), - }), - ); - expect(core.eval.calls).toEqual([ - { method: "setABTestExecutionStatus", args: ["ab-test-1", status, { region: "us-west-2" }] }, - ]); - }); - - test.each(["pause", "resume", "stop"] as const)("%s requires --id", async (command) => { - await expect(run(["eval", "ab-test", command, "--json"])).rejects.toThrow(/--id/); - }); -}); - -describe("eval ab-test delete", () => { - test("deletes by id via Core", async () => { - const { core, stdout } = await run( - ["eval", "ab-test", "delete", "--id", "ab-test-1", "--json"], - (c) => - c.eval.setAbTestDeleteResponse({ - abTestId: "ab-test-1", - abTestArn: "arn:aws:bedrock-agentcore:us-west-2:123456789012:ab-test/ab-test-1", - status: "DELETING", - }), - ); - expect(JSON.parse(stdout).abTestId).toBe("ab-test-1"); - expect(core.eval.calls).toEqual([ - { method: "deleteABTest", args: ["ab-test-1", { region: "us-west-2" }] }, - ]); - }); - - test("requires --id", async () => { - await expect(run(["eval", "ab-test", "delete", "--json"])).rejects.toThrow(/--id/); - }); -}); From 0f31b84f9a48544af20de4f7fb86864818cbb655 Mon Sep 17 00:00:00 2001 From: jariy17 Date: Thu, 27 Aug 2026 21:21:29 +0000 Subject: [PATCH 10/15] test(eval): fold config-bundle run golden into ab-test.fixture.test.tsx --- .../CreateABTestCommand.506cd57a7653b22c.json | 0 .../CreateABTestCommand.a4666d7f80bc7cb0.json | 0 ...urationBundleCommand.e8ee73bc166ad5e4.json | 0 ...luationConfigCommand.ae1ee3532d19f571.json | 0 .../CreateRoleCommand.1fa2ac2a7f0f7fc0.json | 0 .../CreateRoleCommand.7b030b47662eee32.json | 0 .../DeleteRoleCommand.c6a8dc12fb95054d.json | 0 ...eteRolePolicyCommand.3826bd85235b40f0.json | 0 ...tAgentRuntimeCommand.9f77333d1b9dcf5d.json | 0 ...urationBundleCommand.cf3c23a25f6bf298.json | 0 ...urationBundleCommand.ed835ef9d614b3e6.json | 0 .../GetEvaluatorCommand.716589b0884f35c0.json | 0 .../GetGatewayCommand.4216a59651bb046a.json | 0 .../GetRoleCommand.2ca20231e1472584.json | 0 .../GetRoleCommand.c6a8dc12fb95054d.json | 0 ...PutRolePolicyCommand.8bf1be5c5e52a2ec.json | 0 ...PutRolePolicyCommand.bc0b72d6bad3afad.json | 0 ...urationBundleCommand.aca9ba06670aa395.json | 0 ...urationBundleCommand.de17ce40709b64eb.json | 0 .../run-bundle-create.golden.json | 0 .../run-bundle-update.golden.json | 0 .../__fixtures__/run-online-eval.golden.json | 0 .../__fixtures__/run.golden.json | 0 .../eval/ab-test/ab-test.fixture.test.tsx | 185 ++++++++++++++-- .../config-bundle-run.fixture.test.tsx | 198 ------------------ 25 files changed, 170 insertions(+), 213 deletions(-) rename src/handlers/eval/ab-test/{config-bundle => }/__fixtures__/CreateABTestCommand.506cd57a7653b22c.json (100%) rename src/handlers/eval/ab-test/{config-bundle => }/__fixtures__/CreateABTestCommand.a4666d7f80bc7cb0.json (100%) rename src/handlers/eval/ab-test/{config-bundle => }/__fixtures__/CreateConfigurationBundleCommand.e8ee73bc166ad5e4.json (100%) rename src/handlers/eval/ab-test/{config-bundle => }/__fixtures__/CreateOnlineEvaluationConfigCommand.ae1ee3532d19f571.json (100%) rename src/handlers/eval/ab-test/{config-bundle => }/__fixtures__/CreateRoleCommand.1fa2ac2a7f0f7fc0.json (100%) rename src/handlers/eval/ab-test/{config-bundle => }/__fixtures__/CreateRoleCommand.7b030b47662eee32.json (100%) rename src/handlers/eval/ab-test/{config-bundle => }/__fixtures__/DeleteRoleCommand.c6a8dc12fb95054d.json (100%) rename src/handlers/eval/ab-test/{config-bundle => }/__fixtures__/DeleteRolePolicyCommand.3826bd85235b40f0.json (100%) rename src/handlers/eval/ab-test/{config-bundle => }/__fixtures__/GetAgentRuntimeCommand.9f77333d1b9dcf5d.json (100%) rename src/handlers/eval/ab-test/{config-bundle => }/__fixtures__/GetConfigurationBundleCommand.cf3c23a25f6bf298.json (100%) rename src/handlers/eval/ab-test/{config-bundle => }/__fixtures__/GetConfigurationBundleCommand.ed835ef9d614b3e6.json (100%) rename src/handlers/eval/ab-test/{config-bundle => }/__fixtures__/GetEvaluatorCommand.716589b0884f35c0.json (100%) rename src/handlers/eval/ab-test/{config-bundle => }/__fixtures__/GetGatewayCommand.4216a59651bb046a.json (100%) rename src/handlers/eval/ab-test/{config-bundle => }/__fixtures__/GetRoleCommand.2ca20231e1472584.json (100%) rename src/handlers/eval/ab-test/{config-bundle => }/__fixtures__/GetRoleCommand.c6a8dc12fb95054d.json (100%) rename src/handlers/eval/ab-test/{config-bundle => }/__fixtures__/PutRolePolicyCommand.8bf1be5c5e52a2ec.json (100%) rename src/handlers/eval/ab-test/{config-bundle => }/__fixtures__/PutRolePolicyCommand.bc0b72d6bad3afad.json (100%) rename src/handlers/eval/ab-test/{config-bundle => }/__fixtures__/UpdateConfigurationBundleCommand.aca9ba06670aa395.json (100%) rename src/handlers/eval/ab-test/{config-bundle => }/__fixtures__/UpdateConfigurationBundleCommand.de17ce40709b64eb.json (100%) rename src/handlers/eval/ab-test/{config-bundle => }/__fixtures__/run-bundle-create.golden.json (100%) rename src/handlers/eval/ab-test/{config-bundle => }/__fixtures__/run-bundle-update.golden.json (100%) rename src/handlers/eval/ab-test/{config-bundle => }/__fixtures__/run-online-eval.golden.json (100%) rename src/handlers/eval/ab-test/{config-bundle => }/__fixtures__/run.golden.json (100%) delete mode 100644 src/handlers/eval/ab-test/config-bundle/config-bundle-run.fixture.test.tsx diff --git a/src/handlers/eval/ab-test/config-bundle/__fixtures__/CreateABTestCommand.506cd57a7653b22c.json b/src/handlers/eval/ab-test/__fixtures__/CreateABTestCommand.506cd57a7653b22c.json similarity index 100% rename from src/handlers/eval/ab-test/config-bundle/__fixtures__/CreateABTestCommand.506cd57a7653b22c.json rename to src/handlers/eval/ab-test/__fixtures__/CreateABTestCommand.506cd57a7653b22c.json diff --git a/src/handlers/eval/ab-test/config-bundle/__fixtures__/CreateABTestCommand.a4666d7f80bc7cb0.json b/src/handlers/eval/ab-test/__fixtures__/CreateABTestCommand.a4666d7f80bc7cb0.json similarity index 100% rename from src/handlers/eval/ab-test/config-bundle/__fixtures__/CreateABTestCommand.a4666d7f80bc7cb0.json rename to src/handlers/eval/ab-test/__fixtures__/CreateABTestCommand.a4666d7f80bc7cb0.json diff --git a/src/handlers/eval/ab-test/config-bundle/__fixtures__/CreateConfigurationBundleCommand.e8ee73bc166ad5e4.json b/src/handlers/eval/ab-test/__fixtures__/CreateConfigurationBundleCommand.e8ee73bc166ad5e4.json similarity index 100% rename from src/handlers/eval/ab-test/config-bundle/__fixtures__/CreateConfigurationBundleCommand.e8ee73bc166ad5e4.json rename to src/handlers/eval/ab-test/__fixtures__/CreateConfigurationBundleCommand.e8ee73bc166ad5e4.json diff --git a/src/handlers/eval/ab-test/config-bundle/__fixtures__/CreateOnlineEvaluationConfigCommand.ae1ee3532d19f571.json b/src/handlers/eval/ab-test/__fixtures__/CreateOnlineEvaluationConfigCommand.ae1ee3532d19f571.json similarity index 100% rename from src/handlers/eval/ab-test/config-bundle/__fixtures__/CreateOnlineEvaluationConfigCommand.ae1ee3532d19f571.json rename to src/handlers/eval/ab-test/__fixtures__/CreateOnlineEvaluationConfigCommand.ae1ee3532d19f571.json diff --git a/src/handlers/eval/ab-test/config-bundle/__fixtures__/CreateRoleCommand.1fa2ac2a7f0f7fc0.json b/src/handlers/eval/ab-test/__fixtures__/CreateRoleCommand.1fa2ac2a7f0f7fc0.json similarity index 100% rename from src/handlers/eval/ab-test/config-bundle/__fixtures__/CreateRoleCommand.1fa2ac2a7f0f7fc0.json rename to src/handlers/eval/ab-test/__fixtures__/CreateRoleCommand.1fa2ac2a7f0f7fc0.json diff --git a/src/handlers/eval/ab-test/config-bundle/__fixtures__/CreateRoleCommand.7b030b47662eee32.json b/src/handlers/eval/ab-test/__fixtures__/CreateRoleCommand.7b030b47662eee32.json similarity index 100% rename from src/handlers/eval/ab-test/config-bundle/__fixtures__/CreateRoleCommand.7b030b47662eee32.json rename to src/handlers/eval/ab-test/__fixtures__/CreateRoleCommand.7b030b47662eee32.json diff --git a/src/handlers/eval/ab-test/config-bundle/__fixtures__/DeleteRoleCommand.c6a8dc12fb95054d.json b/src/handlers/eval/ab-test/__fixtures__/DeleteRoleCommand.c6a8dc12fb95054d.json similarity index 100% rename from src/handlers/eval/ab-test/config-bundle/__fixtures__/DeleteRoleCommand.c6a8dc12fb95054d.json rename to src/handlers/eval/ab-test/__fixtures__/DeleteRoleCommand.c6a8dc12fb95054d.json diff --git a/src/handlers/eval/ab-test/config-bundle/__fixtures__/DeleteRolePolicyCommand.3826bd85235b40f0.json b/src/handlers/eval/ab-test/__fixtures__/DeleteRolePolicyCommand.3826bd85235b40f0.json similarity index 100% rename from src/handlers/eval/ab-test/config-bundle/__fixtures__/DeleteRolePolicyCommand.3826bd85235b40f0.json rename to src/handlers/eval/ab-test/__fixtures__/DeleteRolePolicyCommand.3826bd85235b40f0.json diff --git a/src/handlers/eval/ab-test/config-bundle/__fixtures__/GetAgentRuntimeCommand.9f77333d1b9dcf5d.json b/src/handlers/eval/ab-test/__fixtures__/GetAgentRuntimeCommand.9f77333d1b9dcf5d.json similarity index 100% rename from src/handlers/eval/ab-test/config-bundle/__fixtures__/GetAgentRuntimeCommand.9f77333d1b9dcf5d.json rename to src/handlers/eval/ab-test/__fixtures__/GetAgentRuntimeCommand.9f77333d1b9dcf5d.json diff --git a/src/handlers/eval/ab-test/config-bundle/__fixtures__/GetConfigurationBundleCommand.cf3c23a25f6bf298.json b/src/handlers/eval/ab-test/__fixtures__/GetConfigurationBundleCommand.cf3c23a25f6bf298.json similarity index 100% rename from src/handlers/eval/ab-test/config-bundle/__fixtures__/GetConfigurationBundleCommand.cf3c23a25f6bf298.json rename to src/handlers/eval/ab-test/__fixtures__/GetConfigurationBundleCommand.cf3c23a25f6bf298.json diff --git a/src/handlers/eval/ab-test/config-bundle/__fixtures__/GetConfigurationBundleCommand.ed835ef9d614b3e6.json b/src/handlers/eval/ab-test/__fixtures__/GetConfigurationBundleCommand.ed835ef9d614b3e6.json similarity index 100% rename from src/handlers/eval/ab-test/config-bundle/__fixtures__/GetConfigurationBundleCommand.ed835ef9d614b3e6.json rename to src/handlers/eval/ab-test/__fixtures__/GetConfigurationBundleCommand.ed835ef9d614b3e6.json diff --git a/src/handlers/eval/ab-test/config-bundle/__fixtures__/GetEvaluatorCommand.716589b0884f35c0.json b/src/handlers/eval/ab-test/__fixtures__/GetEvaluatorCommand.716589b0884f35c0.json similarity index 100% rename from src/handlers/eval/ab-test/config-bundle/__fixtures__/GetEvaluatorCommand.716589b0884f35c0.json rename to src/handlers/eval/ab-test/__fixtures__/GetEvaluatorCommand.716589b0884f35c0.json diff --git a/src/handlers/eval/ab-test/config-bundle/__fixtures__/GetGatewayCommand.4216a59651bb046a.json b/src/handlers/eval/ab-test/__fixtures__/GetGatewayCommand.4216a59651bb046a.json similarity index 100% rename from src/handlers/eval/ab-test/config-bundle/__fixtures__/GetGatewayCommand.4216a59651bb046a.json rename to src/handlers/eval/ab-test/__fixtures__/GetGatewayCommand.4216a59651bb046a.json diff --git a/src/handlers/eval/ab-test/config-bundle/__fixtures__/GetRoleCommand.2ca20231e1472584.json b/src/handlers/eval/ab-test/__fixtures__/GetRoleCommand.2ca20231e1472584.json similarity index 100% rename from src/handlers/eval/ab-test/config-bundle/__fixtures__/GetRoleCommand.2ca20231e1472584.json rename to src/handlers/eval/ab-test/__fixtures__/GetRoleCommand.2ca20231e1472584.json diff --git a/src/handlers/eval/ab-test/config-bundle/__fixtures__/GetRoleCommand.c6a8dc12fb95054d.json b/src/handlers/eval/ab-test/__fixtures__/GetRoleCommand.c6a8dc12fb95054d.json similarity index 100% rename from src/handlers/eval/ab-test/config-bundle/__fixtures__/GetRoleCommand.c6a8dc12fb95054d.json rename to src/handlers/eval/ab-test/__fixtures__/GetRoleCommand.c6a8dc12fb95054d.json diff --git a/src/handlers/eval/ab-test/config-bundle/__fixtures__/PutRolePolicyCommand.8bf1be5c5e52a2ec.json b/src/handlers/eval/ab-test/__fixtures__/PutRolePolicyCommand.8bf1be5c5e52a2ec.json similarity index 100% rename from src/handlers/eval/ab-test/config-bundle/__fixtures__/PutRolePolicyCommand.8bf1be5c5e52a2ec.json rename to src/handlers/eval/ab-test/__fixtures__/PutRolePolicyCommand.8bf1be5c5e52a2ec.json diff --git a/src/handlers/eval/ab-test/config-bundle/__fixtures__/PutRolePolicyCommand.bc0b72d6bad3afad.json b/src/handlers/eval/ab-test/__fixtures__/PutRolePolicyCommand.bc0b72d6bad3afad.json similarity index 100% rename from src/handlers/eval/ab-test/config-bundle/__fixtures__/PutRolePolicyCommand.bc0b72d6bad3afad.json rename to src/handlers/eval/ab-test/__fixtures__/PutRolePolicyCommand.bc0b72d6bad3afad.json diff --git a/src/handlers/eval/ab-test/config-bundle/__fixtures__/UpdateConfigurationBundleCommand.aca9ba06670aa395.json b/src/handlers/eval/ab-test/__fixtures__/UpdateConfigurationBundleCommand.aca9ba06670aa395.json similarity index 100% rename from src/handlers/eval/ab-test/config-bundle/__fixtures__/UpdateConfigurationBundleCommand.aca9ba06670aa395.json rename to src/handlers/eval/ab-test/__fixtures__/UpdateConfigurationBundleCommand.aca9ba06670aa395.json diff --git a/src/handlers/eval/ab-test/config-bundle/__fixtures__/UpdateConfigurationBundleCommand.de17ce40709b64eb.json b/src/handlers/eval/ab-test/__fixtures__/UpdateConfigurationBundleCommand.de17ce40709b64eb.json similarity index 100% rename from src/handlers/eval/ab-test/config-bundle/__fixtures__/UpdateConfigurationBundleCommand.de17ce40709b64eb.json rename to src/handlers/eval/ab-test/__fixtures__/UpdateConfigurationBundleCommand.de17ce40709b64eb.json diff --git a/src/handlers/eval/ab-test/config-bundle/__fixtures__/run-bundle-create.golden.json b/src/handlers/eval/ab-test/__fixtures__/run-bundle-create.golden.json similarity index 100% rename from src/handlers/eval/ab-test/config-bundle/__fixtures__/run-bundle-create.golden.json rename to src/handlers/eval/ab-test/__fixtures__/run-bundle-create.golden.json diff --git a/src/handlers/eval/ab-test/config-bundle/__fixtures__/run-bundle-update.golden.json b/src/handlers/eval/ab-test/__fixtures__/run-bundle-update.golden.json similarity index 100% rename from src/handlers/eval/ab-test/config-bundle/__fixtures__/run-bundle-update.golden.json rename to src/handlers/eval/ab-test/__fixtures__/run-bundle-update.golden.json diff --git a/src/handlers/eval/ab-test/config-bundle/__fixtures__/run-online-eval.golden.json b/src/handlers/eval/ab-test/__fixtures__/run-online-eval.golden.json similarity index 100% rename from src/handlers/eval/ab-test/config-bundle/__fixtures__/run-online-eval.golden.json rename to src/handlers/eval/ab-test/__fixtures__/run-online-eval.golden.json diff --git a/src/handlers/eval/ab-test/config-bundle/__fixtures__/run.golden.json b/src/handlers/eval/ab-test/__fixtures__/run.golden.json similarity index 100% rename from src/handlers/eval/ab-test/config-bundle/__fixtures__/run.golden.json rename to src/handlers/eval/ab-test/__fixtures__/run.golden.json diff --git a/src/handlers/eval/ab-test/ab-test.fixture.test.tsx b/src/handlers/eval/ab-test/ab-test.fixture.test.tsx index 9f8bb97fe..f6f28a6c5 100644 --- a/src/handlers/eval/ab-test/ab-test.fixture.test.tsx +++ b/src/handlers/eval/ab-test/ab-test.fixture.test.tsx @@ -1,32 +1,52 @@ -import { describe, expect, test } from "bun:test"; +import { afterAll, describe, expect, test } from "bun:test"; +import { + DeleteConfigurationBundleCommand, + GetConfigurationBundleCommand, + DeleteOnlineEvaluationConfigCommand, +} from "@aws-sdk/client-bedrock-agentcore-control"; +import { DeleteABTestCommand } from "@aws-sdk/client-bedrock-agentcore"; import { join } from "node:path"; import { CoreClient } from "../../../core"; import { createSilentLogger, fixtureFactories, + isRecording, matchGolden, TestGlobalConfigAccessor, testIO, } from "../../../testing"; +import { createControlClient, createDataClient, createIamClient } from "../../../core/factories"; +import { abTestExecutionRoleName, deleteAbTestRole } from "../../../core/abTestExecutionRole"; import { createRootHandler } from "../../index"; const REGION = "us-west-2"; const FIXTURES = join(import.meta.dir, "__fixtures__"); -// Record with: RECORD=1 bun test src/handlers/eval/ab-test/ab-test.fixture.test.tsx -// -// A/B tests are READ-ONLY here, so — like the batch-evaluation fixture suite — -// this pins pre-existing tests in the fixture account rather than creating one. -// Re-recording requires these ids to still exist; repoint them if they age out. -// -// Exercises the real seam end to end: parsing → handler → CoreClient → -// GetABTest / ListABTest (data plane). GetABTest returns the per-evaluator -// statistical results inline, so there is no CloudWatch seam to record. +// The read describe records against account 725476964917 (a pre-existing target +// based test); the create describe records against 685197708687 (self-created, +// matching the config-bundle fixtures). Replay is offline and account-agnostic; +// re-record each describe under its own account: +// RECORD=1 bun test -t "fixture-backed reads" +// RECORD=1 bun test -t "config-bundle run" const FIXTURE_ABTEST_ID = "abvfylatest_abtargettest-a5f5674e07"; - -// A well-formed but absent id, to reach the not-found path. const MISSING_ABTEST_ID = "missing-abtest-0000000000"; +const RUNTIME_ARN = + "arn:aws:bedrock-agentcore:us-west-2:685197708687:runtime/asdf_MyAgent-3s5axvBC6Q"; +const AGENT_ID = "asdf_MyAgent-3s5axvBC6Q"; +const EVALUATOR_ID = "Builtin.Helpfulness"; +const GATEWAY_ID = "agentcore-cli-gateway-read-fixture-a-l6opkbe2kd"; +const BUNDLE_NAME = "agentcore_cli_abtest_run_bundle"; +const ONLINE_EVAL_NAME = "agentcore_cli_abtest_run_eval"; +const AB_TEST_NAME = "agentcore_cli_abtest_run"; + +const COMPONENTS_V1 = { + [RUNTIME_ARN]: { configuration: { system_prompt: "A/B run fixture v1." } }, +}; +const COMPONENTS_V2 = { + [RUNTIME_ARN]: { configuration: { system_prompt: "A/B run fixture v2." } }, +}; + function createFixtureCore(): CoreClient { const { createControlClient, createDataClient, createIamClient, createLogsClient } = fixtureFactories(FIXTURES); @@ -50,10 +70,13 @@ async function run(args: string[]): Promise { return io.stdout(); } -describe("eval ab-test (fixture-backed)", () => { +async function settle(): Promise { + if (isRecording()) await new Promise((resolve) => setTimeout(resolve, 3000)); +} + +describe("eval ab-test fixture-backed reads", () => { test("get returns the test with per-evaluator metrics inline", async () => { const stdout = await run(["eval", "ab-test", "get", "--id", FIXTURE_ABTEST_ID, "--json"]); - matchGolden(FIXTURES, "get.golden.json", stdout); const detail = JSON.parse(stdout); expect(detail.abTestId).toBe(FIXTURE_ABTEST_ID); @@ -64,7 +87,6 @@ describe("eval ab-test (fixture-backed)", () => { test("list returns the service page", async () => { const stdout = await run(["eval", "ab-test", "list", "--max-results", "3", "--json"]); - matchGolden(FIXTURES, "list.golden.json", stdout); expect(Array.isArray(JSON.parse(stdout).abTests)).toBe(true); }); @@ -75,3 +97,136 @@ describe("eval ab-test (fixture-backed)", () => { ).rejects.toThrow(); }); }); + +const created: { + bundleId?: string; + v1?: string; + v2?: string; + onlineEvalId?: string; + abTestId?: string; +} = {}; + +afterAll(async () => { + if (!isRecording()) return; + const control = createControlClient({ region: REGION }); + const data = createDataClient({ region: REGION }); + if (created.abTestId) { + try { + await data.send(new DeleteABTestCommand({ abTestId: created.abTestId })); + } catch (error) { + console.error("cleanup ab-test:", error); + } + try { + await deleteAbTestRole( + createIamClient({ region: REGION }), + abTestExecutionRoleName(AB_TEST_NAME), + ); + } catch (error) { + console.error("cleanup ab-test role:", error); + } + } + if (created.onlineEvalId) { + try { + await control.send( + new DeleteOnlineEvaluationConfigCommand({ onlineEvaluationConfigId: created.onlineEvalId }), + ); + } catch (error) { + console.error("cleanup online-eval:", error); + } + } + if (created.bundleId) { + try { + await control.send( + new GetConfigurationBundleCommand({ bundleId: created.bundleId, branchName: "mainline" }), + ); + await control.send(new DeleteConfigurationBundleCommand({ bundleId: created.bundleId })); + } catch (error) { + if ((error as Error).name !== "ResourceNotFoundException") { + console.error("cleanup bundle:", error); + } + } + } +}); + +describe("eval ab-test config-bundle run", () => { + test("provisions a bundle with two versions", async () => { + const v1 = await run([ + "eval", + "config-bundle", + "create", + "--name", + BUNDLE_NAME, + "--components", + JSON.stringify(COMPONENTS_V1), + ]); + matchGolden(FIXTURES, "run-bundle-create.golden.json", v1); + const first = JSON.parse(v1); + created.bundleId = first.bundleId; + created.v1 = first.versionId; + + await settle(); + + const v2 = await run([ + "eval", + "config-bundle", + "update", + "--id", + created.bundleId!, + "--components", + JSON.stringify(COMPONENTS_V2), + "--commit-message", + "A/B run fixture v2", + ]); + matchGolden(FIXTURES, "run-bundle-update.golden.json", v2); + created.v2 = JSON.parse(v2).versionId; + expect(created.v2).not.toBe(created.v1); + }, 180_000); + + test("provisions a paused online evaluation config", async () => { + const out = await run([ + "eval", + "online-eval", + "create", + "--name", + ONLINE_EVAL_NAME, + "--agent", + AGENT_ID, + "--evaluator", + EVALUATOR_ID, + "--sampling-rate", + "100", + "--enable-on-create", + "false", + ]); + matchGolden(FIXTURES, "run-online-eval.golden.json", out); + created.onlineEvalId = JSON.parse(out).onlineEvaluationConfigId; + }, 180_000); + + test("runs a paused config-bundle A/B test", async () => { + const out = await run([ + "eval", + "ab-test", + "config-bundle", + "run", + "--name", + AB_TEST_NAME, + "--gateway", + GATEWAY_ID, + "--control", + JSON.stringify({ "config-bundle": created.bundleId, "bundle-version": created.v1 }), + "--treatment", + JSON.stringify({ "config-bundle": created.bundleId, "bundle-version": created.v2 }), + "--online-eval", + created.onlineEvalId!, + "--treatment-weight", + "20", + "--enable-on-create", + "false", + ]); + matchGolden(FIXTURES, "run.golden.json", out); + const abTest = JSON.parse(out); + created.abTestId = abTest.abTestId; + expect(abTest.abTestId).toBeString(); + expect(abTest.executionStatus).toBe("NOT_STARTED"); + }, 180_000); +}); diff --git a/src/handlers/eval/ab-test/config-bundle/config-bundle-run.fixture.test.tsx b/src/handlers/eval/ab-test/config-bundle/config-bundle-run.fixture.test.tsx deleted file mode 100644 index 3b54ca580..000000000 --- a/src/handlers/eval/ab-test/config-bundle/config-bundle-run.fixture.test.tsx +++ /dev/null @@ -1,198 +0,0 @@ -import { afterAll, describe, expect, test } from "bun:test"; -import { - DeleteConfigurationBundleCommand, - GetConfigurationBundleCommand, - DeleteOnlineEvaluationConfigCommand, -} from "@aws-sdk/client-bedrock-agentcore-control"; -import { DeleteABTestCommand } from "@aws-sdk/client-bedrock-agentcore"; -import { join } from "node:path"; -import { CoreClient } from "../../../../core"; -import { - createSilentLogger, - fixtureFactories, - isRecording, - matchGolden, - TestGlobalConfigAccessor, - testIO, -} from "../../../../testing"; -import { createControlClient, createDataClient, createIamClient } from "../../../../core/factories"; -import { abTestExecutionRoleName, deleteAbTestRole } from "../../../../core/abTestExecutionRole"; -import { createRootHandler } from "../../../index"; - -const REGION = "us-west-2"; -const FIXTURES = join(import.meta.dir, "__fixtures__"); - -const RUNTIME_ARN = - "arn:aws:bedrock-agentcore:us-west-2:685197708687:runtime/asdf_MyAgent-3s5axvBC6Q"; -const AGENT_ID = "asdf_MyAgent-3s5axvBC6Q"; -const EVALUATOR_ID = "Builtin.Helpfulness"; -const GATEWAY_ID = "agentcore-cli-gateway-read-fixture-a-l6opkbe2kd"; -const BUNDLE_NAME = "agentcore_cli_abtest_run_bundle"; -const ONLINE_EVAL_NAME = "agentcore_cli_abtest_run_eval"; - -const COMPONENTS_V1 = { - [RUNTIME_ARN]: { configuration: { system_prompt: "A/B run fixture v1." } }, -}; -const COMPONENTS_V2 = { - [RUNTIME_ARN]: { configuration: { system_prompt: "A/B run fixture v2." } }, -}; - -function createFixtureCore(): CoreClient { - const { createControlClient, createDataClient, createIamClient, createLogsClient } = - fixtureFactories(FIXTURES); - return new CoreClient({ - createControlClient, - createDataClient, - createIamClient, - createLogsClient, - logger: createSilentLogger(), - }); -} - -async function run(args: string[]): Promise { - const io = testIO(); - const root = createRootHandler(createFixtureCore(), { - io: io.io, - logger: createSilentLogger(), - globalConfigAccessor: new TestGlobalConfigAccessor(), - }); - await root.route(["node", "agentcore", ...args, "--region", REGION]); - return io.stdout(); -} - -async function settle(): Promise { - if (isRecording()) await new Promise((resolve) => setTimeout(resolve, 3000)); -} - -const created: { - bundleId?: string; - v1?: string; - v2?: string; - onlineEvalId?: string; - abTestId?: string; -} = {}; - -afterAll(async () => { - if (!isRecording()) return; - const control = createControlClient({ region: REGION }); - const data = createDataClient({ region: REGION }); - if (created.abTestId) { - try { - await data.send(new DeleteABTestCommand({ abTestId: created.abTestId })); - } catch (error) { - console.error("cleanup ab-test:", error); - } - try { - await deleteAbTestRole( - createIamClient({ region: REGION }), - abTestExecutionRoleName("agentcore_cli_abtest_run"), - ); - } catch (error) { - console.error("cleanup ab-test role:", error); - } - } - if (created.onlineEvalId) { - try { - await control.send( - new DeleteOnlineEvaluationConfigCommand({ onlineEvaluationConfigId: created.onlineEvalId }), - ); - } catch (error) { - console.error("cleanup online-eval:", error); - } - } - if (created.bundleId) { - try { - await control.send( - new GetConfigurationBundleCommand({ bundleId: created.bundleId, branchName: "mainline" }), - ); - await control.send(new DeleteConfigurationBundleCommand({ bundleId: created.bundleId })); - } catch (error) { - if ((error as Error).name !== "ResourceNotFoundException") { - console.error("cleanup bundle:", error); - } - } - } -}); - -describe("eval ab-test config-bundle run (fixture-backed)", () => { - test("provisions a bundle with two versions", async () => { - const v1 = await run([ - "eval", - "config-bundle", - "create", - "--name", - BUNDLE_NAME, - "--components", - JSON.stringify(COMPONENTS_V1), - ]); - matchGolden(FIXTURES, "run-bundle-create.golden.json", v1); - const first = JSON.parse(v1); - created.bundleId = first.bundleId; - created.v1 = first.versionId; - - await settle(); - - const v2 = await run([ - "eval", - "config-bundle", - "update", - "--id", - created.bundleId!, - "--components", - JSON.stringify(COMPONENTS_V2), - "--commit-message", - "A/B run fixture v2", - ]); - matchGolden(FIXTURES, "run-bundle-update.golden.json", v2); - created.v2 = JSON.parse(v2).versionId; - expect(created.v2).not.toBe(created.v1); - }, 180_000); - - test("provisions a paused online evaluation config", async () => { - const out = await run([ - "eval", - "online-eval", - "create", - "--name", - ONLINE_EVAL_NAME, - "--agent", - AGENT_ID, - "--evaluator", - EVALUATOR_ID, - "--sampling-rate", - "100", - "--enable-on-create", - "false", - ]); - matchGolden(FIXTURES, "run-online-eval.golden.json", out); - created.onlineEvalId = JSON.parse(out).onlineEvaluationConfigId; - }, 180_000); - - test("runs a paused config-bundle A/B test", async () => { - const out = await run([ - "eval", - "ab-test", - "config-bundle", - "run", - "--name", - "agentcore_cli_abtest_run", - "--gateway", - GATEWAY_ID, - "--control", - JSON.stringify({ "config-bundle": created.bundleId, "bundle-version": created.v1 }), - "--treatment", - JSON.stringify({ "config-bundle": created.bundleId, "bundle-version": created.v2 }), - "--online-eval", - created.onlineEvalId!, - "--treatment-weight", - "20", - "--enable-on-create", - "false", - ]); - matchGolden(FIXTURES, "run.golden.json", out); - const abTest = JSON.parse(out); - created.abTestId = abTest.abTestId; - expect(abTest.abTestId).toBeString(); - expect(abTest.executionStatus).toBe("NOT_STARTED"); - }, 180_000); -}); From 2c9274ed51b6cdf75574097970dbd925fbed33a1 Mon Sep 17 00:00:00 2001 From: jariy17 Date: Thu, 27 Aug 2026 21:15:12 +0000 Subject: [PATCH 11/15] feat(eval): add ab-test target-based run (create) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Adds `agentcore eval ab-test target-based run` — an A/B test between two gateway targets and their per-variant online evaluations. --control/--treatment take {gateway-target, online-eval} JSON (inline/file/stdin via SourceResolver); variants use variantConfiguration.target and a perVariantOnlineEvaluationConfig. Extract a shared EvalClient.createABTest helper (GetGateway -> account, role provision + AccessDenied/assume retry + rollback) and drive both config-bundle and target-based create through it, removing the duplicated role/retry block. Consolidated ab-test.test.tsx covers the target-based hierarchy, validation (required flags, mis-shaped JSON, identical targets), and flag->request mapping. Golden fixture deferred (needs a gateway with two wired targets + two online-evals). --- src/core/eval.tsx | 162 +++++++++++++----- src/handlers/eval/ab-test/ab-test.test.tsx | 86 ++++++++++ src/handlers/eval/ab-test/index.tsx | 4 +- .../eval/ab-test/target-based/index.tsx | 10 ++ .../eval/ab-test/target-based/run/index.tsx | 116 +++++++++++++ src/handlers/eval/types.tsx | 17 ++ src/testing/TestCoreClient.tsx | 10 ++ 7 files changed, 357 insertions(+), 48 deletions(-) create mode 100644 src/handlers/eval/ab-test/target-based/index.tsx create mode 100644 src/handlers/eval/ab-test/target-based/run/index.tsx diff --git a/src/core/eval.tsx b/src/core/eval.tsx index 439c67f4f..d438ed26c 100644 --- a/src/core/eval.tsx +++ b/src/core/eval.tsx @@ -76,6 +76,7 @@ import { type EvaluationReferenceInput, type EvaluationResultContent, type EvaluationTarget, + type CreateABTestRequest, type CreateABTestResponse, type GetABTestResponse, type ListABTestsResponse, @@ -123,6 +124,7 @@ import type { CoreEvalClient, CreateConfigurationBundleInput, CreateConfigBundleABTestInput, + CreateTargetBasedABTestInput, CreateDatasetInput, CreateOnlineEvalInput, CreateOnlineInsightInput, @@ -481,65 +483,38 @@ export class EvalClient implements CoreEvalClient { .send(new DeleteABTestCommand({ abTestId: id })); } - async createConfigBundleABTest( - input: CreateConfigBundleABTestInput, + private async createABTest( + name: string, + gateway: string, + callerRoleArn: string | undefined, + build: (context: { + gatewayArn: string; + accountId: string; + roleArn: string; + }) => CreateABTestRequest, options: CoreOptions, ): Promise { const control = this.clients.control(toClientConfig(options)); - const gateway = await control.send(new GetGatewayCommand({ gatewayIdentifier: input.gateway })); - const gatewayArn = gateway.gatewayArn!; + const gatewayArn = (await control.send(new GetGatewayCommand({ gatewayIdentifier: gateway }))) + .gatewayArn!; const accountId = accountIdFromArn(gatewayArn); - const controlBundleArn = `arn:aws:bedrock-agentcore:${options.region}:${accountId}:configuration-bundle/${input.control.configBundle}`; - const treatmentBundleArn = `arn:aws:bedrock-agentcore:${options.region}:${accountId}:configuration-bundle/${input.treatment.configBundle}`; - const onlineEvaluationConfigArn = `arn:aws:bedrock-agentcore:${options.region}:${accountId}:online-evaluation-config/${input.onlineEval}`; - - const treatmentWeight = input.treatmentWeight ?? 50; - const variants = [ - { - name: "C", - weight: 100 - treatmentWeight, - variantConfiguration: { - configurationBundle: { - bundleArn: controlBundleArn, - bundleVersion: input.control.bundleVersion, - }, - }, - }, - { - name: "T1", - weight: treatmentWeight, - variantConfiguration: { - configurationBundle: { - bundleArn: treatmentBundleArn, - bundleVersion: input.treatment.bundleVersion, - }, - }, - }, - ]; - - let roleArn = input.roleArn; + let roleArn = callerRoleArn; let provisionedRoleArn: string | undefined; if (!roleArn) { - const iam = this.clients.iam({ region: options.region }); - const provisioned = await provisionAbTestRole(iam, input.name, gatewayArn, options.region); + const provisioned = await provisionAbTestRole( + this.clients.iam({ region: options.region }), + name, + gatewayArn, + options.region, + ); roleArn = provisioned.roleArn; if (provisioned.created) provisionedRoleArn = provisioned.roleArn; } - const command = new CreateABTestCommand({ - name: input.name, - gatewayArn, - variants, - evaluationConfig: { onlineEvaluationConfigArn }, - roleArn, - gatewayFilter: input.gatewayFilter, - enableOnCreate: input.enableOnCreate ?? true, - clientToken: randomUUID(), - }); - + const command = new CreateABTestCommand(build({ gatewayArn, accountId, roleArn })); try { - return input.roleArn + return callerRoleArn ? await this.clients.data(toClientConfig(options)).send(command) : await retryWhileRolePropagates(() => this.clients.data(toClientConfig(options)).send(command), @@ -556,6 +531,99 @@ export class EvalClient implements CoreEvalClient { } } + async createConfigBundleABTest( + input: CreateConfigBundleABTestInput, + options: CoreOptions, + ): Promise { + const treatmentWeight = input.treatmentWeight ?? 50; + return this.createABTest( + input.name, + input.gateway, + input.roleArn, + ({ gatewayArn, accountId, roleArn }) => { + const bundleArn = (id: string) => + `arn:aws:bedrock-agentcore:${options.region}:${accountId}:configuration-bundle/${id}`; + return { + name: input.name, + gatewayArn, + variants: [ + { + name: "C", + weight: 100 - treatmentWeight, + variantConfiguration: { + configurationBundle: { + bundleArn: bundleArn(input.control.configBundle), + bundleVersion: input.control.bundleVersion, + }, + }, + }, + { + name: "T1", + weight: treatmentWeight, + variantConfiguration: { + configurationBundle: { + bundleArn: bundleArn(input.treatment.configBundle), + bundleVersion: input.treatment.bundleVersion, + }, + }, + }, + ], + evaluationConfig: { + onlineEvaluationConfigArn: `arn:aws:bedrock-agentcore:${options.region}:${accountId}:online-evaluation-config/${input.onlineEval}`, + }, + roleArn, + gatewayFilter: input.gatewayFilter, + enableOnCreate: input.enableOnCreate ?? true, + clientToken: randomUUID(), + }; + }, + options, + ); + } + + async createTargetBasedABTest( + input: CreateTargetBasedABTestInput, + options: CoreOptions, + ): Promise { + const treatmentWeight = input.treatmentWeight ?? 50; + return this.createABTest( + input.name, + input.gateway, + input.roleArn, + ({ gatewayArn, accountId, roleArn }) => { + const evalArn = (id: string) => + `arn:aws:bedrock-agentcore:${options.region}:${accountId}:online-evaluation-config/${id}`; + return { + name: input.name, + gatewayArn, + variants: [ + { + name: "C", + weight: 100 - treatmentWeight, + variantConfiguration: { target: { name: input.control.gatewayTarget } }, + }, + { + name: "T1", + weight: treatmentWeight, + variantConfiguration: { target: { name: input.treatment.gatewayTarget } }, + }, + ], + evaluationConfig: { + perVariantOnlineEvaluationConfig: [ + { name: "C", onlineEvaluationConfigArn: evalArn(input.control.onlineEval) }, + { name: "T1", onlineEvaluationConfigArn: evalArn(input.treatment.onlineEval) }, + ], + }, + roleArn, + gatewayFilter: input.gatewayFilter, + enableOnCreate: input.enableOnCreate ?? true, + clientToken: randomUUID(), + }; + }, + options, + ); + } + async listBatchInsights( nextToken: string | undefined, maxResults: number | undefined, diff --git a/src/handlers/eval/ab-test/ab-test.test.tsx b/src/handlers/eval/ab-test/ab-test.test.tsx index 3f75f8647..73ac93b64 100644 --- a/src/handlers/eval/ab-test/ab-test.test.tsx +++ b/src/handlers/eval/ab-test/ab-test.test.tsx @@ -79,9 +79,12 @@ describe("eval ab-test command hierarchy", () => { "stop", "delete", "config-bundle", + "target-based", ]); const cb = group?.children().find((c) => c.name() === "config-bundle"); expect(cb?.children().map((c) => c.name())).toEqual(["run"]); + const tb = group?.children().find((c) => c.name() === "target-based"); + expect(tb?.children().map((c) => c.name())).toEqual(["run"]); }); }); @@ -264,3 +267,86 @@ describe("eval ab-test config-bundle run validation", () => { }); }); }); + +describe("eval ab-test target-based run validation", () => { + const TB_BASE = [ + "eval", + "ab-test", + "target-based", + "run", + "--name", + "orders-v2-canary", + "--gateway", + "orders-gateway-abc123", + "--control", + '{"gateway-target":"orders-prod-target","online-eval":"prod-quality"}', + "--treatment", + '{"gateway-target":"orders-v2-target","online-eval":"v2-quality"}', + "--json", + ]; + + test.each(["name", "gateway", "control", "treatment"] as const)( + "requires --%s", + async (missing) => { + const args = TB_BASE.filter( + (a, i) => a !== `--${missing}` && TB_BASE[i - 1] !== `--${missing}`, + ); + await expect(run(args)).rejects.toThrow(new RegExp(`--${missing}`)); + }, + ); + + test("rejects a mis-shaped --control object", async () => { + const args = TB_BASE.map((a) => + a === '{"gateway-target":"orders-prod-target","online-eval":"prod-quality"}' + ? '{"wrong":"shape"}' + : a, + ); + await expect(run(args)).rejects.toThrow(/--control must be/); + }); + + test("rejects identical control/treatment targets", async () => { + const same = '{"gateway-target":"t","online-eval":"e"}'; + await expect( + run([ + "eval", + "ab-test", + "target-based", + "run", + "--name", + "x", + "--gateway", + "g", + "--control", + same, + "--treatment", + same, + "--json", + ]), + ).rejects.toThrow(/different gateway targets/); + }); + + test("maps flags to a createTargetBasedABTest call", async () => { + const { core } = await run([...TB_BASE, "--treatment-weight", "20"], (c) => + c.eval.setAbTestCreateResponse({ + abTestId: "x", + abTestArn: ARN, + name: "x", + status: "CREATING", + executionStatus: "RUNNING", + createdAt: new Date("2026-08-26T10:00:00.000Z"), + }), + ); + const call = core.eval.calls.find((c) => c.method === "createTargetBasedABTest"); + expect(call).toBeDefined(); + expect(call!.args[0]).toEqual({ + name: "orders-v2-canary", + gateway: "orders-gateway-abc123", + control: { gatewayTarget: "orders-prod-target", onlineEval: "prod-quality" }, + treatment: { gatewayTarget: "orders-v2-target", onlineEval: "v2-quality" }, + treatmentWeight: 20, + gatewayFilter: undefined, + roleArn: undefined, + enableOnCreate: undefined, + }); + }); +}); diff --git a/src/handlers/eval/ab-test/index.tsx b/src/handlers/eval/ab-test/index.tsx index 4e42ada3a..42b16fd9d 100644 --- a/src/handlers/eval/ab-test/index.tsx +++ b/src/handlers/eval/ab-test/index.tsx @@ -10,6 +10,7 @@ import { createResumeAbTestHandler } from "./resume"; import { createStopAbTestHandler } from "./stop"; import { createDeleteAbTestHandler } from "./delete"; import { createConfigBundleAbTestHandler } from "./config-bundle"; +import { createTargetBasedAbTestHandler } from "./target-based"; export function createAbTestHandler(core: Core, io: AppIO): Router { return new Router("ab-test", "inspect AgentCore A/B tests") @@ -22,7 +23,8 @@ export function createAbTestHandler(core: Core, io: AppIO): Router { .handler(createResumeAbTestHandler(core)) .handler(createStopAbTestHandler(core)) .handler(createDeleteAbTestHandler(core)) - .handler(createConfigBundleAbTestHandler(core, io)); + .handler(createConfigBundleAbTestHandler(core, io)) + .handler(createTargetBasedAbTestHandler(core, io)); } export { AbTestScreen } from "./screen.tsx"; diff --git a/src/handlers/eval/ab-test/target-based/index.tsx b/src/handlers/eval/ab-test/target-based/index.tsx new file mode 100644 index 000000000..e13f20b18 --- /dev/null +++ b/src/handlers/eval/ab-test/target-based/index.tsx @@ -0,0 +1,10 @@ +import { Router } from "../../../../router"; +import type { AppIO } from "../../../../io"; +import type { Core } from "../../../types"; +import { createTargetBasedRunHandler } from "./run"; + +export function createTargetBasedAbTestHandler(core: Core, io: AppIO): Router { + return new Router("target-based", "target-based A/B tests").handler( + createTargetBasedRunHandler(core, io), + ); +} diff --git a/src/handlers/eval/ab-test/target-based/run/index.tsx b/src/handlers/eval/ab-test/target-based/run/index.tsx new file mode 100644 index 000000000..8ef811eb1 --- /dev/null +++ b/src/handlers/eval/ab-test/target-based/run/index.tsx @@ -0,0 +1,116 @@ +import z from "zod"; +import { createHandler, flag } from "../../../../../router"; +import { InputValidationError } from "../../../../../errors"; +import { JsonRendererKey } from "../../../../../tui"; +import { SourceResolver, type AppIO } from "../../../../../io"; +import type { Core } from "../../../../types"; +import type { TargetVariantRef } from "../../../types"; +import { coreOptsFromCtx, parseJsonFlag } from "../../../../utils"; + +const targetRefSchema = z + .object({ + "gateway-target": z.string().min(1), + "online-eval": z.string().min(1), + }) + .strict(); + +function toTargetRef(name: string, raw: unknown): TargetVariantRef { + const parsed = targetRefSchema.safeParse(raw); + if (!parsed.success) { + throw new InputValidationError( + `--${name} must be {"gateway-target": "", "online-eval": ""}`, + ); + } + return { gatewayTarget: parsed.data["gateway-target"], onlineEval: parsed.data["online-eval"] }; +} + +export const createTargetBasedRunHandler = (core: Core, io: AppIO) => + createHandler({ + name: "run", + description: "run an A/B test between two gateway targets and their online evaluations", + flags: [ + flag("name", "the A/B test name", z.string().optional()), + flag("gateway", "deployed gateway id", z.string().optional()), + flag( + "control", + 'control JSON {"gateway-target","online-eval"} (inline, file://, or -)', + z.string().optional(), + ), + flag( + "treatment", + 'treatment JSON {"gateway-target","online-eval"} (inline, file://, or -)', + z.string().optional(), + ), + flag( + "treatment-weight", + "1-99; control weight = 100 - this (default 50)", + z.number().int().optional(), + ), + flag( + "gateway-filter", + 'GatewayFilter JSON, e.g. {"targetPaths":["/orders"]} (inline, file://, or -)', + z.string().optional(), + ), + flag( + "role-arn", + "execution-role override (default: auto-provisioned)", + z.string().optional(), + ), + flag( + "enable-on-create", + "whether to start the test immediately (default true; pass false to create it paused)", + z.enum(["true", "false"]).optional(), + ), + ], + handle: async (ctx, flags) => { + const required = ["name", "gateway", "control", "treatment"] as const; + for (const f of required) { + if (!flags[f]) throw new InputValidationError(`required option '--${f}' not specified`); + } + + const source = new SourceResolver({ stdin: io.stdin }); + const controlRaw = parseJsonFlag( + "control", + await source.resolveText("control", flags["control"]), + ); + const treatmentRaw = parseJsonFlag( + "treatment", + await source.resolveText("treatment", flags["treatment"]), + ); + const gatewayFilter = parseJsonFlag< + import("@aws-sdk/client-bedrock-agentcore").GatewayFilter + >("gateway-filter", await source.resolveText("gateway-filter", flags["gateway-filter"])); + + const control = toTargetRef("control", controlRaw); + const treatment = toTargetRef("treatment", treatmentRaw); + if (control.gatewayTarget === treatment.gatewayTarget) { + throw new InputValidationError( + "control and treatment must reference different gateway targets", + ); + } + + const treatmentWeight = flags["treatment-weight"]; + if (treatmentWeight !== undefined && (treatmentWeight < 1 || treatmentWeight > 99)) { + throw new InputValidationError("--treatment-weight must be between 1 and 99"); + } + + const result = await core.eval.createTargetBasedABTest( + { + name: flags["name"]!, + gateway: flags["gateway"]!, + control, + treatment, + treatmentWeight, + gatewayFilter, + roleArn: flags["role-arn"], + enableOnCreate: + flags["enable-on-create"] === undefined + ? undefined + : flags["enable-on-create"] === "true", + }, + coreOptsFromCtx(ctx), + ); + + ctx.require(JsonRendererKey).renderJson(result); + }, + }); diff --git a/src/handlers/eval/types.tsx b/src/handlers/eval/types.tsx index b13c6f9cd..91a70c705 100644 --- a/src/handlers/eval/types.tsx +++ b/src/handlers/eval/types.tsx @@ -247,6 +247,19 @@ export type CreateConfigBundleABTestInput = { enableOnCreate?: boolean; }; +export type TargetVariantRef = { gatewayTarget: string; onlineEval: string }; + +export type CreateTargetBasedABTestInput = { + name: string; + gateway: string; + control: TargetVariantRef; + treatment: TargetVariantRef; + treatmentWeight?: number; + gatewayFilter?: GatewayFilter; + roleArn?: string; + enableOnCreate?: boolean; +}; + export type CreateDatasetInput = CreateDatasetRequest; export type StartRecommendationInput = { name: string; @@ -447,6 +460,10 @@ export interface CoreEvalClient { input: CreateConfigBundleABTestInput, options: CoreOptions, ): Promise; + createTargetBasedABTest( + input: CreateTargetBasedABTestInput, + options: CoreOptions, + ): Promise; // startBatchEvaluation submits an async, service-side evaluation over sessions // the service gathers from the resolved data source. Returns the durable job id // + RUNNING status; poll with getBatchEvaluation. diff --git a/src/testing/TestCoreClient.tsx b/src/testing/TestCoreClient.tsx index 94a888c47..686bb6bf0 100644 --- a/src/testing/TestCoreClient.tsx +++ b/src/testing/TestCoreClient.tsx @@ -139,6 +139,7 @@ import type { CoreEvalClient, CreateConfigurationBundleInput, CreateConfigBundleABTestInput, + CreateTargetBasedABTestInput, CreateDatasetInput, CreateOnlineEvalInput, CreateOnlineInsightInput, @@ -1912,6 +1913,15 @@ export class TestEvalClient implements CoreEvalClient { return this.abTestCreateResponse; } + async createTargetBasedABTest( + input: CreateTargetBasedABTestInput, + options: CoreOptions, + ): Promise { + this.calls.push({ method: "createTargetBasedABTest", args: [input, options] }); + if (this.error) throw this.error; + return this.abTestCreateResponse; + } + async startBatchEvaluation( input: StartBatchEvaluationInput, options: CoreOptions, From 268305a73b4c3fde2d433c422384e30fb9ebf8ec Mon Sep 17 00:00:00 2001 From: jariy17 Date: Fri, 28 Aug 2026 21:54:53 +0000 Subject: [PATCH 12/15] feat(project): add `project add evaluator code-based` Declarative code-based evaluators via projects. Mode is inferred from flags (mirrors CodeBasedConfigSchema managed XOR external): --lambda-arn -> external (BYO Lambda) --metric -> managed 3P (deepeval/autoevals), scaffolded neither -> managed empty stub you fill in Scaffolds app// from ported evaluator templates (python/deepeval/autoevals lambda), hardcodes codeLocation, and auto-wires additionalPolicies= [execution-role-policy.json]. Also enables `project remove evaluator`. --- .../execution-role-policy.json | 15 ++ .../autoevals-lambda/lambda_function.py | 37 +++ .../autoevals-lambda/pyproject.toml | 22 ++ .../execution-role-policy.json | 15 ++ .../deepeval-lambda/lambda_function.py | 29 ++ .../evaluators/deepeval-lambda/pyproject.toml | 19 ++ .../python-lambda/execution-role-policy.json | 10 + .../python-lambda/lambda_function.py | 19 ++ .../evaluators/python-lambda/pyproject.toml | 15 ++ src/core/project/manager.tsx | 23 +- src/core/project/templates/evaluator.ts | 37 +++ src/core/project/templates/types.ts | 2 + .../add/evaluator/code-based/index.test.ts | 255 ++++++++++++++++++ .../project/add/evaluator/code-based/index.ts | 168 ++++++++++++ src/handlers/project/add/evaluator/index.ts | 2 + src/handlers/project/remove/index.ts | 1 + src/handlers/project/types.ts | 7 + 17 files changed, 675 insertions(+), 1 deletion(-) create mode 100644 src/assets/evaluators/autoevals-lambda/execution-role-policy.json create mode 100644 src/assets/evaluators/autoevals-lambda/lambda_function.py create mode 100644 src/assets/evaluators/autoevals-lambda/pyproject.toml create mode 100644 src/assets/evaluators/deepeval-lambda/execution-role-policy.json create mode 100644 src/assets/evaluators/deepeval-lambda/lambda_function.py create mode 100644 src/assets/evaluators/deepeval-lambda/pyproject.toml create mode 100644 src/assets/evaluators/python-lambda/execution-role-policy.json create mode 100644 src/assets/evaluators/python-lambda/lambda_function.py create mode 100644 src/assets/evaluators/python-lambda/pyproject.toml create mode 100644 src/core/project/templates/evaluator.ts create mode 100644 src/handlers/project/add/evaluator/code-based/index.test.ts create mode 100644 src/handlers/project/add/evaluator/code-based/index.ts diff --git a/src/assets/evaluators/autoevals-lambda/execution-role-policy.json b/src/assets/evaluators/autoevals-lambda/execution-role-policy.json new file mode 100644 index 000000000..6b47af830 --- /dev/null +++ b/src/assets/evaluators/autoevals-lambda/execution-role-policy.json @@ -0,0 +1,15 @@ +{ + "Version": "2012-10-17", + "Statement": [ + { + "Effect": "Allow", + "Action": ["logs:CreateLogGroup", "logs:CreateLogStream", "logs:PutLogEvents"], + "Resource": "arn:*:logs:*:*:log-group:/aws/lambda/*" + }, + { + "Effect": "Allow", + "Action": ["bedrock:InvokeModel"], + "Resource": "*" + } + ] +} diff --git a/src/assets/evaluators/autoevals-lambda/lambda_function.py b/src/assets/evaluators/autoevals-lambda/lambda_function.py new file mode 100644 index 000000000..410f056a6 --- /dev/null +++ b/src/assets/evaluators/autoevals-lambda/lambda_function.py @@ -0,0 +1,37 @@ +{{#if ModelProviderBedrock}} +import os + +# litellm's Bedrock provider reads AWS_REGION_NAME; Lambda only sets AWS_REGION/AWS_DEFAULT_REGION. +os.environ.setdefault("AWS_REGION_NAME", os.environ.get("AWS_REGION", "us-west-2")) + +from autoevals import {{ EvaluatorClass }}, init +from autoevals.litellm import LiteLLMClient + +from bedrock_agentcore.evaluation.custom_code_based_evaluators import ( + EvaluatorInput, + EvaluatorOutput, + custom_code_based_evaluator, +) +from bedrock_agentcore.evaluation.custom_code_based_evaluators.third_party.autoevals import AutoEvalsAdapter + +client = LiteLLMClient() +init(client=client, default_model="{{ Model }}") + +adapter = AutoEvalsAdapter(metric={{ EvaluatorClass }}(client=client, model="{{ Model }}"){{#if EvaluatorParams}}, {{{ EvaluatorParams }}}{{/if}}) +{{else}} +from autoevals import {{ EvaluatorClass }} + +from bedrock_agentcore.evaluation.custom_code_based_evaluators import ( + EvaluatorInput, + EvaluatorOutput, + custom_code_based_evaluator, +) +from bedrock_agentcore.evaluation.custom_code_based_evaluators.third_party.autoevals import AutoEvalsAdapter + +adapter = AutoEvalsAdapter(metric={{ EvaluatorClass }}({{#if Model}}model="{{ Model }}"{{/if}}){{#if EvaluatorParams}}, {{{ EvaluatorParams }}}{{/if}}) +{{/if}} + + +@custom_code_based_evaluator() +def handler(evaluator_input: EvaluatorInput, context) -> EvaluatorOutput: + return adapter(evaluator_input, context) diff --git a/src/assets/evaluators/autoevals-lambda/pyproject.toml b/src/assets/evaluators/autoevals-lambda/pyproject.toml new file mode 100644 index 000000000..b37442fd1 --- /dev/null +++ b/src/assets/evaluators/autoevals-lambda/pyproject.toml @@ -0,0 +1,22 @@ +[build-system] +requires = ["hatchling"] +build-backend = "hatchling.build" + +[project] +name = "{{ Name }}" +version = "0.1.0" +description = "AgentCore Code-Based Evaluator (Autoevals)" +requires-python = ">=3.10" +dependencies = [ + "bedrock-agentcore[autoevals]", + "autoevals>=0.0.80,<1.0.0", +{{#if ModelProviderBedrock}} + # autoevals grades via LiteLLMClient -> Bedrock (Converse); litellm replaces the openai judge + "litellm>=1.60,<1.85", +{{else}} + "openai>=1.0.0", +{{/if}} +] + +[tool.hatch.build.targets.wheel] +packages = ["."] diff --git a/src/assets/evaluators/deepeval-lambda/execution-role-policy.json b/src/assets/evaluators/deepeval-lambda/execution-role-policy.json new file mode 100644 index 000000000..6b47af830 --- /dev/null +++ b/src/assets/evaluators/deepeval-lambda/execution-role-policy.json @@ -0,0 +1,15 @@ +{ + "Version": "2012-10-17", + "Statement": [ + { + "Effect": "Allow", + "Action": ["logs:CreateLogGroup", "logs:CreateLogStream", "logs:PutLogEvents"], + "Resource": "arn:*:logs:*:*:log-group:/aws/lambda/*" + }, + { + "Effect": "Allow", + "Action": ["bedrock:InvokeModel"], + "Resource": "*" + } + ] +} diff --git a/src/assets/evaluators/deepeval-lambda/lambda_function.py b/src/assets/evaluators/deepeval-lambda/lambda_function.py new file mode 100644 index 000000000..a89b22051 --- /dev/null +++ b/src/assets/evaluators/deepeval-lambda/lambda_function.py @@ -0,0 +1,29 @@ +import os + +os.environ.setdefault("DEEPEVAL_RESULTS_FOLDER", "/tmp/.deepeval") +os.environ.setdefault("DEEPEVAL_TELEMETRY_OPT_OUT", "YES") +os.chdir("/tmp") + +{{#if ModelProviderBedrock}} +from deepeval.models import AmazonBedrockModel +{{/if}} +from deepeval.metrics import {{ EvaluatorClass }} + +from bedrock_agentcore.evaluation.custom_code_based_evaluators import ( + EvaluatorInput, + EvaluatorOutput, + custom_code_based_evaluator, +) +from bedrock_agentcore.evaluation.custom_code_based_evaluators.third_party.deepeval import DeepEvalAdapter + +{{#if ModelProviderBedrock}} +model = AmazonBedrockModel(model="{{ Model }}", region=os.environ.get("AWS_REGION", "us-west-2")) +adapter = DeepEvalAdapter(metric={{ EvaluatorClass }}(model=model{{#if EvaluatorParams}}, {{{ EvaluatorParams }}}{{/if}})) +{{else}} +adapter = DeepEvalAdapter(metric={{ EvaluatorClass }}({{{ EvaluatorParams }}})) +{{/if}} + + +@custom_code_based_evaluator() +def handler(evaluator_input: EvaluatorInput, context) -> EvaluatorOutput: + return adapter(evaluator_input, context) diff --git a/src/assets/evaluators/deepeval-lambda/pyproject.toml b/src/assets/evaluators/deepeval-lambda/pyproject.toml new file mode 100644 index 000000000..7385ccfc2 --- /dev/null +++ b/src/assets/evaluators/deepeval-lambda/pyproject.toml @@ -0,0 +1,19 @@ +[build-system] +requires = ["hatchling"] +build-backend = "hatchling.build" + +[project] +name = "{{ Name }}" +version = "0.1.0" +description = "AgentCore Code-Based Evaluator (DeepEval)" +requires-python = ">=3.10" +dependencies = [ + "bedrock-agentcore[deepeval]", + "deepeval>=2.0.0,<3.0.0", +{{#if ModelProviderBedrock}} + "aiobotocore>=2.13.0", +{{/if}} +] + +[tool.hatch.build.targets.wheel] +packages = ["."] diff --git a/src/assets/evaluators/python-lambda/execution-role-policy.json b/src/assets/evaluators/python-lambda/execution-role-policy.json new file mode 100644 index 000000000..b3b98be42 --- /dev/null +++ b/src/assets/evaluators/python-lambda/execution-role-policy.json @@ -0,0 +1,10 @@ +{ + "Version": "2012-10-17", + "Statement": [ + { + "Effect": "Allow", + "Action": ["logs:CreateLogGroup", "logs:CreateLogStream", "logs:PutLogEvents"], + "Resource": "arn:*:logs:*:*:log-group:/aws/lambda/*" + } + ] +} diff --git a/src/assets/evaluators/python-lambda/lambda_function.py b/src/assets/evaluators/python-lambda/lambda_function.py new file mode 100644 index 000000000..0f8bd5c6b --- /dev/null +++ b/src/assets/evaluators/python-lambda/lambda_function.py @@ -0,0 +1,19 @@ +from bedrock_agentcore.evaluation.custom_code_based_evaluators import ( + custom_code_based_evaluator, + EvaluatorInput, + EvaluatorOutput, +) + + +@custom_code_based_evaluator() +def handler(input: EvaluatorInput, context) -> EvaluatorOutput: + """Evaluate agent behavior with custom logic. + + Args: + input: Contains evaluation_level, session_spans, target_trace_id, target_span_id + + Returns: + EvaluatorOutput with value/label for success, or errorCode/errorMessage for failure. + """ + # TODO: Replace with your evaluation logic + return EvaluatorOutput(value=1.0, label="Pass", explanation="Evaluation passed") diff --git a/src/assets/evaluators/python-lambda/pyproject.toml b/src/assets/evaluators/python-lambda/pyproject.toml new file mode 100644 index 000000000..69ad99b43 --- /dev/null +++ b/src/assets/evaluators/python-lambda/pyproject.toml @@ -0,0 +1,15 @@ +[build-system] +requires = ["hatchling"] +build-backend = "hatchling.build" + +[project] +name = "{{ Name }}" +version = "0.1.0" +description = "AgentCore Code-Based Evaluator" +requires-python = ">=3.10" +dependencies = [ + "bedrock-agentcore>=1.6.0", +] + +[tool.hatch.build.targets.wheel] +packages = ["."] diff --git a/src/core/project/manager.tsx b/src/core/project/manager.tsx index 9ede69c1f..5918b94dc 100644 --- a/src/core/project/manager.tsx +++ b/src/core/project/manager.tsx @@ -26,6 +26,7 @@ import { ENV_LOCAL_RELATIVE_PATH, EnvLocalFile } from "./envLocal"; import { getHarnessTemplateResolver } from "./templates/harness"; import { createProjectTree } from "./templates/project"; import { getRuntimeTemplateResolver } from "./templates/runtime"; +import { getEvaluatorTemplateResolver } from "./templates/evaluator"; import { ProjectSpecSchema, type ManagedBy } from "../../projectSchemas/project"; import { ConfigBundleSchema } from "../../projectSchemas/config-bundle"; import { CredentialSchema } from "../../projectSchemas/credential"; @@ -258,7 +259,27 @@ export class FsProjectManager implements ProjectManager { break; } case "evaluator": { - projectSpec.evaluators.push(parseResource(EvaluatorSchema, input.resourceConfig)); + const evaluator = parseResource(EvaluatorSchema, input.resourceConfig); + // Managed code-based evaluators ship generated Lambda source; external + // and llm-as-a-judge evaluators are spec-only. + if (input.scaffold) { + yield { message: "Scaffolding evaluator in project" }; + const outputPath = join(project.rootPath, "app", evaluator.name); + scaffoldedPaths.push(outputPath); + const resolver = getEvaluatorTemplateResolver({ + assetSource: this.assetSource, + templateRenderer: this.templateRenderer, + }); + const result = await resolver.resolve({ + evaluator, + assetDir: input.scaffold.assetDir, + context: input.scaffold.context, + }); + await result.tree.write(dirname(outputPath)); + projectSpec.evaluators.push(...(result.spec.evaluators ?? [])); + } else { + projectSpec.evaluators.push(evaluator); + } break; } case "gateway": diff --git a/src/core/project/templates/evaluator.ts b/src/core/project/templates/evaluator.ts new file mode 100644 index 000000000..5c091e657 --- /dev/null +++ b/src/core/project/templates/evaluator.ts @@ -0,0 +1,37 @@ +import { FsTreeNode } from "./fsTree"; +import type { AssetSource } from "../source"; +import type { Evaluator } from "../../../projectSchemas/evaluator"; +import type { TemplateRenderer, TemplateResolver } from "./types"; + +/** Inputs for scaffolding a managed code-based evaluator's Lambda source. */ +export type EvaluatorScaffoldInput = { + evaluator: Evaluator; + /** Template directory under src/assets/evaluators, e.g. "evaluators/deepeval-lambda". */ + assetDir: string; + /** Handlebars variables for the template (EvaluatorClass, Model, ModelProviderBedrock, ...). */ + context: Record; +}; + +type GetEvaluatorTemplateResolverConfig = { + assetSource: AssetSource; + templateRenderer: TemplateRenderer; +}; + +/** Resolves the template that renders a managed code-based evaluator's code directory. */ +export function getEvaluatorTemplateResolver( + config: GetEvaluatorTemplateResolverConfig, +): TemplateResolver { + return { + async resolve(input) { + const tree = await FsTreeNode.fromAssetSource( + { assetSource: config.assetSource }, + { assetDir: input.assetDir }, + { + rootDirName: input.evaluator.name, + transformContent: (raw) => config.templateRenderer.render(raw, input.context), + }, + ); + return { tree, spec: { evaluators: [input.evaluator] } }; + }, + }; +} diff --git a/src/core/project/templates/types.ts b/src/core/project/templates/types.ts index f472be503..798e4ba4b 100644 --- a/src/core/project/templates/types.ts +++ b/src/core/project/templates/types.ts @@ -3,6 +3,7 @@ import type { ProjectRuntime } from "../../../projectSchemas/runtime"; import type { MemorySchema } from "../../../projectSchemas/memory"; import type { CredentialSchema } from "../../../projectSchemas/credential"; import type { HarnessRegistryEntry } from "../../../projectSchemas/harness"; +import type { Evaluator } from "../../../projectSchemas/evaluator"; import type z from "zod"; /** AgentCore Project Spec Entries that rendered as part of a {@link Template} **/ @@ -11,6 +12,7 @@ export type SpecEntries = { credentials?: z.infer[]; memories?: z.infer[]; harnesses?: HarnessRegistryEntry[]; + evaluators?: Evaluator[]; }; /** A group of files and resources that can be rendered into a project **/ diff --git a/src/handlers/project/add/evaluator/code-based/index.test.ts b/src/handlers/project/add/evaluator/code-based/index.test.ts new file mode 100644 index 000000000..307d15878 --- /dev/null +++ b/src/handlers/project/add/evaluator/code-based/index.test.ts @@ -0,0 +1,255 @@ +import { afterEach, describe, expect, test } from "bun:test"; +import { mkdtemp, rm } from "node:fs/promises"; +import { join } from "node:path"; +import { tmpdir } from "node:os"; +import { createRootHandler } from "../../../../index"; +import { + createSilentLogger, + TestCoreClient, + TestGlobalConfigAccessor, + testIO, +} from "../../../../../testing"; +import { InputValidationError } from "../../../../../errors"; + +const originalCwd = process.cwd(); +const tempDirectories: string[] = []; + +async function inTempDirectory(): Promise { + const directory = await mkdtemp(join(tmpdir(), "agentcore-code-eval-")); + tempDirectories.push(directory); + process.chdir(directory); + return process.cwd(); +} + +afterEach(async () => { + process.chdir(originalCwd); + await Promise.all( + tempDirectories.splice(0).map((directory) => rm(directory, { recursive: true, force: true })), + ); +}); + +async function run(args: string[]) { + const io = testIO(); + const root = createRootHandler(new TestCoreClient(), { + io: io.io, + globalConfigAccessor: new TestGlobalConfigAccessor(), + logger: createSilentLogger(), + }); + await root.route(["node", "agentcore", "project", ...args]); + return { io }; +} + +async function inProject(name = "TestProject"): Promise { + const directory = await inTempDirectory(); + await run(["create", "--name", name, "--skip-install", "--skip-git"]); + const projectRoot = join(directory, name); + process.chdir(projectRoot); + return projectRoot; +} + +const spec = (projectRoot: string) => + Bun.file(join(projectRoot, "agentcore", "agentcore.json")).json(); +const evaluator = async (projectRoot: string, name: string) => + (await spec(projectRoot)).evaluators.find((e: { name: string }) => e.name === name); + +describe("project add evaluator code-based", () => { + test("3P metric → managed config + scaffolded, rendered Lambda source", async () => { + const projectRoot = await inProject(); + await run([ + "add", + "evaluator", + "code-based", + "--name", + "answer_faithfulness", + "--level", + "SESSION", + "--metric", + "deepeval.FaithfulnessMetric", + "--model", + "bedrock/anthropic.claude-3-5-sonnet-20240620-v1:0", + ]); + + expect(await evaluator(projectRoot, "answer_faithfulness")).toMatchObject({ + name: "answer_faithfulness", + level: "SESSION", + config: { + codeBased: { + managed: { + codeLocation: "app/answer_faithfulness", + entrypoint: "lambda_function.handler", + timeoutSeconds: 300, + additionalPolicies: ["execution-role-policy.json"], + }, + }, + }, + }); + + const appDir = join(projectRoot, "app", "answer_faithfulness"); + const handler = await Bun.file(join(appDir, "lambda_function.py")).text(); + expect(handler).toContain("FaithfulnessMetric"); + expect(handler).toContain("AmazonBedrockModel"); + expect(handler).toContain("anthropic.claude-3-5-sonnet-20240620-v1:0"); + expect(await Bun.file(join(appDir, "execution-role-policy.json")).exists()).toBe(true); + // no unrendered Handlebars left behind + expect(handler).not.toContain("{{"); + }); + + test("autoevals metric with default (non-bedrock) model", async () => { + const projectRoot = await inProject(); + await run([ + "add", + "evaluator", + "code-based", + "--name", + "factuality", + "--level", + "TRACE", + "--metric", + "autoevals.Factuality", + ]); + + expect( + (await evaluator(projectRoot, "factuality")).config.codeBased.managed.timeoutSeconds, + ).toBe(60); + const handler = await Bun.file( + join(projectRoot, "app", "factuality", "lambda_function.py"), + ).text(); + expect(handler).toContain("Factuality"); + expect(handler).not.toContain("{{"); + }); + + test("no metric, no lambda → empty managed stub", async () => { + const projectRoot = await inProject(); + await run(["add", "evaluator", "code-based", "--name", "custom_eval", "--level", "TOOL_CALL"]); + + expect((await evaluator(projectRoot, "custom_eval")).config.codeBased.managed).toMatchObject({ + codeLocation: "app/custom_eval", + timeoutSeconds: 60, + }); + const handler = await Bun.file( + join(projectRoot, "app", "custom_eval", "lambda_function.py"), + ).text(); + expect(handler).toContain("TODO"); + expect(handler).toContain("custom_code_based_evaluator"); + }); + + test("--lambda-arn → external config, no scaffold", async () => { + const projectRoot = await inProject(); + const arn = "arn:aws:lambda:us-west-2:123456789012:function:refund-policy"; + await run([ + "add", + "evaluator", + "code-based", + "--name", + "refund_policy", + "--level", + "SESSION", + "--lambda-arn", + arn, + ]); + + expect((await evaluator(projectRoot, "refund_policy")).config).toEqual({ + codeBased: { external: { lambdaArn: arn } }, + }); + expect( + await Bun.file(join(projectRoot, "app", "refund_policy", "lambda_function.py")).exists(), + ).toBe(false); + }); + + test("persists description, kms key, and tags", async () => { + const projectRoot = await inProject(); + const kms = "arn:aws:kms:us-east-1:123456789012:key/12345678-1234-1234-1234-123456789012"; + await run([ + "add", + "evaluator", + "code-based", + "--name", + "full", + "--level", + "SESSION", + "--lambda-arn", + "arn:aws:lambda:us-west-2:123456789012:function:f", + "--description", + "external scorer", + "--kms-key-arn", + kms, + "--tags", + '{"team":"ml"}', + ]); + + expect(await evaluator(projectRoot, "full")).toMatchObject({ + description: "external scorer", + kmsKeyArn: kms, + tags: { team: "ml" }, + }); + }); + + test.each<[string, string[]]>([ + ["missing --name", ["--level", "SESSION"]], + ["missing --level", ["--name", "x"]], + [ + "--metric and --lambda-arn together", + [ + "--name", + "x", + "--level", + "SESSION", + "--metric", + "deepeval.FaithfulnessMetric", + "--lambda-arn", + "arn:aws:lambda:us-west-2:123456789012:function:f", + ], + ], + [ + "unknown metric library", + ["--name", "x", "--level", "SESSION", "--metric", "ragas.Faithfulness"], + ], + ["metric without a class", ["--name", "x", "--level", "SESSION", "--metric", "deepeval"]], + ["--model without --metric", ["--name", "x", "--level", "SESSION", "--model", "bedrock/foo"]], + [ + "managed flag with --lambda-arn", + [ + "--name", + "x", + "--level", + "SESSION", + "--lambda-arn", + "arn:aws:lambda:us-west-2:123456789012:function:f", + "--timeout-seconds", + "30", + ], + ], + ["invalid --level", ["--name", "x", "--level", "NOPE"]], + ["invalid --lambda-arn", ["--name", "x", "--level", "SESSION", "--lambda-arn", "not-an-arn"]], + ])("%s", async (_label, flags) => { + await inProject(); + await expect(run(["add", "evaluator", "code-based", ...flags])).rejects.toBeInstanceOf( + InputValidationError, + ); + }); + + test("rejects a duplicate evaluator name", async () => { + await inProject(); + const flags = [ + "add", + "evaluator", + "code-based", + "--name", + "dup", + "--level", + "SESSION", + "--lambda-arn", + "arn:aws:lambda:us-west-2:123456789012:function:f", + ]; + await run(flags); + await expect(run(flags)).rejects.toBeInstanceOf(InputValidationError); + }); + + test("remove evaluator drops it from the spec", async () => { + const projectRoot = await inProject(); + await run(["add", "evaluator", "code-based", "--name", "gone", "--level", "SESSION"]); + expect(await evaluator(projectRoot, "gone")).toBeDefined(); + await run(["remove", "evaluator", "--name", "gone"]); + expect(await evaluator(projectRoot, "gone")).toBeUndefined(); + }); +}); diff --git a/src/handlers/project/add/evaluator/code-based/index.ts b/src/handlers/project/add/evaluator/code-based/index.ts new file mode 100644 index 000000000..c92da7c42 --- /dev/null +++ b/src/handlers/project/add/evaluator/code-based/index.ts @@ -0,0 +1,168 @@ +import z from "zod"; +import { createHandler, flag, ProjectKey } from "../../../../../router"; +import { InputValidationError } from "../../../../../errors"; +import { EvaluatorSchema } from "../../../../../projectSchemas/evaluator"; +import { TagsSchema } from "../../../../../projectSchemas/tags"; +import { parseJsonFlagWithSchema } from "../../../../utils"; +import type { AddProjectResourceConfig } from "../../types"; + +// 3P evaluator libraries the CLI can scaffold. The metric class is passed +// through to the library (not allowlisted here) — only the library prefix is +// validated. Default timeouts mirror the old CLI's THIRD_PARTY_EVALUATOR_LIBRARIES. +const LIBRARIES: Record = { + deepeval: { assetDir: "evaluators/deepeval-lambda", defaultTimeoutSeconds: 300 }, + autoevals: { assetDir: "evaluators/autoevals-lambda", defaultTimeoutSeconds: 60 }, +}; + +const EMPTY_ASSET_DIR = "evaluators/python-lambda"; +const EMPTY_DEFAULT_TIMEOUT_SECONDS = 60; + +export const createAddCodeBasedEvaluatorHandler = (config: AddProjectResourceConfig) => + createHandler({ + name: "code-based", + description: + "add a code-based evaluator — a Lambda that scores a session. Pass a 3P metric, an existing Lambda, or neither to scaffold an empty evaluator you fill in", + flags: [ + flag("name", "the name of the evaluator", z.string().optional()), + flag("level", "what to score: SESSION, TRACE, or TOOL_CALL", z.string().optional()), + flag( + "metric", + "3P metric to scaffold as , e.g. deepeval.FaithfulnessMetric or autoevals.Factuality", + z.string().optional(), + ), + flag( + "model", + "judge model for the 3P metric, e.g. bedrock/anthropic.claude-3-5-sonnet-20240620-v1:0", + z.string().optional(), + ), + flag("lambda-arn", "ARN of an existing Lambda that scores a session", z.string().optional()), + flag( + "timeout-seconds", + "Lambda timeout in seconds (1-300)", + z.number().int().min(1).max(300).optional(), + ), + flag("description", "a description of what this evaluator measures", z.string().optional()), + flag( + "kms-key-arn", + "customer-managed KMS key ARN to encrypt the evaluator", + z.string().optional(), + ), + flag("tags", "tags to apply (JSON object of key/value strings)", z.string().optional()), + ], + handle: async (ctx, flags) => { + if (!flags["name"]) + throw new InputValidationError("required option '--name ' not specified"); + if (!flags["level"]) + throw new InputValidationError("required option '--level ' not specified"); + + const hasMetric = flags["metric"] !== undefined; + const hasLambda = flags["lambda-arn"] !== undefined; + if (hasMetric && hasLambda) + throw new InputValidationError( + "provide either --metric (managed) or --lambda-arn (external), not both", + ); + + const tags = parseJsonFlagWithSchema("tags", flags["tags"], TagsSchema); + const base = { + name: flags["name"], + level: flags["level"], + description: flags["description"], + kmsKeyArn: flags["kms-key-arn"], + tags, + }; + + // Kept loose so `--level` stays a plain string for EvaluatorSchema to + // validate (mirrors the llm-as-a-judge handler); safeParse narrows it. + let candidate: Record; + let scaffold: { assetDir: string; context: Record } | undefined; + + if (hasLambda) { + if (flags["metric"] || flags["model"] || flags["timeout-seconds"] !== undefined) + throw new InputValidationError( + "--metric, --model, and --timeout-seconds are managed-only and not valid with --lambda-arn", + ); + candidate = { + ...base, + config: { codeBased: { external: { lambdaArn: flags["lambda-arn"] } } }, + }; + } else { + // managed: 3P metric, or empty stub when no metric is given. + if (flags["model"] && !hasMetric) + throw new InputValidationError("--model requires --metric"); + + let assetDir = EMPTY_ASSET_DIR; + let defaultTimeout = EMPTY_DEFAULT_TIMEOUT_SECONDS; + const context: Record = { Name: toPythonPackageName(flags["name"]) }; + + if (hasMetric) { + const raw = flags["metric"]!; + const dot = raw.indexOf("."); + const library = dot > 0 ? raw.slice(0, dot) : ""; + const metricClass = dot > 0 ? raw.slice(dot + 1) : ""; + const lib = library && metricClass ? LIBRARIES[library] : undefined; + if (!lib) + throw new InputValidationError( + `invalid --metric "${raw}": expected where library is one of ${Object.keys(LIBRARIES).join(", ")} (e.g. deepeval.FaithfulnessMetric)`, + ); + assetDir = lib.assetDir; + defaultTimeout = lib.defaultTimeoutSeconds; + + const { modelProviderBedrock, model } = parseModel(flags["model"]); + context["EvaluatorClass"] = metricClass; + context["Model"] = model; + context["ModelProviderBedrock"] = modelProviderBedrock; + context["EvaluatorParams"] = ""; + } + + const timeoutSeconds = flags["timeout-seconds"] ?? defaultTimeout; + candidate = { + ...base, + config: { + codeBased: { + managed: { + codeLocation: `app/${flags["name"]}`, + entrypoint: "lambda_function.handler", + timeoutSeconds, + additionalPolicies: ["execution-role-policy.json"], + }, + }, + }, + }; + scaffold = { assetDir, context }; + } + + const parsed = EvaluatorSchema.safeParse(candidate); + if (!parsed.success) throw new InputValidationError(z.prettifyError(parsed.error)); + + const project = ctx.require(ProjectKey); + for await (const event of config.projectManager.addResource(project, { + resourceType: "evaluator", + resourceConfig: parsed.data, + scaffold, + })) { + config.io.stderr.write(`${event.message}\n`); + } + + config.io.stderr.write(`added evaluator '${flags["name"]}' to '${project.name}'\n`); + }, + }); + +// `bedrock/` selects the Bedrock judge backend (Model = the id); anything +// else (openai/gpt-4o, a bare name, or unset) falls through to the library's +// default model. +function parseModel(model: string | undefined): { modelProviderBedrock: boolean; model: string } { + if (!model) return { modelProviderBedrock: false, model: "" }; + const slash = model.indexOf("/"); + const provider = slash > 0 ? model.slice(0, slash) : ""; + if (provider === "bedrock") return { modelProviderBedrock: true, model: model.slice(slash + 1) }; + return { modelProviderBedrock: false, model }; +} + +// PEP 508 package name: ASCII letters/numbers/period/underscore/hyphen, must +// start and end alphanumeric. Mirrors the runtime template helper. +function toPythonPackageName(name: string): string { + return name + .replace(/[^a-zA-Z0-9._-]/g, "-") + .replace(/^[^a-zA-Z0-9]+/, "") + .replace(/[^a-zA-Z0-9]+$/, ""); +} diff --git a/src/handlers/project/add/evaluator/index.ts b/src/handlers/project/add/evaluator/index.ts index 5dbc413ab..1c05ea3be 100644 --- a/src/handlers/project/add/evaluator/index.ts +++ b/src/handlers/project/add/evaluator/index.ts @@ -1,9 +1,11 @@ import { Router } from "../../../../router"; import type { AddProjectResourceConfig } from "../types"; import { createAddLlmAsAJudgeEvaluatorHandler } from "./llm-as-a-judge"; +import { createAddCodeBasedEvaluatorHandler } from "./code-based"; export function createAddEvaluatorHandler(config: AddProjectResourceConfig): Router { const evaluator = new Router("evaluator", "add a custom evaluator to the current project"); evaluator.handler(createAddLlmAsAJudgeEvaluatorHandler(config)); + evaluator.handler(createAddCodeBasedEvaluatorHandler(config)); return evaluator; } diff --git a/src/handlers/project/remove/index.ts b/src/handlers/project/remove/index.ts index 8d5973d92..308afe1fb 100644 --- a/src/handlers/project/remove/index.ts +++ b/src/handlers/project/remove/index.ts @@ -26,6 +26,7 @@ export const createRemoveProjectHandler = (config: RemoveProjectResourceConfig) .enum([ "harness", "runtime", + "evaluator", "gateway", "gateway-target", "gateway-connector", diff --git a/src/handlers/project/types.ts b/src/handlers/project/types.ts index 19b3a8ef4..2e6f814dd 100644 --- a/src/handlers/project/types.ts +++ b/src/handlers/project/types.ts @@ -160,6 +160,13 @@ export type AddResourceInput = | { resourceType: "evaluator"; resourceConfig: z.input; + /** + * Present only for managed code-based evaluators, whose code the CLI + * generates. `assetDir` picks the template under src/assets/evaluators; + * `context` holds its Handlebars variables. External and llm-as-a-judge + * evaluators omit this and are spec-only. + */ + scaffold?: { assetDir: string; context: Record }; } | { resourceType: "gateway"; From d8eb5227043d0a3543287d76d88c02cda7385a2d Mon Sep 17 00:00:00 2001 From: jariy17 Date: Fri, 28 Aug 2026 22:09:06 +0000 Subject: [PATCH 13/15] fix(project): guard against app/ collisions when scaffolding evaluators Runtimes, harnesses, and evaluators all scaffold into app/, but the duplicate-name guard is per-resource-type and the tree write happens outside the rollback try/catch. An evaluator whose name matches an existing runtime/ harness dir (or a leftover from a removed evaluator) threw a raw 'File already exists' mid-write and orphaned partial files. Fail up front with a clear InputValidationError when app/ already exists. --- src/core/project/manager.tsx | 8 ++++++++ .../add/evaluator/code-based/index.test.ts | 20 +++++++++++++++++-- 2 files changed, 26 insertions(+), 2 deletions(-) diff --git a/src/core/project/manager.tsx b/src/core/project/manager.tsx index 5918b94dc..03ddc1545 100644 --- a/src/core/project/manager.tsx +++ b/src/core/project/manager.tsx @@ -265,6 +265,14 @@ export class FsProjectManager implements ProjectManager { if (input.scaffold) { yield { message: "Scaffolding evaluator in project" }; const outputPath = join(project.rootPath, "app", evaluator.name); + // Runtimes, harnesses, and evaluators all scaffold into app/, + // but the dup-name guard above is per-resource-type. Fail up front + // rather than let FsTreeNode.write throw mid-write (outside the + // rollback try/catch below) and orphan partial files. + if (existsSync(outputPath)) + throw new InputValidationError( + `cannot scaffold evaluator '${evaluator.name}': 'app/${evaluator.name}' already exists (another resource may use this name, or a previous scaffold was left behind)`, + ); scaffoldedPaths.push(outputPath); const resolver = getEvaluatorTemplateResolver({ assetSource: this.assetSource, diff --git a/src/handlers/project/add/evaluator/code-based/index.test.ts b/src/handlers/project/add/evaluator/code-based/index.test.ts index 307d15878..a60f1f0e7 100644 --- a/src/handlers/project/add/evaluator/code-based/index.test.ts +++ b/src/handlers/project/add/evaluator/code-based/index.test.ts @@ -1,5 +1,5 @@ import { afterEach, describe, expect, test } from "bun:test"; -import { mkdtemp, rm } from "node:fs/promises"; +import { mkdir, mkdtemp, rm } from "node:fs/promises"; import { join } from "node:path"; import { tmpdir } from "node:os"; import { createRootHandler } from "../../../../index"; @@ -50,7 +50,7 @@ async function inProject(name = "TestProject"): Promise { const spec = (projectRoot: string) => Bun.file(join(projectRoot, "agentcore", "agentcore.json")).json(); const evaluator = async (projectRoot: string, name: string) => - (await spec(projectRoot)).evaluators.find((e: { name: string }) => e.name === name); + ((await spec(projectRoot)).evaluators ?? []).find((e: { name: string }) => e.name === name); describe("project add evaluator code-based", () => { test("3P metric → managed config + scaffolded, rendered Lambda source", async () => { @@ -245,6 +245,22 @@ describe("project add evaluator code-based", () => { await expect(run(flags)).rejects.toBeInstanceOf(InputValidationError); }); + test("errors before writing when app/ already exists (cross-resource collision)", async () => { + const projectRoot = await inProject(); + const appDir = join(projectRoot, "app", "collide"); + await mkdir(appDir, { recursive: true }); + await Bun.write(join(appDir, "pyproject.toml"), "# pre-existing\n"); + + await expect( + run(["add", "evaluator", "code-based", "--name", "collide", "--level", "SESSION"]), + ).rejects.toBeInstanceOf(InputValidationError); + + // no spec entry, and the pre-existing dir is left untouched (no mid-write orphans) + expect(await evaluator(projectRoot, "collide")).toBeUndefined(); + expect(await Bun.file(join(appDir, "pyproject.toml")).text()).toBe("# pre-existing\n"); + expect(await Bun.file(join(appDir, "lambda_function.py")).exists()).toBe(false); + }); + test("remove evaluator drops it from the spec", async () => { const projectRoot = await inProject(); await run(["add", "evaluator", "code-based", "--name", "gone", "--level", "SESSION"]); From 821b337d2728726f2a86e0c87839aded70343946 Mon Sep 17 00:00:00 2001 From: jariy17 Date: Fri, 28 Aug 2026 22:12:15 +0000 Subject: [PATCH 14/15] fix(project): validate --metric class and require a Bedrock --model for code-based evaluators - Reject a namespaced/multi-dot metric class (e.g. deepeval.metrics.Faithfulness) that would render invalid Python; require a single class identifier. - --model is Bedrock-only: accept a bare model id / inference-profile-or- foundation-model ARN, optionally prefixed with bedrock/, validated via isValidBedrockModelId (same forms the llm-as-a-judge handler accepts). Non-Bedrock or slashless values now error instead of being silently dropped (deepeval) or passed to the wrong client (autoevals). - autoevals template prefixes bedrock/ for litellm routing now that Model is the bare id. --- .../autoevals-lambda/lambda_function.py | 4 +- .../add/evaluator/code-based/index.test.ts | 50 +++++++++++++++++++ .../project/add/evaluator/code-based/index.ts | 39 ++++++++++----- 3 files changed, 78 insertions(+), 15 deletions(-) diff --git a/src/assets/evaluators/autoevals-lambda/lambda_function.py b/src/assets/evaluators/autoevals-lambda/lambda_function.py index 410f056a6..4161dbbf7 100644 --- a/src/assets/evaluators/autoevals-lambda/lambda_function.py +++ b/src/assets/evaluators/autoevals-lambda/lambda_function.py @@ -15,9 +15,9 @@ from bedrock_agentcore.evaluation.custom_code_based_evaluators.third_party.autoevals import AutoEvalsAdapter client = LiteLLMClient() -init(client=client, default_model="{{ Model }}") +init(client=client, default_model="bedrock/{{ Model }}") -adapter = AutoEvalsAdapter(metric={{ EvaluatorClass }}(client=client, model="{{ Model }}"){{#if EvaluatorParams}}, {{{ EvaluatorParams }}}{{/if}}) +adapter = AutoEvalsAdapter(metric={{ EvaluatorClass }}(client=client, model="bedrock/{{ Model }}"){{#if EvaluatorParams}}, {{{ EvaluatorParams }}}{{/if}}) {{else}} from autoevals import {{ EvaluatorClass }} diff --git a/src/handlers/project/add/evaluator/code-based/index.test.ts b/src/handlers/project/add/evaluator/code-based/index.test.ts index a60f1f0e7..846f36194 100644 --- a/src/handlers/project/add/evaluator/code-based/index.test.ts +++ b/src/handlers/project/add/evaluator/code-based/index.test.ts @@ -205,6 +205,36 @@ describe("project add evaluator code-based", () => { ["--name", "x", "--level", "SESSION", "--metric", "ragas.Faithfulness"], ], ["metric without a class", ["--name", "x", "--level", "SESSION", "--metric", "deepeval"]], + [ + "namespaced (multi-dot) metric class", + ["--name", "x", "--level", "SESSION", "--metric", "deepeval.metrics.Faithfulness"], + ], + [ + "non-Bedrock --model", + [ + "--name", + "x", + "--level", + "SESSION", + "--metric", + "deepeval.FaithfulnessMetric", + "--model", + "gpt-4o", + ], + ], + [ + "--model bedrock with no slash/id", + [ + "--name", + "x", + "--level", + "SESSION", + "--metric", + "autoevals.Factuality", + "--model", + "bedrock", + ], + ], ["--model without --metric", ["--name", "x", "--level", "SESSION", "--model", "bedrock/foo"]], [ "managed flag with --lambda-arn", @@ -228,6 +258,26 @@ describe("project add evaluator code-based", () => { ); }); + test("accepts a bare Bedrock inference-profile model id and renders it into the source", async () => { + const projectRoot = await inProject(); + await run([ + "add", + "evaluator", + "code-based", + "--name", + "prof", + "--level", + "SESSION", + "--metric", + "deepeval.FaithfulnessMetric", + "--model", + "us.anthropic.claude-sonnet-4-5-20250929-v1:0", + ]); + expect(await evaluator(projectRoot, "prof")).toBeDefined(); + const src = await Bun.file(join(projectRoot, "app", "prof", "lambda_function.py")).text(); + expect(src).toContain("us.anthropic.claude-sonnet-4-5-20250929-v1:0"); + }); + test("rejects a duplicate evaluator name", async () => { await inProject(); const flags = [ diff --git a/src/handlers/project/add/evaluator/code-based/index.ts b/src/handlers/project/add/evaluator/code-based/index.ts index c92da7c42..3b780707b 100644 --- a/src/handlers/project/add/evaluator/code-based/index.ts +++ b/src/handlers/project/add/evaluator/code-based/index.ts @@ -1,7 +1,7 @@ import z from "zod"; import { createHandler, flag, ProjectKey } from "../../../../../router"; import { InputValidationError } from "../../../../../errors"; -import { EvaluatorSchema } from "../../../../../projectSchemas/evaluator"; +import { EvaluatorSchema, isValidBedrockModelId } from "../../../../../projectSchemas/evaluator"; import { TagsSchema } from "../../../../../projectSchemas/tags"; import { parseJsonFlagWithSchema } from "../../../../utils"; import type { AddProjectResourceConfig } from "../../types"; @@ -104,13 +104,21 @@ export const createAddCodeBasedEvaluatorHandler = (config: AddProjectResourceCon throw new InputValidationError( `invalid --metric "${raw}": expected where library is one of ${Object.keys(LIBRARIES).join(", ")} (e.g. deepeval.FaithfulnessMetric)`, ); + // The class is rendered straight into `from import ` and + // `(...)`, so it must be a single Python identifier. Reject + // dotted/namespaced values (e.g. deepeval.metrics.Faithfulness) that + // would emit invalid Python and only fail at deploy/runtime. + if (!/^[A-Za-z_][A-Za-z0-9_]*$/.test(metricClass)) + throw new InputValidationError( + `invalid metric class "${metricClass}" in --metric "${raw}": expected a single class name like FaithfulnessMetric`, + ); assetDir = lib.assetDir; defaultTimeout = lib.defaultTimeoutSeconds; - const { modelProviderBedrock, model } = parseModel(flags["model"]); + const model = resolveBedrockModel(flags["model"]); context["EvaluatorClass"] = metricClass; - context["Model"] = model; - context["ModelProviderBedrock"] = modelProviderBedrock; + context["Model"] = model ?? ""; + context["ModelProviderBedrock"] = model !== undefined; context["EvaluatorParams"] = ""; } @@ -147,15 +155,20 @@ export const createAddCodeBasedEvaluatorHandler = (config: AddProjectResourceCon }, }); -// `bedrock/` selects the Bedrock judge backend (Model = the id); anything -// else (openai/gpt-4o, a bare name, or unset) falls through to the library's -// default model. -function parseModel(model: string | undefined): { modelProviderBedrock: boolean; model: string } { - if (!model) return { modelProviderBedrock: false, model: "" }; - const slash = model.indexOf("/"); - const provider = slash > 0 ? model.slice(0, slash) : ""; - if (provider === "bedrock") return { modelProviderBedrock: true, model: model.slice(slash + 1) }; - return { modelProviderBedrock: false, model }; +// Judge model for a 3P metric. Bedrock-only: accepts a bare Bedrock model id / +// inference-profile-or-foundation-model ARN, optionally prefixed with +// `bedrock/`, and returns the id with the prefix stripped. Anything else errors +// rather than silently degrading — a non-Bedrock value is dropped by the +// deepeval template and passed to the wrong client by autoevals. Returns +// undefined when no --model was given (library default). +function resolveBedrockModel(model: string | undefined): string | undefined { + if (!model) return undefined; + const id = model.startsWith("bedrock/") ? model.slice("bedrock/".length) : model; + if (!isValidBedrockModelId(id)) + throw new InputValidationError( + `invalid --model "${model}": expected a Bedrock model ID (e.g. anthropic.claude-3-5-sonnet-20240620-v1:0) or an inference-profile/foundation-model ARN, optionally prefixed with "bedrock/"`, + ); + return id; } // PEP 508 package name: ASCII letters/numbers/period/underscore/hyphen, must From 303c8db69d183bf2f11c9eef7ea82e71e7175759 Mon Sep 17 00:00:00 2001 From: jariy17 Date: Fri, 28 Aug 2026 22:13:03 +0000 Subject: [PATCH 15/15] fix(project): echo inferred mode caveats for code-based evaluators Print notes after add: the empty stub returns Pass for every session until implemented, and managed evaluators are scaffolded but not yet provisioned by 'project deploy' (no CDK/L3 support). External (--lambda-arn) prints neither. --- .../add/evaluator/code-based/index.test.ts | 32 +++++++++++++++++++ .../project/add/evaluator/code-based/index.ts | 12 +++++++ 2 files changed, 44 insertions(+) diff --git a/src/handlers/project/add/evaluator/code-based/index.test.ts b/src/handlers/project/add/evaluator/code-based/index.test.ts index 846f36194..3d67f6e79 100644 --- a/src/handlers/project/add/evaluator/code-based/index.test.ts +++ b/src/handlers/project/add/evaluator/code-based/index.test.ts @@ -311,6 +311,38 @@ describe("project add evaluator code-based", () => { expect(await Bun.file(join(appDir, "lambda_function.py")).exists()).toBe(false); }); + test("empty stub warns it returns Pass until implemented, plus the not-deployed note", async () => { + await inProject(); + const { io } = await run([ + "add", + "evaluator", + "code-based", + "--name", + "stub", + "--level", + "SESSION", + ]); + expect(io.stderr()).toContain("returns Pass for every session"); + expect(io.stderr()).toContain("not yet provisioned"); + }); + + test("external mode prints neither managed note", async () => { + await inProject(); + const { io } = await run([ + "add", + "evaluator", + "code-based", + "--name", + "ext", + "--level", + "SESSION", + "--lambda-arn", + "arn:aws:lambda:us-west-2:123456789012:function:f", + ]); + expect(io.stderr()).not.toContain("returns Pass for every session"); + expect(io.stderr()).not.toContain("not yet provisioned"); + }); + test("remove evaluator drops it from the spec", async () => { const projectRoot = await inProject(); await run(["add", "evaluator", "code-based", "--name", "gone", "--level", "SESSION"]); diff --git a/src/handlers/project/add/evaluator/code-based/index.ts b/src/handlers/project/add/evaluator/code-based/index.ts index 3b780707b..60bfe62fc 100644 --- a/src/handlers/project/add/evaluator/code-based/index.ts +++ b/src/handlers/project/add/evaluator/code-based/index.ts @@ -152,6 +152,18 @@ export const createAddCodeBasedEvaluatorHandler = (config: AddProjectResourceCon } config.io.stderr.write(`added evaluator '${flags["name"]}' to '${project.name}'\n`); + // Make the inferred mode and its caveats visible: the empty stub silently + // passes every session, and no managed evaluator is provisioned by deploy + // yet (no CDK/L3 support) — both are silent footguns otherwise. + if (!hasLambda) { + if (!hasMetric) + config.io.stderr.write( + `note: this evaluator returns Pass for every session until you implement app/${flags["name"]}/lambda_function.py\n`, + ); + config.io.stderr.write( + `note: managed code-based evaluators are scaffolded locally but not yet provisioned by 'project deploy' (pending CDK/L3 support)\n`, + ); + } }, });