diff --git a/src/assets/evaluators/autoevals-lambda/execution-role-policy.json b/src/assets/evaluators/autoevals-lambda/execution-role-policy.json new file mode 100644 index 000000000..6b47af830 --- /dev/null +++ b/src/assets/evaluators/autoevals-lambda/execution-role-policy.json @@ -0,0 +1,15 @@ +{ + "Version": "2012-10-17", + "Statement": [ + { + "Effect": "Allow", + "Action": ["logs:CreateLogGroup", "logs:CreateLogStream", "logs:PutLogEvents"], + "Resource": "arn:*:logs:*:*:log-group:/aws/lambda/*" + }, + { + "Effect": "Allow", + "Action": ["bedrock:InvokeModel"], + "Resource": "*" + } + ] +} diff --git a/src/assets/evaluators/autoevals-lambda/lambda_function.py b/src/assets/evaluators/autoevals-lambda/lambda_function.py new file mode 100644 index 000000000..4161dbbf7 --- /dev/null +++ b/src/assets/evaluators/autoevals-lambda/lambda_function.py @@ -0,0 +1,37 @@ +{{#if ModelProviderBedrock}} +import os + +# litellm's Bedrock provider reads AWS_REGION_NAME; Lambda only sets AWS_REGION/AWS_DEFAULT_REGION. +os.environ.setdefault("AWS_REGION_NAME", os.environ.get("AWS_REGION", "us-west-2")) + +from autoevals import {{ EvaluatorClass }}, init +from autoevals.litellm import LiteLLMClient + +from bedrock_agentcore.evaluation.custom_code_based_evaluators import ( + EvaluatorInput, + EvaluatorOutput, + custom_code_based_evaluator, +) +from bedrock_agentcore.evaluation.custom_code_based_evaluators.third_party.autoevals import AutoEvalsAdapter + +client = LiteLLMClient() +init(client=client, default_model="bedrock/{{ Model }}") + +adapter = AutoEvalsAdapter(metric={{ EvaluatorClass }}(client=client, model="bedrock/{{ Model }}"){{#if EvaluatorParams}}, {{{ EvaluatorParams }}}{{/if}}) +{{else}} +from autoevals import {{ EvaluatorClass }} + +from bedrock_agentcore.evaluation.custom_code_based_evaluators import ( + EvaluatorInput, + EvaluatorOutput, + custom_code_based_evaluator, +) +from bedrock_agentcore.evaluation.custom_code_based_evaluators.third_party.autoevals import AutoEvalsAdapter + +adapter = AutoEvalsAdapter(metric={{ EvaluatorClass }}({{#if Model}}model="{{ Model }}"{{/if}}){{#if EvaluatorParams}}, {{{ EvaluatorParams }}}{{/if}}) +{{/if}} + + +@custom_code_based_evaluator() +def handler(evaluator_input: EvaluatorInput, context) -> EvaluatorOutput: + return adapter(evaluator_input, context) diff --git a/src/assets/evaluators/autoevals-lambda/pyproject.toml b/src/assets/evaluators/autoevals-lambda/pyproject.toml new file mode 100644 index 000000000..b37442fd1 --- /dev/null +++ b/src/assets/evaluators/autoevals-lambda/pyproject.toml @@ -0,0 +1,22 @@ +[build-system] +requires = ["hatchling"] +build-backend = "hatchling.build" + +[project] +name = "{{ Name }}" +version = "0.1.0" +description = "AgentCore Code-Based Evaluator (Autoevals)" +requires-python = ">=3.10" +dependencies = [ + "bedrock-agentcore[autoevals]", + "autoevals>=0.0.80,<1.0.0", +{{#if ModelProviderBedrock}} + # autoevals grades via LiteLLMClient -> Bedrock (Converse); litellm replaces the openai judge + "litellm>=1.60,<1.85", +{{else}} + "openai>=1.0.0", +{{/if}} +] + +[tool.hatch.build.targets.wheel] +packages = ["."] diff --git a/src/assets/evaluators/deepeval-lambda/execution-role-policy.json b/src/assets/evaluators/deepeval-lambda/execution-role-policy.json new file mode 100644 index 000000000..6b47af830 --- /dev/null +++ b/src/assets/evaluators/deepeval-lambda/execution-role-policy.json @@ -0,0 +1,15 @@ +{ + "Version": "2012-10-17", + "Statement": [ + { + "Effect": "Allow", + "Action": ["logs:CreateLogGroup", "logs:CreateLogStream", "logs:PutLogEvents"], + "Resource": "arn:*:logs:*:*:log-group:/aws/lambda/*" + }, + { + "Effect": "Allow", + "Action": ["bedrock:InvokeModel"], + "Resource": "*" + } + ] +} diff --git a/src/assets/evaluators/deepeval-lambda/lambda_function.py b/src/assets/evaluators/deepeval-lambda/lambda_function.py new file mode 100644 index 000000000..a89b22051 --- /dev/null +++ b/src/assets/evaluators/deepeval-lambda/lambda_function.py @@ -0,0 +1,29 @@ +import os + +os.environ.setdefault("DEEPEVAL_RESULTS_FOLDER", "/tmp/.deepeval") +os.environ.setdefault("DEEPEVAL_TELEMETRY_OPT_OUT", "YES") +os.chdir("/tmp") + +{{#if ModelProviderBedrock}} +from deepeval.models import AmazonBedrockModel +{{/if}} +from deepeval.metrics import {{ EvaluatorClass }} + +from bedrock_agentcore.evaluation.custom_code_based_evaluators import ( + EvaluatorInput, + EvaluatorOutput, + custom_code_based_evaluator, +) +from bedrock_agentcore.evaluation.custom_code_based_evaluators.third_party.deepeval import DeepEvalAdapter + +{{#if ModelProviderBedrock}} +model = AmazonBedrockModel(model="{{ Model }}", region=os.environ.get("AWS_REGION", "us-west-2")) +adapter = DeepEvalAdapter(metric={{ EvaluatorClass }}(model=model{{#if EvaluatorParams}}, {{{ EvaluatorParams }}}{{/if}})) +{{else}} +adapter = DeepEvalAdapter(metric={{ EvaluatorClass }}({{{ EvaluatorParams }}})) +{{/if}} + + +@custom_code_based_evaluator() +def handler(evaluator_input: EvaluatorInput, context) -> EvaluatorOutput: + return adapter(evaluator_input, context) diff --git a/src/assets/evaluators/deepeval-lambda/pyproject.toml b/src/assets/evaluators/deepeval-lambda/pyproject.toml new file mode 100644 index 000000000..7385ccfc2 --- /dev/null +++ b/src/assets/evaluators/deepeval-lambda/pyproject.toml @@ -0,0 +1,19 @@ +[build-system] +requires = ["hatchling"] +build-backend = "hatchling.build" + +[project] +name = "{{ Name }}" +version = "0.1.0" +description = "AgentCore Code-Based Evaluator (DeepEval)" +requires-python = ">=3.10" +dependencies = [ + "bedrock-agentcore[deepeval]", + "deepeval>=2.0.0,<3.0.0", +{{#if ModelProviderBedrock}} + "aiobotocore>=2.13.0", +{{/if}} +] + +[tool.hatch.build.targets.wheel] +packages = ["."] diff --git a/src/assets/evaluators/python-lambda/execution-role-policy.json b/src/assets/evaluators/python-lambda/execution-role-policy.json new file mode 100644 index 000000000..b3b98be42 --- /dev/null +++ b/src/assets/evaluators/python-lambda/execution-role-policy.json @@ -0,0 +1,10 @@ +{ + "Version": "2012-10-17", + "Statement": [ + { + "Effect": "Allow", + "Action": ["logs:CreateLogGroup", "logs:CreateLogStream", "logs:PutLogEvents"], + "Resource": "arn:*:logs:*:*:log-group:/aws/lambda/*" + } + ] +} diff --git a/src/assets/evaluators/python-lambda/lambda_function.py b/src/assets/evaluators/python-lambda/lambda_function.py new file mode 100644 index 000000000..0f8bd5c6b --- /dev/null +++ b/src/assets/evaluators/python-lambda/lambda_function.py @@ -0,0 +1,19 @@ +from bedrock_agentcore.evaluation.custom_code_based_evaluators import ( + custom_code_based_evaluator, + EvaluatorInput, + EvaluatorOutput, +) + + +@custom_code_based_evaluator() +def handler(input: EvaluatorInput, context) -> EvaluatorOutput: + """Evaluate agent behavior with custom logic. + + Args: + input: Contains evaluation_level, session_spans, target_trace_id, target_span_id + + Returns: + EvaluatorOutput with value/label for success, or errorCode/errorMessage for failure. + """ + # TODO: Replace with your evaluation logic + return EvaluatorOutput(value=1.0, label="Pass", explanation="Evaluation passed") diff --git a/src/assets/evaluators/python-lambda/pyproject.toml b/src/assets/evaluators/python-lambda/pyproject.toml new file mode 100644 index 000000000..69ad99b43 --- /dev/null +++ b/src/assets/evaluators/python-lambda/pyproject.toml @@ -0,0 +1,15 @@ +[build-system] +requires = ["hatchling"] +build-backend = "hatchling.build" + +[project] +name = "{{ Name }}" +version = "0.1.0" +description = "AgentCore Code-Based Evaluator" +requires-python = ">=3.10" +dependencies = [ + "bedrock-agentcore>=1.6.0", +] + +[tool.hatch.build.targets.wheel] +packages = ["."] diff --git a/src/core/abTestExecutionRole.test.ts b/src/core/abTestExecutionRole.test.ts new file mode 100644 index 000000000..7409d61c7 --- /dev/null +++ b/src/core/abTestExecutionRole.test.ts @@ -0,0 +1,100 @@ +import { test, expect, describe } from "bun:test"; +import { CreateRoleCommand, GetRoleCommand, type IAMClient } from "@aws-sdk/client-iam"; +import { + abTestExecutionRoleName, + accountIdFromArn, + provisionAbTestRole, +} from "./abTestExecutionRole"; + +const GATEWAY_ARN = "arn:aws:bedrock-agentcore:us-west-2:123456789012:gateway/orders-gw"; + +type Sent = { name: string; input: unknown }; + +function fakeIam(onGet: "found" | "missing"): { iam: IAMClient; sent: Sent[] } { + const sent: Sent[] = []; + const iam = { + send: async (command: { constructor: { name: string }; input: unknown }) => { + sent.push({ name: command.constructor.name, input: command.input }); + if (command instanceof GetRoleCommand) { + if (onGet === "missing") { + throw Object.assign(new Error("no such entity"), { name: "NoSuchEntityException" }); + } + return { + Role: { + Arn: `arn:aws:iam::123456789012:role/${(command.input as { RoleName: string }).RoleName}`, + }, + }; + } + if (command instanceof CreateRoleCommand) { + return { + Role: { + Arn: `arn:aws:iam::123456789012:role/${(command.input as { RoleName: string }).RoleName}`, + }, + }; + } + return {}; + }, + } as unknown as IAMClient; + return { iam, sent }; +} + +describe("abTestExecutionRoleName", () => { + test("stays within IAM's 64-char limit and is deterministic", () => { + const long = abTestExecutionRoleName("x".repeat(120)); + expect(long.length).toBeLessThanOrEqual(64); + expect(abTestExecutionRoleName("orders")).toBe(abTestExecutionRoleName("orders")); + }); + + test("distinct names for distinct tests", () => { + expect(abTestExecutionRoleName("a")).not.toBe(abTestExecutionRoleName("b")); + }); +}); + +describe("accountIdFromArn", () => { + test("extracts the account segment", () => { + expect(accountIdFromArn(GATEWAY_ARN)).toBe("123456789012"); + }); + test("throws on a malformed ARN", () => { + expect(() => accountIdFromArn("not-an-arn")).toThrow(/account id/); + }); +}); + +describe("provisionAbTestRole", () => { + test("creates the role + inline policy and reports created=true", async () => { + const { iam, sent } = fakeIam("missing"); + const result = await provisionAbTestRole(iam, "orders-v2", GATEWAY_ARN, "us-west-2"); + + expect(result.created).toBe(true); + expect(result.roleArn).toContain(":role/"); + expect(sent.map((s) => s.name)).toEqual([ + "GetRoleCommand", + "CreateRoleCommand", + "PutRolePolicyCommand", + ]); + + const create = sent.find((s) => s.name === "CreateRoleCommand")!.input as { + AssumeRolePolicyDocument: string; + }; + const trust = JSON.parse(create.AssumeRolePolicyDocument); + expect(trust.Statement[0].Principal.Service).toBe("bedrock-agentcore.amazonaws.com"); + expect(trust.Statement[0].Condition.StringEquals["aws:SourceAccount"]).toBe("123456789012"); + expect(trust.Statement[0].Condition.ArnLike["aws:SourceArn"]).toContain(":ab-test/*"); + + const policy = sent.find((s) => s.name === "PutRolePolicyCommand")!.input as { + PolicyDocument: string; + }; + const doc = JSON.parse(policy.PolicyDocument); + const actions = doc.Statement.flatMap((s: { Action: string[] }) => s.Action); + expect(actions).toContain("bedrock-agentcore:GetGateway"); + expect(actions).toContain("bedrock-agentcore:GetConfigurationBundleVersion"); + expect(actions).toContain("bedrock-agentcore:GetOnlineEvaluationConfig"); + }); + + test("reuses an existing role and reports created=false", async () => { + const { iam, sent } = fakeIam("found"); + const result = await provisionAbTestRole(iam, "orders-v2", GATEWAY_ARN, "us-west-2"); + + expect(result.created).toBe(false); + expect(sent.map((s) => s.name)).toEqual(["GetRoleCommand", "PutRolePolicyCommand"]); + }); +}); diff --git a/src/core/abTestExecutionRole.tsx b/src/core/abTestExecutionRole.tsx new file mode 100644 index 000000000..721f4d026 --- /dev/null +++ b/src/core/abTestExecutionRole.tsx @@ -0,0 +1,155 @@ +import { + CreateRoleCommand, + GetRoleCommand, + PutRolePolicyCommand, + DeleteRoleCommand, + DeleteRolePolicyCommand, + type IAMClient, +} from "@aws-sdk/client-iam"; +import { createHash } from "node:crypto"; + +const AB_TEST_POLICY_NAME = "ABTestExecutionPolicy"; + +export function abTestExecutionRoleName(testName: string): string { + const hash = createHash("sha256").update(`ab-test:${testName}`).digest("hex").slice(0, 8); + const base = `AgentCoreABTest-${testName}`; + return `${base.slice(0, 55)}-${hash}`; +} + +export function roleNameFromArn(roleArn: string): string { + const parts = roleArn.split("/"); + return parts[parts.length - 1] ?? roleArn; +} + +export function accountIdFromArn(arn: string): string { + const accountId = arn.split(":")[4]; + if (!accountId) throw new Error(`could not extract account id from ARN: ${arn}`); + return accountId; +} + +function trustPolicy(accountId: string, region: string): string { + return JSON.stringify({ + Version: "2012-10-17", + Statement: [ + { + Effect: "Allow", + Principal: { Service: "bedrock-agentcore.amazonaws.com" }, + Action: "sts:AssumeRole", + Condition: { + StringEquals: { "aws:SourceAccount": accountId }, + ArnLike: { + "aws:SourceArn": `arn:aws:bedrock-agentcore:${region}:${accountId}:ab-test/*`, + }, + }, + }, + ], + }); +} + +function executionPolicy(accountId: string, region: string): string { + return JSON.stringify({ + Version: "2012-10-17", + Statement: [ + { + Sid: "AgentCoreResources", + Effect: "Allow", + Action: [ + "bedrock-agentcore:GetGateway", + "bedrock-agentcore:GetGatewayTarget", + "bedrock-agentcore:ListGatewayTargets", + "bedrock-agentcore:CreateGatewayRule", + "bedrock-agentcore:UpdateGatewayRule", + "bedrock-agentcore:GetGatewayRule", + "bedrock-agentcore:DeleteGatewayRule", + "bedrock-agentcore:ListGatewayRules", + "bedrock-agentcore:GetOnlineEvaluationConfig", + "bedrock-agentcore:GetEvaluator", + "bedrock-agentcore:GetConfigurationBundle", + "bedrock-agentcore:GetConfigurationBundleVersion", + "bedrock-agentcore:ListConfigurationBundleVersions", + ], + Resource: `arn:aws:bedrock-agentcore:${region}:${accountId}:*`, + Condition: { StringEquals: { "aws:ResourceAccount": accountId } }, + }, + { + Sid: "CloudWatchLogsDescribe", + Effect: "Allow", + Action: ["logs:DescribeLogGroups"], + Resource: "*", + }, + { + Sid: "CloudWatchLogs", + Effect: "Allow", + Action: [ + "logs:DescribeIndexPolicies", + "logs:PutIndexPolicy", + "logs:StartQuery", + "logs:GetQueryResults", + "logs:StopQuery", + "logs:FilterLogEvents", + "logs:GetLogEvents", + ], + Resource: [ + `arn:aws:logs:${region}:${accountId}:log-group:/aws/bedrock-agentcore/evaluations/*`, + `arn:aws:logs:${region}:${accountId}:log-group:/aws/bedrock-agentcore/runtimes/*`, + `arn:aws:logs:${region}:${accountId}:log-group:aws/spans`, + `arn:aws:logs:${region}:${accountId}:log-group:aws/spans:*`, + ], + }, + ], + }); +} + +export async function provisionAbTestRole( + iam: IAMClient, + testName: string, + gatewayArn: string, + region: string, +): Promise<{ roleArn: string; created: boolean }> { + const accountId = accountIdFromArn(gatewayArn); + const roleName = abTestExecutionRoleName(testName); + + let roleArn: string; + let created = false; + try { + const existing = await iam.send(new GetRoleCommand({ RoleName: roleName })); + roleArn = existing.Role!.Arn!; + } catch (error) { + if ((error as Error).name !== "NoSuchEntityException") throw error; + const result = await iam.send( + new CreateRoleCommand({ + RoleName: roleName, + AssumeRolePolicyDocument: trustPolicy(accountId, region), + Description: `Execution role for AgentCore A/B test "${testName}" (created by agentcore CLI)`, + }), + ); + roleArn = result.Role!.Arn!; + created = true; + } + + await iam.send( + new PutRolePolicyCommand({ + RoleName: roleName, + PolicyName: AB_TEST_POLICY_NAME, + PolicyDocument: executionPolicy(accountId, region), + }), + ); + + return { roleArn, created }; +} + +export async function deleteAbTestRole(iam: IAMClient, roleArn: string): Promise { + const roleName = roleNameFromArn(roleArn); + try { + await iam.send( + new DeleteRolePolicyCommand({ RoleName: roleName, PolicyName: AB_TEST_POLICY_NAME }), + ); + } catch { + void 0; + } + try { + await iam.send(new DeleteRoleCommand({ RoleName: roleName })); + } catch { + void 0; + } +} diff --git a/src/core/eval.tsx b/src/core/eval.tsx index fd51e04ee..da6458351 100644 --- a/src/core/eval.tsx +++ b/src/core/eval.tsx @@ -16,6 +16,7 @@ import { GetDatasetCommand, GetEvaluatorCommand, GetHarnessCommand, + GetGatewayCommand, GetOnlineEvaluationConfigCommand, ListConfigurationBundlesCommand, ListConfigurationBundleVersionsCommand, @@ -60,6 +61,7 @@ import { import { DeleteRecommendationCommand, EvaluateCommand, + CreateABTestCommand, GetABTestCommand, ListABTestsCommand, UpdateABTestCommand, @@ -74,6 +76,8 @@ import { type EvaluationReferenceInput, type EvaluationResultContent, type EvaluationTarget, + type CreateABTestRequest, + type CreateABTestResponse, type GetABTestResponse, type ListABTestsResponse, type ABTestExecutionStatus, @@ -119,6 +123,8 @@ import type { RoleScopeWarning, CoreEvalClient, CreateConfigurationBundleInput, + CreateConfigBundleABTestInput, + CreateTargetBasedABTestInput, CreateDatasetInput, CreateOnlineEvalInput, CreateOnlineInsightInput, @@ -165,6 +171,7 @@ import { revokeOnlineEvalScope, scopePolicyName, } from "./onlineEvalExecutionRole"; +import { accountIdFromArn, deleteAbTestRole, provisionAbTestRole } from "./abTestExecutionRole"; const DEFAULT_ENDPOINT_QUALIFIER = "DEFAULT"; const DEFAULT_INGESTION_WAIT_MS = 180_000; @@ -216,6 +223,7 @@ export class EvalClient implements CoreEvalClient { // logger for batch-evaluation result-log diagnostics private readonly logger: Logger = noopLogger, private readonly newSessionId: () => string = randomUUID, + private readonly now: () => number = () => Date.now(), ) {} async createEvaluator( @@ -476,6 +484,147 @@ export class EvalClient implements CoreEvalClient { .send(new DeleteABTestCommand({ abTestId: id })); } + private async createABTest( + name: string, + gateway: string, + callerRoleArn: string | undefined, + build: (context: { + gatewayArn: string; + accountId: string; + roleArn: string; + }) => CreateABTestRequest, + options: CoreOptions, + ): Promise { + const control = this.clients.control(toClientConfig(options)); + const gatewayArn = (await control.send(new GetGatewayCommand({ gatewayIdentifier: gateway }))) + .gatewayArn!; + const accountId = accountIdFromArn(gatewayArn); + + let roleArn = callerRoleArn; + let provisionedRoleArn: string | undefined; + if (!roleArn) { + const provisioned = await provisionAbTestRole( + this.clients.iam({ region: options.region }), + name, + gatewayArn, + options.region, + ); + roleArn = provisioned.roleArn; + if (provisioned.created) provisionedRoleArn = provisioned.roleArn; + } + + const command = new CreateABTestCommand(build({ gatewayArn, accountId, roleArn })); + try { + return callerRoleArn + ? await this.clients.data(toClientConfig(options)).send(command) + : await retryWhileRolePropagates(() => + this.clients.data(toClientConfig(options)).send(command), + ); + } catch (error) { + if (provisionedRoleArn) { + try { + await deleteAbTestRole(this.clients.iam({ region: options.region }), provisionedRoleArn); + } catch { + void 0; + } + } + throw error; + } + } + + async createConfigBundleABTest( + input: CreateConfigBundleABTestInput, + options: CoreOptions, + ): Promise { + const treatmentWeight = input.treatmentWeight ?? 50; + return this.createABTest( + input.name, + input.gateway, + input.roleArn, + ({ gatewayArn, accountId, roleArn }) => { + const bundleArn = (id: string) => + `arn:aws:bedrock-agentcore:${options.region}:${accountId}:configuration-bundle/${id}`; + return { + name: input.name, + gatewayArn, + variants: [ + { + name: "C", + weight: 100 - treatmentWeight, + variantConfiguration: { + configurationBundle: { + bundleArn: bundleArn(input.control.configBundle), + bundleVersion: input.control.bundleVersion, + }, + }, + }, + { + name: "T1", + weight: treatmentWeight, + variantConfiguration: { + configurationBundle: { + bundleArn: bundleArn(input.treatment.configBundle), + bundleVersion: input.treatment.bundleVersion, + }, + }, + }, + ], + evaluationConfig: { + onlineEvaluationConfigArn: `arn:aws:bedrock-agentcore:${options.region}:${accountId}:online-evaluation-config/${input.onlineEval}`, + }, + roleArn, + gatewayFilter: input.gatewayFilter, + enableOnCreate: input.enableOnCreate ?? true, + clientToken: randomUUID(), + }; + }, + options, + ); + } + + async createTargetBasedABTest( + input: CreateTargetBasedABTestInput, + options: CoreOptions, + ): Promise { + const treatmentWeight = input.treatmentWeight ?? 50; + return this.createABTest( + input.name, + input.gateway, + input.roleArn, + ({ gatewayArn, accountId, roleArn }) => { + const evalArn = (id: string) => + `arn:aws:bedrock-agentcore:${options.region}:${accountId}:online-evaluation-config/${id}`; + return { + name: input.name, + gatewayArn, + variants: [ + { + name: "C", + weight: 100 - treatmentWeight, + variantConfiguration: { target: { name: input.control.gatewayTarget } }, + }, + { + name: "T1", + weight: treatmentWeight, + variantConfiguration: { target: { name: input.treatment.gatewayTarget } }, + }, + ], + evaluationConfig: { + perVariantOnlineEvaluationConfig: [ + { name: "C", onlineEvaluationConfigArn: evalArn(input.control.onlineEval) }, + { name: "T1", onlineEvaluationConfigArn: evalArn(input.treatment.onlineEval) }, + ], + }, + roleArn, + gatewayFilter: input.gatewayFilter, + enableOnCreate: input.enableOnCreate ?? true, + clientToken: randomUUID(), + }; + }, + options, + ); + } + async listBatchInsights( nextToken: string | undefined, maxResults: number | undefined, @@ -582,7 +731,7 @@ export class EvalClient implements CoreEvalClient { const logGroupName = runtimeLogGroup(runtimeId, qualifier); const serviceName = runtimeServiceName(runtimeName, qualifier); - const endMs = input.window ? +input.window.endTime : Date.now(); + const endMs = input.window ? +input.window.endTime : this.now(); const startMs = input.window ? +input.window.startTime : endMs - SEVEN_DAYS_MS; const startSec = Math.floor(startMs / 1000); const endSec = Math.floor(endMs / 1000); @@ -2024,19 +2173,25 @@ function chunk(items: T[], size: number): T[][] { // is created. It surfaces as one of two messages depending on which part has not // propagated yet. const ROLE_NOT_PROPAGATED = - /role cannot be assumed|does not have permissions to (create log group|access the specified log groups)/i; + /cannot be assumed|unable to assume|does not have permissions to (create log group|access the specified log groups)/i; -// retryWhileRolePropagates retries `send` while the service reports the execution -// role as unusable, which is how a not-yet-propagated role or policy surfaces. -// Bounded and short: propagation is normally a few seconds, and a role that is -// genuinely misconfigured should fail fast rather than hang. async function retryWhileRolePropagates(send: () => Promise): Promise { - const delaysMs = [1_000, 2_000, 4_000, 8_000]; + const delaysMs = [2_000, 4_000, 8_000, 15_000]; for (const delay of delaysMs) { try { return await send(); } catch (error) { - if (!ROLE_NOT_PROPAGATED.test((error as Error).message)) throw error; + const err = error as { + name?: string; + message?: string; + $metadata?: { httpStatusCode?: number }; + }; + const retryable = + err.name === "AccessDeniedException" || + err.$metadata?.httpStatusCode === 403 || + (err.name === "ValidationException" && /assume|role|trust/i.test(err.message ?? "")) || + ROLE_NOT_PROPAGATED.test(err.message ?? ""); + if (!retryable) throw error; await new Promise((resolve) => setTimeout(resolve, delay)); } } diff --git a/src/core/index.tsx b/src/core/index.tsx index 87948c358..19e068f8c 100644 --- a/src/core/index.tsx +++ b/src/core/index.tsx @@ -42,6 +42,7 @@ type CoreClientConfig = { logger: Logger; fetch?: CoreFetch; newSessionId?: () => string; + now?: () => number; }; // CoreClient is the single entry point to the Bedrock AgentCore APIs. It owns the @@ -87,6 +88,7 @@ export class CoreClient implements AwsClients { fetch, this.logger.child({ module: "eval" }), config.newSessionId, + config.now, ); this.projectManager = new FsProjectManager({ diff --git a/src/core/project/manager.tsx b/src/core/project/manager.tsx index 9ede69c1f..03ddc1545 100644 --- a/src/core/project/manager.tsx +++ b/src/core/project/manager.tsx @@ -26,6 +26,7 @@ import { ENV_LOCAL_RELATIVE_PATH, EnvLocalFile } from "./envLocal"; import { getHarnessTemplateResolver } from "./templates/harness"; import { createProjectTree } from "./templates/project"; import { getRuntimeTemplateResolver } from "./templates/runtime"; +import { getEvaluatorTemplateResolver } from "./templates/evaluator"; import { ProjectSpecSchema, type ManagedBy } from "../../projectSchemas/project"; import { ConfigBundleSchema } from "../../projectSchemas/config-bundle"; import { CredentialSchema } from "../../projectSchemas/credential"; @@ -258,7 +259,35 @@ export class FsProjectManager implements ProjectManager { break; } case "evaluator": { - projectSpec.evaluators.push(parseResource(EvaluatorSchema, input.resourceConfig)); + const evaluator = parseResource(EvaluatorSchema, input.resourceConfig); + // Managed code-based evaluators ship generated Lambda source; external + // and llm-as-a-judge evaluators are spec-only. + if (input.scaffold) { + yield { message: "Scaffolding evaluator in project" }; + const outputPath = join(project.rootPath, "app", evaluator.name); + // Runtimes, harnesses, and evaluators all scaffold into app/, + // but the dup-name guard above is per-resource-type. Fail up front + // rather than let FsTreeNode.write throw mid-write (outside the + // rollback try/catch below) and orphan partial files. + if (existsSync(outputPath)) + throw new InputValidationError( + `cannot scaffold evaluator '${evaluator.name}': 'app/${evaluator.name}' already exists (another resource may use this name, or a previous scaffold was left behind)`, + ); + scaffoldedPaths.push(outputPath); + const resolver = getEvaluatorTemplateResolver({ + assetSource: this.assetSource, + templateRenderer: this.templateRenderer, + }); + const result = await resolver.resolve({ + evaluator, + assetDir: input.scaffold.assetDir, + context: input.scaffold.context, + }); + await result.tree.write(dirname(outputPath)); + projectSpec.evaluators.push(...(result.spec.evaluators ?? [])); + } else { + projectSpec.evaluators.push(evaluator); + } break; } case "gateway": diff --git a/src/core/project/templates/evaluator.ts b/src/core/project/templates/evaluator.ts new file mode 100644 index 000000000..5c091e657 --- /dev/null +++ b/src/core/project/templates/evaluator.ts @@ -0,0 +1,37 @@ +import { FsTreeNode } from "./fsTree"; +import type { AssetSource } from "../source"; +import type { Evaluator } from "../../../projectSchemas/evaluator"; +import type { TemplateRenderer, TemplateResolver } from "./types"; + +/** Inputs for scaffolding a managed code-based evaluator's Lambda source. */ +export type EvaluatorScaffoldInput = { + evaluator: Evaluator; + /** Template directory under src/assets/evaluators, e.g. "evaluators/deepeval-lambda". */ + assetDir: string; + /** Handlebars variables for the template (EvaluatorClass, Model, ModelProviderBedrock, ...). */ + context: Record; +}; + +type GetEvaluatorTemplateResolverConfig = { + assetSource: AssetSource; + templateRenderer: TemplateRenderer; +}; + +/** Resolves the template that renders a managed code-based evaluator's code directory. */ +export function getEvaluatorTemplateResolver( + config: GetEvaluatorTemplateResolverConfig, +): TemplateResolver { + return { + async resolve(input) { + const tree = await FsTreeNode.fromAssetSource( + { assetSource: config.assetSource }, + { assetDir: input.assetDir }, + { + rootDirName: input.evaluator.name, + transformContent: (raw) => config.templateRenderer.render(raw, input.context), + }, + ); + return { tree, spec: { evaluators: [input.evaluator] } }; + }, + }; +} diff --git a/src/core/project/templates/types.ts b/src/core/project/templates/types.ts index f472be503..798e4ba4b 100644 --- a/src/core/project/templates/types.ts +++ b/src/core/project/templates/types.ts @@ -3,6 +3,7 @@ import type { ProjectRuntime } from "../../../projectSchemas/runtime"; import type { MemorySchema } from "../../../projectSchemas/memory"; import type { CredentialSchema } from "../../../projectSchemas/credential"; import type { HarnessRegistryEntry } from "../../../projectSchemas/harness"; +import type { Evaluator } from "../../../projectSchemas/evaluator"; import type z from "zod"; /** AgentCore Project Spec Entries that rendered as part of a {@link Template} **/ @@ -11,6 +12,7 @@ export type SpecEntries = { credentials?: z.infer[]; memories?: z.infer[]; harnesses?: HarnessRegistryEntry[]; + evaluators?: Evaluator[]; }; /** A group of files and resources that can be rendered into a project **/ diff --git a/src/handlers/eval/ab-test/__fixtures__/CreateABTestCommand.506cd57a7653b22c.json b/src/handlers/eval/ab-test/__fixtures__/CreateABTestCommand.506cd57a7653b22c.json new file mode 100644 index 000000000..51aff6399 --- /dev/null +++ b/src/handlers/eval/ab-test/__fixtures__/CreateABTestCommand.506cd57a7653b22c.json @@ -0,0 +1,6 @@ +{ + "$error": { + "name": "ValidationException", + "message": "Unable to assume the provided IAM role. Verify the role exists and its trust policy allows bedrock-agentcore.amazonaws.com to assume it." + } +} \ No newline at end of file diff --git a/src/handlers/eval/ab-test/__fixtures__/CreateABTestCommand.a4666d7f80bc7cb0.json b/src/handlers/eval/ab-test/__fixtures__/CreateABTestCommand.a4666d7f80bc7cb0.json new file mode 100644 index 000000000..971f7115e --- /dev/null +++ b/src/handlers/eval/ab-test/__fixtures__/CreateABTestCommand.a4666d7f80bc7cb0.json @@ -0,0 +1,10 @@ +{ + "abTestId": "agentcore_cli_abtest_run-8e10bf2f27", + "abTestArn": "arn:aws:bedrock-agentcore:us-west-2:685197708687:ab-test/agentcore_cli_abtest_run-8e10bf2f27", + "status": "CREATING", + "executionStatus": "NOT_STARTED", + "createdAt": { + "$date": "2026-08-27T21:03:19.535Z" + }, + "name": "agentcore_cli_abtest_run" +} \ No newline at end of file diff --git a/src/handlers/eval/ab-test/__fixtures__/CreateConfigurationBundleCommand.e8ee73bc166ad5e4.json b/src/handlers/eval/ab-test/__fixtures__/CreateConfigurationBundleCommand.e8ee73bc166ad5e4.json new file mode 100644 index 000000000..265aebbc3 --- /dev/null +++ b/src/handlers/eval/ab-test/__fixtures__/CreateConfigurationBundleCommand.e8ee73bc166ad5e4.json @@ -0,0 +1,8 @@ +{ + "bundleArn": "arn:aws:bedrock-agentcore:us-west-2:685197708687:configuration-bundle/agentcore_cli_abtest_run_bundle-QTjmqQ9zT6", + "bundleId": "agentcore_cli_abtest_run_bundle-QTjmqQ9zT6", + "versionId": "e3c144d3-22ce-4a82-9413-cc9be04eb8a5", + "createdAt": { + "$date": "2026-08-27T21:02:58.851Z" + } +} \ No newline at end of file diff --git a/src/handlers/eval/ab-test/__fixtures__/CreateOnlineEvaluationConfigCommand.ae1ee3532d19f571.json b/src/handlers/eval/ab-test/__fixtures__/CreateOnlineEvaluationConfigCommand.ae1ee3532d19f571.json new file mode 100644 index 000000000..1d2638815 --- /dev/null +++ b/src/handlers/eval/ab-test/__fixtures__/CreateOnlineEvaluationConfigCommand.ae1ee3532d19f571.json @@ -0,0 +1,14 @@ +{ + "onlineEvaluationConfigArn": "arn:aws:bedrock-agentcore:us-west-2:685197708687:online-evaluation-config/agentcore_cli_abtest_run_eval-i7s3ryDRdt", + "onlineEvaluationConfigId": "agentcore_cli_abtest_run_eval-i7s3ryDRdt", + "createdAt": { + "$date": "2026-08-27T21:03:03.249Z" + }, + "status": "CREATING", + "executionStatus": "DISABLED", + "outputConfig": { + "cloudWatchConfig": { + "logGroupName": "/aws/bedrock-agentcore/evaluations/results/agentcore_cli_abtest_run_eval-i7s3ryDRdt" + } + } +} \ No newline at end of file diff --git a/src/handlers/eval/ab-test/__fixtures__/CreateRoleCommand.1fa2ac2a7f0f7fc0.json b/src/handlers/eval/ab-test/__fixtures__/CreateRoleCommand.1fa2ac2a7f0f7fc0.json new file mode 100644 index 000000000..40fb878c0 --- /dev/null +++ b/src/handlers/eval/ab-test/__fixtures__/CreateRoleCommand.1fa2ac2a7f0f7fc0.json @@ -0,0 +1,12 @@ +{ + "Role": { + "Path": "/", + "RoleName": "AgentCoreABTest-agentcore_cli_abtest_run-1cc7229b", + "RoleId": "AROAZ7CHXJWHZBY6C35TH", + "Arn": "arn:aws:iam::685197708687:role/AgentCoreABTest-agentcore_cli_abtest_run-1cc7229b", + "CreateDate": { + "$date": "2026-08-27T21:03:03.000Z" + }, + "AssumeRolePolicyDocument": "%7B%22Version%22%3A%222012-10-17%22%2C%22Statement%22%3A%5B%7B%22Effect%22%3A%22Allow%22%2C%22Principal%22%3A%7B%22Service%22%3A%22bedrock-agentcore.amazonaws.com%22%7D%2C%22Action%22%3A%22sts%3AAssumeRole%22%2C%22Condition%22%3A%7B%22StringEquals%22%3A%7B%22aws%3ASourceAccount%22%3A%22685197708687%22%7D%2C%22ArnLike%22%3A%7B%22aws%3ASourceArn%22%3A%22arn%3Aaws%3Abedrock-agentcore%3Aus-west-2%3A685197708687%3Aab-test%2F%2A%22%7D%7D%7D%5D%7D" + } +} \ No newline at end of file diff --git a/src/handlers/eval/ab-test/__fixtures__/CreateRoleCommand.7b030b47662eee32.json b/src/handlers/eval/ab-test/__fixtures__/CreateRoleCommand.7b030b47662eee32.json new file mode 100644 index 000000000..1968c9437 --- /dev/null +++ b/src/handlers/eval/ab-test/__fixtures__/CreateRoleCommand.7b030b47662eee32.json @@ -0,0 +1,12 @@ +{ + "Role": { + "Path": "/", + "RoleName": "AgentCoreOnlineEval-agentcore_cli_abtest_run_eval", + "RoleId": "AROAZ7CHXJWH3HARLZYI3", + "Arn": "arn:aws:iam::685197708687:role/AgentCoreOnlineEval-agentcore_cli_abtest_run_eval", + "CreateDate": { + "$date": "2026-08-27T21:01:46.000Z" + }, + "AssumeRolePolicyDocument": "%7B%22Version%22%3A%222012-10-17%22%2C%22Statement%22%3A%5B%7B%22Effect%22%3A%22Allow%22%2C%22Principal%22%3A%7B%22Service%22%3A%22bedrock-agentcore.amazonaws.com%22%7D%2C%22Action%22%3A%22sts%3AAssumeRole%22%7D%5D%7D" + } +} \ No newline at end of file diff --git a/src/handlers/eval/ab-test/__fixtures__/DeleteRoleCommand.c6a8dc12fb95054d.json b/src/handlers/eval/ab-test/__fixtures__/DeleteRoleCommand.c6a8dc12fb95054d.json new file mode 100644 index 000000000..9e26dfeeb --- /dev/null +++ b/src/handlers/eval/ab-test/__fixtures__/DeleteRoleCommand.c6a8dc12fb95054d.json @@ -0,0 +1 @@ +{} \ No newline at end of file diff --git a/src/handlers/eval/ab-test/__fixtures__/DeleteRolePolicyCommand.3826bd85235b40f0.json b/src/handlers/eval/ab-test/__fixtures__/DeleteRolePolicyCommand.3826bd85235b40f0.json new file mode 100644 index 000000000..9e26dfeeb --- /dev/null +++ b/src/handlers/eval/ab-test/__fixtures__/DeleteRolePolicyCommand.3826bd85235b40f0.json @@ -0,0 +1 @@ +{} \ No newline at end of file diff --git a/src/handlers/eval/ab-test/__fixtures__/GetAgentRuntimeCommand.9f77333d1b9dcf5d.json b/src/handlers/eval/ab-test/__fixtures__/GetAgentRuntimeCommand.9f77333d1b9dcf5d.json new file mode 100644 index 000000000..2d3b5e713 --- /dev/null +++ b/src/handlers/eval/ab-test/__fixtures__/GetAgentRuntimeCommand.9f77333d1b9dcf5d.json @@ -0,0 +1,47 @@ +{ + "agentRuntimeArn": "arn:aws:bedrock-agentcore:us-west-2:685197708687:runtime/asdf_MyAgent-3s5axvBC6Q", + "agentRuntimeName": "asdf_MyAgent", + "agentRuntimeId": "asdf_MyAgent-3s5axvBC6Q", + "agentRuntimeVersion": "1", + "createdAt": { + "$date": "2026-04-23T21:17:21.895Z" + }, + "lastUpdatedAt": { + "$date": "2026-04-23T21:17:35.159Z" + }, + "roleArn": "arn:aws:iam::685197708687:role/AgentCore-asdf-default-ApplicationAgentMyAgentRunti-KdyUbgImzDRK", + "networkConfiguration": { + "networkMode": "PUBLIC" + }, + "status": "READY", + "lifecycleConfiguration": { + "idleRuntimeSessionTimeout": 900, + "maxLifetime": 28800 + }, + "description": "AgentCore Runtime: asdf_MyAgent", + "workloadIdentityDetails": { + "workloadIdentityArn": "arn:aws:bedrock-agentcore:us-west-2:685197708687:workload-identity-directory/default/workload-identity/asdf_MyAgent-3s5axvBC6Q" + }, + "agentRuntimeArtifact": { + "codeConfiguration": { + "code": { + "s3": { + "bucket": "cdk-hnb659fds-assets-685197708687-us-west-2", + "prefix": "a07977786dda1e2e5be304cb7485237a19ed24d5e05b02e73ca91a43fd2e7280.zip" + } + }, + "runtime": "PYTHON_3_13", + "entryPoint": [ + "opentelemetry-instrument", + "main.py" + ] + } + }, + "environmentVariables": { + "AGENTCORE_GATEWAY_BUGBASHGW1776978672_AUTH_TYPE": "NONE", + "AGENTCORE_GATEWAY_BUGBASHGW1776978672_URL": "https://bugbashgw1776978672-zsy8cbqwts.gateway.bedrock-agentcore.us-west-2.amazonaws.com/mcp" + }, + "metadataConfiguration": { + "requireMMDSV2": true + } +} \ No newline at end of file diff --git a/src/handlers/eval/ab-test/__fixtures__/GetConfigurationBundleCommand.cf3c23a25f6bf298.json b/src/handlers/eval/ab-test/__fixtures__/GetConfigurationBundleCommand.cf3c23a25f6bf298.json new file mode 100644 index 000000000..7876da211 --- /dev/null +++ b/src/handlers/eval/ab-test/__fixtures__/GetConfigurationBundleCommand.cf3c23a25f6bf298.json @@ -0,0 +1,23 @@ +{ + "bundleArn": "arn:aws:bedrock-agentcore:us-west-2:685197708687:configuration-bundle/agentcore_cli_abtest_run_bundle-QTjmqQ9zT6", + "bundleId": "agentcore_cli_abtest_run_bundle-QTjmqQ9zT6", + "bundleName": "agentcore_cli_abtest_run_bundle", + "versionId": "e3c144d3-22ce-4a82-9413-cc9be04eb8a5", + "components": { + "arn:aws:bedrock-agentcore:us-west-2:685197708687:runtime/asdf_MyAgent-3s5axvBC6Q": { + "configuration": { + "system_prompt": "A/B run fixture v1." + } + } + }, + "createdAt": { + "$date": "2026-08-27T21:02:58.851Z" + }, + "updatedAt": { + "$date": "2026-08-27T21:02:58.851Z" + }, + "lineageMetadata": { + "parentVersionIds": [], + "branchName": "mainline" + } +} \ No newline at end of file diff --git a/src/handlers/eval/ab-test/__fixtures__/GetConfigurationBundleCommand.ed835ef9d614b3e6.json b/src/handlers/eval/ab-test/__fixtures__/GetConfigurationBundleCommand.ed835ef9d614b3e6.json new file mode 100644 index 000000000..489607a72 --- /dev/null +++ b/src/handlers/eval/ab-test/__fixtures__/GetConfigurationBundleCommand.ed835ef9d614b3e6.json @@ -0,0 +1,23 @@ +{ + "bundleArn": "arn:aws:bedrock-agentcore:us-west-2:685197708687:configuration-bundle/agentcore_cli_abtest_run_bundle-Fcuyfm8hxE", + "bundleId": "agentcore_cli_abtest_run_bundle-Fcuyfm8hxE", + "bundleName": "agentcore_cli_abtest_run_bundle", + "versionId": "ee7a2803-a60c-4b93-ac4d-5c20f32856da", + "components": { + "arn:aws:bedrock-agentcore:us-west-2:685197708687:runtime/asdf_MyAgent-3s5axvBC6Q": { + "configuration": { + "system_prompt": "A/B run fixture v1." + } + } + }, + "createdAt": { + "$date": "2026-08-27T21:01:43.000Z" + }, + "updatedAt": { + "$date": "2026-08-27T21:01:43.000Z" + }, + "lineageMetadata": { + "parentVersionIds": [], + "branchName": "mainline" + } +} \ No newline at end of file diff --git a/src/handlers/eval/ab-test/__fixtures__/GetEvaluatorCommand.716589b0884f35c0.json b/src/handlers/eval/ab-test/__fixtures__/GetEvaluatorCommand.716589b0884f35c0.json new file mode 100644 index 000000000..ebf428698 --- /dev/null +++ b/src/handlers/eval/ab-test/__fixtures__/GetEvaluatorCommand.716589b0884f35c0.json @@ -0,0 +1,72 @@ +{ + "evaluatorArn": "arn:aws:bedrock-agentcore:::evaluator/Builtin.Helpfulness", + "evaluatorId": "Builtin.Helpfulness", + "evaluatorName": "Builtin.Helpfulness", + "evaluatorConfig": { + "llmAsAJudge": { + "ratingScale": { + "numerical": [ + { + "value": { + "string": "0.0", + "type": "bigDecimal" + }, + "label": "Not helpful at all" + }, + { + "value": { + "string": "1.0", + "type": "bigDecimal" + }, + "label": "Very unhelpful" + }, + { + "value": { + "string": "2.0", + "type": "bigDecimal" + }, + "label": "Somewhat unhelpful" + }, + { + "value": { + "string": "3.0", + "type": "bigDecimal" + }, + "label": "Neutral/Mixed" + }, + { + "value": { + "string": "4.0", + "type": "bigDecimal" + }, + "label": "Somewhat helpful" + }, + { + "value": { + "string": "5.0", + "type": "bigDecimal" + }, + "label": "Very helpful" + }, + { + "value": { + "string": "6.0", + "type": "bigDecimal" + }, + "label": "Above and beyond" + } + ] + } + } + }, + "level": "TRACE", + "status": "ACTIVE", + "createdAt": { + "$date": "2024-10-22T00:00:00.000Z" + }, + "updatedAt": { + "$date": "2024-10-22T00:00:00.000Z" + }, + "description": "Response Quality Metric. Evaluates from user's perspective how useful and valuable the agent's response is", + "lockedForModification": true +} \ No newline at end of file diff --git a/src/handlers/eval/ab-test/__fixtures__/GetGatewayCommand.4216a59651bb046a.json b/src/handlers/eval/ab-test/__fixtures__/GetGatewayCommand.4216a59651bb046a.json new file mode 100644 index 000000000..1007a1d2f --- /dev/null +++ b/src/handlers/eval/ab-test/__fixtures__/GetGatewayCommand.4216a59651bb046a.json @@ -0,0 +1,20 @@ +{ + "gatewayArn": "arn:aws:bedrock-agentcore:us-west-2:685197708687:gateway/agentcore-cli-gateway-read-fixture-a-l6opkbe2kd", + "gatewayId": "agentcore-cli-gateway-read-fixture-a-l6opkbe2kd", + "createdAt": { + "$date": "2026-07-29T22:19:37.409Z" + }, + "updatedAt": { + "$date": "2026-07-29T22:19:37.971Z" + }, + "status": "READY", + "name": "agentcore-cli-gateway-read-fixture-a", + "authorizerType": "NONE", + "gatewayUrl": "https://agentcore-cli-gateway-read-fixture-a-l6opkbe2kd.gateway.bedrock-agentcore.us-west-2.amazonaws.com/mcp", + "description": "AgentCore CLI persistent Gateway read fixture", + "roleArn": "arn:aws:iam::685197708687:role/AgentCoreCliGatewayReadFixtureRole", + "protocolType": "MCP", + "workloadIdentityDetails": { + "workloadIdentityArn": "arn:aws:bedrock-agentcore:us-west-2:685197708687:workload-identity-directory/default/workload-identity/agentcore-cli-gateway-read-fixture-a-l6opkbe2kd" + } +} \ No newline at end of file diff --git a/src/handlers/eval/ab-test/__fixtures__/GetRoleCommand.2ca20231e1472584.json b/src/handlers/eval/ab-test/__fixtures__/GetRoleCommand.2ca20231e1472584.json new file mode 100644 index 000000000..afe04004e --- /dev/null +++ b/src/handlers/eval/ab-test/__fixtures__/GetRoleCommand.2ca20231e1472584.json @@ -0,0 +1,15 @@ +{ + "Role": { + "Path": "/", + "RoleName": "AgentCoreOnlineEval-agentcore_cli_abtest_run_eval", + "RoleId": "AROAZ7CHXJWH3HARLZYI3", + "Arn": "arn:aws:iam::685197708687:role/AgentCoreOnlineEval-agentcore_cli_abtest_run_eval", + "CreateDate": { + "$date": "2026-08-27T21:01:46.000Z" + }, + "AssumeRolePolicyDocument": "%7B%22Version%22%3A%222012-10-17%22%2C%22Statement%22%3A%5B%7B%22Effect%22%3A%22Allow%22%2C%22Principal%22%3A%7B%22Service%22%3A%22bedrock-agentcore.amazonaws.com%22%7D%2C%22Action%22%3A%22sts%3AAssumeRole%22%7D%5D%7D", + "Description": "Default execution role for the AgentCore online evaluation config \"agentcore_cli_abtest_run_eval\" (created by the agentcore CLI)", + "MaxSessionDuration": 3600, + "RoleLastUsed": {} + } +} \ No newline at end of file diff --git a/src/handlers/eval/ab-test/__fixtures__/GetRoleCommand.c6a8dc12fb95054d.json b/src/handlers/eval/ab-test/__fixtures__/GetRoleCommand.c6a8dc12fb95054d.json new file mode 100644 index 000000000..8bda1283f --- /dev/null +++ b/src/handlers/eval/ab-test/__fixtures__/GetRoleCommand.c6a8dc12fb95054d.json @@ -0,0 +1,6 @@ +{ + "$error": { + "name": "NoSuchEntityException", + "message": "The role with name AgentCoreABTest-agentcore_cli_abtest_run-1cc7229b cannot be found." + } +} \ No newline at end of file diff --git a/src/handlers/eval/ab-test/__fixtures__/PutRolePolicyCommand.8bf1be5c5e52a2ec.json b/src/handlers/eval/ab-test/__fixtures__/PutRolePolicyCommand.8bf1be5c5e52a2ec.json new file mode 100644 index 000000000..9e26dfeeb --- /dev/null +++ b/src/handlers/eval/ab-test/__fixtures__/PutRolePolicyCommand.8bf1be5c5e52a2ec.json @@ -0,0 +1 @@ +{} \ No newline at end of file diff --git a/src/handlers/eval/ab-test/__fixtures__/PutRolePolicyCommand.bc0b72d6bad3afad.json b/src/handlers/eval/ab-test/__fixtures__/PutRolePolicyCommand.bc0b72d6bad3afad.json new file mode 100644 index 000000000..9e26dfeeb --- /dev/null +++ b/src/handlers/eval/ab-test/__fixtures__/PutRolePolicyCommand.bc0b72d6bad3afad.json @@ -0,0 +1 @@ +{} \ No newline at end of file diff --git a/src/handlers/eval/ab-test/__fixtures__/UpdateConfigurationBundleCommand.aca9ba06670aa395.json b/src/handlers/eval/ab-test/__fixtures__/UpdateConfigurationBundleCommand.aca9ba06670aa395.json new file mode 100644 index 000000000..6284b216a --- /dev/null +++ b/src/handlers/eval/ab-test/__fixtures__/UpdateConfigurationBundleCommand.aca9ba06670aa395.json @@ -0,0 +1,8 @@ +{ + "bundleArn": "arn:aws:bedrock-agentcore:us-west-2:685197708687:configuration-bundle/agentcore_cli_abtest_run_bundle-QTjmqQ9zT6", + "bundleId": "agentcore_cli_abtest_run_bundle-QTjmqQ9zT6", + "versionId": "d9379d05-8c1c-4764-afcb-8b5f852d6830", + "updatedAt": { + "$date": "2026-08-27T21:03:02.235Z" + } +} \ No newline at end of file diff --git a/src/handlers/eval/ab-test/__fixtures__/UpdateConfigurationBundleCommand.de17ce40709b64eb.json b/src/handlers/eval/ab-test/__fixtures__/UpdateConfigurationBundleCommand.de17ce40709b64eb.json new file mode 100644 index 000000000..8772cde4d --- /dev/null +++ b/src/handlers/eval/ab-test/__fixtures__/UpdateConfigurationBundleCommand.de17ce40709b64eb.json @@ -0,0 +1,8 @@ +{ + "bundleArn": "arn:aws:bedrock-agentcore:us-west-2:685197708687:configuration-bundle/agentcore_cli_abtest_run_bundle-Fcuyfm8hxE", + "bundleId": "agentcore_cli_abtest_run_bundle-Fcuyfm8hxE", + "versionId": "e4d7ad4b-764b-41a5-a007-7b7f0ddac0df", + "updatedAt": { + "$date": "2026-08-27T21:01:46.324Z" + } +} \ No newline at end of file diff --git a/src/handlers/eval/ab-test/__fixtures__/run-bundle-create.golden.json b/src/handlers/eval/ab-test/__fixtures__/run-bundle-create.golden.json new file mode 100644 index 000000000..af43b1ef8 --- /dev/null +++ b/src/handlers/eval/ab-test/__fixtures__/run-bundle-create.golden.json @@ -0,0 +1,6 @@ +{ + "bundleArn": "arn:aws:bedrock-agentcore:us-west-2:685197708687:configuration-bundle/agentcore_cli_abtest_run_bundle-QTjmqQ9zT6", + "bundleId": "agentcore_cli_abtest_run_bundle-QTjmqQ9zT6", + "versionId": "e3c144d3-22ce-4a82-9413-cc9be04eb8a5", + "createdAt": "2026-08-27T21:02:58.851Z" +} \ No newline at end of file diff --git a/src/handlers/eval/ab-test/__fixtures__/run-bundle-update.golden.json b/src/handlers/eval/ab-test/__fixtures__/run-bundle-update.golden.json new file mode 100644 index 000000000..f60044fae --- /dev/null +++ b/src/handlers/eval/ab-test/__fixtures__/run-bundle-update.golden.json @@ -0,0 +1,6 @@ +{ + "bundleArn": "arn:aws:bedrock-agentcore:us-west-2:685197708687:configuration-bundle/agentcore_cli_abtest_run_bundle-QTjmqQ9zT6", + "bundleId": "agentcore_cli_abtest_run_bundle-QTjmqQ9zT6", + "versionId": "d9379d05-8c1c-4764-afcb-8b5f852d6830", + "updatedAt": "2026-08-27T21:03:02.235Z" +} \ No newline at end of file diff --git a/src/handlers/eval/ab-test/__fixtures__/run-online-eval.golden.json b/src/handlers/eval/ab-test/__fixtures__/run-online-eval.golden.json new file mode 100644 index 000000000..654a825b1 --- /dev/null +++ b/src/handlers/eval/ab-test/__fixtures__/run-online-eval.golden.json @@ -0,0 +1,12 @@ +{ + "onlineEvaluationConfigArn": "arn:aws:bedrock-agentcore:us-west-2:685197708687:online-evaluation-config/agentcore_cli_abtest_run_eval-i7s3ryDRdt", + "onlineEvaluationConfigId": "agentcore_cli_abtest_run_eval-i7s3ryDRdt", + "createdAt": "2026-08-27T21:03:03.249Z", + "status": "CREATING", + "executionStatus": "DISABLED", + "outputConfig": { + "cloudWatchConfig": { + "logGroupName": "/aws/bedrock-agentcore/evaluations/results/agentcore_cli_abtest_run_eval-i7s3ryDRdt" + } + } +} \ No newline at end of file diff --git a/src/handlers/eval/ab-test/__fixtures__/run.golden.json b/src/handlers/eval/ab-test/__fixtures__/run.golden.json new file mode 100644 index 000000000..66b3dfd1e --- /dev/null +++ b/src/handlers/eval/ab-test/__fixtures__/run.golden.json @@ -0,0 +1,8 @@ +{ + "abTestId": "agentcore_cli_abtest_run-8e10bf2f27", + "abTestArn": "arn:aws:bedrock-agentcore:us-west-2:685197708687:ab-test/agentcore_cli_abtest_run-8e10bf2f27", + "status": "CREATING", + "executionStatus": "NOT_STARTED", + "createdAt": "2026-08-27T21:03:19.535Z", + "name": "agentcore_cli_abtest_run" +} \ No newline at end of file diff --git a/src/handlers/eval/ab-test/ab-test.fixture.test.tsx b/src/handlers/eval/ab-test/ab-test.fixture.test.tsx index 9f8bb97fe..f6f28a6c5 100644 --- a/src/handlers/eval/ab-test/ab-test.fixture.test.tsx +++ b/src/handlers/eval/ab-test/ab-test.fixture.test.tsx @@ -1,32 +1,52 @@ -import { describe, expect, test } from "bun:test"; +import { afterAll, describe, expect, test } from "bun:test"; +import { + DeleteConfigurationBundleCommand, + GetConfigurationBundleCommand, + DeleteOnlineEvaluationConfigCommand, +} from "@aws-sdk/client-bedrock-agentcore-control"; +import { DeleteABTestCommand } from "@aws-sdk/client-bedrock-agentcore"; import { join } from "node:path"; import { CoreClient } from "../../../core"; import { createSilentLogger, fixtureFactories, + isRecording, matchGolden, TestGlobalConfigAccessor, testIO, } from "../../../testing"; +import { createControlClient, createDataClient, createIamClient } from "../../../core/factories"; +import { abTestExecutionRoleName, deleteAbTestRole } from "../../../core/abTestExecutionRole"; import { createRootHandler } from "../../index"; const REGION = "us-west-2"; const FIXTURES = join(import.meta.dir, "__fixtures__"); -// Record with: RECORD=1 bun test src/handlers/eval/ab-test/ab-test.fixture.test.tsx -// -// A/B tests are READ-ONLY here, so — like the batch-evaluation fixture suite — -// this pins pre-existing tests in the fixture account rather than creating one. -// Re-recording requires these ids to still exist; repoint them if they age out. -// -// Exercises the real seam end to end: parsing → handler → CoreClient → -// GetABTest / ListABTest (data plane). GetABTest returns the per-evaluator -// statistical results inline, so there is no CloudWatch seam to record. +// The read describe records against account 725476964917 (a pre-existing target +// based test); the create describe records against 685197708687 (self-created, +// matching the config-bundle fixtures). Replay is offline and account-agnostic; +// re-record each describe under its own account: +// RECORD=1 bun test -t "fixture-backed reads" +// RECORD=1 bun test -t "config-bundle run" const FIXTURE_ABTEST_ID = "abvfylatest_abtargettest-a5f5674e07"; - -// A well-formed but absent id, to reach the not-found path. const MISSING_ABTEST_ID = "missing-abtest-0000000000"; +const RUNTIME_ARN = + "arn:aws:bedrock-agentcore:us-west-2:685197708687:runtime/asdf_MyAgent-3s5axvBC6Q"; +const AGENT_ID = "asdf_MyAgent-3s5axvBC6Q"; +const EVALUATOR_ID = "Builtin.Helpfulness"; +const GATEWAY_ID = "agentcore-cli-gateway-read-fixture-a-l6opkbe2kd"; +const BUNDLE_NAME = "agentcore_cli_abtest_run_bundle"; +const ONLINE_EVAL_NAME = "agentcore_cli_abtest_run_eval"; +const AB_TEST_NAME = "agentcore_cli_abtest_run"; + +const COMPONENTS_V1 = { + [RUNTIME_ARN]: { configuration: { system_prompt: "A/B run fixture v1." } }, +}; +const COMPONENTS_V2 = { + [RUNTIME_ARN]: { configuration: { system_prompt: "A/B run fixture v2." } }, +}; + function createFixtureCore(): CoreClient { const { createControlClient, createDataClient, createIamClient, createLogsClient } = fixtureFactories(FIXTURES); @@ -50,10 +70,13 @@ async function run(args: string[]): Promise { return io.stdout(); } -describe("eval ab-test (fixture-backed)", () => { +async function settle(): Promise { + if (isRecording()) await new Promise((resolve) => setTimeout(resolve, 3000)); +} + +describe("eval ab-test fixture-backed reads", () => { test("get returns the test with per-evaluator metrics inline", async () => { const stdout = await run(["eval", "ab-test", "get", "--id", FIXTURE_ABTEST_ID, "--json"]); - matchGolden(FIXTURES, "get.golden.json", stdout); const detail = JSON.parse(stdout); expect(detail.abTestId).toBe(FIXTURE_ABTEST_ID); @@ -64,7 +87,6 @@ describe("eval ab-test (fixture-backed)", () => { test("list returns the service page", async () => { const stdout = await run(["eval", "ab-test", "list", "--max-results", "3", "--json"]); - matchGolden(FIXTURES, "list.golden.json", stdout); expect(Array.isArray(JSON.parse(stdout).abTests)).toBe(true); }); @@ -75,3 +97,136 @@ describe("eval ab-test (fixture-backed)", () => { ).rejects.toThrow(); }); }); + +const created: { + bundleId?: string; + v1?: string; + v2?: string; + onlineEvalId?: string; + abTestId?: string; +} = {}; + +afterAll(async () => { + if (!isRecording()) return; + const control = createControlClient({ region: REGION }); + const data = createDataClient({ region: REGION }); + if (created.abTestId) { + try { + await data.send(new DeleteABTestCommand({ abTestId: created.abTestId })); + } catch (error) { + console.error("cleanup ab-test:", error); + } + try { + await deleteAbTestRole( + createIamClient({ region: REGION }), + abTestExecutionRoleName(AB_TEST_NAME), + ); + } catch (error) { + console.error("cleanup ab-test role:", error); + } + } + if (created.onlineEvalId) { + try { + await control.send( + new DeleteOnlineEvaluationConfigCommand({ onlineEvaluationConfigId: created.onlineEvalId }), + ); + } catch (error) { + console.error("cleanup online-eval:", error); + } + } + if (created.bundleId) { + try { + await control.send( + new GetConfigurationBundleCommand({ bundleId: created.bundleId, branchName: "mainline" }), + ); + await control.send(new DeleteConfigurationBundleCommand({ bundleId: created.bundleId })); + } catch (error) { + if ((error as Error).name !== "ResourceNotFoundException") { + console.error("cleanup bundle:", error); + } + } + } +}); + +describe("eval ab-test config-bundle run", () => { + test("provisions a bundle with two versions", async () => { + const v1 = await run([ + "eval", + "config-bundle", + "create", + "--name", + BUNDLE_NAME, + "--components", + JSON.stringify(COMPONENTS_V1), + ]); + matchGolden(FIXTURES, "run-bundle-create.golden.json", v1); + const first = JSON.parse(v1); + created.bundleId = first.bundleId; + created.v1 = first.versionId; + + await settle(); + + const v2 = await run([ + "eval", + "config-bundle", + "update", + "--id", + created.bundleId!, + "--components", + JSON.stringify(COMPONENTS_V2), + "--commit-message", + "A/B run fixture v2", + ]); + matchGolden(FIXTURES, "run-bundle-update.golden.json", v2); + created.v2 = JSON.parse(v2).versionId; + expect(created.v2).not.toBe(created.v1); + }, 180_000); + + test("provisions a paused online evaluation config", async () => { + const out = await run([ + "eval", + "online-eval", + "create", + "--name", + ONLINE_EVAL_NAME, + "--agent", + AGENT_ID, + "--evaluator", + EVALUATOR_ID, + "--sampling-rate", + "100", + "--enable-on-create", + "false", + ]); + matchGolden(FIXTURES, "run-online-eval.golden.json", out); + created.onlineEvalId = JSON.parse(out).onlineEvaluationConfigId; + }, 180_000); + + test("runs a paused config-bundle A/B test", async () => { + const out = await run([ + "eval", + "ab-test", + "config-bundle", + "run", + "--name", + AB_TEST_NAME, + "--gateway", + GATEWAY_ID, + "--control", + JSON.stringify({ "config-bundle": created.bundleId, "bundle-version": created.v1 }), + "--treatment", + JSON.stringify({ "config-bundle": created.bundleId, "bundle-version": created.v2 }), + "--online-eval", + created.onlineEvalId!, + "--treatment-weight", + "20", + "--enable-on-create", + "false", + ]); + matchGolden(FIXTURES, "run.golden.json", out); + const abTest = JSON.parse(out); + created.abTestId = abTest.abTestId; + expect(abTest.abTestId).toBeString(); + expect(abTest.executionStatus).toBe("NOT_STARTED"); + }, 180_000); +}); diff --git a/src/handlers/eval/ab-test/ab-test.test.tsx b/src/handlers/eval/ab-test/ab-test.test.tsx new file mode 100644 index 000000000..73ac93b64 --- /dev/null +++ b/src/handlers/eval/ab-test/ab-test.test.tsx @@ -0,0 +1,352 @@ +import { test, expect, describe } from "bun:test"; +import type { GetABTestResponse, ListABTestsResponse } from "@aws-sdk/client-bedrock-agentcore"; +import { createRootHandler } from "../../index"; +import { createSilentLogger, TestCoreClient, testIO } from "../../../testing"; +import { TestGlobalConfigAccessor } from "../../../testing/"; + +async function run(args: string[], configure?: (core: TestCoreClient) => void) { + const core = new TestCoreClient(); + configure?.(core); + const io = testIO(); + const root = createRootHandler(core, { + io: io.io, + logger: createSilentLogger(), + globalConfigAccessor: new TestGlobalConfigAccessor(), + }); + await root.route(["node", "agentcore", ...args, "--region", "us-west-2"]); + return { core, stdout: io.stdout() }; +} + +const ARN = "arn:aws:bedrock-agentcore:us-west-2:123456789012:ab-test/ab-test-1"; + +const GET_RESPONSE = { + abTestId: "ab-test-1", + abTestArn: ARN, + name: "orders-v2", + status: "ACTIVE", + executionStatus: "RUNNING", + gatewayArn: "arn:aws:bedrock-agentcore:us-west-2:123456789012:gateway/orders", + variants: [], + evaluationConfig: { + onlineEvaluationConfigArn: + "arn:aws:bedrock-agentcore:us-west-2:123456789012:online-evaluation-config/x", + }, + createdAt: new Date("2026-07-19T01:02:03.000Z"), + updatedAt: new Date("2026-07-20T12:34:56.000Z"), +} satisfies GetABTestResponse; + +const LIST_RESPONSE = { + abTests: [{ abTestId: "ab-test-1", status: "ACTIVE" }], + nextToken: "next", +} as ListABTestsResponse; + +const RUN_BASE = [ + "eval", + "ab-test", + "config-bundle", + "run", + "--name", + "orders-v2", + "--gateway", + "orders-gateway-abc123", + "--control", + '{"config-bundle":"orders-prompt-abc","bundle-version":"1111"}', + "--treatment", + '{"config-bundle":"orders-prompt-abc","bundle-version":"2222"}', + "--online-eval", + "online-eval-abc123", + "--json", +]; + +describe("eval ab-test command hierarchy", () => { + test("registers get, list, pause, resume, stop, delete, config-bundle", () => { + const io = testIO(); + const root = createRootHandler(new TestCoreClient(), { + io: io.io, + logger: createSilentLogger(), + globalConfigAccessor: new TestGlobalConfigAccessor(), + }); + const group = root + .children() + .find((c) => c.name() === "eval") + ?.children() + .find((c) => c.name() === "ab-test"); + expect(group?.children().map((c) => c.name())).toEqual([ + "get", + "list", + "pause", + "resume", + "stop", + "delete", + "config-bundle", + "target-based", + ]); + const cb = group?.children().find((c) => c.name() === "config-bundle"); + expect(cb?.children().map((c) => c.name())).toEqual(["run"]); + const tb = group?.children().find((c) => c.name() === "target-based"); + expect(tb?.children().map((c) => c.name())).toEqual(["run"]); + }); +}); + +describe("eval ab-test get", () => { + test("returns the test by id", async () => { + const { core, stdout } = await run( + ["eval", "ab-test", "get", "--id", "ab-test-1", "--json"], + (c) => c.eval.setAbTestGetResponse(GET_RESPONSE), + ); + expect(JSON.parse(stdout).abTestId).toBe("ab-test-1"); + expect(core.eval.calls).toEqual([ + { method: "getABTest", args: ["ab-test-1", { region: "us-west-2" }] }, + ]); + }); + + test("requires --id", async () => { + await expect(run(["eval", "ab-test", "get", "--json"])).rejects.toThrow(/--id/); + }); + + test("surfaces a Core error", async () => { + await expect( + run(["eval", "ab-test", "get", "--id", "missing", "--json"], (c) => + c.eval.setError(new Error("ResourceNotFound")), + ), + ).rejects.toThrow(/ResourceNotFound/); + }); +}); + +describe("eval ab-test list", () => { + test("passes pagination through", async () => { + const { core, stdout } = await run( + ["eval", "ab-test", "list", "--max-results", "10", "--json"], + (c) => c.eval.setAbTestListResponse(LIST_RESPONSE), + ); + expect(JSON.parse(stdout).nextToken).toBe("next"); + expect(core.eval.calls[0]?.args).toEqual([undefined, 10, { region: "us-west-2" }]); + }); + + test("surfaces a Core error", async () => { + await expect( + run(["eval", "ab-test", "list", "--json"], (c) => c.eval.setError(new Error("boom"))), + ).rejects.toThrow(/boom/); + }); +}); + +describe("eval ab-test transitions", () => { + test.each([ + ["pause", "PAUSED"], + ["resume", "RUNNING"], + ["stop", "STOPPED"], + ] as const)("%s sets executionStatus %s via Core", async (command, status) => { + const { core } = await run(["eval", "ab-test", command, "--id", "ab-test-1", "--json"], (c) => + c.eval.setAbTestUpdateResponse({ + abTestId: "ab-test-1", + abTestArn: ARN, + status: "ACTIVE", + executionStatus: status, + updatedAt: new Date("2026-07-20T12:34:56.000Z"), + }), + ); + expect(core.eval.calls).toEqual([ + { method: "setABTestExecutionStatus", args: ["ab-test-1", status, { region: "us-west-2" }] }, + ]); + }); + + test.each(["pause", "resume", "stop"] as const)("%s requires --id", async (command) => { + await expect(run(["eval", "ab-test", command, "--json"])).rejects.toThrow(/--id/); + }); + + test.each(["pause", "resume", "stop"] as const)("%s surfaces a Core error", async (command) => { + await expect( + run(["eval", "ab-test", command, "--id", "ab-test-1", "--json"], (c) => + c.eval.setError(new Error("invalid transition")), + ), + ).rejects.toThrow(/invalid transition/); + }); +}); + +describe("eval ab-test delete", () => { + test("deletes by id via Core", async () => { + const { core, stdout } = await run( + ["eval", "ab-test", "delete", "--id", "ab-test-1", "--json"], + (c) => + c.eval.setAbTestDeleteResponse({ + abTestId: "ab-test-1", + abTestArn: ARN, + status: "DELETING", + }), + ); + expect(JSON.parse(stdout).abTestId).toBe("ab-test-1"); + expect(core.eval.calls).toEqual([ + { method: "deleteABTest", args: ["ab-test-1", { region: "us-west-2" }] }, + ]); + }); + + test("requires --id", async () => { + await expect(run(["eval", "ab-test", "delete", "--json"])).rejects.toThrow(/--id/); + }); + + test("surfaces a Core error (e.g. not stopped)", async () => { + await expect( + run(["eval", "ab-test", "delete", "--id", "ab-test-1", "--json"], (c) => + c.eval.setError(new Error("must be stopped")), + ), + ).rejects.toThrow(/must be stopped/); + }); +}); + +describe("eval ab-test config-bundle run validation", () => { + test.each(["name", "gateway", "control", "treatment", "online-eval"] as const)( + "requires --%s", + async (missing) => { + const args = RUN_BASE.filter( + (a, i) => a !== `--${missing}` && RUN_BASE[i - 1] !== `--${missing}`, + ); + await expect(run(args)).rejects.toThrow(new RegExp(`--${missing}`)); + }, + ); + + test("rejects malformed --control JSON", async () => { + const args = RUN_BASE.map((a) => + a === '{"config-bundle":"orders-prompt-abc","bundle-version":"1111"}' ? "notjson" : a, + ); + await expect(run(args)).rejects.toThrow(/Invalid JSON/); + }); + + test("rejects a mis-shaped --control object", async () => { + const args = RUN_BASE.map((a) => + a === '{"config-bundle":"orders-prompt-abc","bundle-version":"1111"}' + ? '{"wrong":"shape"}' + : a, + ); + await expect(run(args)).rejects.toThrow(/--control must be/); + }); + + test("rejects identical control/treatment", async () => { + const same = '{"config-bundle":"b","bundle-version":"same"}'; + await expect( + run([ + "eval", + "ab-test", + "config-bundle", + "run", + "--name", + "x", + "--gateway", + "g", + "--control", + same, + "--treatment", + same, + "--online-eval", + "o", + "--json", + ]), + ).rejects.toThrow(/must reference a different/); + }); + + test.each(["0", "100"])("rejects --treatment-weight %s", async (w) => { + await expect(run([...RUN_BASE, "--treatment-weight", w])).rejects.toThrow(/1 and 99/); + }); + + test("passes --gateway-filter through as a GatewayFilter", async () => { + const { core } = await run( + [...RUN_BASE, "--gateway-filter", '{"targetPaths":["/orders/checkout"]}'], + (c) => + c.eval.setAbTestCreateResponse({ + abTestId: "x", + abTestArn: ARN, + name: "x", + status: "CREATING", + executionStatus: "NOT_STARTED", + createdAt: new Date("2026-08-26T10:00:00.000Z"), + }), + ); + const call = core.eval.calls.find((c) => c.method === "createConfigBundleABTest"); + expect(call).toBeDefined(); + expect((call!.args[0] as { gatewayFilter?: unknown }).gatewayFilter).toEqual({ + targetPaths: ["/orders/checkout"], + }); + }); +}); + +describe("eval ab-test target-based run validation", () => { + const TB_BASE = [ + "eval", + "ab-test", + "target-based", + "run", + "--name", + "orders-v2-canary", + "--gateway", + "orders-gateway-abc123", + "--control", + '{"gateway-target":"orders-prod-target","online-eval":"prod-quality"}', + "--treatment", + '{"gateway-target":"orders-v2-target","online-eval":"v2-quality"}', + "--json", + ]; + + test.each(["name", "gateway", "control", "treatment"] as const)( + "requires --%s", + async (missing) => { + const args = TB_BASE.filter( + (a, i) => a !== `--${missing}` && TB_BASE[i - 1] !== `--${missing}`, + ); + await expect(run(args)).rejects.toThrow(new RegExp(`--${missing}`)); + }, + ); + + test("rejects a mis-shaped --control object", async () => { + const args = TB_BASE.map((a) => + a === '{"gateway-target":"orders-prod-target","online-eval":"prod-quality"}' + ? '{"wrong":"shape"}' + : a, + ); + await expect(run(args)).rejects.toThrow(/--control must be/); + }); + + test("rejects identical control/treatment targets", async () => { + const same = '{"gateway-target":"t","online-eval":"e"}'; + await expect( + run([ + "eval", + "ab-test", + "target-based", + "run", + "--name", + "x", + "--gateway", + "g", + "--control", + same, + "--treatment", + same, + "--json", + ]), + ).rejects.toThrow(/different gateway targets/); + }); + + test("maps flags to a createTargetBasedABTest call", async () => { + const { core } = await run([...TB_BASE, "--treatment-weight", "20"], (c) => + c.eval.setAbTestCreateResponse({ + abTestId: "x", + abTestArn: ARN, + name: "x", + status: "CREATING", + executionStatus: "RUNNING", + createdAt: new Date("2026-08-26T10:00:00.000Z"), + }), + ); + const call = core.eval.calls.find((c) => c.method === "createTargetBasedABTest"); + expect(call).toBeDefined(); + expect(call!.args[0]).toEqual({ + name: "orders-v2-canary", + gateway: "orders-gateway-abc123", + control: { gatewayTarget: "orders-prod-target", onlineEval: "prod-quality" }, + treatment: { gatewayTarget: "orders-v2-target", onlineEval: "v2-quality" }, + treatmentWeight: 20, + gatewayFilter: undefined, + roleArn: undefined, + enableOnCreate: undefined, + }); + }); +}); diff --git a/src/handlers/eval/ab-test/ab-test.write.test.tsx b/src/handlers/eval/ab-test/ab-test.write.test.tsx deleted file mode 100644 index 31116a1a5..000000000 --- a/src/handlers/eval/ab-test/ab-test.write.test.tsx +++ /dev/null @@ -1,88 +0,0 @@ -import { test, expect, describe } from "bun:test"; -import { createRootHandler } from "../../index"; -import { createSilentLogger, TestCoreClient, testIO } from "../../../testing"; -import { TestGlobalConfigAccessor } from "../../../testing/"; - -async function run(args: string[], configure?: (core: TestCoreClient) => void) { - const core = new TestCoreClient(); - configure?.(core); - const io = testIO(); - const root = createRootHandler(core, { - io: io.io, - logger: createSilentLogger(), - globalConfigAccessor: new TestGlobalConfigAccessor(), - }); - await root.route(["node", "agentcore", ...args, "--region", "us-west-2"]); - return { core, stdout: io.stdout() }; -} - -describe("eval ab-test command hierarchy", () => { - test("registers get, list, pause, resume, stop, delete", () => { - const io = testIO(); - const root = createRootHandler(new TestCoreClient(), { - io: io.io, - logger: createSilentLogger(), - globalConfigAccessor: new TestGlobalConfigAccessor(), - }); - const group = root - .children() - .find((c) => c.name() === "eval") - ?.children() - .find((c) => c.name() === "ab-test"); - expect(group?.children().map((c) => c.name())).toEqual([ - "get", - "list", - "pause", - "resume", - "stop", - "delete", - ]); - }); -}); - -describe("eval ab-test transitions", () => { - test.each([ - ["pause", "PAUSED"], - ["resume", "RUNNING"], - ["stop", "STOPPED"], - ] as const)("%s sets executionStatus %s via Core", async (command, status) => { - const { core } = await run(["eval", "ab-test", command, "--id", "ab-test-1", "--json"], (c) => - c.eval.setAbTestUpdateResponse({ - abTestId: "ab-test-1", - abTestArn: "arn:aws:bedrock-agentcore:us-west-2:123456789012:ab-test/ab-test-1", - status: "ACTIVE", - executionStatus: status, - updatedAt: new Date("2026-07-20T12:34:56.000Z"), - }), - ); - expect(core.eval.calls).toEqual([ - { method: "setABTestExecutionStatus", args: ["ab-test-1", status, { region: "us-west-2" }] }, - ]); - }); - - test.each(["pause", "resume", "stop"] as const)("%s requires --id", async (command) => { - await expect(run(["eval", "ab-test", command, "--json"])).rejects.toThrow(/--id/); - }); -}); - -describe("eval ab-test delete", () => { - test("deletes by id via Core", async () => { - const { core, stdout } = await run( - ["eval", "ab-test", "delete", "--id", "ab-test-1", "--json"], - (c) => - c.eval.setAbTestDeleteResponse({ - abTestId: "ab-test-1", - abTestArn: "arn:aws:bedrock-agentcore:us-west-2:123456789012:ab-test/ab-test-1", - status: "DELETING", - }), - ); - expect(JSON.parse(stdout).abTestId).toBe("ab-test-1"); - expect(core.eval.calls).toEqual([ - { method: "deleteABTest", args: ["ab-test-1", { region: "us-west-2" }] }, - ]); - }); - - test("requires --id", async () => { - await expect(run(["eval", "ab-test", "delete", "--json"])).rejects.toThrow(/--id/); - }); -}); diff --git a/src/handlers/eval/ab-test/config-bundle/index.tsx b/src/handlers/eval/ab-test/config-bundle/index.tsx new file mode 100644 index 000000000..1c04c571c --- /dev/null +++ b/src/handlers/eval/ab-test/config-bundle/index.tsx @@ -0,0 +1,10 @@ +import { Router } from "../../../../router"; +import type { AppIO } from "../../../../io"; +import type { Core } from "../../../types"; +import { createConfigBundleRunHandler } from "./run"; + +export function createConfigBundleAbTestHandler(core: Core, io: AppIO): Router { + return new Router("config-bundle", "config-bundle A/B tests").handler( + createConfigBundleRunHandler(core, io), + ); +} diff --git a/src/handlers/eval/ab-test/config-bundle/run/index.tsx b/src/handlers/eval/ab-test/config-bundle/run/index.tsx new file mode 100644 index 000000000..a7abd1e5d --- /dev/null +++ b/src/handlers/eval/ab-test/config-bundle/run/index.tsx @@ -0,0 +1,127 @@ +import type { GatewayFilter } from "@aws-sdk/client-bedrock-agentcore"; +import z from "zod"; +import { createHandler, flag } from "../../../../../router"; +import { InputValidationError } from "../../../../../errors"; +import { JsonRendererKey } from "../../../../../tui"; +import { SourceResolver, type AppIO } from "../../../../../io"; +import type { Core } from "../../../../types"; +import type { BundleRef } from "../../../types"; +import { coreOptsFromCtx } from "../../../../utils"; +import { parseJsonFlag } from "../../../../utils"; + +const bundleRefSchema = z + .object({ + "config-bundle": z.string().min(1), + "bundle-version": z.string().min(1), + }) + .strict(); + +function toBundleRef(name: string, raw: unknown): BundleRef { + const parsed = bundleRefSchema.safeParse(raw); + if (!parsed.success) { + throw new InputValidationError( + `--${name} must be {"config-bundle": "", "bundle-version": ""}`, + ); + } + return { + configBundle: parsed.data["config-bundle"], + bundleVersion: parsed.data["bundle-version"], + }; +} + +export const createConfigBundleRunHandler = (core: Core, io: AppIO) => + createHandler({ + name: "run", + description: "run an A/B test between two config-bundle versions on one gateway", + flags: [ + flag("name", "the A/B test name", z.string().optional()), + flag("gateway", "deployed gateway id", z.string().optional()), + flag( + "control", + 'control JSON {"config-bundle","bundle-version"} (inline, file://, or -)', + z.string().optional(), + ), + flag( + "treatment", + 'treatment JSON {"config-bundle","bundle-version"} (inline, file://, or -)', + z.string().optional(), + ), + flag("online-eval", "online-evaluation config id", z.string().optional()), + flag( + "treatment-weight", + "1-99; control weight = 100 - this (default 50)", + z.number().int().optional(), + ), + flag( + "gateway-filter", + 'GatewayFilter JSON, e.g. {"targetPaths":["/orders"]} (inline, file://, or -)', + z.string().optional(), + ), + flag( + "role-arn", + "execution-role override (default: auto-provisioned)", + z.string().optional(), + ), + flag( + "enable-on-create", + "whether to start the test immediately (default true; pass false to create it paused)", + z.enum(["true", "false"]).optional(), + ), + ], + handle: async (ctx, flags) => { + const required = ["name", "gateway", "control", "treatment", "online-eval"] as const; + for (const f of required) { + if (!flags[f]) throw new InputValidationError(`required option '--${f}' not specified`); + } + + const source = new SourceResolver({ stdin: io.stdin }); + const controlRaw = parseJsonFlag( + "control", + await source.resolveText("control", flags["control"]), + ); + const treatmentRaw = parseJsonFlag( + "treatment", + await source.resolveText("treatment", flags["treatment"]), + ); + const gatewayFilter = parseJsonFlag( + "gateway-filter", + await source.resolveText("gateway-filter", flags["gateway-filter"]), + ); + + const control = toBundleRef("control", controlRaw); + const treatment = toBundleRef("treatment", treatmentRaw); + if ( + control.configBundle === treatment.configBundle && + control.bundleVersion === treatment.bundleVersion + ) { + throw new InputValidationError( + "control and treatment must reference a different config-bundle or bundle-version", + ); + } + + const treatmentWeight = flags["treatment-weight"]; + if (treatmentWeight !== undefined && (treatmentWeight < 1 || treatmentWeight > 99)) { + throw new InputValidationError("--treatment-weight must be between 1 and 99"); + } + + const result = await core.eval.createConfigBundleABTest( + { + name: flags["name"]!, + gateway: flags["gateway"]!, + control, + treatment, + onlineEval: flags["online-eval"]!, + treatmentWeight, + gatewayFilter, + roleArn: flags["role-arn"], + enableOnCreate: + flags["enable-on-create"] === undefined + ? undefined + : flags["enable-on-create"] === "true", + }, + coreOptsFromCtx(ctx), + ); + + ctx.require(JsonRendererKey).renderJson(result); + }, + }); diff --git a/src/handlers/eval/ab-test/index.tsx b/src/handlers/eval/ab-test/index.tsx index 6bccf91fc..42b16fd9d 100644 --- a/src/handlers/eval/ab-test/index.tsx +++ b/src/handlers/eval/ab-test/index.tsx @@ -9,6 +9,8 @@ import { createPauseAbTestHandler } from "./pause"; import { createResumeAbTestHandler } from "./resume"; import { createStopAbTestHandler } from "./stop"; import { createDeleteAbTestHandler } from "./delete"; +import { createConfigBundleAbTestHandler } from "./config-bundle"; +import { createTargetBasedAbTestHandler } from "./target-based"; export function createAbTestHandler(core: Core, io: AppIO): Router { return new Router("ab-test", "inspect AgentCore A/B tests") @@ -20,7 +22,9 @@ export function createAbTestHandler(core: Core, io: AppIO): Router { .handler(createPauseAbTestHandler(core)) .handler(createResumeAbTestHandler(core)) .handler(createStopAbTestHandler(core)) - .handler(createDeleteAbTestHandler(core)); + .handler(createDeleteAbTestHandler(core)) + .handler(createConfigBundleAbTestHandler(core, io)) + .handler(createTargetBasedAbTestHandler(core, io)); } export { AbTestScreen } from "./screen.tsx"; diff --git a/src/handlers/eval/ab-test/target-based/index.tsx b/src/handlers/eval/ab-test/target-based/index.tsx new file mode 100644 index 000000000..e13f20b18 --- /dev/null +++ b/src/handlers/eval/ab-test/target-based/index.tsx @@ -0,0 +1,10 @@ +import { Router } from "../../../../router"; +import type { AppIO } from "../../../../io"; +import type { Core } from "../../../types"; +import { createTargetBasedRunHandler } from "./run"; + +export function createTargetBasedAbTestHandler(core: Core, io: AppIO): Router { + return new Router("target-based", "target-based A/B tests").handler( + createTargetBasedRunHandler(core, io), + ); +} diff --git a/src/handlers/eval/ab-test/target-based/run/index.tsx b/src/handlers/eval/ab-test/target-based/run/index.tsx new file mode 100644 index 000000000..8ef811eb1 --- /dev/null +++ b/src/handlers/eval/ab-test/target-based/run/index.tsx @@ -0,0 +1,116 @@ +import z from "zod"; +import { createHandler, flag } from "../../../../../router"; +import { InputValidationError } from "../../../../../errors"; +import { JsonRendererKey } from "../../../../../tui"; +import { SourceResolver, type AppIO } from "../../../../../io"; +import type { Core } from "../../../../types"; +import type { TargetVariantRef } from "../../../types"; +import { coreOptsFromCtx, parseJsonFlag } from "../../../../utils"; + +const targetRefSchema = z + .object({ + "gateway-target": z.string().min(1), + "online-eval": z.string().min(1), + }) + .strict(); + +function toTargetRef(name: string, raw: unknown): TargetVariantRef { + const parsed = targetRefSchema.safeParse(raw); + if (!parsed.success) { + throw new InputValidationError( + `--${name} must be {"gateway-target": "", "online-eval": ""}`, + ); + } + return { gatewayTarget: parsed.data["gateway-target"], onlineEval: parsed.data["online-eval"] }; +} + +export const createTargetBasedRunHandler = (core: Core, io: AppIO) => + createHandler({ + name: "run", + description: "run an A/B test between two gateway targets and their online evaluations", + flags: [ + flag("name", "the A/B test name", z.string().optional()), + flag("gateway", "deployed gateway id", z.string().optional()), + flag( + "control", + 'control JSON {"gateway-target","online-eval"} (inline, file://, or -)', + z.string().optional(), + ), + flag( + "treatment", + 'treatment JSON {"gateway-target","online-eval"} (inline, file://, or -)', + z.string().optional(), + ), + flag( + "treatment-weight", + "1-99; control weight = 100 - this (default 50)", + z.number().int().optional(), + ), + flag( + "gateway-filter", + 'GatewayFilter JSON, e.g. {"targetPaths":["/orders"]} (inline, file://, or -)', + z.string().optional(), + ), + flag( + "role-arn", + "execution-role override (default: auto-provisioned)", + z.string().optional(), + ), + flag( + "enable-on-create", + "whether to start the test immediately (default true; pass false to create it paused)", + z.enum(["true", "false"]).optional(), + ), + ], + handle: async (ctx, flags) => { + const required = ["name", "gateway", "control", "treatment"] as const; + for (const f of required) { + if (!flags[f]) throw new InputValidationError(`required option '--${f}' not specified`); + } + + const source = new SourceResolver({ stdin: io.stdin }); + const controlRaw = parseJsonFlag( + "control", + await source.resolveText("control", flags["control"]), + ); + const treatmentRaw = parseJsonFlag( + "treatment", + await source.resolveText("treatment", flags["treatment"]), + ); + const gatewayFilter = parseJsonFlag< + import("@aws-sdk/client-bedrock-agentcore").GatewayFilter + >("gateway-filter", await source.resolveText("gateway-filter", flags["gateway-filter"])); + + const control = toTargetRef("control", controlRaw); + const treatment = toTargetRef("treatment", treatmentRaw); + if (control.gatewayTarget === treatment.gatewayTarget) { + throw new InputValidationError( + "control and treatment must reference different gateway targets", + ); + } + + const treatmentWeight = flags["treatment-weight"]; + if (treatmentWeight !== undefined && (treatmentWeight < 1 || treatmentWeight > 99)) { + throw new InputValidationError("--treatment-weight must be between 1 and 99"); + } + + const result = await core.eval.createTargetBasedABTest( + { + name: flags["name"]!, + gateway: flags["gateway"]!, + control, + treatment, + treatmentWeight, + gatewayFilter, + roleArn: flags["role-arn"], + enableOnCreate: + flags["enable-on-create"] === undefined + ? undefined + : flags["enable-on-create"] === "true", + }, + coreOptsFromCtx(ctx), + ); + + ctx.require(JsonRendererKey).renderJson(result); + }, + }); diff --git a/src/handlers/eval/ondemand/__fixtures__/EvaluateCommand.6867137a17d29ce5.json b/src/handlers/eval/ondemand/__fixtures__/EvaluateCommand.6867137a17d29ce5.json new file mode 100644 index 000000000..b8c9304a2 --- /dev/null +++ b/src/handlers/eval/ondemand/__fixtures__/EvaluateCommand.6867137a17d29ce5.json @@ -0,0 +1,114 @@ +{ + "evaluationResults": [ + { + "evaluatorArn": "arn:aws:bedrock-agentcore:::evaluator/Builtin.Helpfulness", + "evaluatorId": "Builtin.Helpfulness", + "evaluatorName": "Builtin.Helpfulness", + "context": { + "spanContext": { + "sessionId": "00000000-0000-4000-8000-000000000001", + "traceId": "6a8f5b461983df572e2aca3264d32d4d" + } + }, + "explanation": "The user simply said 'hi', which is a greeting with no specific goal expressed. The assistant responded with a polite greeting and an open invitation to help. This is appropriate conversational behavior that maintains the flow of interaction and invites the user to share their actual needs. Since the user hasn't expressed a specific goal yet, the assistant's response is a standard, appropriate reply that keeps the conversation open. It doesn't advance any specific goal (since none exists yet), but it doesn't hinder progress either. This falls into the 'Neutral/Mixed' category as it's appropriate chit-chat for conversation flow with no specific goal to advance.", + "value": 0.5, + "label": "Neutral/Mixed", + "tokenUsage": { + "inputTokens": 815, + "outputTokens": 155, + "totalTokens": 970 + }, + "ignoredReferenceInputFields": [ + "assertions" + ] + }, + { + "evaluatorArn": "arn:aws:bedrock-agentcore:::evaluator/Builtin.Helpfulness", + "evaluatorId": "Builtin.Helpfulness", + "evaluatorName": "Builtin.Helpfulness", + "context": { + "spanContext": { + "sessionId": "00000000-0000-4000-8000-000000000001", + "traceId": "6a8f5bbc1882c05547b64a9246f17ca4" + } + }, + "explanation": "The user has said 'hi' twice without providing any specific goal or request. The assistant's response 'Hi there! How can I assist you today?' is a standard greeting that keeps the conversation open and invites the user to share their needs. Since the user hasn't expressed a specific goal yet, the assistant can't do much more than respond to the greeting and prompt the user to share what they need. This response is appropriate for the conversational context - it's a polite acknowledgment that maintains the conversation flow and opens the door for the user to state their actual needs. It doesn't advance any specific goal (since none has been stated), but it doesn't hinder progress either. This falls into the 'Neutral/Mixed' category as it's appropriate chit-chat for conversation flow with no specific goal to advance.", + "value": 0.5, + "label": "Neutral/Mixed", + "tokenUsage": { + "inputTokens": 887, + "outputTokens": 194, + "totalTokens": 1081 + }, + "ignoredReferenceInputFields": [ + "assertions" + ] + }, + { + "evaluatorArn": "arn:aws:bedrock-agentcore:::evaluator/Builtin.Helpfulness", + "evaluatorId": "Builtin.Helpfulness", + "evaluatorName": "Builtin.Helpfulness", + "context": { + "spanContext": { + "sessionId": "00000000-0000-4000-8000-000000000001", + "traceId": "6a8f639540ebe6cb6f30d5d80762c2e2" + } + }, + "explanation": "The user has sent 'hi' three times in a row without providing any specific request or goal. The assistant's response 'Hello! How can I help you today?' is a standard greeting that keeps the conversation open and invites the user to share their needs. This is the third identical exchange, and the assistant is simply repeating the same greeting. While the response is appropriate and doesn't obstruct any goal, it also doesn't advance any specific goal since the user hasn't expressed one yet. The response is essentially neutral - it's appropriate chit-chat that maintains conversation flow without moving toward any particular goal (since no goal has been stated). The assistant could potentially note that the user has greeted multiple times and ask if they need help with something specific, which would be slightly more proactive, but the current response is still a reasonable reply to a simple greeting.", + "value": 0.5, + "label": "Neutral/Mixed", + "tokenUsage": { + "inputTokens": 958, + "outputTokens": 207, + "totalTokens": 1165 + }, + "ignoredReferenceInputFields": [ + "assertions" + ] + }, + { + "evaluatorArn": "arn:aws:bedrock-agentcore:::evaluator/Builtin.Helpfulness", + "evaluatorId": "Builtin.Helpfulness", + "evaluatorName": "Builtin.Helpfulness", + "context": { + "spanContext": { + "sessionId": "00000000-0000-4000-8000-000000000001", + "traceId": "6a90a5987755c6e457ada5616a16dddb" + } + }, + "explanation": "The user's final request was 'Say hello and offer help.' The assistant's response directly fulfills this request by:\n1. Saying hello with a greeting and wave emoji\n2. Offering help by listing specific capabilities it can assist with\n\nThe response is friendly, clear, and directly addresses what the user asked for. It goes slightly beyond the minimal requirement by providing specific examples of what it can help with (web search, fetching webpages, calculations), which gives the user actionable information about how to proceed.\n\nHowever, the listed capabilities (web search, fetch webpages, perform calculations) seem oddly specific and somewhat limiting - a general AI assistant can help with many more things like writing, analysis, coding, answering questions, etc. This specificity might actually mislead the user about the assistant's full range of capabilities.\n\nDespite this minor issue, the response does exactly what was requested - says hello and offers help - and does so in a clear, organized manner. The user's goal was simple and the assistant met it directly.", + "value": 0.83, + "label": "Very Helpful", + "tokenUsage": { + "inputTokens": 1112, + "outputTokens": 258, + "totalTokens": 1370 + }, + "ignoredReferenceInputFields": [ + "assertions" + ] + }, + { + "evaluatorArn": "arn:aws:bedrock-agentcore:::evaluator/Builtin.Helpfulness", + "evaluatorId": "Builtin.Helpfulness", + "evaluatorName": "Builtin.Helpfulness", + "context": { + "spanContext": { + "sessionId": "00000000-0000-4000-8000-000000000001", + "traceId": "6a90a74750f33f567076943f7bce7966" + } + }, + "explanation": "The user's request is simple and explicit: 'Say hello and offer help.' The assistant's response directly fulfills this request by greeting the user with 'Hello! 👋 Welcome!' and then offering help with a clear, organized list of capabilities (web searches, reading webpages, and calculations). The response is well-formatted, friendly, and ends with an open invitation for the user to specify what they need. This is essentially the same response as the previous turn (with minor variations like adding 'Welcome!'), which is appropriate since the user repeated the same request. The response fully satisfies the user's stated goal of having the assistant say hello and offer help. It's comprehensive and actionable, clearly communicating what the assistant can do. There's nothing missing or problematic about this response given the user's simple, direct request.", + "value": 0.83, + "label": "Very Helpful", + "tokenUsage": { + "inputTokens": 1267, + "outputTokens": 203, + "totalTokens": 1470 + }, + "ignoredReferenceInputFields": [ + "assertions" + ] + } + ] +} \ No newline at end of file diff --git a/src/handlers/eval/ondemand/__fixtures__/GetQueryResultsCommand.2e6a36ed61668bc0.json b/src/handlers/eval/ondemand/__fixtures__/GetQueryResultsCommand.2e6a36ed61668bc0.json new file mode 100644 index 000000000..12a4f7f3f --- /dev/null +++ b/src/handlers/eval/ondemand/__fixtures__/GetQueryResultsCommand.2e6a36ed61668bc0.json @@ -0,0 +1,344 @@ +{ + "queryLanguage": "CWLI", + "results": [ + [ + { + "field": "@message", + "value": "{\"resource\":{\"attributes\":{\"telemetry.sdk.language\":\"python\",\"telemetry.sdk.name\":\"opentelemetry\",\"telemetry.sdk.version\":\"1.40.0\",\"service.name\":\"asdf_MyAgent.DEFAULT\",\"aws.log.group.names\":\"/aws/bedrock-agentcore/runtimes/asdf_MyAgent-3s5axvBC6Q-DEFAULT\",\"aws.log.stream.names\":\"otel-rt-logs\",\"deployment.environment.name\":\"bedrock-agentcore:default\",\"cloud.resource_id\":\"arn:aws:bedrock-agentcore:us-west-2:685197708687:runtime/asdf_MyAgent-3s5axvBC6Q/runtime-endpoint/DEFAULT:DEFAULT\",\"cloud.platform\":\"aws_bedrock_agentcore\",\"cloud.provider\":\"aws\",\"cloud.region\":\"us-west-2\",\"telemetry.auto.version\":\"0.17.0-aws\",\"aws.local.service\":\"asdf_MyAgent.DEFAULT\",\"aws.service.type\":\"gen_ai_agent\"}},\"scope\":{\"name\":\"strands.telemetry.tracer\"},\"timeUnixNano\":1787779917882748907,\"observedTimeUnixNano\":1787779918378293959,\"severityNumber\":9,\"severityText\":\"\",\"body\":{\"input\":{\"messages\":[{\"role\":\"system\",\"content\":{\"content\":\"[{\\\"text\\\": \\\"\\\\n You are a helpful assistant. Use tools when appropriate.\\\\n \\\"}]\"}},{\"role\":\"user\",\"content\":{\"content\":\"[{\\\"text\\\": \\\"hi\\\"}]\"}}]},\"output\":{\"messages\":[{\"role\":\"assistant\",\"content\":{\"finish_reason\":\"end_turn\",\"message\":\"[{\\\"text\\\": \\\"Hello! How can I help you today?\\\"}]\"}}]}},\"attributes\":{\"session.id\":\"00000000-0000-4000-8000-000000000001\",\"event.name\":\"strands.telemetry.tracer\"},\"flags\":1,\"traceId\":\"6a8f5b461983df572e2aca3264d32d4d\",\"spanId\":\"ac9653f70866483d\"}" + }, + { + "field": "sessionId", + "value": "00000000-0000-4000-8000-000000000001" + }, + { + "field": "traceId", + "value": "6a8f5b461983df572e2aca3264d32d4d" + }, + { + "field": "spanId", + "value": "ac9653f70866483d" + }, + { + "field": "@ptr", + "value": "CswBCo8BCkw2ODUxOTc3MDg2ODc6L2F3cy9iZWRyb2NrLWFnZW50Y29yZS9ydW50aW1lcy9hc2RmX015QWdlbnQtM3M1YXh2QkM2US1ERUZBVUxUEAAaJGY2ZGY3NDIyLWVlZTQtNDhkNS05NDg3LTE0NTZkYjdkZGE1YiIOCICQ+IyCNBDnn6qthDRA0cHR4dszSAASNhoYAgakLY/QAAAAAD0Mr1AABqj1tJAAAAQCIAEo/63y/4M0MK/U8v+DNDgfQKbVAUikaFCmRyACEBoYAQ==" + } + ], + [ + { + "field": "@message", + "value": "{\"resource\":{\"attributes\":{\"telemetry.sdk.language\":\"python\",\"telemetry.sdk.name\":\"opentelemetry\",\"telemetry.sdk.version\":\"1.40.0\",\"service.name\":\"asdf_MyAgent.DEFAULT\",\"aws.log.group.names\":\"/aws/bedrock-agentcore/runtimes/asdf_MyAgent-3s5axvBC6Q-DEFAULT\",\"aws.log.stream.names\":\"otel-rt-logs\",\"deployment.environment.name\":\"bedrock-agentcore:default\",\"cloud.resource_id\":\"arn:aws:bedrock-agentcore:us-west-2:685197708687:runtime/asdf_MyAgent-3s5axvBC6Q/runtime-endpoint/DEFAULT:DEFAULT\",\"cloud.platform\":\"aws_bedrock_agentcore\",\"cloud.provider\":\"aws\",\"cloud.region\":\"us-west-2\",\"telemetry.auto.version\":\"0.17.0-aws\",\"aws.local.service\":\"asdf_MyAgent.DEFAULT\",\"aws.service.type\":\"gen_ai_agent\"}},\"scope\":{\"name\":\"strands.telemetry.tracer\"},\"timeUnixNano\":1787779917883212081,\"observedTimeUnixNano\":1787779918378532682,\"severityNumber\":9,\"severityText\":\"\",\"body\":{\"input\":{\"messages\":[{\"role\":\"system\",\"content\":\"\\n You are a helpful assistant. Use tools when appropriate.\\n \"},{\"role\":\"user\",\"content\":{\"content\":\"[{\\\"text\\\": \\\"hi\\\"}]\"}}]},\"output\":{\"messages\":[{\"role\":\"assistant\",\"content\":{\"message\":\"Hello! How can I help you today?\\n\",\"finish_reason\":\"end_turn\"}}]}},\"attributes\":{\"session.id\":\"00000000-0000-4000-8000-000000000001\",\"event.name\":\"strands.telemetry.tracer\"},\"flags\":1,\"traceId\":\"6a8f5b461983df572e2aca3264d32d4d\",\"spanId\":\"82fe397271c56f75\"}" + }, + { + "field": "sessionId", + "value": "00000000-0000-4000-8000-000000000001" + }, + { + "field": "traceId", + "value": "6a8f5b461983df572e2aca3264d32d4d" + }, + { + "field": "spanId", + "value": "82fe397271c56f75" + }, + { + "field": "@ptr", + "value": "CswBCo8BCkw2ODUxOTc3MDg2ODc6L2F3cy9iZWRyb2NrLWFnZW50Y29yZS9ydW50aW1lcy9hc2RmX015QWdlbnQtM3M1YXh2QkM2US1ERUZBVUxUEAAaJGY2ZGY3NDIyLWVlZTQtNDhkNS05NDg3LTE0NTZkYjdkZGE1YiIOCICQ+IyCNBDnn6qthDRA0cHR4dszSAASNhoYAgakLY/QAAAAAD0Mr1AABqj1tJAAAAQCIAEo/63y/4M0MK/U8v+DNDgfQKbVAUikaFCmRyACEBwYAQ==" + } + ], + [ + { + "field": "@message", + "value": "{\"resource\":{\"attributes\":{\"telemetry.sdk.language\":\"python\",\"telemetry.sdk.name\":\"opentelemetry\",\"telemetry.sdk.version\":\"1.40.0\",\"service.name\":\"asdf_MyAgent.DEFAULT\",\"aws.log.group.names\":\"/aws/bedrock-agentcore/runtimes/asdf_MyAgent-3s5axvBC6Q-DEFAULT\",\"aws.log.stream.names\":\"otel-rt-logs\",\"deployment.environment.name\":\"bedrock-agentcore:default\",\"cloud.resource_id\":\"arn:aws:bedrock-agentcore:us-west-2:685197708687:runtime/asdf_MyAgent-3s5axvBC6Q/runtime-endpoint/DEFAULT:DEFAULT\",\"cloud.platform\":\"aws_bedrock_agentcore\",\"cloud.provider\":\"aws\",\"cloud.region\":\"us-west-2\",\"telemetry.auto.version\":\"0.17.0-aws\",\"aws.local.service\":\"asdf_MyAgent.DEFAULT\",\"aws.service.type\":\"gen_ai_agent\"}},\"scope\":{\"name\":\"strands.telemetry.tracer\"},\"timeUnixNano\":1787779917883063841,\"observedTimeUnixNano\":1787779918378428749,\"severityNumber\":9,\"severityText\":\"\",\"body\":{\"input\":{\"messages\":[{\"role\":\"user\",\"content\":{\"content\":\"[{\\\"text\\\": \\\"hi\\\"}]\"}}]}},\"attributes\":{\"session.id\":\"00000000-0000-4000-8000-000000000001\",\"event.name\":\"strands.telemetry.tracer\"},\"flags\":1,\"traceId\":\"6a8f5b461983df572e2aca3264d32d4d\",\"spanId\":\"c9722c3dbb4e781e\"}" + }, + { + "field": "sessionId", + "value": "00000000-0000-4000-8000-000000000001" + }, + { + "field": "traceId", + "value": "6a8f5b461983df572e2aca3264d32d4d" + }, + { + "field": "spanId", + "value": "c9722c3dbb4e781e" + }, + { + "field": "@ptr", + "value": "CswBCo8BCkw2ODUxOTc3MDg2ODc6L2F3cy9iZWRyb2NrLWFnZW50Y29yZS9ydW50aW1lcy9hc2RmX015QWdlbnQtM3M1YXh2QkM2US1ERUZBVUxUEAAaJGY2ZGY3NDIyLWVlZTQtNDhkNS05NDg3LTE0NTZkYjdkZGE1YiIOCICQ+IyCNBDnn6qthDRA0cHR4dszSAASNhoYAgakLY/QAAAAAD0Mr1AABqj1tJAAAAQCIAEo/63y/4M0MK/U8v+DNDgfQKbVAUikaFCmRyACEBsYAQ==" + } + ], + [ + { + "field": "@message", + "value": "{\"resource\":{\"attributes\":{\"telemetry.sdk.language\":\"python\",\"telemetry.sdk.name\":\"opentelemetry\",\"telemetry.sdk.version\":\"1.40.0\",\"service.name\":\"asdf_MyAgent.DEFAULT\",\"aws.log.group.names\":\"/aws/bedrock-agentcore/runtimes/asdf_MyAgent-3s5axvBC6Q-DEFAULT\",\"aws.log.stream.names\":\"otel-rt-logs\",\"deployment.environment.name\":\"bedrock-agentcore:default\",\"cloud.resource_id\":\"arn:aws:bedrock-agentcore:us-west-2:685197708687:runtime/asdf_MyAgent-3s5axvBC6Q/runtime-endpoint/DEFAULT:DEFAULT\",\"cloud.platform\":\"aws_bedrock_agentcore\",\"cloud.provider\":\"aws\",\"cloud.region\":\"us-west-2\",\"telemetry.auto.version\":\"0.17.0-aws\",\"aws.local.service\":\"asdf_MyAgent.DEFAULT\",\"aws.service.type\":\"gen_ai_agent\"}},\"scope\":{\"name\":\"strands.telemetry.tracer\"},\"timeUnixNano\":1787780030855720915,\"observedTimeUnixNano\":1787780033471228591,\"severityNumber\":9,\"severityText\":\"\",\"body\":{\"input\":{\"messages\":[{\"role\":\"system\",\"content\":{\"content\":\"[{\\\"text\\\": \\\"\\\\n You are a helpful assistant. Use tools when appropriate.\\\\n \\\"}]\"}},{\"role\":\"user\",\"content\":{\"content\":\"[{\\\"text\\\": \\\"hi\\\"}]\"}}]},\"output\":{\"messages\":[{\"role\":\"assistant\",\"content\":{\"content\":\"[{\\\"text\\\": \\\"Hello! How can I help you today?\\\"}]\"}},{\"role\":\"assistant\",\"content\":{\"finish_reason\":\"end_turn\",\"message\":\"[{\\\"text\\\": \\\"Hi there! How can I assist you today?\\\"}]\"}}]}},\"attributes\":{\"session.id\":\"00000000-0000-4000-8000-000000000001\",\"event.name\":\"strands.telemetry.tracer\"},\"flags\":1,\"traceId\":\"6a8f5bbc1882c05547b64a9246f17ca4\",\"spanId\":\"2b53da16274ce9f5\"}" + }, + { + "field": "sessionId", + "value": "00000000-0000-4000-8000-000000000001" + }, + { + "field": "traceId", + "value": "6a8f5bbc1882c05547b64a9246f17ca4" + }, + { + "field": "spanId", + "value": "2b53da16274ce9f5" + }, + { + "field": "@ptr", + "value": "Cs4BCo8BCkw2ODUxOTc3MDg2ODc6L2F3cy9iZWRyb2NrLWFnZW50Y29yZS9ydW50aW1lcy9hc2RmX015QWdlbnQtM3M1YXh2QkM2US1ERUZBVUxUEAAaJGY2ZGY3NDIyLWVlZTQtNDhkNS05NDg3LTE0NTZkYjdkZGE1YiIOCICQ+IyCNBDnn6qthDRA0cHR4dszSAASOBoYAgakLY/QAAAAAD0NEwIABqj1vAAAAAQCIAEo1IL2/4M0MIjD+f+DNDgYQKiOAkj11wFQ9oABIAIQFRgB" + } + ], + [ + { + "field": "@message", + "value": "{\"resource\":{\"attributes\":{\"telemetry.sdk.language\":\"python\",\"telemetry.sdk.name\":\"opentelemetry\",\"telemetry.sdk.version\":\"1.40.0\",\"service.name\":\"asdf_MyAgent.DEFAULT\",\"aws.log.group.names\":\"/aws/bedrock-agentcore/runtimes/asdf_MyAgent-3s5axvBC6Q-DEFAULT\",\"aws.log.stream.names\":\"otel-rt-logs\",\"deployment.environment.name\":\"bedrock-agentcore:default\",\"cloud.resource_id\":\"arn:aws:bedrock-agentcore:us-west-2:685197708687:runtime/asdf_MyAgent-3s5axvBC6Q/runtime-endpoint/DEFAULT:DEFAULT\",\"cloud.platform\":\"aws_bedrock_agentcore\",\"cloud.provider\":\"aws\",\"cloud.region\":\"us-west-2\",\"telemetry.auto.version\":\"0.17.0-aws\",\"aws.local.service\":\"asdf_MyAgent.DEFAULT\",\"aws.service.type\":\"gen_ai_agent\"}},\"scope\":{\"name\":\"strands.telemetry.tracer\"},\"timeUnixNano\":1787780030855979475,\"observedTimeUnixNano\":1787780033471374339,\"severityNumber\":9,\"severityText\":\"\",\"body\":{\"input\":{\"messages\":[{\"role\":\"user\",\"content\":{\"content\":\"[{\\\"text\\\": \\\"hi\\\"}]\"}}]},\"output\":{\"messages\":[{\"role\":\"assistant\",\"content\":{\"content\":\"[{\\\"text\\\": \\\"Hello! How can I help you today?\\\"}]\"}}]}},\"attributes\":{\"session.id\":\"00000000-0000-4000-8000-000000000001\",\"event.name\":\"strands.telemetry.tracer\"},\"flags\":1,\"traceId\":\"6a8f5bbc1882c05547b64a9246f17ca4\",\"spanId\":\"43f809568e52d1ca\"}" + }, + { + "field": "sessionId", + "value": "00000000-0000-4000-8000-000000000001" + }, + { + "field": "traceId", + "value": "6a8f5bbc1882c05547b64a9246f17ca4" + }, + { + "field": "spanId", + "value": "43f809568e52d1ca" + }, + { + "field": "@ptr", + "value": "Cs4BCo8BCkw2ODUxOTc3MDg2ODc6L2F3cy9iZWRyb2NrLWFnZW50Y29yZS9ydW50aW1lcy9hc2RmX015QWdlbnQtM3M1YXh2QkM2US1ERUZBVUxUEAAaJGY2ZGY3NDIyLWVlZTQtNDhkNS05NDg3LTE0NTZkYjdkZGE1YiIOCICQ+IyCNBDnn6qthDRA0cHR4dszSAASOBoYAgakLY/QAAAAAD0NEwIABqj1vAAAAAQCIAEo1IL2/4M0MIjD+f+DNDgYQKiOAkj11wFQ9oABIAIQFhgB" + } + ], + [ + { + "field": "@message", + "value": "{\"resource\":{\"attributes\":{\"telemetry.sdk.language\":\"python\",\"telemetry.sdk.name\":\"opentelemetry\",\"telemetry.sdk.version\":\"1.40.0\",\"service.name\":\"asdf_MyAgent.DEFAULT\",\"aws.log.group.names\":\"/aws/bedrock-agentcore/runtimes/asdf_MyAgent-3s5axvBC6Q-DEFAULT\",\"aws.log.stream.names\":\"otel-rt-logs\",\"deployment.environment.name\":\"bedrock-agentcore:default\",\"cloud.resource_id\":\"arn:aws:bedrock-agentcore:us-west-2:685197708687:runtime/asdf_MyAgent-3s5axvBC6Q/runtime-endpoint/DEFAULT:DEFAULT\",\"cloud.platform\":\"aws_bedrock_agentcore\",\"cloud.provider\":\"aws\",\"cloud.region\":\"us-west-2\",\"telemetry.auto.version\":\"0.17.0-aws\",\"aws.local.service\":\"asdf_MyAgent.DEFAULT\",\"aws.service.type\":\"gen_ai_agent\"}},\"scope\":{\"name\":\"strands.telemetry.tracer\"},\"timeUnixNano\":1787780030856119726,\"observedTimeUnixNano\":1787780033471475805,\"severityNumber\":9,\"severityText\":\"\",\"body\":{\"input\":{\"messages\":[{\"role\":\"system\",\"content\":\"\\n You are a helpful assistant. Use tools when appropriate.\\n \"},{\"role\":\"user\",\"content\":{\"content\":\"[{\\\"text\\\": \\\"hi\\\"}]\"}}]},\"output\":{\"messages\":[{\"role\":\"assistant\",\"content\":{\"message\":\"Hi there! How can I assist you today?\\n\",\"finish_reason\":\"end_turn\"}}]}},\"attributes\":{\"session.id\":\"00000000-0000-4000-8000-000000000001\",\"event.name\":\"strands.telemetry.tracer\"},\"flags\":1,\"traceId\":\"6a8f5bbc1882c05547b64a9246f17ca4\",\"spanId\":\"324e3a835d1af5c4\"}" + }, + { + "field": "sessionId", + "value": "00000000-0000-4000-8000-000000000001" + }, + { + "field": "traceId", + "value": "6a8f5bbc1882c05547b64a9246f17ca4" + }, + { + "field": "spanId", + "value": "324e3a835d1af5c4" + }, + { + "field": "@ptr", + "value": "Cs4BCo8BCkw2ODUxOTc3MDg2ODc6L2F3cy9iZWRyb2NrLWFnZW50Y29yZS9ydW50aW1lcy9hc2RmX015QWdlbnQtM3M1YXh2QkM2US1ERUZBVUxUEAAaJGY2ZGY3NDIyLWVlZTQtNDhkNS05NDg3LTE0NTZkYjdkZGE1YiIOCICQ+IyCNBDnn6qthDRA0cHR4dszSAASOBoYAgakLY/QAAAAAD0NEwIABqj1vAAAAAQCIAEo1IL2/4M0MIjD+f+DNDgYQKiOAkj11wFQ9oABIAIQFxgB" + } + ], + [ + { + "field": "@message", + "value": "{\"resource\":{\"attributes\":{\"telemetry.sdk.language\":\"python\",\"telemetry.sdk.name\":\"opentelemetry\",\"telemetry.sdk.version\":\"1.40.0\",\"service.name\":\"asdf_MyAgent.DEFAULT\",\"aws.log.group.names\":\"/aws/bedrock-agentcore/runtimes/asdf_MyAgent-3s5axvBC6Q-DEFAULT\",\"aws.log.stream.names\":\"otel-rt-logs\",\"deployment.environment.name\":\"bedrock-agentcore:default\",\"cloud.resource_id\":\"arn:aws:bedrock-agentcore:us-west-2:685197708687:runtime/asdf_MyAgent-3s5axvBC6Q/runtime-endpoint/DEFAULT:DEFAULT\",\"cloud.platform\":\"aws_bedrock_agentcore\",\"cloud.provider\":\"aws\",\"cloud.region\":\"us-west-2\",\"telemetry.auto.version\":\"0.17.0-aws\",\"aws.local.service\":\"asdf_MyAgent.DEFAULT\",\"aws.service.type\":\"gen_ai_agent\"}},\"scope\":{\"name\":\"strands.telemetry.tracer\"},\"timeUnixNano\":1787782045239528397,\"observedTimeUnixNano\":1787782046488355037,\"severityNumber\":9,\"severityText\":\"\",\"body\":{\"input\":{\"messages\":[{\"role\":\"user\",\"content\":{\"content\":\"[{\\\"text\\\": \\\"hi\\\"}]\"}}]}},\"attributes\":{\"session.id\":\"00000000-0000-4000-8000-000000000001\",\"event.name\":\"strands.telemetry.tracer\"},\"flags\":1,\"traceId\":\"6a8f639540ebe6cb6f30d5d80762c2e2\",\"spanId\":\"fc189f3b80a0c1f4\"}" + }, + { + "field": "sessionId", + "value": "00000000-0000-4000-8000-000000000001" + }, + { + "field": "traceId", + "value": "6a8f639540ebe6cb6f30d5d80762c2e2" + }, + { + "field": "spanId", + "value": "fc189f3b80a0c1f4" + }, + { + "field": "@ptr", + "value": "CswBCo8BCkw2ODUxOTc3MDg2ODc6L2F3cy9iZWRyb2NrLWFnZW50Y29yZS9ydW50aW1lcy9hc2RmX015QWdlbnQtM3M1YXh2QkM2US1ERUZBVUxUEAAaJGY2ZGY3NDIyLWVlZTQtNDhkNS05NDg3LTE0NTZkYjdkZGE1YiIOCICQ+IyCNBDnn6qthDRA0cHR4dszSAASNhoYAgakLY/QAAAAAD0VMD0ABqj2OYAAAAQCIAEo7Z/0gIQ0MJ3G9ICENDggQLfZAUisaFCuRyACEB0YAQ==" + } + ], + [ + { + "field": "@message", + "value": "{\"resource\":{\"attributes\":{\"telemetry.sdk.language\":\"python\",\"telemetry.sdk.name\":\"opentelemetry\",\"telemetry.sdk.version\":\"1.40.0\",\"service.name\":\"asdf_MyAgent.DEFAULT\",\"aws.log.group.names\":\"/aws/bedrock-agentcore/runtimes/asdf_MyAgent-3s5axvBC6Q-DEFAULT\",\"aws.log.stream.names\":\"otel-rt-logs\",\"deployment.environment.name\":\"bedrock-agentcore:default\",\"cloud.resource_id\":\"arn:aws:bedrock-agentcore:us-west-2:685197708687:runtime/asdf_MyAgent-3s5axvBC6Q/runtime-endpoint/DEFAULT:DEFAULT\",\"cloud.platform\":\"aws_bedrock_agentcore\",\"cloud.provider\":\"aws\",\"cloud.region\":\"us-west-2\",\"telemetry.auto.version\":\"0.17.0-aws\",\"aws.local.service\":\"asdf_MyAgent.DEFAULT\",\"aws.service.type\":\"gen_ai_agent\"}},\"scope\":{\"name\":\"strands.telemetry.tracer\"},\"timeUnixNano\":1787782045239678560,\"observedTimeUnixNano\":1787782046488465849,\"severityNumber\":9,\"severityText\":\"\",\"body\":{\"input\":{\"messages\":[{\"role\":\"system\",\"content\":\"\\n You are a helpful assistant. Use tools when appropriate.\\n \"},{\"role\":\"user\",\"content\":{\"content\":\"[{\\\"text\\\": \\\"hi\\\"}]\"}}]},\"output\":{\"messages\":[{\"role\":\"assistant\",\"content\":{\"message\":\"Hello! How can I help you today?\\n\",\"finish_reason\":\"end_turn\"}}]}},\"attributes\":{\"session.id\":\"00000000-0000-4000-8000-000000000001\",\"event.name\":\"strands.telemetry.tracer\"},\"flags\":1,\"traceId\":\"6a8f639540ebe6cb6f30d5d80762c2e2\",\"spanId\":\"6f925a9a78a2fe62\"}" + }, + { + "field": "sessionId", + "value": "00000000-0000-4000-8000-000000000001" + }, + { + "field": "traceId", + "value": "6a8f639540ebe6cb6f30d5d80762c2e2" + }, + { + "field": "spanId", + "value": "6f925a9a78a2fe62" + }, + { + "field": "@ptr", + "value": "CswBCo8BCkw2ODUxOTc3MDg2ODc6L2F3cy9iZWRyb2NrLWFnZW50Y29yZS9ydW50aW1lcy9hc2RmX015QWdlbnQtM3M1YXh2QkM2US1ERUZBVUxUEAAaJGY2ZGY3NDIyLWVlZTQtNDhkNS05NDg3LTE0NTZkYjdkZGE1YiIOCICQ+IyCNBDnn6qthDRA0cHR4dszSAASNhoYAgakLY/QAAAAAD0VMD0ABqj2OYAAAAQCIAEo7Z/0gIQ0MJ3G9ICENDggQLfZAUisaFCuRyACEB4YAQ==" + } + ], + [ + { + "field": "@message", + "value": "{\"resource\":{\"attributes\":{\"telemetry.sdk.language\":\"python\",\"telemetry.sdk.name\":\"opentelemetry\",\"telemetry.sdk.version\":\"1.40.0\",\"service.name\":\"asdf_MyAgent.DEFAULT\",\"aws.log.group.names\":\"/aws/bedrock-agentcore/runtimes/asdf_MyAgent-3s5axvBC6Q-DEFAULT\",\"aws.log.stream.names\":\"otel-rt-logs\",\"deployment.environment.name\":\"bedrock-agentcore:default\",\"cloud.resource_id\":\"arn:aws:bedrock-agentcore:us-west-2:685197708687:runtime/asdf_MyAgent-3s5axvBC6Q/runtime-endpoint/DEFAULT:DEFAULT\",\"cloud.platform\":\"aws_bedrock_agentcore\",\"cloud.provider\":\"aws\",\"cloud.region\":\"us-west-2\",\"telemetry.auto.version\":\"0.17.0-aws\",\"aws.local.service\":\"asdf_MyAgent.DEFAULT\",\"aws.service.type\":\"gen_ai_agent\"}},\"scope\":{\"name\":\"strands.telemetry.tracer\"},\"timeUnixNano\":1787782045239215762,\"observedTimeUnixNano\":1787782046488222333,\"severityNumber\":9,\"severityText\":\"\",\"body\":{\"input\":{\"messages\":[{\"role\":\"system\",\"content\":{\"content\":\"[{\\\"text\\\": \\\"\\\\n You are a helpful assistant. Use tools when appropriate.\\\\n \\\"}]\"}},{\"role\":\"user\",\"content\":{\"content\":\"[{\\\"text\\\": \\\"hi\\\"}]\"}}]},\"output\":{\"messages\":[{\"role\":\"assistant\",\"content\":{\"finish_reason\":\"end_turn\",\"message\":\"[{\\\"text\\\": \\\"Hello! How can I help you today?\\\"}]\"}}]}},\"attributes\":{\"session.id\":\"00000000-0000-4000-8000-000000000001\",\"event.name\":\"strands.telemetry.tracer\"},\"flags\":1,\"traceId\":\"6a8f639540ebe6cb6f30d5d80762c2e2\",\"spanId\":\"0925d4cc59be00d9\"}" + }, + { + "field": "sessionId", + "value": "00000000-0000-4000-8000-000000000001" + }, + { + "field": "traceId", + "value": "6a8f639540ebe6cb6f30d5d80762c2e2" + }, + { + "field": "spanId", + "value": "0925d4cc59be00d9" + }, + { + "field": "@ptr", + "value": "CswBCo8BCkw2ODUxOTc3MDg2ODc6L2F3cy9iZWRyb2NrLWFnZW50Y29yZS9ydW50aW1lcy9hc2RmX015QWdlbnQtM3M1YXh2QkM2US1ERUZBVUxUEAAaJGY2ZGY3NDIyLWVlZTQtNDhkNS05NDg3LTE0NTZkYjdkZGE1YiIOCICQ+IyCNBDnn6qthDRA0cHR4dszSAASNhoYAgakLY/QAAAAAD0VMD0ABqj2OYAAAAQCIAEo7Z/0gIQ0MJ3G9ICENDggQLfZAUisaFCuRyACEBwYAQ==" + } + ], + [ + { + "field": "@message", + "value": "{\"resource\":{\"attributes\":{\"telemetry.sdk.language\":\"python\",\"telemetry.sdk.name\":\"opentelemetry\",\"telemetry.sdk.version\":\"1.40.0\",\"service.name\":\"asdf_MyAgent.DEFAULT\",\"aws.log.group.names\":\"/aws/bedrock-agentcore/runtimes/asdf_MyAgent-3s5axvBC6Q-DEFAULT\",\"aws.log.stream.names\":\"otel-rt-logs\",\"deployment.environment.name\":\"bedrock-agentcore:default\",\"cloud.resource_id\":\"arn:aws:bedrock-agentcore:us-west-2:685197708687:runtime/asdf_MyAgent-3s5axvBC6Q/runtime-endpoint/DEFAULT:DEFAULT\",\"cloud.platform\":\"aws_bedrock_agentcore\",\"cloud.provider\":\"aws\",\"cloud.region\":\"us-west-2\",\"telemetry.auto.version\":\"0.17.0-aws\",\"aws.local.service\":\"asdf_MyAgent.DEFAULT\",\"aws.service.type\":\"gen_ai_agent\"}},\"scope\":{\"name\":\"strands.telemetry.tracer\"},\"timeUnixNano\":1787864481907136303,\"observedTimeUnixNano\":1787864486789333679,\"severityNumber\":9,\"severityText\":\"\",\"body\":{\"input\":{\"messages\":[{\"role\":\"system\",\"content\":{\"content\":\"[{\\\"text\\\": \\\"\\\\n You are a helpful assistant. Use tools when appropriate.\\\\n \\\"}]\"}},{\"role\":\"user\",\"content\":{\"content\":\"[{\\\"text\\\": \\\"Say hello and offer help\\\"}]\"}}]},\"output\":{\"messages\":[{\"role\":\"assistant\",\"content\":{\"finish_reason\":\"end_turn\",\"message\":\"[{\\\"text\\\": \\\"Hello! 👋 \\\\n\\\\nI'm here to help you with a variety of tasks. I can:\\\\n\\\\n- **Search the web** for current information, news, facts, or answers to questions\\\\n- **Fetch and read webpages** to get detailed content from specific URLs\\\\n- **Perform calculations** like adding numbers together\\\\n\\\\nWhat can I help you with today?\\\"}]\"}}]}},\"attributes\":{\"session.id\":\"00000000-0000-4000-8000-000000000001\",\"event.name\":\"strands.telemetry.tracer\"},\"flags\":1,\"traceId\":\"6a90a5987755c6e457ada5616a16dddb\",\"spanId\":\"d473e096a2db85a8\"}" + }, + { + "field": "sessionId", + "value": "00000000-0000-4000-8000-000000000001" + }, + { + "field": "traceId", + "value": "6a90a5987755c6e457ada5616a16dddb" + }, + { + "field": "spanId", + "value": "d473e096a2db85a8" + }, + { + "field": "@ptr", + "value": "CswBCo8BCkw2ODUxOTc3MDg2ODc6L2F3cy9iZWRyb2NrLWFnZW50Y29yZS9ydW50aW1lcy9hc2RmX015QWdlbnQtM3M1YXh2QkM2US1ERUZBVUxUEAAaJGY2ZGY3NDIyLWVlZTQtNDhkNS05NDg3LTE0NTZkYjdkZGE1YiIOCICQ+IyCNBDnn6qthDRA0cHR4dszSAASNhoYAgahOL2DAAAAAJJ5XU0ABqkKWdAAAAYyIAEojNybqIQ0MPOAnKiENDgkQJLgAUj9blD9TSACECEYAQ==" + } + ], + [ + { + "field": "@message", + "value": "{\"resource\":{\"attributes\":{\"telemetry.sdk.language\":\"python\",\"telemetry.sdk.name\":\"opentelemetry\",\"telemetry.sdk.version\":\"1.40.0\",\"service.name\":\"asdf_MyAgent.DEFAULT\",\"aws.log.group.names\":\"/aws/bedrock-agentcore/runtimes/asdf_MyAgent-3s5axvBC6Q-DEFAULT\",\"aws.log.stream.names\":\"otel-rt-logs\",\"deployment.environment.name\":\"bedrock-agentcore:default\",\"cloud.resource_id\":\"arn:aws:bedrock-agentcore:us-west-2:685197708687:runtime/asdf_MyAgent-3s5axvBC6Q/runtime-endpoint/DEFAULT:DEFAULT\",\"cloud.platform\":\"aws_bedrock_agentcore\",\"cloud.provider\":\"aws\",\"cloud.region\":\"us-west-2\",\"telemetry.auto.version\":\"0.17.0-aws\",\"aws.local.service\":\"asdf_MyAgent.DEFAULT\",\"aws.service.type\":\"gen_ai_agent\"}},\"scope\":{\"name\":\"strands.telemetry.tracer\"},\"timeUnixNano\":1787864481907743241,\"observedTimeUnixNano\":1787864486789635040,\"severityNumber\":9,\"severityText\":\"\",\"body\":{\"input\":{\"messages\":[{\"role\":\"system\",\"content\":\"\\n You are a helpful assistant. Use tools when appropriate.\\n \"},{\"role\":\"user\",\"content\":{\"content\":\"[{\\\"text\\\": \\\"Say hello and offer help\\\"}]\"}}]},\"output\":{\"messages\":[{\"role\":\"assistant\",\"content\":{\"message\":\"Hello! 👋 \\n\\nI'm here to help you with a variety of tasks. I can:\\n\\n- **Search the web** for current information, news, facts, or answers to questions\\n- **Fetch and read webpages** to get detailed content from specific URLs\\n- **Perform calculations** like adding numbers together\\n\\nWhat can I help you with today?\\n\",\"finish_reason\":\"end_turn\"}}]}},\"attributes\":{\"session.id\":\"00000000-0000-4000-8000-000000000001\",\"event.name\":\"strands.telemetry.tracer\"},\"flags\":1,\"traceId\":\"6a90a5987755c6e457ada5616a16dddb\",\"spanId\":\"2619914761138b83\"}" + }, + { + "field": "sessionId", + "value": "00000000-0000-4000-8000-000000000001" + }, + { + "field": "traceId", + "value": "6a90a5987755c6e457ada5616a16dddb" + }, + { + "field": "spanId", + "value": "2619914761138b83" + }, + { + "field": "@ptr", + "value": "CswBCo8BCkw2ODUxOTc3MDg2ODc6L2F3cy9iZWRyb2NrLWFnZW50Y29yZS9ydW50aW1lcy9hc2RmX015QWdlbnQtM3M1YXh2QkM2US1ERUZBVUxUEAAaJGY2ZGY3NDIyLWVlZTQtNDhkNS05NDg3LTE0NTZkYjdkZGE1YiIOCICQ+IyCNBDnn6qthDRA0cHR4dszSAASNhoYAgahOL2DAAAAAJJ5XU0ABqkKWdAAAAYyIAEojNybqIQ0MPOAnKiENDgkQJLgAUj9blD9TSACECMYAQ==" + } + ], + [ + { + "field": "@message", + "value": "{\"resource\":{\"attributes\":{\"telemetry.sdk.language\":\"python\",\"telemetry.sdk.name\":\"opentelemetry\",\"telemetry.sdk.version\":\"1.40.0\",\"service.name\":\"asdf_MyAgent.DEFAULT\",\"aws.log.group.names\":\"/aws/bedrock-agentcore/runtimes/asdf_MyAgent-3s5axvBC6Q-DEFAULT\",\"aws.log.stream.names\":\"otel-rt-logs\",\"deployment.environment.name\":\"bedrock-agentcore:default\",\"cloud.resource_id\":\"arn:aws:bedrock-agentcore:us-west-2:685197708687:runtime/asdf_MyAgent-3s5axvBC6Q/runtime-endpoint/DEFAULT:DEFAULT\",\"cloud.platform\":\"aws_bedrock_agentcore\",\"cloud.provider\":\"aws\",\"cloud.region\":\"us-west-2\",\"telemetry.auto.version\":\"0.17.0-aws\",\"aws.local.service\":\"asdf_MyAgent.DEFAULT\",\"aws.service.type\":\"gen_ai_agent\"}},\"scope\":{\"name\":\"strands.telemetry.tracer\"},\"timeUnixNano\":1787864481907561373,\"observedTimeUnixNano\":1787864486789498361,\"severityNumber\":9,\"severityText\":\"\",\"body\":{\"input\":{\"messages\":[{\"role\":\"user\",\"content\":{\"content\":\"[{\\\"text\\\": \\\"Say hello and offer help\\\"}]\"}}]}},\"attributes\":{\"session.id\":\"00000000-0000-4000-8000-000000000001\",\"event.name\":\"strands.telemetry.tracer\"},\"flags\":1,\"traceId\":\"6a90a5987755c6e457ada5616a16dddb\",\"spanId\":\"17fb5fcf047ef303\"}" + }, + { + "field": "sessionId", + "value": "00000000-0000-4000-8000-000000000001" + }, + { + "field": "traceId", + "value": "6a90a5987755c6e457ada5616a16dddb" + }, + { + "field": "spanId", + "value": "17fb5fcf047ef303" + }, + { + "field": "@ptr", + "value": "CswBCo8BCkw2ODUxOTc3MDg2ODc6L2F3cy9iZWRyb2NrLWFnZW50Y29yZS9ydW50aW1lcy9hc2RmX015QWdlbnQtM3M1YXh2QkM2US1ERUZBVUxUEAAaJGY2ZGY3NDIyLWVlZTQtNDhkNS05NDg3LTE0NTZkYjdkZGE1YiIOCICQ+IyCNBDnn6qthDRA0cHR4dszSAASNhoYAgahOL2DAAAAAJJ5XU0ABqkKWdAAAAYyIAEojNybqIQ0MPOAnKiENDgkQJLgAUj9blD9TSACECIYAQ==" + } + ], + [ + { + "field": "@message", + "value": "{\"resource\":{\"attributes\":{\"telemetry.sdk.language\":\"python\",\"telemetry.sdk.name\":\"opentelemetry\",\"telemetry.sdk.version\":\"1.40.0\",\"service.name\":\"asdf_MyAgent.DEFAULT\",\"aws.log.group.names\":\"/aws/bedrock-agentcore/runtimes/asdf_MyAgent-3s5axvBC6Q-DEFAULT\",\"aws.log.stream.names\":\"otel-rt-logs\",\"deployment.environment.name\":\"bedrock-agentcore:default\",\"cloud.resource_id\":\"arn:aws:bedrock-agentcore:us-west-2:685197708687:runtime/asdf_MyAgent-3s5axvBC6Q/runtime-endpoint/DEFAULT:DEFAULT\",\"cloud.platform\":\"aws_bedrock_agentcore\",\"cloud.provider\":\"aws\",\"cloud.region\":\"us-west-2\",\"telemetry.auto.version\":\"0.17.0-aws\",\"aws.local.service\":\"asdf_MyAgent.DEFAULT\",\"aws.service.type\":\"gen_ai_agent\"}},\"scope\":{\"name\":\"strands.telemetry.tracer\"},\"timeUnixNano\":1787864908109701143,\"observedTimeUnixNano\":1787864911841960802,\"severityNumber\":9,\"severityText\":\"\",\"body\":{\"input\":{\"messages\":[{\"role\":\"system\",\"content\":{\"content\":\"[{\\\"text\\\": \\\"\\\\n You are a helpful assistant. Use tools when appropriate.\\\\n \\\"}]\"}},{\"role\":\"user\",\"content\":{\"content\":\"[{\\\"text\\\": \\\"Say hello and offer help\\\"}]\"}}]},\"output\":{\"messages\":[{\"role\":\"assistant\",\"content\":{\"content\":\"[{\\\"text\\\": \\\"Hello! 👋 \\\\n\\\\nI'm here to help you with a variety of tasks. I can:\\\\n\\\\n- **Search the web** for current information, news, facts, or answers to questions\\\\n- **Fetch and read webpages** to get detailed content from specific URLs\\\\n- **Perform calculations** like adding numbers together\\\\n\\\\nWhat can I help you with today?\\\"}]\"}},{\"role\":\"assistant\",\"content\":{\"finish_reason\":\"end_turn\",\"message\":\"[{\\\"text\\\": \\\"Hello! 👋 Welcome!\\\\n\\\\nI'm here to assist you today. I can help you with:\\\\n\\\\n- **Web searches** - Find current information, news, research, or answers to questions on any topic\\\\n- **Reading webpages** - Extract and read content from specific URLs or articles\\\\n- **Simple calculations** - Add numbers together\\\\n\\\\nWhat would you like help with?\\\"}]\"}}]}},\"attributes\":{\"session.id\":\"00000000-0000-4000-8000-000000000001\",\"event.name\":\"strands.telemetry.tracer\"},\"flags\":1,\"traceId\":\"6a90a74750f33f567076943f7bce7966\",\"spanId\":\"3bf0732214c6e32c\"}" + }, + { + "field": "sessionId", + "value": "00000000-0000-4000-8000-000000000001" + }, + { + "field": "traceId", + "value": "6a90a74750f33f567076943f7bce7966" + }, + { + "field": "spanId", + "value": "3bf0732214c6e32c" + }, + { + "field": "@ptr", + "value": "CssBCo8BCkw2ODUxOTc3MDg2ODc6L2F3cy9iZWRyb2NrLWFnZW50Y29yZS9ydW50aW1lcy9hc2RmX015QWdlbnQtM3M1YXh2QkM2US1ERUZBVUxUEAAaJGY2ZGY3NDIyLWVlZTQtNDhkNS05NDg3LTE0NTZkYjdkZGE1YiIOCICQ+IyCNBDnn6qthDRA0cHR4dszSAASNRoYAgahOL2DAAAAAJJ6V08ABqkKdOAAAAYyIAEon/i1qIQ0MM6CtqiENDgGQIk5SIhKUNMvIAIQARgB" + } + ], + [ + { + "field": "@message", + "value": "{\"resource\":{\"attributes\":{\"telemetry.sdk.language\":\"python\",\"telemetry.sdk.name\":\"opentelemetry\",\"telemetry.sdk.version\":\"1.40.0\",\"service.name\":\"asdf_MyAgent.DEFAULT\",\"aws.log.group.names\":\"/aws/bedrock-agentcore/runtimes/asdf_MyAgent-3s5axvBC6Q-DEFAULT\",\"aws.log.stream.names\":\"otel-rt-logs\",\"deployment.environment.name\":\"bedrock-agentcore:default\",\"cloud.resource_id\":\"arn:aws:bedrock-agentcore:us-west-2:685197708687:runtime/asdf_MyAgent-3s5axvBC6Q/runtime-endpoint/DEFAULT:DEFAULT\",\"cloud.platform\":\"aws_bedrock_agentcore\",\"cloud.provider\":\"aws\",\"cloud.region\":\"us-west-2\",\"telemetry.auto.version\":\"0.17.0-aws\",\"aws.local.service\":\"asdf_MyAgent.DEFAULT\",\"aws.service.type\":\"gen_ai_agent\"}},\"scope\":{\"name\":\"strands.telemetry.tracer\"},\"timeUnixNano\":1787864908110047680,\"observedTimeUnixNano\":1787864911842132631,\"severityNumber\":9,\"severityText\":\"\",\"body\":{\"input\":{\"messages\":[{\"role\":\"user\",\"content\":{\"content\":\"[{\\\"text\\\": \\\"Say hello and offer help\\\"}]\"}}]},\"output\":{\"messages\":[{\"role\":\"assistant\",\"content\":{\"content\":\"[{\\\"text\\\": \\\"Hello! 👋 \\\\n\\\\nI'm here to help you with a variety of tasks. I can:\\\\n\\\\n- **Search the web** for current information, news, facts, or answers to questions\\\\n- **Fetch and read webpages** to get detailed content from specific URLs\\\\n- **Perform calculations** like adding numbers together\\\\n\\\\nWhat can I help you with today?\\\"}]\"}}]}},\"attributes\":{\"session.id\":\"00000000-0000-4000-8000-000000000001\",\"event.name\":\"strands.telemetry.tracer\"},\"flags\":1,\"traceId\":\"6a90a74750f33f567076943f7bce7966\",\"spanId\":\"5e744ea5e19f1053\"}" + }, + { + "field": "sessionId", + "value": "00000000-0000-4000-8000-000000000001" + }, + { + "field": "traceId", + "value": "6a90a74750f33f567076943f7bce7966" + }, + { + "field": "spanId", + "value": "5e744ea5e19f1053" + }, + { + "field": "@ptr", + "value": "CssBCo8BCkw2ODUxOTc3MDg2ODc6L2F3cy9iZWRyb2NrLWFnZW50Y29yZS9ydW50aW1lcy9hc2RmX015QWdlbnQtM3M1YXh2QkM2US1ERUZBVUxUEAAaJGY2ZGY3NDIyLWVlZTQtNDhkNS05NDg3LTE0NTZkYjdkZGE1YiIOCICQ+IyCNBDnn6qthDRA0cHR4dszSAASNRoYAgahOL2DAAAAAJJ6V08ABqkKdOAAAAYyIAEon/i1qIQ0MM6CtqiENDgGQIk5SIhKUNMvIAIQAhgB" + } + ], + [ + { + "field": "@message", + "value": "{\"resource\":{\"attributes\":{\"telemetry.sdk.language\":\"python\",\"telemetry.sdk.name\":\"opentelemetry\",\"telemetry.sdk.version\":\"1.40.0\",\"service.name\":\"asdf_MyAgent.DEFAULT\",\"aws.log.group.names\":\"/aws/bedrock-agentcore/runtimes/asdf_MyAgent-3s5axvBC6Q-DEFAULT\",\"aws.log.stream.names\":\"otel-rt-logs\",\"deployment.environment.name\":\"bedrock-agentcore:default\",\"cloud.resource_id\":\"arn:aws:bedrock-agentcore:us-west-2:685197708687:runtime/asdf_MyAgent-3s5axvBC6Q/runtime-endpoint/DEFAULT:DEFAULT\",\"cloud.platform\":\"aws_bedrock_agentcore\",\"cloud.provider\":\"aws\",\"cloud.region\":\"us-west-2\",\"telemetry.auto.version\":\"0.17.0-aws\",\"aws.local.service\":\"asdf_MyAgent.DEFAULT\",\"aws.service.type\":\"gen_ai_agent\"}},\"scope\":{\"name\":\"strands.telemetry.tracer\"},\"timeUnixNano\":1787864908110229781,\"observedTimeUnixNano\":1787864911842264993,\"severityNumber\":9,\"severityText\":\"\",\"body\":{\"input\":{\"messages\":[{\"role\":\"system\",\"content\":\"\\n You are a helpful assistant. Use tools when appropriate.\\n \"},{\"role\":\"user\",\"content\":{\"content\":\"[{\\\"text\\\": \\\"Say hello and offer help\\\"}]\"}}]},\"output\":{\"messages\":[{\"role\":\"assistant\",\"content\":{\"message\":\"Hello! 👋 Welcome!\\n\\nI'm here to assist you today. I can help you with:\\n\\n- **Web searches** - Find current information, news, research, or answers to questions on any topic\\n- **Reading webpages** - Extract and read content from specific URLs or articles\\n- **Simple calculations** - Add numbers together\\n\\nWhat would you like help with?\\n\",\"finish_reason\":\"end_turn\"}}]}},\"attributes\":{\"session.id\":\"00000000-0000-4000-8000-000000000001\",\"event.name\":\"strands.telemetry.tracer\"},\"flags\":1,\"traceId\":\"6a90a74750f33f567076943f7bce7966\",\"spanId\":\"b970895264868438\"}" + }, + { + "field": "sessionId", + "value": "00000000-0000-4000-8000-000000000001" + }, + { + "field": "traceId", + "value": "6a90a74750f33f567076943f7bce7966" + }, + { + "field": "spanId", + "value": "b970895264868438" + }, + { + "field": "@ptr", + "value": "CssBCo8BCkw2ODUxOTc3MDg2ODc6L2F3cy9iZWRyb2NrLWFnZW50Y29yZS9ydW50aW1lcy9hc2RmX015QWdlbnQtM3M1YXh2QkM2US1ERUZBVUxUEAAaJGY2ZGY3NDIyLWVlZTQtNDhkNS05NDg3LTE0NTZkYjdkZGE1YiIOCICQ+IyCNBDnn6qthDRA0cHR4dszSAASNRoYAgahOL2DAAAAAJJ6V08ABqkKdOAAAAYyIAEon/i1qIQ0MM6CtqiENDgGQIk5SIhKUNMvIAIQAxgB" + } + ] + ], + "statistics": { + "recordsMatched": 15, + "recordsScanned": 142, + "estimatedRecordsSkipped": 20, + "bytesScanned": 134730, + "estimatedBytesSkipped": 23963, + "logGroupsScanned": 1 + }, + "status": "Complete" +} \ No newline at end of file diff --git a/src/handlers/eval/ondemand/__fixtures__/GetQueryResultsCommand.70382c1aa9a94e3a.json b/src/handlers/eval/ondemand/__fixtures__/GetQueryResultsCommand.70382c1aa9a94e3a.json new file mode 100644 index 000000000..9e456cd44 --- /dev/null +++ b/src/handlers/eval/ondemand/__fixtures__/GetQueryResultsCommand.70382c1aa9a94e3a.json @@ -0,0 +1,960 @@ +{ + "queryLanguage": "CWLI", + "results": [ + [ + { + "field": "@message", + "value": "{\"resource\":{\"attributes\":{\"deployment.environment.name\":\"bedrock-agentcore:default\",\"aws.local.service\":\"asdf_MyAgent.DEFAULT\",\"service.name\":\"asdf_MyAgent.DEFAULT\",\"cloud.region\":\"us-west-2\",\"aws.log.stream.names\":\"otel-rt-logs\",\"telemetry.sdk.name\":\"opentelemetry\",\"aws.service.type\":\"gen_ai_agent\",\"telemetry.sdk.language\":\"python\",\"cloud.provider\":\"aws\",\"cloud.resource_id\":\"arn:aws:bedrock-agentcore:us-west-2:685197708687:runtime/asdf_MyAgent-3s5axvBC6Q/runtime-endpoint/DEFAULT:DEFAULT\",\"aws.log.group.names\":\"/aws/bedrock-agentcore/runtimes/asdf_MyAgent-3s5axvBC6Q-DEFAULT\",\"telemetry.sdk.version\":\"1.40.0\",\"cloud.platform\":\"aws_bedrock_agentcore\",\"telemetry.auto.version\":\"0.17.0-aws\"}},\"scope\":{\"name\":\"opentelemetry.instrumentation.urllib3\",\"version\":\"0.61b0\"},\"traceId\":\"6a8f5b461983df572e2aca3264d32d4d\",\"spanId\":\"e4198cd154238815\",\"parentSpanId\":\"3497dea65b2e2550\",\"flags\":256,\"name\":\"PUT\",\"kind\":\"CLIENT\",\"startTimeUnixNano\":1787779915187794321,\"endTimeUnixNano\":1787779915188812325,\"durationNano\":1018004,\"attributes\":{\"aws.local.service\":\"asdf_MyAgent.DEFAULT\",\"telemetry.extended\":\"true\",\"http.url\":\"http://169.254.169.254/latest/api/token\",\"aws.remote.service\":\"169.254.169.254\",\"aws.local.environment\":\"bedrock-agentcore:default\",\"aws.remote.operation\":\"PUT /latest\",\"http.status_code\":200,\"aws.local.operation\":\"UnmappedOperation\",\"aws.span.kind\":\"CLIENT\",\"PlatformType\":\"AWS::BedrockAgentCore\",\"http.method\":\"PUT\",\"http.response.status_code\":200,\"session.id\":\"00000000-0000-4000-8000-000000000001\"},\"status\":{\"code\":\"UNSET\"}}" + }, + { + "field": "sessionId", + "value": "00000000-0000-4000-8000-000000000001" + }, + { + "field": "traceId", + "value": "6a8f5b461983df572e2aca3264d32d4d" + }, + { + "field": "spanId", + "value": "e4198cd154238815" + }, + { + "field": "@ptr", + "value": "CpwBCmAKFjY4NTE5NzcwODY4Nzphd3Mvc3BhbnMQABokZTE0NTI5MzgtZjVjZi00OTc1LWE0N2QtYzk3YzMwZmFlY2ZlIg4IgJD4jII0EOefqq2ENDjv37W+yTNAyO+wktMzSAASNhoYAganxnVsAAAAACC2BcQABqj1tNAAAAXCIAEotLvy/4M0MLvQ8v+DNDgLQNCZAUijZ1COPCACEAAYAQ==" + } + ], + [ + { + "field": "@message", + "value": "{\"resource\":{\"attributes\":{\"deployment.environment.name\":\"bedrock-agentcore:default\",\"aws.local.service\":\"asdf_MyAgent.DEFAULT\",\"service.name\":\"asdf_MyAgent.DEFAULT\",\"cloud.region\":\"us-west-2\",\"aws.log.stream.names\":\"otel-rt-logs\",\"telemetry.sdk.name\":\"opentelemetry\",\"aws.service.type\":\"gen_ai_agent\",\"telemetry.sdk.language\":\"python\",\"cloud.provider\":\"aws\",\"cloud.resource_id\":\"arn:aws:bedrock-agentcore:us-west-2:685197708687:runtime/asdf_MyAgent-3s5axvBC6Q/runtime-endpoint/DEFAULT:DEFAULT\",\"aws.log.group.names\":\"/aws/bedrock-agentcore/runtimes/asdf_MyAgent-3s5axvBC6Q-DEFAULT\",\"telemetry.sdk.version\":\"1.40.0\",\"cloud.platform\":\"aws_bedrock_agentcore\",\"telemetry.auto.version\":\"0.17.0-aws\"}},\"scope\":{\"name\":\"opentelemetry.instrumentation.urllib3\",\"version\":\"0.61b0\"},\"traceId\":\"6a8f5b461983df572e2aca3264d32d4d\",\"spanId\":\"664e729f0f3e33fe\",\"parentSpanId\":\"3497dea65b2e2550\",\"flags\":256,\"name\":\"GET\",\"kind\":\"CLIENT\",\"startTimeUnixNano\":1787779915189208828,\"endTimeUnixNano\":1787779915189765731,\"durationNano\":556903,\"attributes\":{\"aws.local.service\":\"asdf_MyAgent.DEFAULT\",\"telemetry.extended\":\"true\",\"http.url\":\"http://169.254.169.254/latest/meta-data/iam/security-credentials/\",\"aws.remote.service\":\"169.254.169.254\",\"aws.local.environment\":\"bedrock-agentcore:default\",\"aws.remote.operation\":\"GET /latest\",\"http.status_code\":200,\"aws.local.operation\":\"UnmappedOperation\",\"aws.span.kind\":\"CLIENT\",\"PlatformType\":\"AWS::BedrockAgentCore\",\"http.method\":\"GET\",\"http.response.status_code\":200,\"session.id\":\"00000000-0000-4000-8000-000000000001\"},\"status\":{\"code\":\"UNSET\"}}" + }, + { + "field": "sessionId", + "value": "00000000-0000-4000-8000-000000000001" + }, + { + "field": "traceId", + "value": "6a8f5b461983df572e2aca3264d32d4d" + }, + { + "field": "spanId", + "value": "664e729f0f3e33fe" + }, + { + "field": "@ptr", + "value": "CpwBCmAKFjY4NTE5NzcwODY4Nzphd3Mvc3BhbnMQABokZTE0NTI5MzgtZjVjZi00OTc1LWE0N2QtYzk3YzMwZmFlY2ZlIg4IgJD4jII0EOefqq2ENDjv37W+yTNAyO+wktMzSAASNhoYAganxnVsAAAAACC2BcQABqj1tNAAAAXCIAEotLvy/4M0MLvQ8v+DNDgLQNCZAUijZ1COPCACEAEYAQ==" + } + ], + [ + { + "field": "@message", + "value": "{\"resource\":{\"attributes\":{\"deployment.environment.name\":\"bedrock-agentcore:default\",\"aws.local.service\":\"asdf_MyAgent.DEFAULT\",\"service.name\":\"asdf_MyAgent.DEFAULT\",\"cloud.region\":\"us-west-2\",\"aws.log.stream.names\":\"otel-rt-logs\",\"telemetry.sdk.name\":\"opentelemetry\",\"aws.service.type\":\"gen_ai_agent\",\"telemetry.sdk.language\":\"python\",\"cloud.provider\":\"aws\",\"cloud.resource_id\":\"arn:aws:bedrock-agentcore:us-west-2:685197708687:runtime/asdf_MyAgent-3s5axvBC6Q/runtime-endpoint/DEFAULT:DEFAULT\",\"aws.log.group.names\":\"/aws/bedrock-agentcore/runtimes/asdf_MyAgent-3s5axvBC6Q-DEFAULT\",\"telemetry.sdk.version\":\"1.40.0\",\"cloud.platform\":\"aws_bedrock_agentcore\",\"telemetry.auto.version\":\"0.17.0-aws\"}},\"scope\":{\"name\":\"opentelemetry.instrumentation.urllib3\",\"version\":\"0.61b0\"},\"traceId\":\"6a8f5b461983df572e2aca3264d32d4d\",\"spanId\":\"8a9482865c52e8fc\",\"parentSpanId\":\"3497dea65b2e2550\",\"flags\":256,\"name\":\"GET\",\"kind\":\"CLIENT\",\"startTimeUnixNano\":1787779915190080195,\"endTimeUnixNano\":1787779915190585314,\"durationNano\":505119,\"attributes\":{\"aws.local.service\":\"asdf_MyAgent.DEFAULT\",\"telemetry.extended\":\"true\",\"http.url\":\"http://169.254.169.254/latest/meta-data/iam/security-credentials/execution_role\",\"aws.remote.service\":\"169.254.169.254\",\"aws.local.environment\":\"bedrock-agentcore:default\",\"aws.remote.operation\":\"GET /latest\",\"http.status_code\":200,\"aws.local.operation\":\"UnmappedOperation\",\"aws.span.kind\":\"CLIENT\",\"PlatformType\":\"AWS::BedrockAgentCore\",\"http.method\":\"GET\",\"http.response.status_code\":200,\"session.id\":\"00000000-0000-4000-8000-000000000001\"},\"status\":{\"code\":\"UNSET\"}}" + }, + { + "field": "sessionId", + "value": "00000000-0000-4000-8000-000000000001" + }, + { + "field": "traceId", + "value": "6a8f5b461983df572e2aca3264d32d4d" + }, + { + "field": "spanId", + "value": "8a9482865c52e8fc" + }, + { + "field": "@ptr", + "value": "CpwBCmAKFjY4NTE5NzcwODY4Nzphd3Mvc3BhbnMQABokZTE0NTI5MzgtZjVjZi00OTc1LWE0N2QtYzk3YzMwZmFlY2ZlIg4IgJD4jII0EOefqq2ENDjv37W+yTNAyO+wktMzSAASNhoYAganxnVsAAAAACC2BcQABqj1tNAAAAXCIAEotLvy/4M0MLvQ8v+DNDgLQNCZAUijZ1COPCACEAIYAQ==" + } + ], + [ + { + "field": "@message", + "value": "{\"resource\":{\"attributes\":{\"deployment.environment.name\":\"bedrock-agentcore:default\",\"aws.local.service\":\"asdf_MyAgent.DEFAULT\",\"service.name\":\"asdf_MyAgent.DEFAULT\",\"cloud.region\":\"us-west-2\",\"aws.log.stream.names\":\"otel-rt-logs\",\"telemetry.sdk.name\":\"opentelemetry\",\"aws.service.type\":\"gen_ai_agent\",\"telemetry.sdk.language\":\"python\",\"cloud.provider\":\"aws\",\"cloud.resource_id\":\"arn:aws:bedrock-agentcore:us-west-2:685197708687:runtime/asdf_MyAgent-3s5axvBC6Q/runtime-endpoint/DEFAULT:DEFAULT\",\"aws.log.group.names\":\"/aws/bedrock-agentcore/runtimes/asdf_MyAgent-3s5axvBC6Q-DEFAULT\",\"telemetry.sdk.version\":\"1.40.0\",\"cloud.platform\":\"aws_bedrock_agentcore\",\"telemetry.auto.version\":\"0.17.0-aws\"}},\"scope\":{\"name\":\"opentelemetry.instrumentation.httpx\",\"version\":\"0.61b0\"},\"traceId\":\"6a8f5b461983df572e2aca3264d32d4d\",\"spanId\":\"8aa39d663ee028ef\",\"parentSpanId\":\"3497dea65b2e2550\",\"flags\":256,\"name\":\"POST\",\"kind\":\"CLIENT\",\"startTimeUnixNano\":1787779915296590217,\"endTimeUnixNano\":1787779915340065584,\"durationNano\":43475367,\"attributes\":{\"aws.local.service\":\"asdf_MyAgent.DEFAULT\",\"telemetry.extended\":\"true\",\"http.url\":\"https://bugbashgw1776978672-zsy8cbqwts.gateway.bedrock-agentcore.us-west-2.amazonaws.com/mcp\",\"aws.remote.service\":\"bugbashgw1776978672-zsy8cbqwts.gateway.bedrock-agentcore.us-west-2.amazonaws.com\",\"aws.local.environment\":\"bedrock-agentcore:default\",\"aws.remote.operation\":\"POST /mcp\",\"http.status_code\":200,\"aws.local.operation\":\"UnmappedOperation\",\"aws.span.kind\":\"CLIENT\",\"PlatformType\":\"AWS::BedrockAgentCore\",\"http.method\":\"POST\",\"http.response.status_code\":200,\"session.id\":\"00000000-0000-4000-8000-000000000001\"},\"status\":{\"code\":\"UNSET\"}}" + }, + { + "field": "sessionId", + "value": "00000000-0000-4000-8000-000000000001" + }, + { + "field": "traceId", + "value": "6a8f5b461983df572e2aca3264d32d4d" + }, + { + "field": "spanId", + "value": "8aa39d663ee028ef" + }, + { + "field": "@ptr", + "value": "CpwBCmAKFjY4NTE5NzcwODY4Nzphd3Mvc3BhbnMQABokZTE0NTI5MzgtZjVjZi00OTc1LWE0N2QtYzk3YzMwZmFlY2ZlIg4IgJD4jII0EOefqq2ENDjv37W+yTNAyO+wktMzSAASNhoYAganxnVsAAAAACC2BcQABqj1tNAAAAXCIAEotLvy/4M0MLvQ8v+DNDgLQNCZAUijZ1COPCACEAMYAQ==" + } + ], + [ + { + "field": "@message", + "value": "{\"resource\":{\"attributes\":{\"deployment.environment.name\":\"bedrock-agentcore:default\",\"aws.local.service\":\"asdf_MyAgent.DEFAULT\",\"service.name\":\"asdf_MyAgent.DEFAULT\",\"cloud.region\":\"us-west-2\",\"aws.log.stream.names\":\"otel-rt-logs\",\"telemetry.sdk.name\":\"opentelemetry\",\"aws.service.type\":\"gen_ai_agent\",\"telemetry.sdk.language\":\"python\",\"cloud.provider\":\"aws\",\"cloud.resource_id\":\"arn:aws:bedrock-agentcore:us-west-2:685197708687:runtime/asdf_MyAgent-3s5axvBC6Q/runtime-endpoint/DEFAULT:DEFAULT\",\"aws.log.group.names\":\"/aws/bedrock-agentcore/runtimes/asdf_MyAgent-3s5axvBC6Q-DEFAULT\",\"telemetry.sdk.version\":\"1.40.0\",\"cloud.platform\":\"aws_bedrock_agentcore\",\"telemetry.auto.version\":\"0.17.0-aws\"}},\"scope\":{\"name\":\"opentelemetry.instrumentation.httpx\",\"version\":\"0.61b0\"},\"traceId\":\"6a8f5b461983df572e2aca3264d32d4d\",\"spanId\":\"c3411a35a4406d31\",\"parentSpanId\":\"3497dea65b2e2550\",\"flags\":256,\"name\":\"POST\",\"kind\":\"CLIENT\",\"startTimeUnixNano\":1787779915342535411,\"endTimeUnixNano\":1787779915372977571,\"durationNano\":30442160,\"attributes\":{\"aws.local.service\":\"asdf_MyAgent.DEFAULT\",\"telemetry.extended\":\"true\",\"http.url\":\"https://bugbashgw1776978672-zsy8cbqwts.gateway.bedrock-agentcore.us-west-2.amazonaws.com/mcp\",\"aws.remote.service\":\"bugbashgw1776978672-zsy8cbqwts.gateway.bedrock-agentcore.us-west-2.amazonaws.com\",\"aws.local.environment\":\"bedrock-agentcore:default\",\"aws.remote.operation\":\"POST /mcp\",\"http.status_code\":202,\"aws.local.operation\":\"UnmappedOperation\",\"aws.span.kind\":\"CLIENT\",\"PlatformType\":\"AWS::BedrockAgentCore\",\"http.method\":\"POST\",\"http.response.status_code\":202,\"session.id\":\"00000000-0000-4000-8000-000000000001\"},\"status\":{\"code\":\"UNSET\"}}" + }, + { + "field": "sessionId", + "value": "00000000-0000-4000-8000-000000000001" + }, + { + "field": "traceId", + "value": "6a8f5b461983df572e2aca3264d32d4d" + }, + { + "field": "spanId", + "value": "c3411a35a4406d31" + }, + { + "field": "@ptr", + "value": "CpwBCmAKFjY4NTE5NzcwODY4Nzphd3Mvc3BhbnMQABokZTE0NTI5MzgtZjVjZi00OTc1LWE0N2QtYzk3YzMwZmFlY2ZlIg4IgJD4jII0EOefqq2ENDjv37W+yTNAyO+wktMzSAASNhoYAganxnVsAAAAACC2BcQABqj1tNAAAAXCIAEotLvy/4M0MLvQ8v+DNDgLQNCZAUijZ1COPCACEAQYAQ==" + } + ], + [ + { + "field": "@message", + "value": "{\"resource\":{\"attributes\":{\"deployment.environment.name\":\"bedrock-agentcore:default\",\"aws.local.service\":\"asdf_MyAgent.DEFAULT\",\"service.name\":\"asdf_MyAgent.DEFAULT\",\"cloud.region\":\"us-west-2\",\"aws.log.stream.names\":\"otel-rt-logs\",\"telemetry.sdk.name\":\"opentelemetry\",\"aws.service.type\":\"gen_ai_agent\",\"telemetry.sdk.language\":\"python\",\"cloud.provider\":\"aws\",\"cloud.resource_id\":\"arn:aws:bedrock-agentcore:us-west-2:685197708687:runtime/asdf_MyAgent-3s5axvBC6Q/runtime-endpoint/DEFAULT:DEFAULT\",\"aws.log.group.names\":\"/aws/bedrock-agentcore/runtimes/asdf_MyAgent-3s5axvBC6Q-DEFAULT\",\"telemetry.sdk.version\":\"1.40.0\",\"cloud.platform\":\"aws_bedrock_agentcore\",\"telemetry.auto.version\":\"0.17.0-aws\"}},\"scope\":{\"name\":\"opentelemetry.instrumentation.httpx\",\"version\":\"0.61b0\"},\"traceId\":\"6a8f5b461983df572e2aca3264d32d4d\",\"spanId\":\"92319b090266de53\",\"parentSpanId\":\"3497dea65b2e2550\",\"flags\":256,\"name\":\"POST\",\"kind\":\"CLIENT\",\"startTimeUnixNano\":1787779915374228814,\"endTimeUnixNano\":1787779915441287083,\"durationNano\":67058269,\"attributes\":{\"aws.local.service\":\"asdf_MyAgent.DEFAULT\",\"telemetry.extended\":\"true\",\"http.url\":\"https://bugbashgw1776978672-zsy8cbqwts.gateway.bedrock-agentcore.us-west-2.amazonaws.com/mcp\",\"aws.remote.service\":\"bugbashgw1776978672-zsy8cbqwts.gateway.bedrock-agentcore.us-west-2.amazonaws.com\",\"aws.local.environment\":\"bedrock-agentcore:default\",\"aws.remote.operation\":\"POST /mcp\",\"http.status_code\":200,\"aws.local.operation\":\"UnmappedOperation\",\"aws.span.kind\":\"CLIENT\",\"PlatformType\":\"AWS::BedrockAgentCore\",\"http.method\":\"POST\",\"http.response.status_code\":200,\"session.id\":\"00000000-0000-4000-8000-000000000001\"},\"status\":{\"code\":\"UNSET\"}}" + }, + { + "field": "sessionId", + "value": "00000000-0000-4000-8000-000000000001" + }, + { + "field": "traceId", + "value": "6a8f5b461983df572e2aca3264d32d4d" + }, + { + "field": "spanId", + "value": "92319b090266de53" + }, + { + "field": "@ptr", + "value": "CpwBCmAKFjY4NTE5NzcwODY4Nzphd3Mvc3BhbnMQABokZTE0NTI5MzgtZjVjZi00OTc1LWE0N2QtYzk3YzMwZmFlY2ZlIg4IgJD4jII0EOefqq2ENDjv37W+yTNAyO+wktMzSAASNhoYAganxnVsAAAAACC2BcQABqj1tNAAAAXCIAEotLvy/4M0MLvQ8v+DNDgLQNCZAUijZ1COPCACEAUYAQ==" + } + ], + [ + { + "field": "@message", + "value": "{\"resource\":{\"attributes\":{\"deployment.environment.name\":\"bedrock-agentcore:default\",\"aws.local.service\":\"asdf_MyAgent.DEFAULT\",\"service.name\":\"asdf_MyAgent.DEFAULT\",\"cloud.region\":\"us-west-2\",\"aws.log.stream.names\":\"otel-rt-logs\",\"telemetry.sdk.name\":\"opentelemetry\",\"aws.service.type\":\"gen_ai_agent\",\"telemetry.sdk.language\":\"python\",\"cloud.provider\":\"aws\",\"cloud.resource_id\":\"arn:aws:bedrock-agentcore:us-west-2:685197708687:runtime/asdf_MyAgent-3s5axvBC6Q/runtime-endpoint/DEFAULT:DEFAULT\",\"aws.log.group.names\":\"/aws/bedrock-agentcore/runtimes/asdf_MyAgent-3s5axvBC6Q-DEFAULT\",\"telemetry.sdk.version\":\"1.40.0\",\"cloud.platform\":\"aws_bedrock_agentcore\",\"telemetry.auto.version\":\"0.17.0-aws\"}},\"scope\":{\"name\":\"strands.telemetry.tracer\",\"version\":\"\"},\"traceId\":\"6a8f5b461983df572e2aca3264d32d4d\",\"spanId\":\"ac9653f70866483d\",\"parentSpanId\":\"c9722c3dbb4e781e\",\"flags\":256,\"name\":\"chat\",\"kind\":\"INTERNAL\",\"startTimeUnixNano\":1787779915444785352,\"endTimeUnixNano\":1787779917882748907,\"durationNano\":2437963555,\"attributes\":{\"aws.local.service\":\"asdf_MyAgent.DEFAULT\",\"gen_ai.usage.prompt_tokens\":1199,\"gen_ai.usage.output_tokens\":12,\"gen_ai.server.request.duration\":2404,\"gen_ai.usage.total_tokens\":1211,\"gen_ai.usage.completion_tokens\":12,\"aws.genai.span_kind\":\"LLM\",\"gen_ai.event.start_time\":\"2026-08-26T21:31:55.444793+00:00\",\"gen_ai.server.time_to_first_token\":2292,\"aws.local.environment\":\"bedrock-agentcore:default\",\"gen_ai.provider.name\":\"strands-agents\",\"gen_ai.operation.name\":\"chat\",\"gen_ai.event.end_time\":\"2026-08-26T21:31:57.882714+00:00\",\"gen_ai.usage.input_tokens\":1199,\"aws.genai.token_count_total\":1211,\"gen_ai.request.model\":\"global.anthropic.claude-sonnet-4-5-20250929-v1:0\",\"PlatformType\":\"AWS::BedrockAgentCore\",\"session.id\":\"00000000-0000-4000-8000-000000000001\",\"gen_ai.system\":\"strands-agents\"},\"status\":{\"code\":\"OK\"}}" + }, + { + "field": "sessionId", + "value": "00000000-0000-4000-8000-000000000001" + }, + { + "field": "traceId", + "value": "6a8f5b461983df572e2aca3264d32d4d" + }, + { + "field": "spanId", + "value": "ac9653f70866483d" + }, + { + "field": "@ptr", + "value": "CpwBCmAKFjY4NTE5NzcwODY4Nzphd3Mvc3BhbnMQABokZTE0NTI5MzgtZjVjZi00OTc1LWE0N2QtYzk3YzMwZmFlY2ZlIg4IgJD4jII0EOefqq2ENDjv37W+yTNAyO+wktMzSAASNhoYAganxnVsAAAAACC2BcQABqj1tNAAAAXCIAEotLvy/4M0MLvQ8v+DNDgLQNCZAUijZ1COPCACEAcYAQ==" + } + ], + [ + { + "field": "@message", + "value": "{\"resource\":{\"attributes\":{\"deployment.environment.name\":\"bedrock-agentcore:default\",\"aws.local.service\":\"asdf_MyAgent.DEFAULT\",\"service.name\":\"asdf_MyAgent.DEFAULT\",\"cloud.region\":\"us-west-2\",\"aws.log.stream.names\":\"otel-rt-logs\",\"telemetry.sdk.name\":\"opentelemetry\",\"aws.service.type\":\"gen_ai_agent\",\"telemetry.sdk.language\":\"python\",\"cloud.provider\":\"aws\",\"cloud.resource_id\":\"arn:aws:bedrock-agentcore:us-west-2:685197708687:runtime/asdf_MyAgent-3s5axvBC6Q/runtime-endpoint/DEFAULT:DEFAULT\",\"aws.log.group.names\":\"/aws/bedrock-agentcore/runtimes/asdf_MyAgent-3s5axvBC6Q-DEFAULT\",\"telemetry.sdk.version\":\"1.40.0\",\"cloud.platform\":\"aws_bedrock_agentcore\",\"telemetry.auto.version\":\"0.17.0-aws\"}},\"scope\":{\"name\":\"opentelemetry.instrumentation.botocore.bedrock-runtime\",\"version\":\"0.61b0\"},\"traceId\":\"6a8f5b461983df572e2aca3264d32d4d\",\"spanId\":\"ac5f4cb63375480b\",\"parentSpanId\":\"ac9653f70866483d\",\"flags\":256,\"name\":\"chat global.anthropic.claude-sonnet-4-5-20250929-v1:0\",\"kind\":\"CLIENT\",\"startTimeUnixNano\":1787779915457699487,\"endTimeUnixNano\":1787779917882096447,\"durationNano\":2424396960,\"attributes\":{\"aws.local.service\":\"asdf_MyAgent.DEFAULT\",\"rpc.service\":\"Bedrock Runtime\",\"aws.remote.resource.identifier\":\"global.anthropic.claude-sonnet-4-5-20250929-v1:0\",\"aws.local.environment\":\"bedrock-agentcore:default\",\"aws.remote.operation\":\"ConverseStream\",\"gen_ai.provider.name\":\"aws.bedrock\",\"server.address\":\"bedrock-runtime.us-west-2.amazonaws.com\",\"aws.request_id\":\"abfa0ec8-9efc-4f12-aa53-1a4e19dbfb25\",\"aws.local.operation\":\"UnmappedOperation\",\"aws.span.kind\":\"CLIENT\",\"aws.auth.region\":\"us-west-2\",\"rpc.method\":\"ConverseStream\",\"gen_ai.response.finish_reasons\":[\"end_turn\"],\"server.port\":443,\"gen_ai.request.model\":\"global.anthropic.claude-sonnet-4-5-20250929-v1:0\",\"http.response.status_code\":200,\"gen_ai.system\":\"aws.bedrock\",\"telemetry.extended\":\"true\",\"gen_ai.usage.output_tokens\":12,\"aws.genai.span_kind\":\"LLM\",\"rpc.system\":\"aws-api\",\"aws.remote.service\":\"AWS::BedrockRuntime\",\"http.status_code\":200,\"aws.region\":\"us-west-2\",\"aws.remote.resource.type\":\"AWS::Bedrock::Model\",\"gen_ai.operation.name\":\"chat\",\"gen_ai.usage.input_tokens\":1199,\"aws.genai.token_count_total\":1211,\"retry_attempts\":0,\"PlatformType\":\"AWS::BedrockAgentCore\",\"aws.auth.account.access_key\":\"ASIAZ7CHXJWHTC5VPKVH\",\"session.id\":\"00000000-0000-4000-8000-000000000001\"},\"status\":{\"code\":\"UNSET\"}}" + }, + { + "field": "sessionId", + "value": "00000000-0000-4000-8000-000000000001" + }, + { + "field": "traceId", + "value": "6a8f5b461983df572e2aca3264d32d4d" + }, + { + "field": "spanId", + "value": "ac5f4cb63375480b" + }, + { + "field": "@ptr", + "value": "CpwBCmAKFjY4NTE5NzcwODY4Nzphd3Mvc3BhbnMQABokZTE0NTI5MzgtZjVjZi00OTc1LWE0N2QtYzk3YzMwZmFlY2ZlIg4IgJD4jII0EOefqq2ENDjv37W+yTNAyO+wktMzSAASNhoYAganxnVsAAAAACC2BcQABqj1tNAAAAXCIAEotLvy/4M0MLvQ8v+DNDgLQNCZAUijZ1COPCACEAYYAQ==" + } + ], + [ + { + "field": "@message", + "value": "{\"resource\":{\"attributes\":{\"deployment.environment.name\":\"bedrock-agentcore:default\",\"aws.local.service\":\"asdf_MyAgent.DEFAULT\",\"service.name\":\"asdf_MyAgent.DEFAULT\",\"cloud.region\":\"us-west-2\",\"aws.log.stream.names\":\"otel-rt-logs\",\"telemetry.sdk.name\":\"opentelemetry\",\"aws.service.type\":\"gen_ai_agent\",\"telemetry.sdk.language\":\"python\",\"cloud.provider\":\"aws\",\"cloud.resource_id\":\"arn:aws:bedrock-agentcore:us-west-2:685197708687:runtime/asdf_MyAgent-3s5axvBC6Q/runtime-endpoint/DEFAULT:DEFAULT\",\"aws.log.group.names\":\"/aws/bedrock-agentcore/runtimes/asdf_MyAgent-3s5axvBC6Q-DEFAULT\",\"telemetry.sdk.version\":\"1.40.0\",\"cloud.platform\":\"aws_bedrock_agentcore\",\"telemetry.auto.version\":\"0.17.0-aws\"}},\"scope\":{\"name\":\"opentelemetry.instrumentation.starlette\",\"version\":\"0.61b0\"},\"traceId\":\"6a8f5b461983df572e2aca3264d32d4d\",\"spanId\":\"3497dea65b2e2550\",\"parentSpanId\":\"6f895a24f81f3726\",\"flags\":768,\"name\":\"POST /invocations\",\"kind\":\"SERVER\",\"startTimeUnixNano\":1787779915147134575,\"endTimeUnixNano\":1787779917883512213,\"durationNano\":2736377638,\"attributes\":{\"aws.local.service\":\"asdf_MyAgent.DEFAULT\",\"net.peer.port\":48806,\"telemetry.extended\":\"true\",\"http.target\":\"/invocations\",\"http.flavor\":\"1.1\",\"http.url\":\"http://cell01.us-west-2.prod.arp.kepler-analytics.aws.dev/invocations\",\"net.peer.ip\":\"127.0.0.1\",\"http.host\":\"127.0.0.1:8080\",\"aws.local.environment\":\"bedrock-agentcore:default\",\"http.status_code\":200,\"aws.local.operation\":\"POST /invocations\",\"aws.span.kind\":\"SERVER\",\"http.server_name\":\"cell01.us-west-2.prod.arp.kepler-analytics.aws.dev\",\"net.host.port\":8080,\"http.route\":\"/invocations\",\"PlatformType\":\"AWS::BedrockAgentCore\",\"http.method\":\"POST\",\"http.response.status_code\":200,\"session.id\":\"00000000-0000-4000-8000-000000000001\",\"http.scheme\":\"http\"},\"status\":{\"code\":\"UNSET\"}}" + }, + { + "field": "sessionId", + "value": "00000000-0000-4000-8000-000000000001" + }, + { + "field": "traceId", + "value": "6a8f5b461983df572e2aca3264d32d4d" + }, + { + "field": "spanId", + "value": "3497dea65b2e2550" + }, + { + "field": "@ptr", + "value": "CpwBCmAKFjY4NTE5NzcwODY4Nzphd3Mvc3BhbnMQABokZTE0NTI5MzgtZjVjZi00OTc1LWE0N2QtYzk3YzMwZmFlY2ZlIg4IgJD4jII0EOefqq2ENDjv37W+yTNAyO+wktMzSAASNhoYAganxnVsAAAAACC2BcQABqj1tNAAAAXCIAEotLvy/4M0MLvQ8v+DNDgLQNCZAUijZ1COPCACEAoYAQ==" + } + ], + [ + { + "field": "@message", + "value": "{\"resource\":{\"attributes\":{\"deployment.environment.name\":\"bedrock-agentcore:default\",\"aws.local.service\":\"asdf_MyAgent.DEFAULT\",\"service.name\":\"asdf_MyAgent.DEFAULT\",\"cloud.region\":\"us-west-2\",\"aws.log.stream.names\":\"otel-rt-logs\",\"telemetry.sdk.name\":\"opentelemetry\",\"aws.service.type\":\"gen_ai_agent\",\"telemetry.sdk.language\":\"python\",\"cloud.provider\":\"aws\",\"cloud.resource_id\":\"arn:aws:bedrock-agentcore:us-west-2:685197708687:runtime/asdf_MyAgent-3s5axvBC6Q/runtime-endpoint/DEFAULT:DEFAULT\",\"aws.log.group.names\":\"/aws/bedrock-agentcore/runtimes/asdf_MyAgent-3s5axvBC6Q-DEFAULT\",\"telemetry.sdk.version\":\"1.40.0\",\"cloud.platform\":\"aws_bedrock_agentcore\",\"telemetry.auto.version\":\"0.17.0-aws\"}},\"scope\":{\"name\":\"strands.telemetry.tracer\",\"version\":\"\"},\"traceId\":\"6a8f5b461983df572e2aca3264d32d4d\",\"spanId\":\"c9722c3dbb4e781e\",\"parentSpanId\":\"82fe397271c56f75\",\"flags\":256,\"name\":\"execute_event_loop_cycle\",\"kind\":\"INTERNAL\",\"startTimeUnixNano\":1787779915444636525,\"endTimeUnixNano\":1787779917883063841,\"durationNano\":2438427316,\"attributes\":{\"aws.local.service\":\"asdf_MyAgent.DEFAULT\",\"gen_ai.operation.name\":\"execute_event_loop_cycle\",\"gen_ai.event.end_time\":\"2026-08-26T21:31:57.883048+00:00\",\"event_loop.cycle_id\":\"3eac5258-e59c-44d1-afa9-02a1a42a10c3\",\"gen_ai.event.start_time\":\"2026-08-26T21:31:55.444648+00:00\",\"PlatformType\":\"AWS::BedrockAgentCore\",\"session.id\":\"00000000-0000-4000-8000-000000000001\",\"gen_ai.system\":\"strands-agents\",\"aws.local.environment\":\"bedrock-agentcore:default\",\"gen_ai.provider.name\":\"strands-agents\"},\"status\":{\"code\":\"OK\"}}" + }, + { + "field": "sessionId", + "value": "00000000-0000-4000-8000-000000000001" + }, + { + "field": "traceId", + "value": "6a8f5b461983df572e2aca3264d32d4d" + }, + { + "field": "spanId", + "value": "c9722c3dbb4e781e" + }, + { + "field": "@ptr", + "value": "CpwBCmAKFjY4NTE5NzcwODY4Nzphd3Mvc3BhbnMQABokZTE0NTI5MzgtZjVjZi00OTc1LWE0N2QtYzk3YzMwZmFlY2ZlIg4IgJD4jII0EOefqq2ENDjv37W+yTNAyO+wktMzSAASNhoYAganxnVsAAAAACC2BcQABqj1tNAAAAXCIAEotLvy/4M0MLvQ8v+DNDgLQNCZAUijZ1COPCACEAgYAQ==" + } + ], + [ + { + "field": "@message", + "value": "{\"resource\":{\"attributes\":{\"deployment.environment.name\":\"bedrock-agentcore:default\",\"aws.local.service\":\"asdf_MyAgent.DEFAULT\",\"service.name\":\"asdf_MyAgent.DEFAULT\",\"cloud.region\":\"us-west-2\",\"aws.log.stream.names\":\"otel-rt-logs\",\"telemetry.sdk.name\":\"opentelemetry\",\"aws.service.type\":\"gen_ai_agent\",\"telemetry.sdk.language\":\"python\",\"cloud.provider\":\"aws\",\"cloud.resource_id\":\"arn:aws:bedrock-agentcore:us-west-2:685197708687:runtime/asdf_MyAgent-3s5axvBC6Q/runtime-endpoint/DEFAULT:DEFAULT\",\"aws.log.group.names\":\"/aws/bedrock-agentcore/runtimes/asdf_MyAgent-3s5axvBC6Q-DEFAULT\",\"telemetry.sdk.version\":\"1.40.0\",\"cloud.platform\":\"aws_bedrock_agentcore\",\"telemetry.auto.version\":\"0.17.0-aws\"}},\"scope\":{\"name\":\"strands.telemetry.tracer\",\"version\":\"\"},\"traceId\":\"6a8f5b461983df572e2aca3264d32d4d\",\"spanId\":\"82fe397271c56f75\",\"parentSpanId\":\"3497dea65b2e2550\",\"flags\":256,\"name\":\"invoke_agent Strands Agents\",\"kind\":\"INTERNAL\",\"startTimeUnixNano\":1787779915443995518,\"endTimeUnixNano\":1787779917883212081,\"durationNano\":2439216563,\"attributes\":{\"aws.local.service\":\"asdf_MyAgent.DEFAULT\",\"gen_ai.usage.prompt_tokens\":1199,\"gen_ai.usage.output_tokens\":12,\"gen_ai.usage.cache_write_input_tokens\":0,\"gen_ai.agent.name\":\"Strands Agents\",\"gen_ai.usage.total_tokens\":1211,\"gen_ai.usage.completion_tokens\":12,\"aws.genai.span_kind\":\"AGENT\",\"gen_ai.event.start_time\":\"2026-08-26T21:31:55.444014+00:00\",\"aws.local.environment\":\"bedrock-agentcore:default\",\"gen_ai.provider.name\":\"strands-agents\",\"gen_ai.operation.name\":\"invoke_agent\",\"gen_ai.event.end_time\":\"2026-08-26T21:31:57.883192+00:00\",\"gen_ai.usage.input_tokens\":1199,\"aws.genai.token_count_total\":1211,\"gen_ai.request.model\":\"global.anthropic.claude-sonnet-4-5-20250929-v1:0\",\"gen_ai.usage.cache_read_input_tokens\":0,\"gen_ai.agent.tools\":\"[\\\"add_numbers\\\", \\\"x_amz_bedrock_agentcore_search\\\", \\\"mcpTarget___web_fetch_exa\\\", \\\"mcpTarget___web_search_exa\\\"]\",\"PlatformType\":\"AWS::BedrockAgentCore\",\"session.id\":\"00000000-0000-4000-8000-000000000001\",\"gen_ai.system\":\"strands-agents\"},\"status\":{\"code\":\"OK\"}}" + }, + { + "field": "sessionId", + "value": "00000000-0000-4000-8000-000000000001" + }, + { + "field": "traceId", + "value": "6a8f5b461983df572e2aca3264d32d4d" + }, + { + "field": "spanId", + "value": "82fe397271c56f75" + }, + { + "field": "@ptr", + "value": "CpwBCmAKFjY4NTE5NzcwODY4Nzphd3Mvc3BhbnMQABokZTE0NTI5MzgtZjVjZi00OTc1LWE0N2QtYzk3YzMwZmFlY2ZlIg4IgJD4jII0EOefqq2ENDjv37W+yTNAyO+wktMzSAASNhoYAganxnVsAAAAACC2BcQABqj1tNAAAAXCIAEotLvy/4M0MLvQ8v+DNDgLQNCZAUijZ1COPCACEAkYAQ==" + } + ], + [ + { + "field": "@message", + "value": "{\"resource\":{\"attributes\":{\"deployment.environment.name\":\"bedrock-agentcore:default\",\"aws.local.service\":\"asdf_MyAgent.DEFAULT\",\"service.name\":\"asdf_MyAgent.DEFAULT\",\"cloud.region\":\"us-west-2\",\"aws.log.stream.names\":\"otel-rt-logs\",\"telemetry.sdk.name\":\"opentelemetry\",\"aws.service.type\":\"gen_ai_agent\",\"telemetry.sdk.language\":\"python\",\"cloud.provider\":\"aws\",\"cloud.resource_id\":\"arn:aws:bedrock-agentcore:us-west-2:685197708687:runtime/asdf_MyAgent-3s5axvBC6Q/runtime-endpoint/DEFAULT:DEFAULT\",\"aws.log.group.names\":\"/aws/bedrock-agentcore/runtimes/asdf_MyAgent-3s5axvBC6Q-DEFAULT\",\"telemetry.sdk.version\":\"1.40.0\",\"cloud.platform\":\"aws_bedrock_agentcore\",\"telemetry.auto.version\":\"0.17.0-aws\"}},\"scope\":{\"name\":\"opentelemetry.instrumentation.botocore.bedrock-runtime\",\"version\":\"0.61b0\"},\"traceId\":\"6a8f5bbc1882c05547b64a9246f17ca4\",\"spanId\":\"c6fce016050b427c\",\"parentSpanId\":\"2b53da16274ce9f5\",\"flags\":256,\"name\":\"chat global.anthropic.claude-sonnet-4-5-20250929-v1:0\",\"kind\":\"CLIENT\",\"startTimeUnixNano\":1787780028788804860,\"endTimeUnixNano\":1787780030855083391,\"durationNano\":2066278531,\"attributes\":{\"aws.local.service\":\"asdf_MyAgent.DEFAULT\",\"rpc.service\":\"Bedrock Runtime\",\"aws.remote.resource.identifier\":\"global.anthropic.claude-sonnet-4-5-20250929-v1:0\",\"aws.local.environment\":\"bedrock-agentcore:default\",\"aws.remote.operation\":\"ConverseStream\",\"gen_ai.provider.name\":\"aws.bedrock\",\"server.address\":\"bedrock-runtime.us-west-2.amazonaws.com\",\"aws.request_id\":\"ff6866c3-28d5-400d-8947-249c9a7576fb\",\"aws.local.operation\":\"UnmappedOperation\",\"aws.span.kind\":\"CLIENT\",\"aws.auth.region\":\"us-west-2\",\"rpc.method\":\"ConverseStream\",\"gen_ai.response.finish_reasons\":[\"end_turn\"],\"server.port\":443,\"gen_ai.request.model\":\"global.anthropic.claude-sonnet-4-5-20250929-v1:0\",\"http.response.status_code\":200,\"gen_ai.system\":\"aws.bedrock\",\"telemetry.extended\":\"true\",\"gen_ai.usage.output_tokens\":13,\"aws.genai.span_kind\":\"LLM\",\"rpc.system\":\"aws-api\",\"aws.remote.service\":\"AWS::BedrockRuntime\",\"http.status_code\":200,\"aws.region\":\"us-west-2\",\"aws.remote.resource.type\":\"AWS::Bedrock::Model\",\"gen_ai.operation.name\":\"chat\",\"gen_ai.usage.input_tokens\":1215,\"aws.genai.token_count_total\":1228,\"retry_attempts\":0,\"PlatformType\":\"AWS::BedrockAgentCore\",\"aws.auth.account.access_key\":\"ASIAZ7CHXJWHTC5VPKVH\",\"session.id\":\"00000000-0000-4000-8000-000000000001\"},\"status\":{\"code\":\"UNSET\"}}" + }, + { + "field": "sessionId", + "value": "00000000-0000-4000-8000-000000000001" + }, + { + "field": "traceId", + "value": "6a8f5bbc1882c05547b64a9246f17ca4" + }, + { + "field": "spanId", + "value": "c6fce016050b427c" + }, + { + "field": "@ptr", + "value": "CpsBCmAKFjY4NTE5NzcwODY4Nzphd3Mvc3BhbnMQABokZTE0NTI5MzgtZjVjZi00OTc1LWE0N2QtYzk3YzMwZmFlY2ZlIg4IgJD4jII0EOefqq2ENDjv37W+yTNAyO+wktMzSAASNRoYAganxnVsAAAAACC2y4gABqj1vAAAAAXCIAEoh8P5/4M0MIjD+f+DNDgFQK9MSKleUJYzIAIQABgB" + } + ], + [ + { + "field": "@message", + "value": "{\"resource\":{\"attributes\":{\"deployment.environment.name\":\"bedrock-agentcore:default\",\"aws.local.service\":\"asdf_MyAgent.DEFAULT\",\"service.name\":\"asdf_MyAgent.DEFAULT\",\"cloud.region\":\"us-west-2\",\"aws.log.stream.names\":\"otel-rt-logs\",\"telemetry.sdk.name\":\"opentelemetry\",\"aws.service.type\":\"gen_ai_agent\",\"telemetry.sdk.language\":\"python\",\"cloud.provider\":\"aws\",\"cloud.resource_id\":\"arn:aws:bedrock-agentcore:us-west-2:685197708687:runtime/asdf_MyAgent-3s5axvBC6Q/runtime-endpoint/DEFAULT:DEFAULT\",\"aws.log.group.names\":\"/aws/bedrock-agentcore/runtimes/asdf_MyAgent-3s5axvBC6Q-DEFAULT\",\"telemetry.sdk.version\":\"1.40.0\",\"cloud.platform\":\"aws_bedrock_agentcore\",\"telemetry.auto.version\":\"0.17.0-aws\"}},\"scope\":{\"name\":\"strands.telemetry.tracer\",\"version\":\"\"},\"traceId\":\"6a8f5bbc1882c05547b64a9246f17ca4\",\"spanId\":\"2b53da16274ce9f5\",\"parentSpanId\":\"43f809568e52d1ca\",\"flags\":256,\"name\":\"chat\",\"kind\":\"INTERNAL\",\"startTimeUnixNano\":1787780028788254696,\"endTimeUnixNano\":1787780030855720915,\"durationNano\":2067466219,\"attributes\":{\"aws.local.service\":\"asdf_MyAgent.DEFAULT\",\"gen_ai.usage.prompt_tokens\":1215,\"gen_ai.usage.output_tokens\":13,\"gen_ai.server.request.duration\":2060,\"gen_ai.usage.total_tokens\":1228,\"gen_ai.usage.completion_tokens\":13,\"aws.genai.span_kind\":\"LLM\",\"gen_ai.event.start_time\":\"2026-08-26T21:33:48.788261+00:00\",\"gen_ai.server.time_to_first_token\":1710,\"aws.local.environment\":\"bedrock-agentcore:default\",\"gen_ai.provider.name\":\"strands-agents\",\"gen_ai.operation.name\":\"chat\",\"gen_ai.event.end_time\":\"2026-08-26T21:33:50.855686+00:00\",\"gen_ai.usage.input_tokens\":1215,\"aws.genai.token_count_total\":1228,\"gen_ai.request.model\":\"global.anthropic.claude-sonnet-4-5-20250929-v1:0\",\"PlatformType\":\"AWS::BedrockAgentCore\",\"session.id\":\"00000000-0000-4000-8000-000000000001\",\"gen_ai.system\":\"strands-agents\"},\"status\":{\"code\":\"OK\"}}" + }, + { + "field": "sessionId", + "value": "00000000-0000-4000-8000-000000000001" + }, + { + "field": "traceId", + "value": "6a8f5bbc1882c05547b64a9246f17ca4" + }, + { + "field": "spanId", + "value": "2b53da16274ce9f5" + }, + { + "field": "@ptr", + "value": "CpsBCmAKFjY4NTE5NzcwODY4Nzphd3Mvc3BhbnMQABokZTE0NTI5MzgtZjVjZi00OTc1LWE0N2QtYzk3YzMwZmFlY2ZlIg4IgJD4jII0EOefqq2ENDjv37W+yTNAyO+wktMzSAASNRoYAganxnVsAAAAACC2y4gABqj1vAAAAAXCIAEoh8P5/4M0MIjD+f+DNDgFQK9MSKleUJYzIAIQARgB" + } + ], + [ + { + "field": "@message", + "value": "{\"resource\":{\"attributes\":{\"deployment.environment.name\":\"bedrock-agentcore:default\",\"aws.local.service\":\"asdf_MyAgent.DEFAULT\",\"service.name\":\"asdf_MyAgent.DEFAULT\",\"cloud.region\":\"us-west-2\",\"aws.log.stream.names\":\"otel-rt-logs\",\"telemetry.sdk.name\":\"opentelemetry\",\"aws.service.type\":\"gen_ai_agent\",\"telemetry.sdk.language\":\"python\",\"cloud.provider\":\"aws\",\"cloud.resource_id\":\"arn:aws:bedrock-agentcore:us-west-2:685197708687:runtime/asdf_MyAgent-3s5axvBC6Q/runtime-endpoint/DEFAULT:DEFAULT\",\"aws.log.group.names\":\"/aws/bedrock-agentcore/runtimes/asdf_MyAgent-3s5axvBC6Q-DEFAULT\",\"telemetry.sdk.version\":\"1.40.0\",\"cloud.platform\":\"aws_bedrock_agentcore\",\"telemetry.auto.version\":\"0.17.0-aws\"}},\"scope\":{\"name\":\"strands.telemetry.tracer\",\"version\":\"\"},\"traceId\":\"6a8f5bbc1882c05547b64a9246f17ca4\",\"spanId\":\"43f809568e52d1ca\",\"parentSpanId\":\"324e3a835d1af5c4\",\"flags\":256,\"name\":\"execute_event_loop_cycle\",\"kind\":\"INTERNAL\",\"startTimeUnixNano\":1787780028788110333,\"endTimeUnixNano\":1787780030855979475,\"durationNano\":2067869142,\"attributes\":{\"aws.local.service\":\"asdf_MyAgent.DEFAULT\",\"gen_ai.operation.name\":\"execute_event_loop_cycle\",\"gen_ai.event.end_time\":\"2026-08-26T21:33:50.855965+00:00\",\"event_loop.cycle_id\":\"6e6ce2c7-3386-4efc-a0d4-f65e3832a565\",\"gen_ai.event.start_time\":\"2026-08-26T21:33:48.788119+00:00\",\"PlatformType\":\"AWS::BedrockAgentCore\",\"session.id\":\"00000000-0000-4000-8000-000000000001\",\"gen_ai.system\":\"strands-agents\",\"aws.local.environment\":\"bedrock-agentcore:default\",\"gen_ai.provider.name\":\"strands-agents\"},\"status\":{\"code\":\"OK\"}}" + }, + { + "field": "sessionId", + "value": "00000000-0000-4000-8000-000000000001" + }, + { + "field": "traceId", + "value": "6a8f5bbc1882c05547b64a9246f17ca4" + }, + { + "field": "spanId", + "value": "43f809568e52d1ca" + }, + { + "field": "@ptr", + "value": "CpsBCmAKFjY4NTE5NzcwODY4Nzphd3Mvc3BhbnMQABokZTE0NTI5MzgtZjVjZi00OTc1LWE0N2QtYzk3YzMwZmFlY2ZlIg4IgJD4jII0EOefqq2ENDjv37W+yTNAyO+wktMzSAASNRoYAganxnVsAAAAACC2y4gABqj1vAAAAAXCIAEoh8P5/4M0MIjD+f+DNDgFQK9MSKleUJYzIAIQAhgB" + } + ], + [ + { + "field": "@message", + "value": "{\"resource\":{\"attributes\":{\"deployment.environment.name\":\"bedrock-agentcore:default\",\"aws.local.service\":\"asdf_MyAgent.DEFAULT\",\"service.name\":\"asdf_MyAgent.DEFAULT\",\"cloud.region\":\"us-west-2\",\"aws.log.stream.names\":\"otel-rt-logs\",\"telemetry.sdk.name\":\"opentelemetry\",\"aws.service.type\":\"gen_ai_agent\",\"telemetry.sdk.language\":\"python\",\"cloud.provider\":\"aws\",\"cloud.resource_id\":\"arn:aws:bedrock-agentcore:us-west-2:685197708687:runtime/asdf_MyAgent-3s5axvBC6Q/runtime-endpoint/DEFAULT:DEFAULT\",\"aws.log.group.names\":\"/aws/bedrock-agentcore/runtimes/asdf_MyAgent-3s5axvBC6Q-DEFAULT\",\"telemetry.sdk.version\":\"1.40.0\",\"cloud.platform\":\"aws_bedrock_agentcore\",\"telemetry.auto.version\":\"0.17.0-aws\"}},\"scope\":{\"name\":\"strands.telemetry.tracer\",\"version\":\"\"},\"traceId\":\"6a8f5bbc1882c05547b64a9246f17ca4\",\"spanId\":\"324e3a835d1af5c4\",\"parentSpanId\":\"eba2750a08d48390\",\"flags\":256,\"name\":\"invoke_agent Strands Agents\",\"kind\":\"INTERNAL\",\"startTimeUnixNano\":1787780028787818615,\"endTimeUnixNano\":1787780030856119726,\"durationNano\":2068301111,\"attributes\":{\"aws.local.service\":\"asdf_MyAgent.DEFAULT\",\"gen_ai.usage.prompt_tokens\":2414,\"gen_ai.usage.output_tokens\":25,\"gen_ai.usage.cache_write_input_tokens\":0,\"gen_ai.agent.name\":\"Strands Agents\",\"gen_ai.usage.total_tokens\":2439,\"gen_ai.usage.completion_tokens\":25,\"aws.genai.span_kind\":\"AGENT\",\"gen_ai.event.start_time\":\"2026-08-26T21:33:48.787832+00:00\",\"aws.local.environment\":\"bedrock-agentcore:default\",\"gen_ai.provider.name\":\"strands-agents\",\"gen_ai.operation.name\":\"invoke_agent\",\"gen_ai.event.end_time\":\"2026-08-26T21:33:50.856099+00:00\",\"gen_ai.usage.input_tokens\":2414,\"aws.genai.token_count_total\":2439,\"gen_ai.request.model\":\"global.anthropic.claude-sonnet-4-5-20250929-v1:0\",\"gen_ai.usage.cache_read_input_tokens\":0,\"gen_ai.agent.tools\":\"[\\\"add_numbers\\\", \\\"x_amz_bedrock_agentcore_search\\\", \\\"mcpTarget___web_fetch_exa\\\", \\\"mcpTarget___web_search_exa\\\"]\",\"PlatformType\":\"AWS::BedrockAgentCore\",\"session.id\":\"00000000-0000-4000-8000-000000000001\",\"gen_ai.system\":\"strands-agents\"},\"status\":{\"code\":\"OK\"}}" + }, + { + "field": "sessionId", + "value": "00000000-0000-4000-8000-000000000001" + }, + { + "field": "traceId", + "value": "6a8f5bbc1882c05547b64a9246f17ca4" + }, + { + "field": "spanId", + "value": "324e3a835d1af5c4" + }, + { + "field": "@ptr", + "value": "CpsBCmAKFjY4NTE5NzcwODY4Nzphd3Mvc3BhbnMQABokZTE0NTI5MzgtZjVjZi00OTc1LWE0N2QtYzk3YzMwZmFlY2ZlIg4IgJD4jII0EOefqq2ENDjv37W+yTNAyO+wktMzSAASNRoYAganxnVsAAAAACC2y4gABqj1vAAAAAXCIAEoh8P5/4M0MIjD+f+DNDgFQK9MSKleUJYzIAIQAxgB" + } + ], + [ + { + "field": "@message", + "value": "{\"resource\":{\"attributes\":{\"deployment.environment.name\":\"bedrock-agentcore:default\",\"aws.local.service\":\"asdf_MyAgent.DEFAULT\",\"service.name\":\"asdf_MyAgent.DEFAULT\",\"cloud.region\":\"us-west-2\",\"aws.log.stream.names\":\"otel-rt-logs\",\"telemetry.sdk.name\":\"opentelemetry\",\"aws.service.type\":\"gen_ai_agent\",\"telemetry.sdk.language\":\"python\",\"cloud.provider\":\"aws\",\"cloud.resource_id\":\"arn:aws:bedrock-agentcore:us-west-2:685197708687:runtime/asdf_MyAgent-3s5axvBC6Q/runtime-endpoint/DEFAULT:DEFAULT\",\"aws.log.group.names\":\"/aws/bedrock-agentcore/runtimes/asdf_MyAgent-3s5axvBC6Q-DEFAULT\",\"telemetry.sdk.version\":\"1.40.0\",\"cloud.platform\":\"aws_bedrock_agentcore\",\"telemetry.auto.version\":\"0.17.0-aws\"}},\"scope\":{\"name\":\"opentelemetry.instrumentation.starlette\",\"version\":\"0.61b0\"},\"traceId\":\"6a8f5bbc1882c05547b64a9246f17ca4\",\"spanId\":\"eba2750a08d48390\",\"parentSpanId\":\"4c77c79e708e39db\",\"flags\":768,\"name\":\"POST /invocations\",\"kind\":\"SERVER\",\"startTimeUnixNano\":1787780028786050715,\"endTimeUnixNano\":1787780030856458693,\"durationNano\":2070407978,\"attributes\":{\"aws.local.service\":\"asdf_MyAgent.DEFAULT\",\"net.peer.port\":37368,\"telemetry.extended\":\"true\",\"http.target\":\"/invocations\",\"http.flavor\":\"1.1\",\"http.url\":\"http://cell01.us-west-2.prod.arp.kepler-analytics.aws.dev/invocations\",\"net.peer.ip\":\"127.0.0.1\",\"http.host\":\"127.0.0.1:8080\",\"aws.local.environment\":\"bedrock-agentcore:default\",\"http.status_code\":200,\"aws.local.operation\":\"POST /invocations\",\"aws.span.kind\":\"SERVER\",\"http.server_name\":\"cell01.us-west-2.prod.arp.kepler-analytics.aws.dev\",\"net.host.port\":8080,\"http.route\":\"/invocations\",\"PlatformType\":\"AWS::BedrockAgentCore\",\"http.method\":\"POST\",\"http.response.status_code\":200,\"session.id\":\"00000000-0000-4000-8000-000000000001\",\"http.scheme\":\"http\"},\"status\":{\"code\":\"UNSET\"}}" + }, + { + "field": "sessionId", + "value": "00000000-0000-4000-8000-000000000001" + }, + { + "field": "traceId", + "value": "6a8f5bbc1882c05547b64a9246f17ca4" + }, + { + "field": "spanId", + "value": "eba2750a08d48390" + }, + { + "field": "@ptr", + "value": "CpsBCmAKFjY4NTE5NzcwODY4Nzphd3Mvc3BhbnMQABokZTE0NTI5MzgtZjVjZi00OTc1LWE0N2QtYzk3YzMwZmFlY2ZlIg4IgJD4jII0EOefqq2ENDjv37W+yTNAyO+wktMzSAASNRoYAganxnVsAAAAACC2y4gABqj1vAAAAAXCIAEoh8P5/4M0MIjD+f+DNDgFQK9MSKleUJYzIAIQBBgB" + } + ], + [ + { + "field": "@message", + "value": "{\"resource\":{\"attributes\":{\"deployment.environment.name\":\"bedrock-agentcore:default\",\"aws.local.service\":\"asdf_MyAgent.DEFAULT\",\"service.name\":\"asdf_MyAgent.DEFAULT\",\"cloud.region\":\"us-west-2\",\"aws.log.stream.names\":\"otel-rt-logs\",\"telemetry.sdk.name\":\"opentelemetry\",\"aws.service.type\":\"gen_ai_agent\",\"telemetry.sdk.language\":\"python\",\"cloud.provider\":\"aws\",\"cloud.resource_id\":\"arn:aws:bedrock-agentcore:us-west-2:685197708687:runtime/asdf_MyAgent-3s5axvBC6Q/runtime-endpoint/DEFAULT:DEFAULT\",\"aws.log.group.names\":\"/aws/bedrock-agentcore/runtimes/asdf_MyAgent-3s5axvBC6Q-DEFAULT\",\"telemetry.sdk.version\":\"1.40.0\",\"cloud.platform\":\"aws_bedrock_agentcore\",\"telemetry.auto.version\":\"0.17.0-aws\"}},\"scope\":{\"name\":\"opentelemetry.instrumentation.urllib3\",\"version\":\"0.61b0\"},\"traceId\":\"6a8f639540ebe6cb6f30d5d80762c2e2\",\"spanId\":\"09f7e5c0e6870466\",\"parentSpanId\":\"9a98ae2cca8efc3d\",\"flags\":256,\"name\":\"PUT\",\"kind\":\"CLIENT\",\"startTimeUnixNano\":1787782043375063062,\"endTimeUnixNano\":1787782043376082417,\"durationNano\":1019355,\"attributes\":{\"aws.local.service\":\"asdf_MyAgent.DEFAULT\",\"telemetry.extended\":\"true\",\"http.url\":\"http://169.254.169.254/latest/api/token\",\"aws.remote.service\":\"169.254.169.254\",\"aws.local.environment\":\"bedrock-agentcore:default\",\"aws.remote.operation\":\"PUT /latest\",\"http.status_code\":200,\"aws.local.operation\":\"UnmappedOperation\",\"aws.span.kind\":\"CLIENT\",\"PlatformType\":\"AWS::BedrockAgentCore\",\"http.method\":\"PUT\",\"http.response.status_code\":200,\"session.id\":\"00000000-0000-4000-8000-000000000001\"},\"status\":{\"code\":\"UNSET\"}}" + }, + { + "field": "sessionId", + "value": "00000000-0000-4000-8000-000000000001" + }, + { + "field": "traceId", + "value": "6a8f639540ebe6cb6f30d5d80762c2e2" + }, + { + "field": "spanId", + "value": "09f7e5c0e6870466" + }, + { + "field": "@ptr", + "value": "CpwBCmAKFjY4NTE5NzcwODY4Nzphd3Mvc3BhbnMQABokZTE0NTI5MzgtZjVjZi00OTc1LWE0N2QtYzk3YzMwZmFlY2ZlIg4IgJD4jII0EOefqq2ENDjv37W+yTNAyO+wktMzSAASNhoYAganxnVsAAAAACDEC4EABqj2OeAAAAXCIAEo8K30gIQ0MLe89ICENDgLQNCZAUidZ1CIPCACEAAYAQ==" + } + ], + [ + { + "field": "@message", + "value": "{\"resource\":{\"attributes\":{\"deployment.environment.name\":\"bedrock-agentcore:default\",\"aws.local.service\":\"asdf_MyAgent.DEFAULT\",\"service.name\":\"asdf_MyAgent.DEFAULT\",\"cloud.region\":\"us-west-2\",\"aws.log.stream.names\":\"otel-rt-logs\",\"telemetry.sdk.name\":\"opentelemetry\",\"aws.service.type\":\"gen_ai_agent\",\"telemetry.sdk.language\":\"python\",\"cloud.provider\":\"aws\",\"cloud.resource_id\":\"arn:aws:bedrock-agentcore:us-west-2:685197708687:runtime/asdf_MyAgent-3s5axvBC6Q/runtime-endpoint/DEFAULT:DEFAULT\",\"aws.log.group.names\":\"/aws/bedrock-agentcore/runtimes/asdf_MyAgent-3s5axvBC6Q-DEFAULT\",\"telemetry.sdk.version\":\"1.40.0\",\"cloud.platform\":\"aws_bedrock_agentcore\",\"telemetry.auto.version\":\"0.17.0-aws\"}},\"scope\":{\"name\":\"opentelemetry.instrumentation.urllib3\",\"version\":\"0.61b0\"},\"traceId\":\"6a8f639540ebe6cb6f30d5d80762c2e2\",\"spanId\":\"f7718c78b1f483d8\",\"parentSpanId\":\"9a98ae2cca8efc3d\",\"flags\":256,\"name\":\"GET\",\"kind\":\"CLIENT\",\"startTimeUnixNano\":1787782043376507637,\"endTimeUnixNano\":1787782043377035089,\"durationNano\":527452,\"attributes\":{\"aws.local.service\":\"asdf_MyAgent.DEFAULT\",\"telemetry.extended\":\"true\",\"http.url\":\"http://169.254.169.254/latest/meta-data/iam/security-credentials/\",\"aws.remote.service\":\"169.254.169.254\",\"aws.local.environment\":\"bedrock-agentcore:default\",\"aws.remote.operation\":\"GET /latest\",\"http.status_code\":200,\"aws.local.operation\":\"UnmappedOperation\",\"aws.span.kind\":\"CLIENT\",\"PlatformType\":\"AWS::BedrockAgentCore\",\"http.method\":\"GET\",\"http.response.status_code\":200,\"session.id\":\"00000000-0000-4000-8000-000000000001\"},\"status\":{\"code\":\"UNSET\"}}" + }, + { + "field": "sessionId", + "value": "00000000-0000-4000-8000-000000000001" + }, + { + "field": "traceId", + "value": "6a8f639540ebe6cb6f30d5d80762c2e2" + }, + { + "field": "spanId", + "value": "f7718c78b1f483d8" + }, + { + "field": "@ptr", + "value": "CpwBCmAKFjY4NTE5NzcwODY4Nzphd3Mvc3BhbnMQABokZTE0NTI5MzgtZjVjZi00OTc1LWE0N2QtYzk3YzMwZmFlY2ZlIg4IgJD4jII0EOefqq2ENDjv37W+yTNAyO+wktMzSAASNhoYAganxnVsAAAAACDEC4EABqj2OeAAAAXCIAEo8K30gIQ0MLe89ICENDgLQNCZAUidZ1CIPCACEAEYAQ==" + } + ], + [ + { + "field": "@message", + "value": "{\"resource\":{\"attributes\":{\"deployment.environment.name\":\"bedrock-agentcore:default\",\"aws.local.service\":\"asdf_MyAgent.DEFAULT\",\"service.name\":\"asdf_MyAgent.DEFAULT\",\"cloud.region\":\"us-west-2\",\"aws.log.stream.names\":\"otel-rt-logs\",\"telemetry.sdk.name\":\"opentelemetry\",\"aws.service.type\":\"gen_ai_agent\",\"telemetry.sdk.language\":\"python\",\"cloud.provider\":\"aws\",\"cloud.resource_id\":\"arn:aws:bedrock-agentcore:us-west-2:685197708687:runtime/asdf_MyAgent-3s5axvBC6Q/runtime-endpoint/DEFAULT:DEFAULT\",\"aws.log.group.names\":\"/aws/bedrock-agentcore/runtimes/asdf_MyAgent-3s5axvBC6Q-DEFAULT\",\"telemetry.sdk.version\":\"1.40.0\",\"cloud.platform\":\"aws_bedrock_agentcore\",\"telemetry.auto.version\":\"0.17.0-aws\"}},\"scope\":{\"name\":\"opentelemetry.instrumentation.urllib3\",\"version\":\"0.61b0\"},\"traceId\":\"6a8f639540ebe6cb6f30d5d80762c2e2\",\"spanId\":\"4f5b2a32c8aa0586\",\"parentSpanId\":\"9a98ae2cca8efc3d\",\"flags\":256,\"name\":\"GET\",\"kind\":\"CLIENT\",\"startTimeUnixNano\":1787782043377350818,\"endTimeUnixNano\":1787782043377867116,\"durationNano\":516298,\"attributes\":{\"aws.local.service\":\"asdf_MyAgent.DEFAULT\",\"telemetry.extended\":\"true\",\"http.url\":\"http://169.254.169.254/latest/meta-data/iam/security-credentials/execution_role\",\"aws.remote.service\":\"169.254.169.254\",\"aws.local.environment\":\"bedrock-agentcore:default\",\"aws.remote.operation\":\"GET /latest\",\"http.status_code\":200,\"aws.local.operation\":\"UnmappedOperation\",\"aws.span.kind\":\"CLIENT\",\"PlatformType\":\"AWS::BedrockAgentCore\",\"http.method\":\"GET\",\"http.response.status_code\":200,\"session.id\":\"00000000-0000-4000-8000-000000000001\"},\"status\":{\"code\":\"UNSET\"}}" + }, + { + "field": "sessionId", + "value": "00000000-0000-4000-8000-000000000001" + }, + { + "field": "traceId", + "value": "6a8f639540ebe6cb6f30d5d80762c2e2" + }, + { + "field": "spanId", + "value": "4f5b2a32c8aa0586" + }, + { + "field": "@ptr", + "value": "CpwBCmAKFjY4NTE5NzcwODY4Nzphd3Mvc3BhbnMQABokZTE0NTI5MzgtZjVjZi00OTc1LWE0N2QtYzk3YzMwZmFlY2ZlIg4IgJD4jII0EOefqq2ENDjv37W+yTNAyO+wktMzSAASNhoYAganxnVsAAAAACDEC4EABqj2OeAAAAXCIAEo8K30gIQ0MLe89ICENDgLQNCZAUidZ1CIPCACEAIYAQ==" + } + ], + [ + { + "field": "@message", + "value": "{\"resource\":{\"attributes\":{\"deployment.environment.name\":\"bedrock-agentcore:default\",\"aws.local.service\":\"asdf_MyAgent.DEFAULT\",\"service.name\":\"asdf_MyAgent.DEFAULT\",\"cloud.region\":\"us-west-2\",\"aws.log.stream.names\":\"otel-rt-logs\",\"telemetry.sdk.name\":\"opentelemetry\",\"aws.service.type\":\"gen_ai_agent\",\"telemetry.sdk.language\":\"python\",\"cloud.provider\":\"aws\",\"cloud.resource_id\":\"arn:aws:bedrock-agentcore:us-west-2:685197708687:runtime/asdf_MyAgent-3s5axvBC6Q/runtime-endpoint/DEFAULT:DEFAULT\",\"aws.log.group.names\":\"/aws/bedrock-agentcore/runtimes/asdf_MyAgent-3s5axvBC6Q-DEFAULT\",\"telemetry.sdk.version\":\"1.40.0\",\"cloud.platform\":\"aws_bedrock_agentcore\",\"telemetry.auto.version\":\"0.17.0-aws\"}},\"scope\":{\"name\":\"opentelemetry.instrumentation.httpx\",\"version\":\"0.61b0\"},\"traceId\":\"6a8f639540ebe6cb6f30d5d80762c2e2\",\"spanId\":\"7bad4d2a822b9d46\",\"parentSpanId\":\"9a98ae2cca8efc3d\",\"flags\":256,\"name\":\"POST\",\"kind\":\"CLIENT\",\"startTimeUnixNano\":1787782043483277501,\"endTimeUnixNano\":1787782043527189690,\"durationNano\":43912189,\"attributes\":{\"aws.local.service\":\"asdf_MyAgent.DEFAULT\",\"telemetry.extended\":\"true\",\"http.url\":\"https://bugbashgw1776978672-zsy8cbqwts.gateway.bedrock-agentcore.us-west-2.amazonaws.com/mcp\",\"aws.remote.service\":\"bugbashgw1776978672-zsy8cbqwts.gateway.bedrock-agentcore.us-west-2.amazonaws.com\",\"aws.local.environment\":\"bedrock-agentcore:default\",\"aws.remote.operation\":\"POST /mcp\",\"http.status_code\":200,\"aws.local.operation\":\"UnmappedOperation\",\"aws.span.kind\":\"CLIENT\",\"PlatformType\":\"AWS::BedrockAgentCore\",\"http.method\":\"POST\",\"http.response.status_code\":200,\"session.id\":\"00000000-0000-4000-8000-000000000001\"},\"status\":{\"code\":\"UNSET\"}}" + }, + { + "field": "sessionId", + "value": "00000000-0000-4000-8000-000000000001" + }, + { + "field": "traceId", + "value": "6a8f639540ebe6cb6f30d5d80762c2e2" + }, + { + "field": "spanId", + "value": "7bad4d2a822b9d46" + }, + { + "field": "@ptr", + "value": "CpwBCmAKFjY4NTE5NzcwODY4Nzphd3Mvc3BhbnMQABokZTE0NTI5MzgtZjVjZi00OTc1LWE0N2QtYzk3YzMwZmFlY2ZlIg4IgJD4jII0EOefqq2ENDjv37W+yTNAyO+wktMzSAASNhoYAganxnVsAAAAACDEC4EABqj2OeAAAAXCIAEo8K30gIQ0MLe89ICENDgLQNCZAUidZ1CIPCACEAMYAQ==" + } + ], + [ + { + "field": "@message", + "value": "{\"resource\":{\"attributes\":{\"deployment.environment.name\":\"bedrock-agentcore:default\",\"aws.local.service\":\"asdf_MyAgent.DEFAULT\",\"service.name\":\"asdf_MyAgent.DEFAULT\",\"cloud.region\":\"us-west-2\",\"aws.log.stream.names\":\"otel-rt-logs\",\"telemetry.sdk.name\":\"opentelemetry\",\"aws.service.type\":\"gen_ai_agent\",\"telemetry.sdk.language\":\"python\",\"cloud.provider\":\"aws\",\"cloud.resource_id\":\"arn:aws:bedrock-agentcore:us-west-2:685197708687:runtime/asdf_MyAgent-3s5axvBC6Q/runtime-endpoint/DEFAULT:DEFAULT\",\"aws.log.group.names\":\"/aws/bedrock-agentcore/runtimes/asdf_MyAgent-3s5axvBC6Q-DEFAULT\",\"telemetry.sdk.version\":\"1.40.0\",\"cloud.platform\":\"aws_bedrock_agentcore\",\"telemetry.auto.version\":\"0.17.0-aws\"}},\"scope\":{\"name\":\"opentelemetry.instrumentation.httpx\",\"version\":\"0.61b0\"},\"traceId\":\"6a8f639540ebe6cb6f30d5d80762c2e2\",\"spanId\":\"d14fb48aa6858b3d\",\"parentSpanId\":\"9a98ae2cca8efc3d\",\"flags\":256,\"name\":\"POST\",\"kind\":\"CLIENT\",\"startTimeUnixNano\":1787782043529850664,\"endTimeUnixNano\":1787782043562515163,\"durationNano\":32664499,\"attributes\":{\"aws.local.service\":\"asdf_MyAgent.DEFAULT\",\"telemetry.extended\":\"true\",\"http.url\":\"https://bugbashgw1776978672-zsy8cbqwts.gateway.bedrock-agentcore.us-west-2.amazonaws.com/mcp\",\"aws.remote.service\":\"bugbashgw1776978672-zsy8cbqwts.gateway.bedrock-agentcore.us-west-2.amazonaws.com\",\"aws.local.environment\":\"bedrock-agentcore:default\",\"aws.remote.operation\":\"POST /mcp\",\"http.status_code\":202,\"aws.local.operation\":\"UnmappedOperation\",\"aws.span.kind\":\"CLIENT\",\"PlatformType\":\"AWS::BedrockAgentCore\",\"http.method\":\"POST\",\"http.response.status_code\":202,\"session.id\":\"00000000-0000-4000-8000-000000000001\"},\"status\":{\"code\":\"UNSET\"}}" + }, + { + "field": "sessionId", + "value": "00000000-0000-4000-8000-000000000001" + }, + { + "field": "traceId", + "value": "6a8f639540ebe6cb6f30d5d80762c2e2" + }, + { + "field": "spanId", + "value": "d14fb48aa6858b3d" + }, + { + "field": "@ptr", + "value": "CpwBCmAKFjY4NTE5NzcwODY4Nzphd3Mvc3BhbnMQABokZTE0NTI5MzgtZjVjZi00OTc1LWE0N2QtYzk3YzMwZmFlY2ZlIg4IgJD4jII0EOefqq2ENDjv37W+yTNAyO+wktMzSAASNhoYAganxnVsAAAAACDEC4EABqj2OeAAAAXCIAEo8K30gIQ0MLe89ICENDgLQNCZAUidZ1CIPCACEAQYAQ==" + } + ], + [ + { + "field": "@message", + "value": "{\"resource\":{\"attributes\":{\"deployment.environment.name\":\"bedrock-agentcore:default\",\"aws.local.service\":\"asdf_MyAgent.DEFAULT\",\"service.name\":\"asdf_MyAgent.DEFAULT\",\"cloud.region\":\"us-west-2\",\"aws.log.stream.names\":\"otel-rt-logs\",\"telemetry.sdk.name\":\"opentelemetry\",\"aws.service.type\":\"gen_ai_agent\",\"telemetry.sdk.language\":\"python\",\"cloud.provider\":\"aws\",\"cloud.resource_id\":\"arn:aws:bedrock-agentcore:us-west-2:685197708687:runtime/asdf_MyAgent-3s5axvBC6Q/runtime-endpoint/DEFAULT:DEFAULT\",\"aws.log.group.names\":\"/aws/bedrock-agentcore/runtimes/asdf_MyAgent-3s5axvBC6Q-DEFAULT\",\"telemetry.sdk.version\":\"1.40.0\",\"cloud.platform\":\"aws_bedrock_agentcore\",\"telemetry.auto.version\":\"0.17.0-aws\"}},\"scope\":{\"name\":\"opentelemetry.instrumentation.httpx\",\"version\":\"0.61b0\"},\"traceId\":\"6a8f639540ebe6cb6f30d5d80762c2e2\",\"spanId\":\"d779e95172d4d1ba\",\"parentSpanId\":\"9a98ae2cca8efc3d\",\"flags\":256,\"name\":\"POST\",\"kind\":\"CLIENT\",\"startTimeUnixNano\":1787782043564209768,\"endTimeUnixNano\":1787782043624521554,\"durationNano\":60311786,\"attributes\":{\"aws.local.service\":\"asdf_MyAgent.DEFAULT\",\"telemetry.extended\":\"true\",\"http.url\":\"https://bugbashgw1776978672-zsy8cbqwts.gateway.bedrock-agentcore.us-west-2.amazonaws.com/mcp\",\"aws.remote.service\":\"bugbashgw1776978672-zsy8cbqwts.gateway.bedrock-agentcore.us-west-2.amazonaws.com\",\"aws.local.environment\":\"bedrock-agentcore:default\",\"aws.remote.operation\":\"POST /mcp\",\"http.status_code\":200,\"aws.local.operation\":\"UnmappedOperation\",\"aws.span.kind\":\"CLIENT\",\"PlatformType\":\"AWS::BedrockAgentCore\",\"http.method\":\"POST\",\"http.response.status_code\":200,\"session.id\":\"00000000-0000-4000-8000-000000000001\"},\"status\":{\"code\":\"UNSET\"}}" + }, + { + "field": "sessionId", + "value": "00000000-0000-4000-8000-000000000001" + }, + { + "field": "traceId", + "value": "6a8f639540ebe6cb6f30d5d80762c2e2" + }, + { + "field": "spanId", + "value": "d779e95172d4d1ba" + }, + { + "field": "@ptr", + "value": "CpwBCmAKFjY4NTE5NzcwODY4Nzphd3Mvc3BhbnMQABokZTE0NTI5MzgtZjVjZi00OTc1LWE0N2QtYzk3YzMwZmFlY2ZlIg4IgJD4jII0EOefqq2ENDjv37W+yTNAyO+wktMzSAASNhoYAganxnVsAAAAACDEC4EABqj2OeAAAAXCIAEo8K30gIQ0MLe89ICENDgLQNCZAUidZ1CIPCACEAUYAQ==" + } + ], + [ + { + "field": "@message", + "value": "{\"resource\":{\"attributes\":{\"deployment.environment.name\":\"bedrock-agentcore:default\",\"aws.local.service\":\"asdf_MyAgent.DEFAULT\",\"service.name\":\"asdf_MyAgent.DEFAULT\",\"cloud.region\":\"us-west-2\",\"aws.log.stream.names\":\"otel-rt-logs\",\"telemetry.sdk.name\":\"opentelemetry\",\"aws.service.type\":\"gen_ai_agent\",\"telemetry.sdk.language\":\"python\",\"cloud.provider\":\"aws\",\"cloud.resource_id\":\"arn:aws:bedrock-agentcore:us-west-2:685197708687:runtime/asdf_MyAgent-3s5axvBC6Q/runtime-endpoint/DEFAULT:DEFAULT\",\"aws.log.group.names\":\"/aws/bedrock-agentcore/runtimes/asdf_MyAgent-3s5axvBC6Q-DEFAULT\",\"telemetry.sdk.version\":\"1.40.0\",\"cloud.platform\":\"aws_bedrock_agentcore\",\"telemetry.auto.version\":\"0.17.0-aws\"}},\"scope\":{\"name\":\"opentelemetry.instrumentation.botocore.bedrock-runtime\",\"version\":\"0.61b0\"},\"traceId\":\"6a8f639540ebe6cb6f30d5d80762c2e2\",\"spanId\":\"0e31289cb4c82723\",\"parentSpanId\":\"0925d4cc59be00d9\",\"flags\":256,\"name\":\"chat global.anthropic.claude-sonnet-4-5-20250929-v1:0\",\"kind\":\"CLIENT\",\"startTimeUnixNano\":1787782043640469865,\"endTimeUnixNano\":1787782045238681022,\"durationNano\":1598211157,\"attributes\":{\"aws.local.service\":\"asdf_MyAgent.DEFAULT\",\"rpc.service\":\"Bedrock Runtime\",\"aws.remote.resource.identifier\":\"global.anthropic.claude-sonnet-4-5-20250929-v1:0\",\"aws.local.environment\":\"bedrock-agentcore:default\",\"aws.remote.operation\":\"ConverseStream\",\"gen_ai.provider.name\":\"aws.bedrock\",\"server.address\":\"bedrock-runtime.us-west-2.amazonaws.com\",\"aws.request_id\":\"45f112bf-ae3f-4cd2-8110-8f6199d04114\",\"aws.local.operation\":\"UnmappedOperation\",\"aws.span.kind\":\"CLIENT\",\"aws.auth.region\":\"us-west-2\",\"rpc.method\":\"ConverseStream\",\"gen_ai.response.finish_reasons\":[\"end_turn\"],\"server.port\":443,\"gen_ai.request.model\":\"global.anthropic.claude-sonnet-4-5-20250929-v1:0\",\"http.response.status_code\":200,\"gen_ai.system\":\"aws.bedrock\",\"telemetry.extended\":\"true\",\"gen_ai.usage.output_tokens\":12,\"aws.genai.span_kind\":\"LLM\",\"rpc.system\":\"aws-api\",\"aws.remote.service\":\"AWS::BedrockRuntime\",\"http.status_code\":200,\"aws.region\":\"us-west-2\",\"aws.remote.resource.type\":\"AWS::Bedrock::Model\",\"gen_ai.operation.name\":\"chat\",\"gen_ai.usage.input_tokens\":1199,\"aws.genai.token_count_total\":1211,\"retry_attempts\":0,\"PlatformType\":\"AWS::BedrockAgentCore\",\"aws.auth.account.access_key\":\"ASIAZ7CHXJWHWD7B2RCF\",\"session.id\":\"00000000-0000-4000-8000-000000000001\"},\"status\":{\"code\":\"UNSET\"}}" + }, + { + "field": "sessionId", + "value": "00000000-0000-4000-8000-000000000001" + }, + { + "field": "traceId", + "value": "6a8f639540ebe6cb6f30d5d80762c2e2" + }, + { + "field": "spanId", + "value": "0e31289cb4c82723" + }, + { + "field": "@ptr", + "value": "CpwBCmAKFjY4NTE5NzcwODY4Nzphd3Mvc3BhbnMQABokZTE0NTI5MzgtZjVjZi00OTc1LWE0N2QtYzk3YzMwZmFlY2ZlIg4IgJD4jII0EOefqq2ENDjv37W+yTNAyO+wktMzSAASNhoYAganxnVsAAAAACDEC4EABqj2OeAAAAXCIAEo8K30gIQ0MLe89ICENDgLQNCZAUidZ1CIPCACEAYYAQ==" + } + ], + [ + { + "field": "@message", + "value": "{\"resource\":{\"attributes\":{\"deployment.environment.name\":\"bedrock-agentcore:default\",\"aws.local.service\":\"asdf_MyAgent.DEFAULT\",\"service.name\":\"asdf_MyAgent.DEFAULT\",\"cloud.region\":\"us-west-2\",\"aws.log.stream.names\":\"otel-rt-logs\",\"telemetry.sdk.name\":\"opentelemetry\",\"aws.service.type\":\"gen_ai_agent\",\"telemetry.sdk.language\":\"python\",\"cloud.provider\":\"aws\",\"cloud.resource_id\":\"arn:aws:bedrock-agentcore:us-west-2:685197708687:runtime/asdf_MyAgent-3s5axvBC6Q/runtime-endpoint/DEFAULT:DEFAULT\",\"aws.log.group.names\":\"/aws/bedrock-agentcore/runtimes/asdf_MyAgent-3s5axvBC6Q-DEFAULT\",\"telemetry.sdk.version\":\"1.40.0\",\"cloud.platform\":\"aws_bedrock_agentcore\",\"telemetry.auto.version\":\"0.17.0-aws\"}},\"scope\":{\"name\":\"strands.telemetry.tracer\",\"version\":\"\"},\"traceId\":\"6a8f639540ebe6cb6f30d5d80762c2e2\",\"spanId\":\"6f925a9a78a2fe62\",\"parentSpanId\":\"9a98ae2cca8efc3d\",\"flags\":256,\"name\":\"invoke_agent Strands Agents\",\"kind\":\"INTERNAL\",\"startTimeUnixNano\":1787782043627084668,\"endTimeUnixNano\":1787782045239678560,\"durationNano\":1612593892,\"attributes\":{\"aws.local.service\":\"asdf_MyAgent.DEFAULT\",\"gen_ai.usage.prompt_tokens\":1199,\"gen_ai.usage.output_tokens\":12,\"gen_ai.usage.cache_write_input_tokens\":0,\"gen_ai.agent.name\":\"Strands Agents\",\"gen_ai.usage.total_tokens\":1211,\"gen_ai.usage.completion_tokens\":12,\"aws.genai.span_kind\":\"AGENT\",\"gen_ai.event.start_time\":\"2026-08-26T22:07:23.627100+00:00\",\"aws.local.environment\":\"bedrock-agentcore:default\",\"gen_ai.provider.name\":\"strands-agents\",\"gen_ai.operation.name\":\"invoke_agent\",\"gen_ai.event.end_time\":\"2026-08-26T22:07:25.239658+00:00\",\"gen_ai.usage.input_tokens\":1199,\"aws.genai.token_count_total\":1211,\"gen_ai.request.model\":\"global.anthropic.claude-sonnet-4-5-20250929-v1:0\",\"gen_ai.usage.cache_read_input_tokens\":0,\"gen_ai.agent.tools\":\"[\\\"add_numbers\\\", \\\"x_amz_bedrock_agentcore_search\\\", \\\"mcpTarget___web_fetch_exa\\\", \\\"mcpTarget___web_search_exa\\\"]\",\"PlatformType\":\"AWS::BedrockAgentCore\",\"session.id\":\"00000000-0000-4000-8000-000000000001\",\"gen_ai.system\":\"strands-agents\"},\"status\":{\"code\":\"OK\"}}" + }, + { + "field": "sessionId", + "value": "00000000-0000-4000-8000-000000000001" + }, + { + "field": "traceId", + "value": "6a8f639540ebe6cb6f30d5d80762c2e2" + }, + { + "field": "spanId", + "value": "6f925a9a78a2fe62" + }, + { + "field": "@ptr", + "value": "CpwBCmAKFjY4NTE5NzcwODY4Nzphd3Mvc3BhbnMQABokZTE0NTI5MzgtZjVjZi00OTc1LWE0N2QtYzk3YzMwZmFlY2ZlIg4IgJD4jII0EOefqq2ENDjv37W+yTNAyO+wktMzSAASNhoYAganxnVsAAAAACDEC4EABqj2OeAAAAXCIAEo8K30gIQ0MLe89ICENDgLQNCZAUidZ1CIPCACEAkYAQ==" + } + ], + [ + { + "field": "@message", + "value": "{\"resource\":{\"attributes\":{\"deployment.environment.name\":\"bedrock-agentcore:default\",\"aws.local.service\":\"asdf_MyAgent.DEFAULT\",\"service.name\":\"asdf_MyAgent.DEFAULT\",\"cloud.region\":\"us-west-2\",\"aws.log.stream.names\":\"otel-rt-logs\",\"telemetry.sdk.name\":\"opentelemetry\",\"aws.service.type\":\"gen_ai_agent\",\"telemetry.sdk.language\":\"python\",\"cloud.provider\":\"aws\",\"cloud.resource_id\":\"arn:aws:bedrock-agentcore:us-west-2:685197708687:runtime/asdf_MyAgent-3s5axvBC6Q/runtime-endpoint/DEFAULT:DEFAULT\",\"aws.log.group.names\":\"/aws/bedrock-agentcore/runtimes/asdf_MyAgent-3s5axvBC6Q-DEFAULT\",\"telemetry.sdk.version\":\"1.40.0\",\"cloud.platform\":\"aws_bedrock_agentcore\",\"telemetry.auto.version\":\"0.17.0-aws\"}},\"scope\":{\"name\":\"strands.telemetry.tracer\",\"version\":\"\"},\"traceId\":\"6a8f639540ebe6cb6f30d5d80762c2e2\",\"spanId\":\"fc189f3b80a0c1f4\",\"parentSpanId\":\"6f925a9a78a2fe62\",\"flags\":256,\"name\":\"execute_event_loop_cycle\",\"kind\":\"INTERNAL\",\"startTimeUnixNano\":1787782043627671178,\"endTimeUnixNano\":1787782045239528397,\"durationNano\":1611857219,\"attributes\":{\"aws.local.service\":\"asdf_MyAgent.DEFAULT\",\"gen_ai.operation.name\":\"execute_event_loop_cycle\",\"gen_ai.event.end_time\":\"2026-08-26T22:07:25.239513+00:00\",\"event_loop.cycle_id\":\"cc82280b-018e-4cea-ac5a-526dae34b998\",\"gen_ai.event.start_time\":\"2026-08-26T22:07:23.627681+00:00\",\"PlatformType\":\"AWS::BedrockAgentCore\",\"session.id\":\"00000000-0000-4000-8000-000000000001\",\"gen_ai.system\":\"strands-agents\",\"aws.local.environment\":\"bedrock-agentcore:default\",\"gen_ai.provider.name\":\"strands-agents\"},\"status\":{\"code\":\"OK\"}}" + }, + { + "field": "sessionId", + "value": "00000000-0000-4000-8000-000000000001" + }, + { + "field": "traceId", + "value": "6a8f639540ebe6cb6f30d5d80762c2e2" + }, + { + "field": "spanId", + "value": "fc189f3b80a0c1f4" + }, + { + "field": "@ptr", + "value": "CpwBCmAKFjY4NTE5NzcwODY4Nzphd3Mvc3BhbnMQABokZTE0NTI5MzgtZjVjZi00OTc1LWE0N2QtYzk3YzMwZmFlY2ZlIg4IgJD4jII0EOefqq2ENDjv37W+yTNAyO+wktMzSAASNhoYAganxnVsAAAAACDEC4EABqj2OeAAAAXCIAEo8K30gIQ0MLe89ICENDgLQNCZAUidZ1CIPCACEAgYAQ==" + } + ], + [ + { + "field": "@message", + "value": "{\"resource\":{\"attributes\":{\"deployment.environment.name\":\"bedrock-agentcore:default\",\"aws.local.service\":\"asdf_MyAgent.DEFAULT\",\"service.name\":\"asdf_MyAgent.DEFAULT\",\"cloud.region\":\"us-west-2\",\"aws.log.stream.names\":\"otel-rt-logs\",\"telemetry.sdk.name\":\"opentelemetry\",\"aws.service.type\":\"gen_ai_agent\",\"telemetry.sdk.language\":\"python\",\"cloud.provider\":\"aws\",\"cloud.resource_id\":\"arn:aws:bedrock-agentcore:us-west-2:685197708687:runtime/asdf_MyAgent-3s5axvBC6Q/runtime-endpoint/DEFAULT:DEFAULT\",\"aws.log.group.names\":\"/aws/bedrock-agentcore/runtimes/asdf_MyAgent-3s5axvBC6Q-DEFAULT\",\"telemetry.sdk.version\":\"1.40.0\",\"cloud.platform\":\"aws_bedrock_agentcore\",\"telemetry.auto.version\":\"0.17.0-aws\"}},\"scope\":{\"name\":\"opentelemetry.instrumentation.starlette\",\"version\":\"0.61b0\"},\"traceId\":\"6a8f639540ebe6cb6f30d5d80762c2e2\",\"spanId\":\"9a98ae2cca8efc3d\",\"parentSpanId\":\"6efc239e6bd4139f\",\"flags\":768,\"name\":\"POST /invocations\",\"kind\":\"SERVER\",\"startTimeUnixNano\":1787782043334606170,\"endTimeUnixNano\":1787782045239993499,\"durationNano\":1905387329,\"attributes\":{\"aws.local.service\":\"asdf_MyAgent.DEFAULT\",\"net.peer.port\":34872,\"telemetry.extended\":\"true\",\"http.target\":\"/invocations\",\"http.flavor\":\"1.1\",\"http.url\":\"http://cell01.us-west-2.prod.arp.kepler-analytics.aws.dev/invocations\",\"net.peer.ip\":\"127.0.0.1\",\"http.host\":\"127.0.0.1:8080\",\"aws.local.environment\":\"bedrock-agentcore:default\",\"http.status_code\":200,\"aws.local.operation\":\"POST /invocations\",\"aws.span.kind\":\"SERVER\",\"http.server_name\":\"cell01.us-west-2.prod.arp.kepler-analytics.aws.dev\",\"net.host.port\":8080,\"http.route\":\"/invocations\",\"PlatformType\":\"AWS::BedrockAgentCore\",\"http.method\":\"POST\",\"http.response.status_code\":200,\"session.id\":\"00000000-0000-4000-8000-000000000001\",\"http.scheme\":\"http\"},\"status\":{\"code\":\"UNSET\"}}" + }, + { + "field": "sessionId", + "value": "00000000-0000-4000-8000-000000000001" + }, + { + "field": "traceId", + "value": "6a8f639540ebe6cb6f30d5d80762c2e2" + }, + { + "field": "spanId", + "value": "9a98ae2cca8efc3d" + }, + { + "field": "@ptr", + "value": "CpwBCmAKFjY4NTE5NzcwODY4Nzphd3Mvc3BhbnMQABokZTE0NTI5MzgtZjVjZi00OTc1LWE0N2QtYzk3YzMwZmFlY2ZlIg4IgJD4jII0EOefqq2ENDjv37W+yTNAyO+wktMzSAASNhoYAganxnVsAAAAACDEC4EABqj2OeAAAAXCIAEo8K30gIQ0MLe89ICENDgLQNCZAUidZ1CIPCACEAoYAQ==" + } + ], + [ + { + "field": "@message", + "value": "{\"resource\":{\"attributes\":{\"deployment.environment.name\":\"bedrock-agentcore:default\",\"aws.local.service\":\"asdf_MyAgent.DEFAULT\",\"service.name\":\"asdf_MyAgent.DEFAULT\",\"cloud.region\":\"us-west-2\",\"aws.log.stream.names\":\"otel-rt-logs\",\"telemetry.sdk.name\":\"opentelemetry\",\"aws.service.type\":\"gen_ai_agent\",\"telemetry.sdk.language\":\"python\",\"cloud.provider\":\"aws\",\"cloud.resource_id\":\"arn:aws:bedrock-agentcore:us-west-2:685197708687:runtime/asdf_MyAgent-3s5axvBC6Q/runtime-endpoint/DEFAULT:DEFAULT\",\"aws.log.group.names\":\"/aws/bedrock-agentcore/runtimes/asdf_MyAgent-3s5axvBC6Q-DEFAULT\",\"telemetry.sdk.version\":\"1.40.0\",\"cloud.platform\":\"aws_bedrock_agentcore\",\"telemetry.auto.version\":\"0.17.0-aws\"}},\"scope\":{\"name\":\"strands.telemetry.tracer\",\"version\":\"\"},\"traceId\":\"6a8f639540ebe6cb6f30d5d80762c2e2\",\"spanId\":\"0925d4cc59be00d9\",\"parentSpanId\":\"fc189f3b80a0c1f4\",\"flags\":256,\"name\":\"chat\",\"kind\":\"INTERNAL\",\"startTimeUnixNano\":1787782043627802055,\"endTimeUnixNano\":1787782045239215762,\"durationNano\":1611413707,\"attributes\":{\"aws.local.service\":\"asdf_MyAgent.DEFAULT\",\"gen_ai.usage.prompt_tokens\":1199,\"gen_ai.usage.output_tokens\":12,\"gen_ai.server.request.duration\":1579,\"gen_ai.usage.total_tokens\":1211,\"gen_ai.usage.completion_tokens\":12,\"aws.genai.span_kind\":\"LLM\",\"gen_ai.event.start_time\":\"2026-08-26T22:07:23.627809+00:00\",\"gen_ai.server.time_to_first_token\":1504,\"aws.local.environment\":\"bedrock-agentcore:default\",\"gen_ai.provider.name\":\"strands-agents\",\"gen_ai.operation.name\":\"chat\",\"gen_ai.event.end_time\":\"2026-08-26T22:07:25.239178+00:00\",\"gen_ai.usage.input_tokens\":1199,\"aws.genai.token_count_total\":1211,\"gen_ai.request.model\":\"global.anthropic.claude-sonnet-4-5-20250929-v1:0\",\"PlatformType\":\"AWS::BedrockAgentCore\",\"session.id\":\"00000000-0000-4000-8000-000000000001\",\"gen_ai.system\":\"strands-agents\"},\"status\":{\"code\":\"OK\"}}" + }, + { + "field": "sessionId", + "value": "00000000-0000-4000-8000-000000000001" + }, + { + "field": "traceId", + "value": "6a8f639540ebe6cb6f30d5d80762c2e2" + }, + { + "field": "spanId", + "value": "0925d4cc59be00d9" + }, + { + "field": "@ptr", + "value": "CpwBCmAKFjY4NTE5NzcwODY4Nzphd3Mvc3BhbnMQABokZTE0NTI5MzgtZjVjZi00OTc1LWE0N2QtYzk3YzMwZmFlY2ZlIg4IgJD4jII0EOefqq2ENDjv37W+yTNAyO+wktMzSAASNhoYAganxnVsAAAAACDEC4EABqj2OeAAAAXCIAEo8K30gIQ0MLe89ICENDgLQNCZAUidZ1CIPCACEAcYAQ==" + } + ], + [ + { + "field": "@message", + "value": "{\"resource\":{\"attributes\":{\"deployment.environment.name\":\"bedrock-agentcore:default\",\"aws.local.service\":\"asdf_MyAgent.DEFAULT\",\"service.name\":\"asdf_MyAgent.DEFAULT\",\"cloud.region\":\"us-west-2\",\"aws.log.stream.names\":\"otel-rt-logs\",\"telemetry.sdk.name\":\"opentelemetry\",\"aws.service.type\":\"gen_ai_agent\",\"telemetry.sdk.language\":\"python\",\"cloud.provider\":\"aws\",\"cloud.resource_id\":\"arn:aws:bedrock-agentcore:us-west-2:685197708687:runtime/asdf_MyAgent-3s5axvBC6Q/runtime-endpoint/DEFAULT:DEFAULT\",\"aws.log.group.names\":\"/aws/bedrock-agentcore/runtimes/asdf_MyAgent-3s5axvBC6Q-DEFAULT\",\"telemetry.sdk.version\":\"1.40.0\",\"cloud.platform\":\"aws_bedrock_agentcore\",\"telemetry.auto.version\":\"0.17.0-aws\"}},\"scope\":{\"name\":\"opentelemetry.instrumentation.urllib3\",\"version\":\"0.61b0\"},\"traceId\":\"6a90a5987755c6e457ada5616a16dddb\",\"spanId\":\"dc14ad43d77bfb36\",\"parentSpanId\":\"98dc1b886c48967e\",\"flags\":256,\"name\":\"PUT\",\"kind\":\"CLIENT\",\"startTimeUnixNano\":1787864478709074216,\"endTimeUnixNano\":1787864478710322708,\"durationNano\":1248492,\"attributes\":{\"aws.local.service\":\"asdf_MyAgent.DEFAULT\",\"telemetry.extended\":\"true\",\"http.url\":\"http://169.254.169.254/latest/api/token\",\"aws.remote.service\":\"169.254.169.254\",\"aws.local.environment\":\"bedrock-agentcore:default\",\"aws.remote.operation\":\"PUT /latest\",\"http.status_code\":200,\"aws.local.operation\":\"UnmappedOperation\",\"aws.span.kind\":\"CLIENT\",\"PlatformType\":\"AWS::BedrockAgentCore\",\"http.method\":\"PUT\",\"http.response.status_code\":200,\"session.id\":\"00000000-0000-4000-8000-000000000001\"},\"status\":{\"code\":\"UNSET\"}}" + }, + { + "field": "sessionId", + "value": "00000000-0000-4000-8000-000000000001" + }, + { + "field": "traceId", + "value": "6a90a5987755c6e457ada5616a16dddb" + }, + { + "field": "spanId", + "value": "dc14ad43d77bfb36" + }, + { + "field": "@ptr", + "value": "CpsBCmAKFjY4NTE5NzcwODY4Nzphd3Mvc3BhbnMQABokZTE0NTI5MzgtZjVjZi00OTc1LWE0N2QtYzk3YzMwZmFlY2ZlIg4IgJD4jII0EOefqq2ENDjv37W+yTNAyO+wktMzSAASNRoYAgakSqFXAAAAAiX3oQcABqkKWgAAAANyIAEo9uebqIQ0MLrqm6iENDgGQKFNSLk1UOgfIAIQABgB" + } + ], + [ + { + "field": "@message", + "value": "{\"resource\":{\"attributes\":{\"deployment.environment.name\":\"bedrock-agentcore:default\",\"aws.local.service\":\"asdf_MyAgent.DEFAULT\",\"service.name\":\"asdf_MyAgent.DEFAULT\",\"cloud.region\":\"us-west-2\",\"aws.log.stream.names\":\"otel-rt-logs\",\"telemetry.sdk.name\":\"opentelemetry\",\"aws.service.type\":\"gen_ai_agent\",\"telemetry.sdk.language\":\"python\",\"cloud.provider\":\"aws\",\"cloud.resource_id\":\"arn:aws:bedrock-agentcore:us-west-2:685197708687:runtime/asdf_MyAgent-3s5axvBC6Q/runtime-endpoint/DEFAULT:DEFAULT\",\"aws.log.group.names\":\"/aws/bedrock-agentcore/runtimes/asdf_MyAgent-3s5axvBC6Q-DEFAULT\",\"telemetry.sdk.version\":\"1.40.0\",\"cloud.platform\":\"aws_bedrock_agentcore\",\"telemetry.auto.version\":\"0.17.0-aws\"}},\"scope\":{\"name\":\"opentelemetry.instrumentation.urllib3\",\"version\":\"0.61b0\"},\"traceId\":\"6a90a5987755c6e457ada5616a16dddb\",\"spanId\":\"cc7a992ec6c59a21\",\"parentSpanId\":\"98dc1b886c48967e\",\"flags\":256,\"name\":\"GET\",\"kind\":\"CLIENT\",\"startTimeUnixNano\":1787864478710808895,\"endTimeUnixNano\":1787864478711561018,\"durationNano\":752123,\"attributes\":{\"aws.local.service\":\"asdf_MyAgent.DEFAULT\",\"telemetry.extended\":\"true\",\"http.url\":\"http://169.254.169.254/latest/meta-data/iam/security-credentials/\",\"aws.remote.service\":\"169.254.169.254\",\"aws.local.environment\":\"bedrock-agentcore:default\",\"aws.remote.operation\":\"GET /latest\",\"http.status_code\":200,\"aws.local.operation\":\"UnmappedOperation\",\"aws.span.kind\":\"CLIENT\",\"PlatformType\":\"AWS::BedrockAgentCore\",\"http.method\":\"GET\",\"http.response.status_code\":200,\"session.id\":\"00000000-0000-4000-8000-000000000001\"},\"status\":{\"code\":\"UNSET\"}}" + }, + { + "field": "sessionId", + "value": "00000000-0000-4000-8000-000000000001" + }, + { + "field": "traceId", + "value": "6a90a5987755c6e457ada5616a16dddb" + }, + { + "field": "spanId", + "value": "cc7a992ec6c59a21" + }, + { + "field": "@ptr", + "value": "CpsBCmAKFjY4NTE5NzcwODY4Nzphd3Mvc3BhbnMQABokZTE0NTI5MzgtZjVjZi00OTc1LWE0N2QtYzk3YzMwZmFlY2ZlIg4IgJD4jII0EOefqq2ENDjv37W+yTNAyO+wktMzSAASNRoYAgakSqFXAAAAAiX3oQcABqkKWgAAAANyIAEo9uebqIQ0MLrqm6iENDgGQKFNSLk1UOgfIAIQARgB" + } + ], + [ + { + "field": "@message", + "value": "{\"resource\":{\"attributes\":{\"deployment.environment.name\":\"bedrock-agentcore:default\",\"aws.local.service\":\"asdf_MyAgent.DEFAULT\",\"service.name\":\"asdf_MyAgent.DEFAULT\",\"cloud.region\":\"us-west-2\",\"aws.log.stream.names\":\"otel-rt-logs\",\"telemetry.sdk.name\":\"opentelemetry\",\"aws.service.type\":\"gen_ai_agent\",\"telemetry.sdk.language\":\"python\",\"cloud.provider\":\"aws\",\"cloud.resource_id\":\"arn:aws:bedrock-agentcore:us-west-2:685197708687:runtime/asdf_MyAgent-3s5axvBC6Q/runtime-endpoint/DEFAULT:DEFAULT\",\"aws.log.group.names\":\"/aws/bedrock-agentcore/runtimes/asdf_MyAgent-3s5axvBC6Q-DEFAULT\",\"telemetry.sdk.version\":\"1.40.0\",\"cloud.platform\":\"aws_bedrock_agentcore\",\"telemetry.auto.version\":\"0.17.0-aws\"}},\"scope\":{\"name\":\"opentelemetry.instrumentation.urllib3\",\"version\":\"0.61b0\"},\"traceId\":\"6a90a5987755c6e457ada5616a16dddb\",\"spanId\":\"f155589448edb57d\",\"parentSpanId\":\"98dc1b886c48967e\",\"flags\":256,\"name\":\"GET\",\"kind\":\"CLIENT\",\"startTimeUnixNano\":1787864478711972677,\"endTimeUnixNano\":1787864478712747135,\"durationNano\":774458,\"attributes\":{\"aws.local.service\":\"asdf_MyAgent.DEFAULT\",\"telemetry.extended\":\"true\",\"http.url\":\"http://169.254.169.254/latest/meta-data/iam/security-credentials/execution_role\",\"aws.remote.service\":\"169.254.169.254\",\"aws.local.environment\":\"bedrock-agentcore:default\",\"aws.remote.operation\":\"GET /latest\",\"http.status_code\":200,\"aws.local.operation\":\"UnmappedOperation\",\"aws.span.kind\":\"CLIENT\",\"PlatformType\":\"AWS::BedrockAgentCore\",\"http.method\":\"GET\",\"http.response.status_code\":200,\"session.id\":\"00000000-0000-4000-8000-000000000001\"},\"status\":{\"code\":\"UNSET\"}}" + }, + { + "field": "sessionId", + "value": "00000000-0000-4000-8000-000000000001" + }, + { + "field": "traceId", + "value": "6a90a5987755c6e457ada5616a16dddb" + }, + { + "field": "spanId", + "value": "f155589448edb57d" + }, + { + "field": "@ptr", + "value": "CpsBCmAKFjY4NTE5NzcwODY4Nzphd3Mvc3BhbnMQABokZTE0NTI5MzgtZjVjZi00OTc1LWE0N2QtYzk3YzMwZmFlY2ZlIg4IgJD4jII0EOefqq2ENDjv37W+yTNAyO+wktMzSAASNRoYAgakSqFXAAAAAiX3oQcABqkKWgAAAANyIAEo9uebqIQ0MLrqm6iENDgGQKFNSLk1UOgfIAIQAhgB" + } + ], + [ + { + "field": "@message", + "value": "{\"resource\":{\"attributes\":{\"deployment.environment.name\":\"bedrock-agentcore:default\",\"aws.local.service\":\"asdf_MyAgent.DEFAULT\",\"service.name\":\"asdf_MyAgent.DEFAULT\",\"cloud.region\":\"us-west-2\",\"aws.log.stream.names\":\"otel-rt-logs\",\"telemetry.sdk.name\":\"opentelemetry\",\"aws.service.type\":\"gen_ai_agent\",\"telemetry.sdk.language\":\"python\",\"cloud.provider\":\"aws\",\"cloud.resource_id\":\"arn:aws:bedrock-agentcore:us-west-2:685197708687:runtime/asdf_MyAgent-3s5axvBC6Q/runtime-endpoint/DEFAULT:DEFAULT\",\"aws.log.group.names\":\"/aws/bedrock-agentcore/runtimes/asdf_MyAgent-3s5axvBC6Q-DEFAULT\",\"telemetry.sdk.version\":\"1.40.0\",\"cloud.platform\":\"aws_bedrock_agentcore\",\"telemetry.auto.version\":\"0.17.0-aws\"}},\"scope\":{\"name\":\"opentelemetry.instrumentation.httpx\",\"version\":\"0.61b0\"},\"traceId\":\"6a90a5987755c6e457ada5616a16dddb\",\"spanId\":\"e350f049b9ee9a83\",\"parentSpanId\":\"98dc1b886c48967e\",\"flags\":256,\"name\":\"POST\",\"kind\":\"CLIENT\",\"startTimeUnixNano\":1787864478853179249,\"endTimeUnixNano\":1787864478920600443,\"durationNano\":67421194,\"attributes\":{\"aws.local.service\":\"asdf_MyAgent.DEFAULT\",\"telemetry.extended\":\"true\",\"http.url\":\"https://bugbashgw1776978672-zsy8cbqwts.gateway.bedrock-agentcore.us-west-2.amazonaws.com/mcp\",\"aws.remote.service\":\"bugbashgw1776978672-zsy8cbqwts.gateway.bedrock-agentcore.us-west-2.amazonaws.com\",\"aws.local.environment\":\"bedrock-agentcore:default\",\"aws.remote.operation\":\"POST /mcp\",\"http.status_code\":200,\"aws.local.operation\":\"UnmappedOperation\",\"aws.span.kind\":\"CLIENT\",\"PlatformType\":\"AWS::BedrockAgentCore\",\"http.method\":\"POST\",\"http.response.status_code\":200,\"session.id\":\"00000000-0000-4000-8000-000000000001\"},\"status\":{\"code\":\"UNSET\"}}" + }, + { + "field": "sessionId", + "value": "00000000-0000-4000-8000-000000000001" + }, + { + "field": "traceId", + "value": "6a90a5987755c6e457ada5616a16dddb" + }, + { + "field": "spanId", + "value": "e350f049b9ee9a83" + }, + { + "field": "@ptr", + "value": "CpsBCmAKFjY4NTE5NzcwODY4Nzphd3Mvc3BhbnMQABokZTE0NTI5MzgtZjVjZi00OTc1LWE0N2QtYzk3YzMwZmFlY2ZlIg4IgJD4jII0EOefqq2ENDjv37W+yTNAyO+wktMzSAASNRoYAgakSqFXAAAAAiX3oQcABqkKWgAAAANyIAEo9uebqIQ0MLrqm6iENDgGQKFNSLk1UOgfIAIQAxgB" + } + ], + [ + { + "field": "@message", + "value": "{\"resource\":{\"attributes\":{\"deployment.environment.name\":\"bedrock-agentcore:default\",\"aws.local.service\":\"asdf_MyAgent.DEFAULT\",\"service.name\":\"asdf_MyAgent.DEFAULT\",\"cloud.region\":\"us-west-2\",\"aws.log.stream.names\":\"otel-rt-logs\",\"telemetry.sdk.name\":\"opentelemetry\",\"aws.service.type\":\"gen_ai_agent\",\"telemetry.sdk.language\":\"python\",\"cloud.provider\":\"aws\",\"cloud.resource_id\":\"arn:aws:bedrock-agentcore:us-west-2:685197708687:runtime/asdf_MyAgent-3s5axvBC6Q/runtime-endpoint/DEFAULT:DEFAULT\",\"aws.log.group.names\":\"/aws/bedrock-agentcore/runtimes/asdf_MyAgent-3s5axvBC6Q-DEFAULT\",\"telemetry.sdk.version\":\"1.40.0\",\"cloud.platform\":\"aws_bedrock_agentcore\",\"telemetry.auto.version\":\"0.17.0-aws\"}},\"scope\":{\"name\":\"opentelemetry.instrumentation.httpx\",\"version\":\"0.61b0\"},\"traceId\":\"6a90a5987755c6e457ada5616a16dddb\",\"spanId\":\"c8a7a6467d7a7d7c\",\"parentSpanId\":\"98dc1b886c48967e\",\"flags\":256,\"name\":\"POST\",\"kind\":\"CLIENT\",\"startTimeUnixNano\":1787864478923846540,\"endTimeUnixNano\":1787864478960458576,\"durationNano\":36612036,\"attributes\":{\"aws.local.service\":\"asdf_MyAgent.DEFAULT\",\"telemetry.extended\":\"true\",\"http.url\":\"https://bugbashgw1776978672-zsy8cbqwts.gateway.bedrock-agentcore.us-west-2.amazonaws.com/mcp\",\"aws.remote.service\":\"bugbashgw1776978672-zsy8cbqwts.gateway.bedrock-agentcore.us-west-2.amazonaws.com\",\"aws.local.environment\":\"bedrock-agentcore:default\",\"aws.remote.operation\":\"POST /mcp\",\"http.status_code\":202,\"aws.local.operation\":\"UnmappedOperation\",\"aws.span.kind\":\"CLIENT\",\"PlatformType\":\"AWS::BedrockAgentCore\",\"http.method\":\"POST\",\"http.response.status_code\":202,\"session.id\":\"00000000-0000-4000-8000-000000000001\"},\"status\":{\"code\":\"UNSET\"}}" + }, + { + "field": "sessionId", + "value": "00000000-0000-4000-8000-000000000001" + }, + { + "field": "traceId", + "value": "6a90a5987755c6e457ada5616a16dddb" + }, + { + "field": "spanId", + "value": "c8a7a6467d7a7d7c" + }, + { + "field": "@ptr", + "value": "CpsBCmAKFjY4NTE5NzcwODY4Nzphd3Mvc3BhbnMQABokZTE0NTI5MzgtZjVjZi00OTc1LWE0N2QtYzk3YzMwZmFlY2ZlIg4IgJD4jII0EOefqq2ENDjv37W+yTNAyO+wktMzSAASNRoYAgakSqFXAAAAAiX3oQcABqkKWgAAAANyIAEo9uebqIQ0MLrqm6iENDgGQKFNSLk1UOgfIAIQBBgB" + } + ], + [ + { + "field": "@message", + "value": "{\"resource\":{\"attributes\":{\"deployment.environment.name\":\"bedrock-agentcore:default\",\"aws.local.service\":\"asdf_MyAgent.DEFAULT\",\"service.name\":\"asdf_MyAgent.DEFAULT\",\"cloud.region\":\"us-west-2\",\"aws.log.stream.names\":\"otel-rt-logs\",\"telemetry.sdk.name\":\"opentelemetry\",\"aws.service.type\":\"gen_ai_agent\",\"telemetry.sdk.language\":\"python\",\"cloud.provider\":\"aws\",\"cloud.resource_id\":\"arn:aws:bedrock-agentcore:us-west-2:685197708687:runtime/asdf_MyAgent-3s5axvBC6Q/runtime-endpoint/DEFAULT:DEFAULT\",\"aws.log.group.names\":\"/aws/bedrock-agentcore/runtimes/asdf_MyAgent-3s5axvBC6Q-DEFAULT\",\"telemetry.sdk.version\":\"1.40.0\",\"cloud.platform\":\"aws_bedrock_agentcore\",\"telemetry.auto.version\":\"0.17.0-aws\"}},\"scope\":{\"name\":\"opentelemetry.instrumentation.httpx\",\"version\":\"0.61b0\"},\"traceId\":\"6a90a5987755c6e457ada5616a16dddb\",\"spanId\":\"29da3bc6bf4e4025\",\"parentSpanId\":\"98dc1b886c48967e\",\"flags\":256,\"name\":\"POST\",\"kind\":\"CLIENT\",\"startTimeUnixNano\":1787864478962011733,\"endTimeUnixNano\":1787864479034750244,\"durationNano\":72738511,\"attributes\":{\"aws.local.service\":\"asdf_MyAgent.DEFAULT\",\"telemetry.extended\":\"true\",\"http.url\":\"https://bugbashgw1776978672-zsy8cbqwts.gateway.bedrock-agentcore.us-west-2.amazonaws.com/mcp\",\"aws.remote.service\":\"bugbashgw1776978672-zsy8cbqwts.gateway.bedrock-agentcore.us-west-2.amazonaws.com\",\"aws.local.environment\":\"bedrock-agentcore:default\",\"aws.remote.operation\":\"POST /mcp\",\"http.status_code\":200,\"aws.local.operation\":\"UnmappedOperation\",\"aws.span.kind\":\"CLIENT\",\"PlatformType\":\"AWS::BedrockAgentCore\",\"http.method\":\"POST\",\"http.response.status_code\":200,\"session.id\":\"00000000-0000-4000-8000-000000000001\"},\"status\":{\"code\":\"UNSET\"}}" + }, + { + "field": "sessionId", + "value": "00000000-0000-4000-8000-000000000001" + }, + { + "field": "traceId", + "value": "6a90a5987755c6e457ada5616a16dddb" + }, + { + "field": "spanId", + "value": "29da3bc6bf4e4025" + }, + { + "field": "@ptr", + "value": "CpsBCmAKFjY4NTE5NzcwODY4Nzphd3Mvc3BhbnMQABokZTE0NTI5MzgtZjVjZi00OTc1LWE0N2QtYzk3YzMwZmFlY2ZlIg4IgJD4jII0EOefqq2ENDjv37W+yTNAyO+wktMzSAASNRoYAgakSqFXAAAAAiX3oQcABqkKWgAAAANyIAEo9uebqIQ0MLrqm6iENDgGQKFNSLk1UOgfIAIQBRgB" + } + ], + [ + { + "field": "@message", + "value": "{\"resource\":{\"attributes\":{\"deployment.environment.name\":\"bedrock-agentcore:default\",\"aws.local.service\":\"asdf_MyAgent.DEFAULT\",\"service.name\":\"asdf_MyAgent.DEFAULT\",\"cloud.region\":\"us-west-2\",\"aws.log.stream.names\":\"otel-rt-logs\",\"telemetry.sdk.name\":\"opentelemetry\",\"aws.service.type\":\"gen_ai_agent\",\"telemetry.sdk.language\":\"python\",\"cloud.provider\":\"aws\",\"cloud.resource_id\":\"arn:aws:bedrock-agentcore:us-west-2:685197708687:runtime/asdf_MyAgent-3s5axvBC6Q/runtime-endpoint/DEFAULT:DEFAULT\",\"aws.log.group.names\":\"/aws/bedrock-agentcore/runtimes/asdf_MyAgent-3s5axvBC6Q-DEFAULT\",\"telemetry.sdk.version\":\"1.40.0\",\"cloud.platform\":\"aws_bedrock_agentcore\",\"telemetry.auto.version\":\"0.17.0-aws\"}},\"scope\":{\"name\":\"opentelemetry.instrumentation.botocore.bedrock-runtime\",\"version\":\"0.61b0\"},\"traceId\":\"6a90a5987755c6e457ada5616a16dddb\",\"spanId\":\"12835ac323e13dde\",\"parentSpanId\":\"d473e096a2db85a8\",\"flags\":256,\"name\":\"chat global.anthropic.claude-sonnet-4-5-20250929-v1:0\",\"kind\":\"CLIENT\",\"startTimeUnixNano\":1787864479055516061,\"endTimeUnixNano\":1787864481906556152,\"durationNano\":2851040091,\"attributes\":{\"aws.local.service\":\"asdf_MyAgent.DEFAULT\",\"rpc.service\":\"Bedrock Runtime\",\"aws.remote.resource.identifier\":\"global.anthropic.claude-sonnet-4-5-20250929-v1:0\",\"aws.local.environment\":\"bedrock-agentcore:default\",\"aws.remote.operation\":\"ConverseStream\",\"gen_ai.provider.name\":\"aws.bedrock\",\"server.address\":\"bedrock-runtime.us-west-2.amazonaws.com\",\"aws.request_id\":\"8ce89304-4900-46ba-8429-2815f2f4922c\",\"aws.local.operation\":\"UnmappedOperation\",\"aws.span.kind\":\"CLIENT\",\"aws.auth.region\":\"us-west-2\",\"rpc.method\":\"ConverseStream\",\"gen_ai.response.finish_reasons\":[\"end_turn\"],\"server.port\":443,\"gen_ai.request.model\":\"global.anthropic.claude-sonnet-4-5-20250929-v1:0\",\"http.response.status_code\":200,\"gen_ai.system\":\"aws.bedrock\",\"telemetry.extended\":\"true\",\"gen_ai.usage.output_tokens\":82,\"aws.genai.span_kind\":\"LLM\",\"rpc.system\":\"aws-api\",\"aws.remote.service\":\"AWS::BedrockRuntime\",\"http.status_code\":200,\"aws.region\":\"us-west-2\",\"aws.remote.resource.type\":\"AWS::Bedrock::Model\",\"gen_ai.operation.name\":\"chat\",\"gen_ai.usage.input_tokens\":1203,\"aws.genai.token_count_total\":1285,\"retry_attempts\":0,\"PlatformType\":\"AWS::BedrockAgentCore\",\"aws.auth.account.access_key\":\"ASIAZ7CHXJWH4OBTTP6T\",\"session.id\":\"00000000-0000-4000-8000-000000000001\"},\"status\":{\"code\":\"UNSET\"}}" + }, + { + "field": "sessionId", + "value": "00000000-0000-4000-8000-000000000001" + }, + { + "field": "traceId", + "value": "6a90a5987755c6e457ada5616a16dddb" + }, + { + "field": "spanId", + "value": "12835ac323e13dde" + }, + { + "field": "@ptr", + "value": "CpsBCmAKFjY4NTE5NzcwODY4Nzphd3Mvc3BhbnMQABokZTE0NTI5MzgtZjVjZi00OTc1LWE0N2QtYzk3YzMwZmFlY2ZlIg4IgJD4jII0EOefqq2ENDjv37W+yTNAyO+wktMzSAASNRoYAgakSqFXAAAAAiX3ojcABqkKWmAAAANyIAEo8oCcqIQ0MPSAnKiENDgFQK9MSKdkUJQ5IAIQABgB" + } + ], + [ + { + "field": "@message", + "value": "{\"resource\":{\"attributes\":{\"deployment.environment.name\":\"bedrock-agentcore:default\",\"aws.local.service\":\"asdf_MyAgent.DEFAULT\",\"service.name\":\"asdf_MyAgent.DEFAULT\",\"cloud.region\":\"us-west-2\",\"aws.log.stream.names\":\"otel-rt-logs\",\"telemetry.sdk.name\":\"opentelemetry\",\"aws.service.type\":\"gen_ai_agent\",\"telemetry.sdk.language\":\"python\",\"cloud.provider\":\"aws\",\"cloud.resource_id\":\"arn:aws:bedrock-agentcore:us-west-2:685197708687:runtime/asdf_MyAgent-3s5axvBC6Q/runtime-endpoint/DEFAULT:DEFAULT\",\"aws.log.group.names\":\"/aws/bedrock-agentcore/runtimes/asdf_MyAgent-3s5axvBC6Q-DEFAULT\",\"telemetry.sdk.version\":\"1.40.0\",\"cloud.platform\":\"aws_bedrock_agentcore\",\"telemetry.auto.version\":\"0.17.0-aws\"}},\"scope\":{\"name\":\"strands.telemetry.tracer\",\"version\":\"\"},\"traceId\":\"6a90a5987755c6e457ada5616a16dddb\",\"spanId\":\"17fb5fcf047ef303\",\"parentSpanId\":\"2619914761138b83\",\"flags\":256,\"name\":\"execute_event_loop_cycle\",\"kind\":\"INTERNAL\",\"startTimeUnixNano\":1787864479038684800,\"endTimeUnixNano\":1787864481907561373,\"durationNano\":2868876573,\"attributes\":{\"aws.local.service\":\"asdf_MyAgent.DEFAULT\",\"gen_ai.operation.name\":\"execute_event_loop_cycle\",\"gen_ai.event.end_time\":\"2026-08-27T21:01:21.907543+00:00\",\"event_loop.cycle_id\":\"7b2c4a18-cab6-4ea0-931d-07a005ba1f88\",\"gen_ai.event.start_time\":\"2026-08-27T21:01:19.038699+00:00\",\"PlatformType\":\"AWS::BedrockAgentCore\",\"session.id\":\"00000000-0000-4000-8000-000000000001\",\"gen_ai.system\":\"strands-agents\",\"aws.local.environment\":\"bedrock-agentcore:default\",\"gen_ai.provider.name\":\"strands-agents\"},\"status\":{\"code\":\"OK\"}}" + }, + { + "field": "sessionId", + "value": "00000000-0000-4000-8000-000000000001" + }, + { + "field": "traceId", + "value": "6a90a5987755c6e457ada5616a16dddb" + }, + { + "field": "spanId", + "value": "17fb5fcf047ef303" + }, + { + "field": "@ptr", + "value": "CpsBCmAKFjY4NTE5NzcwODY4Nzphd3Mvc3BhbnMQABokZTE0NTI5MzgtZjVjZi00OTc1LWE0N2QtYzk3YzMwZmFlY2ZlIg4IgJD4jII0EOefqq2ENDjv37W+yTNAyO+wktMzSAASNRoYAgakSqFXAAAAAiX3ojcABqkKWmAAAANyIAEo8oCcqIQ0MPSAnKiENDgFQK9MSKdkUJQ5IAIQAhgB" + } + ], + [ + { + "field": "@message", + "value": "{\"resource\":{\"attributes\":{\"deployment.environment.name\":\"bedrock-agentcore:default\",\"aws.local.service\":\"asdf_MyAgent.DEFAULT\",\"service.name\":\"asdf_MyAgent.DEFAULT\",\"cloud.region\":\"us-west-2\",\"aws.log.stream.names\":\"otel-rt-logs\",\"telemetry.sdk.name\":\"opentelemetry\",\"aws.service.type\":\"gen_ai_agent\",\"telemetry.sdk.language\":\"python\",\"cloud.provider\":\"aws\",\"cloud.resource_id\":\"arn:aws:bedrock-agentcore:us-west-2:685197708687:runtime/asdf_MyAgent-3s5axvBC6Q/runtime-endpoint/DEFAULT:DEFAULT\",\"aws.log.group.names\":\"/aws/bedrock-agentcore/runtimes/asdf_MyAgent-3s5axvBC6Q-DEFAULT\",\"telemetry.sdk.version\":\"1.40.0\",\"cloud.platform\":\"aws_bedrock_agentcore\",\"telemetry.auto.version\":\"0.17.0-aws\"}},\"scope\":{\"name\":\"strands.telemetry.tracer\",\"version\":\"\"},\"traceId\":\"6a90a5987755c6e457ada5616a16dddb\",\"spanId\":\"2619914761138b83\",\"parentSpanId\":\"98dc1b886c48967e\",\"flags\":256,\"name\":\"invoke_agent Strands Agents\",\"kind\":\"INTERNAL\",\"startTimeUnixNano\":1787864479037932324,\"endTimeUnixNano\":1787864481907743241,\"durationNano\":2869810917,\"attributes\":{\"aws.local.service\":\"asdf_MyAgent.DEFAULT\",\"gen_ai.usage.prompt_tokens\":1203,\"gen_ai.usage.output_tokens\":82,\"gen_ai.usage.cache_write_input_tokens\":0,\"gen_ai.agent.name\":\"Strands Agents\",\"gen_ai.usage.total_tokens\":1285,\"gen_ai.usage.completion_tokens\":82,\"aws.genai.span_kind\":\"AGENT\",\"gen_ai.event.start_time\":\"2026-08-27T21:01:19.037951+00:00\",\"aws.local.environment\":\"bedrock-agentcore:default\",\"gen_ai.provider.name\":\"strands-agents\",\"gen_ai.operation.name\":\"invoke_agent\",\"gen_ai.event.end_time\":\"2026-08-27T21:01:21.907716+00:00\",\"gen_ai.usage.input_tokens\":1203,\"aws.genai.token_count_total\":1285,\"gen_ai.request.model\":\"global.anthropic.claude-sonnet-4-5-20250929-v1:0\",\"gen_ai.usage.cache_read_input_tokens\":0,\"gen_ai.agent.tools\":\"[\\\"add_numbers\\\", \\\"x_amz_bedrock_agentcore_search\\\", \\\"mcpTarget___web_fetch_exa\\\", \\\"mcpTarget___web_search_exa\\\"]\",\"PlatformType\":\"AWS::BedrockAgentCore\",\"session.id\":\"00000000-0000-4000-8000-000000000001\",\"gen_ai.system\":\"strands-agents\"},\"status\":{\"code\":\"OK\"}}" + }, + { + "field": "sessionId", + "value": "00000000-0000-4000-8000-000000000001" + }, + { + "field": "traceId", + "value": "6a90a5987755c6e457ada5616a16dddb" + }, + { + "field": "spanId", + "value": "2619914761138b83" + }, + { + "field": "@ptr", + "value": "CpsBCmAKFjY4NTE5NzcwODY4Nzphd3Mvc3BhbnMQABokZTE0NTI5MzgtZjVjZi00OTc1LWE0N2QtYzk3YzMwZmFlY2ZlIg4IgJD4jII0EOefqq2ENDjv37W+yTNAyO+wktMzSAASNRoYAgakSqFXAAAAAiX3ojcABqkKWmAAAANyIAEo8oCcqIQ0MPSAnKiENDgFQK9MSKdkUJQ5IAIQAxgB" + } + ], + [ + { + "field": "@message", + "value": "{\"resource\":{\"attributes\":{\"deployment.environment.name\":\"bedrock-agentcore:default\",\"aws.local.service\":\"asdf_MyAgent.DEFAULT\",\"service.name\":\"asdf_MyAgent.DEFAULT\",\"cloud.region\":\"us-west-2\",\"aws.log.stream.names\":\"otel-rt-logs\",\"telemetry.sdk.name\":\"opentelemetry\",\"aws.service.type\":\"gen_ai_agent\",\"telemetry.sdk.language\":\"python\",\"cloud.provider\":\"aws\",\"cloud.resource_id\":\"arn:aws:bedrock-agentcore:us-west-2:685197708687:runtime/asdf_MyAgent-3s5axvBC6Q/runtime-endpoint/DEFAULT:DEFAULT\",\"aws.log.group.names\":\"/aws/bedrock-agentcore/runtimes/asdf_MyAgent-3s5axvBC6Q-DEFAULT\",\"telemetry.sdk.version\":\"1.40.0\",\"cloud.platform\":\"aws_bedrock_agentcore\",\"telemetry.auto.version\":\"0.17.0-aws\"}},\"scope\":{\"name\":\"strands.telemetry.tracer\",\"version\":\"\"},\"traceId\":\"6a90a5987755c6e457ada5616a16dddb\",\"spanId\":\"d473e096a2db85a8\",\"parentSpanId\":\"17fb5fcf047ef303\",\"flags\":256,\"name\":\"chat\",\"kind\":\"INTERNAL\",\"startTimeUnixNano\":1787864479038852619,\"endTimeUnixNano\":1787864481907136303,\"durationNano\":2868283684,\"attributes\":{\"aws.local.service\":\"asdf_MyAgent.DEFAULT\",\"gen_ai.usage.prompt_tokens\":1203,\"gen_ai.usage.output_tokens\":82,\"gen_ai.server.request.duration\":2813,\"gen_ai.usage.total_tokens\":1285,\"gen_ai.usage.completion_tokens\":82,\"aws.genai.span_kind\":\"LLM\",\"gen_ai.event.start_time\":\"2026-08-27T21:01:19.038861+00:00\",\"gen_ai.server.time_to_first_token\":1508,\"aws.local.environment\":\"bedrock-agentcore:default\",\"gen_ai.provider.name\":\"strands-agents\",\"gen_ai.operation.name\":\"chat\",\"gen_ai.event.end_time\":\"2026-08-27T21:01:21.907090+00:00\",\"gen_ai.usage.input_tokens\":1203,\"aws.genai.token_count_total\":1285,\"gen_ai.request.model\":\"global.anthropic.claude-sonnet-4-5-20250929-v1:0\",\"PlatformType\":\"AWS::BedrockAgentCore\",\"session.id\":\"00000000-0000-4000-8000-000000000001\",\"gen_ai.system\":\"strands-agents\"},\"status\":{\"code\":\"OK\"}}" + }, + { + "field": "sessionId", + "value": "00000000-0000-4000-8000-000000000001" + }, + { + "field": "traceId", + "value": "6a90a5987755c6e457ada5616a16dddb" + }, + { + "field": "spanId", + "value": "d473e096a2db85a8" + }, + { + "field": "@ptr", + "value": "CpsBCmAKFjY4NTE5NzcwODY4Nzphd3Mvc3BhbnMQABokZTE0NTI5MzgtZjVjZi00OTc1LWE0N2QtYzk3YzMwZmFlY2ZlIg4IgJD4jII0EOefqq2ENDjv37W+yTNAyO+wktMzSAASNRoYAgakSqFXAAAAAiX3ojcABqkKWmAAAANyIAEo8oCcqIQ0MPSAnKiENDgFQK9MSKdkUJQ5IAIQARgB" + } + ], + [ + { + "field": "@message", + "value": "{\"resource\":{\"attributes\":{\"deployment.environment.name\":\"bedrock-agentcore:default\",\"aws.local.service\":\"asdf_MyAgent.DEFAULT\",\"service.name\":\"asdf_MyAgent.DEFAULT\",\"cloud.region\":\"us-west-2\",\"aws.log.stream.names\":\"otel-rt-logs\",\"telemetry.sdk.name\":\"opentelemetry\",\"aws.service.type\":\"gen_ai_agent\",\"telemetry.sdk.language\":\"python\",\"cloud.provider\":\"aws\",\"cloud.resource_id\":\"arn:aws:bedrock-agentcore:us-west-2:685197708687:runtime/asdf_MyAgent-3s5axvBC6Q/runtime-endpoint/DEFAULT:DEFAULT\",\"aws.log.group.names\":\"/aws/bedrock-agentcore/runtimes/asdf_MyAgent-3s5axvBC6Q-DEFAULT\",\"telemetry.sdk.version\":\"1.40.0\",\"cloud.platform\":\"aws_bedrock_agentcore\",\"telemetry.auto.version\":\"0.17.0-aws\"}},\"scope\":{\"name\":\"opentelemetry.instrumentation.starlette\",\"version\":\"0.61b0\"},\"traceId\":\"6a90a5987755c6e457ada5616a16dddb\",\"spanId\":\"98dc1b886c48967e\",\"parentSpanId\":\"1305dbef52b28faa\",\"flags\":768,\"name\":\"POST /invocations\",\"kind\":\"SERVER\",\"startTimeUnixNano\":1787864478654367295,\"endTimeUnixNano\":1787864481908108495,\"durationNano\":3253741200,\"attributes\":{\"aws.local.service\":\"asdf_MyAgent.DEFAULT\",\"net.peer.port\":50084,\"telemetry.extended\":\"true\",\"http.target\":\"/invocations\",\"http.flavor\":\"1.1\",\"http.url\":\"http://cell01.us-west-2.prod.arp.kepler-analytics.aws.dev/invocations\",\"net.peer.ip\":\"127.0.0.1\",\"http.host\":\"127.0.0.1:8080\",\"aws.local.environment\":\"bedrock-agentcore:default\",\"http.status_code\":200,\"aws.local.operation\":\"POST /invocations\",\"aws.span.kind\":\"SERVER\",\"http.server_name\":\"cell01.us-west-2.prod.arp.kepler-analytics.aws.dev\",\"net.host.port\":8080,\"http.route\":\"/invocations\",\"PlatformType\":\"AWS::BedrockAgentCore\",\"http.method\":\"POST\",\"http.response.status_code\":200,\"session.id\":\"00000000-0000-4000-8000-000000000001\",\"http.scheme\":\"http\"},\"status\":{\"code\":\"UNSET\"}}" + }, + { + "field": "sessionId", + "value": "00000000-0000-4000-8000-000000000001" + }, + { + "field": "traceId", + "value": "6a90a5987755c6e457ada5616a16dddb" + }, + { + "field": "spanId", + "value": "98dc1b886c48967e" + }, + { + "field": "@ptr", + "value": "CpsBCmAKFjY4NTE5NzcwODY4Nzphd3Mvc3BhbnMQABokZTE0NTI5MzgtZjVjZi00OTc1LWE0N2QtYzk3YzMwZmFlY2ZlIg4IgJD4jII0EOefqq2ENDjv37W+yTNAyO+wktMzSAASNRoYAgakSqFXAAAAAiX3ojcABqkKWmAAAANyIAEo8oCcqIQ0MPSAnKiENDgFQK9MSKdkUJQ5IAIQBBgB" + } + ], + [ + { + "field": "@message", + "value": "{\"resource\":{\"attributes\":{\"deployment.environment.name\":\"bedrock-agentcore:default\",\"aws.local.service\":\"asdf_MyAgent.DEFAULT\",\"service.name\":\"asdf_MyAgent.DEFAULT\",\"cloud.region\":\"us-west-2\",\"aws.log.stream.names\":\"otel-rt-logs\",\"telemetry.sdk.name\":\"opentelemetry\",\"aws.service.type\":\"gen_ai_agent\",\"telemetry.sdk.language\":\"python\",\"cloud.provider\":\"aws\",\"cloud.resource_id\":\"arn:aws:bedrock-agentcore:us-west-2:685197708687:runtime/asdf_MyAgent-3s5axvBC6Q/runtime-endpoint/DEFAULT:DEFAULT\",\"aws.log.group.names\":\"/aws/bedrock-agentcore/runtimes/asdf_MyAgent-3s5axvBC6Q-DEFAULT\",\"telemetry.sdk.version\":\"1.40.0\",\"cloud.platform\":\"aws_bedrock_agentcore\",\"telemetry.auto.version\":\"0.17.0-aws\"}},\"scope\":{\"name\":\"opentelemetry.instrumentation.botocore.bedrock-runtime\",\"version\":\"0.61b0\"},\"traceId\":\"6a90a74750f33f567076943f7bce7966\",\"spanId\":\"8214bd2dfefe43fc\",\"parentSpanId\":\"3bf0732214c6e32c\",\"flags\":256,\"name\":\"chat global.anthropic.claude-sonnet-4-5-20250929-v1:0\",\"kind\":\"CLIENT\",\"startTimeUnixNano\":1787864903784381474,\"endTimeUnixNano\":1787864908108981863,\"durationNano\":4324600389,\"attributes\":{\"aws.local.service\":\"asdf_MyAgent.DEFAULT\",\"rpc.service\":\"Bedrock Runtime\",\"aws.remote.resource.identifier\":\"global.anthropic.claude-sonnet-4-5-20250929-v1:0\",\"aws.local.environment\":\"bedrock-agentcore:default\",\"aws.remote.operation\":\"ConverseStream\",\"gen_ai.provider.name\":\"aws.bedrock\",\"server.address\":\"bedrock-runtime.us-west-2.amazonaws.com\",\"aws.request_id\":\"26c3b497-0da3-49ff-bc10-3eee9841d247\",\"aws.local.operation\":\"UnmappedOperation\",\"aws.span.kind\":\"CLIENT\",\"aws.auth.region\":\"us-west-2\",\"rpc.method\":\"ConverseStream\",\"gen_ai.response.finish_reasons\":[\"end_turn\"],\"server.port\":443,\"gen_ai.request.model\":\"global.anthropic.claude-sonnet-4-5-20250929-v1:0\",\"http.response.status_code\":200,\"gen_ai.system\":\"aws.bedrock\",\"telemetry.extended\":\"true\",\"gen_ai.usage.output_tokens\":84,\"aws.genai.span_kind\":\"LLM\",\"rpc.system\":\"aws-api\",\"aws.remote.service\":\"AWS::BedrockRuntime\",\"http.status_code\":200,\"aws.region\":\"us-west-2\",\"aws.remote.resource.type\":\"AWS::Bedrock::Model\",\"gen_ai.operation.name\":\"chat\",\"gen_ai.usage.input_tokens\":1293,\"aws.genai.token_count_total\":1377,\"retry_attempts\":1,\"PlatformType\":\"AWS::BedrockAgentCore\",\"aws.auth.account.access_key\":\"ASIAZ7CHXJWH4OBTTP6T\",\"session.id\":\"00000000-0000-4000-8000-000000000001\"},\"status\":{\"code\":\"UNSET\"}}" + }, + { + "field": "sessionId", + "value": "00000000-0000-4000-8000-000000000001" + }, + { + "field": "traceId", + "value": "6a90a74750f33f567076943f7bce7966" + }, + { + "field": "spanId", + "value": "8214bd2dfefe43fc" + }, + { + "field": "@ptr", + "value": "CpsBCmAKFjY4NTE5NzcwODY4Nzphd3Mvc3BhbnMQABokZTE0NTI5MzgtZjVjZi00OTc1LWE0N2QtYzk3YzMwZmFlY2ZlIg4IgJD4jII0EOefqq2ENDjv37W+yTNAyO+wktMzSAASNRoYAgakSqFXAAAAAiX4KqgABqkKdPAAAANyIAEozIK2qIQ0MM6CtqiENDgFQLFMSLpkUKc5IAIQABgB" + } + ], + [ + { + "field": "@message", + "value": "{\"resource\":{\"attributes\":{\"deployment.environment.name\":\"bedrock-agentcore:default\",\"aws.local.service\":\"asdf_MyAgent.DEFAULT\",\"service.name\":\"asdf_MyAgent.DEFAULT\",\"cloud.region\":\"us-west-2\",\"aws.log.stream.names\":\"otel-rt-logs\",\"telemetry.sdk.name\":\"opentelemetry\",\"aws.service.type\":\"gen_ai_agent\",\"telemetry.sdk.language\":\"python\",\"cloud.provider\":\"aws\",\"cloud.resource_id\":\"arn:aws:bedrock-agentcore:us-west-2:685197708687:runtime/asdf_MyAgent-3s5axvBC6Q/runtime-endpoint/DEFAULT:DEFAULT\",\"aws.log.group.names\":\"/aws/bedrock-agentcore/runtimes/asdf_MyAgent-3s5axvBC6Q-DEFAULT\",\"telemetry.sdk.version\":\"1.40.0\",\"cloud.platform\":\"aws_bedrock_agentcore\",\"telemetry.auto.version\":\"0.17.0-aws\"}},\"scope\":{\"name\":\"strands.telemetry.tracer\",\"version\":\"\"},\"traceId\":\"6a90a74750f33f567076943f7bce7966\",\"spanId\":\"3bf0732214c6e32c\",\"parentSpanId\":\"5e744ea5e19f1053\",\"flags\":256,\"name\":\"chat\",\"kind\":\"INTERNAL\",\"startTimeUnixNano\":1787864903783684675,\"endTimeUnixNano\":1787864908109701143,\"durationNano\":4326016468,\"attributes\":{\"aws.local.service\":\"asdf_MyAgent.DEFAULT\",\"gen_ai.usage.prompt_tokens\":1293,\"gen_ai.usage.output_tokens\":84,\"gen_ai.server.request.duration\":3357,\"gen_ai.usage.total_tokens\":1377,\"gen_ai.usage.completion_tokens\":84,\"aws.genai.span_kind\":\"LLM\",\"gen_ai.event.start_time\":\"2026-08-27T21:08:23.783693+00:00\",\"gen_ai.server.time_to_first_token\":2101,\"aws.local.environment\":\"bedrock-agentcore:default\",\"gen_ai.provider.name\":\"strands-agents\",\"gen_ai.operation.name\":\"chat\",\"gen_ai.event.end_time\":\"2026-08-27T21:08:28.109656+00:00\",\"gen_ai.usage.input_tokens\":1293,\"aws.genai.token_count_total\":1377,\"gen_ai.request.model\":\"global.anthropic.claude-sonnet-4-5-20250929-v1:0\",\"PlatformType\":\"AWS::BedrockAgentCore\",\"session.id\":\"00000000-0000-4000-8000-000000000001\",\"gen_ai.system\":\"strands-agents\"},\"status\":{\"code\":\"OK\"}}" + }, + { + "field": "sessionId", + "value": "00000000-0000-4000-8000-000000000001" + }, + { + "field": "traceId", + "value": "6a90a74750f33f567076943f7bce7966" + }, + { + "field": "spanId", + "value": "3bf0732214c6e32c" + }, + { + "field": "@ptr", + "value": "CpsBCmAKFjY4NTE5NzcwODY4Nzphd3Mvc3BhbnMQABokZTE0NTI5MzgtZjVjZi00OTc1LWE0N2QtYzk3YzMwZmFlY2ZlIg4IgJD4jII0EOefqq2ENDjv37W+yTNAyO+wktMzSAASNRoYAgakSqFXAAAAAiX4KqgABqkKdPAAAANyIAEozIK2qIQ0MM6CtqiENDgFQLFMSLpkUKc5IAIQARgB" + } + ], + [ + { + "field": "@message", + "value": "{\"resource\":{\"attributes\":{\"deployment.environment.name\":\"bedrock-agentcore:default\",\"aws.local.service\":\"asdf_MyAgent.DEFAULT\",\"service.name\":\"asdf_MyAgent.DEFAULT\",\"cloud.region\":\"us-west-2\",\"aws.log.stream.names\":\"otel-rt-logs\",\"telemetry.sdk.name\":\"opentelemetry\",\"aws.service.type\":\"gen_ai_agent\",\"telemetry.sdk.language\":\"python\",\"cloud.provider\":\"aws\",\"cloud.resource_id\":\"arn:aws:bedrock-agentcore:us-west-2:685197708687:runtime/asdf_MyAgent-3s5axvBC6Q/runtime-endpoint/DEFAULT:DEFAULT\",\"aws.log.group.names\":\"/aws/bedrock-agentcore/runtimes/asdf_MyAgent-3s5axvBC6Q-DEFAULT\",\"telemetry.sdk.version\":\"1.40.0\",\"cloud.platform\":\"aws_bedrock_agentcore\",\"telemetry.auto.version\":\"0.17.0-aws\"}},\"scope\":{\"name\":\"strands.telemetry.tracer\",\"version\":\"\"},\"traceId\":\"6a90a74750f33f567076943f7bce7966\",\"spanId\":\"b970895264868438\",\"parentSpanId\":\"e98a035e27f5b117\",\"flags\":256,\"name\":\"invoke_agent Strands Agents\",\"kind\":\"INTERNAL\",\"startTimeUnixNano\":1787864903783116063,\"endTimeUnixNano\":1787864908110229781,\"durationNano\":4327113718,\"attributes\":{\"aws.local.service\":\"asdf_MyAgent.DEFAULT\",\"gen_ai.usage.prompt_tokens\":2496,\"gen_ai.usage.output_tokens\":166,\"gen_ai.usage.cache_write_input_tokens\":0,\"gen_ai.agent.name\":\"Strands Agents\",\"gen_ai.usage.total_tokens\":2662,\"gen_ai.usage.completion_tokens\":166,\"aws.genai.span_kind\":\"AGENT\",\"gen_ai.event.start_time\":\"2026-08-27T21:08:23.783133+00:00\",\"aws.local.environment\":\"bedrock-agentcore:default\",\"gen_ai.provider.name\":\"strands-agents\",\"gen_ai.operation.name\":\"invoke_agent\",\"gen_ai.event.end_time\":\"2026-08-27T21:08:28.110203+00:00\",\"gen_ai.usage.input_tokens\":2496,\"aws.genai.token_count_total\":2662,\"gen_ai.request.model\":\"global.anthropic.claude-sonnet-4-5-20250929-v1:0\",\"gen_ai.usage.cache_read_input_tokens\":0,\"gen_ai.agent.tools\":\"[\\\"add_numbers\\\", \\\"x_amz_bedrock_agentcore_search\\\", \\\"mcpTarget___web_fetch_exa\\\", \\\"mcpTarget___web_search_exa\\\"]\",\"PlatformType\":\"AWS::BedrockAgentCore\",\"session.id\":\"00000000-0000-4000-8000-000000000001\",\"gen_ai.system\":\"strands-agents\"},\"status\":{\"code\":\"OK\"}}" + }, + { + "field": "sessionId", + "value": "00000000-0000-4000-8000-000000000001" + }, + { + "field": "traceId", + "value": "6a90a74750f33f567076943f7bce7966" + }, + { + "field": "spanId", + "value": "b970895264868438" + }, + { + "field": "@ptr", + "value": "CpsBCmAKFjY4NTE5NzcwODY4Nzphd3Mvc3BhbnMQABokZTE0NTI5MzgtZjVjZi00OTc1LWE0N2QtYzk3YzMwZmFlY2ZlIg4IgJD4jII0EOefqq2ENDjv37W+yTNAyO+wktMzSAASNRoYAgakSqFXAAAAAiX4KqgABqkKdPAAAANyIAEozIK2qIQ0MM6CtqiENDgFQLFMSLpkUKc5IAIQAxgB" + } + ], + [ + { + "field": "@message", + "value": "{\"resource\":{\"attributes\":{\"deployment.environment.name\":\"bedrock-agentcore:default\",\"aws.local.service\":\"asdf_MyAgent.DEFAULT\",\"service.name\":\"asdf_MyAgent.DEFAULT\",\"cloud.region\":\"us-west-2\",\"aws.log.stream.names\":\"otel-rt-logs\",\"telemetry.sdk.name\":\"opentelemetry\",\"aws.service.type\":\"gen_ai_agent\",\"telemetry.sdk.language\":\"python\",\"cloud.provider\":\"aws\",\"cloud.resource_id\":\"arn:aws:bedrock-agentcore:us-west-2:685197708687:runtime/asdf_MyAgent-3s5axvBC6Q/runtime-endpoint/DEFAULT:DEFAULT\",\"aws.log.group.names\":\"/aws/bedrock-agentcore/runtimes/asdf_MyAgent-3s5axvBC6Q-DEFAULT\",\"telemetry.sdk.version\":\"1.40.0\",\"cloud.platform\":\"aws_bedrock_agentcore\",\"telemetry.auto.version\":\"0.17.0-aws\"}},\"scope\":{\"name\":\"opentelemetry.instrumentation.starlette\",\"version\":\"0.61b0\"},\"traceId\":\"6a90a74750f33f567076943f7bce7966\",\"spanId\":\"e98a035e27f5b117\",\"parentSpanId\":\"f9316ed3e092f921\",\"flags\":768,\"name\":\"POST /invocations\",\"kind\":\"SERVER\",\"startTimeUnixNano\":1787864903781172232,\"endTimeUnixNano\":1787864908110619156,\"durationNano\":4329446924,\"attributes\":{\"aws.local.service\":\"asdf_MyAgent.DEFAULT\",\"net.peer.port\":56230,\"telemetry.extended\":\"true\",\"http.target\":\"/invocations\",\"http.flavor\":\"1.1\",\"http.url\":\"http://cell01.us-west-2.prod.arp.kepler-analytics.aws.dev/invocations\",\"net.peer.ip\":\"127.0.0.1\",\"http.host\":\"127.0.0.1:8080\",\"aws.local.environment\":\"bedrock-agentcore:default\",\"http.status_code\":200,\"aws.local.operation\":\"POST /invocations\",\"aws.span.kind\":\"SERVER\",\"http.server_name\":\"cell01.us-west-2.prod.arp.kepler-analytics.aws.dev\",\"net.host.port\":8080,\"http.route\":\"/invocations\",\"PlatformType\":\"AWS::BedrockAgentCore\",\"http.method\":\"POST\",\"http.response.status_code\":200,\"session.id\":\"00000000-0000-4000-8000-000000000001\",\"http.scheme\":\"http\"},\"status\":{\"code\":\"UNSET\"}}" + }, + { + "field": "sessionId", + "value": "00000000-0000-4000-8000-000000000001" + }, + { + "field": "traceId", + "value": "6a90a74750f33f567076943f7bce7966" + }, + { + "field": "spanId", + "value": "e98a035e27f5b117" + }, + { + "field": "@ptr", + "value": "CpsBCmAKFjY4NTE5NzcwODY4Nzphd3Mvc3BhbnMQABokZTE0NTI5MzgtZjVjZi00OTc1LWE0N2QtYzk3YzMwZmFlY2ZlIg4IgJD4jII0EOefqq2ENDjv37W+yTNAyO+wktMzSAASNRoYAgakSqFXAAAAAiX4KqgABqkKdPAAAANyIAEozIK2qIQ0MM6CtqiENDgFQLFMSLpkUKc5IAIQBBgB" + } + ], + [ + { + "field": "@message", + "value": "{\"resource\":{\"attributes\":{\"deployment.environment.name\":\"bedrock-agentcore:default\",\"aws.local.service\":\"asdf_MyAgent.DEFAULT\",\"service.name\":\"asdf_MyAgent.DEFAULT\",\"cloud.region\":\"us-west-2\",\"aws.log.stream.names\":\"otel-rt-logs\",\"telemetry.sdk.name\":\"opentelemetry\",\"aws.service.type\":\"gen_ai_agent\",\"telemetry.sdk.language\":\"python\",\"cloud.provider\":\"aws\",\"cloud.resource_id\":\"arn:aws:bedrock-agentcore:us-west-2:685197708687:runtime/asdf_MyAgent-3s5axvBC6Q/runtime-endpoint/DEFAULT:DEFAULT\",\"aws.log.group.names\":\"/aws/bedrock-agentcore/runtimes/asdf_MyAgent-3s5axvBC6Q-DEFAULT\",\"telemetry.sdk.version\":\"1.40.0\",\"cloud.platform\":\"aws_bedrock_agentcore\",\"telemetry.auto.version\":\"0.17.0-aws\"}},\"scope\":{\"name\":\"strands.telemetry.tracer\",\"version\":\"\"},\"traceId\":\"6a90a74750f33f567076943f7bce7966\",\"spanId\":\"5e744ea5e19f1053\",\"parentSpanId\":\"b970895264868438\",\"flags\":256,\"name\":\"execute_event_loop_cycle\",\"kind\":\"INTERNAL\",\"startTimeUnixNano\":1787864903783489151,\"endTimeUnixNano\":1787864908110047680,\"durationNano\":4326558529,\"attributes\":{\"aws.local.service\":\"asdf_MyAgent.DEFAULT\",\"gen_ai.operation.name\":\"execute_event_loop_cycle\",\"gen_ai.event.end_time\":\"2026-08-27T21:08:28.110029+00:00\",\"event_loop.cycle_id\":\"8071f25d-d51e-465d-a206-3c93648afe25\",\"gen_ai.event.start_time\":\"2026-08-27T21:08:23.783500+00:00\",\"PlatformType\":\"AWS::BedrockAgentCore\",\"session.id\":\"00000000-0000-4000-8000-000000000001\",\"gen_ai.system\":\"strands-agents\",\"aws.local.environment\":\"bedrock-agentcore:default\",\"gen_ai.provider.name\":\"strands-agents\"},\"status\":{\"code\":\"OK\"}}" + }, + { + "field": "sessionId", + "value": "00000000-0000-4000-8000-000000000001" + }, + { + "field": "traceId", + "value": "6a90a74750f33f567076943f7bce7966" + }, + { + "field": "spanId", + "value": "5e744ea5e19f1053" + }, + { + "field": "@ptr", + "value": "CpsBCmAKFjY4NTE5NzcwODY4Nzphd3Mvc3BhbnMQABokZTE0NTI5MzgtZjVjZi00OTc1LWE0N2QtYzk3YzMwZmFlY2ZlIg4IgJD4jII0EOefqq2ENDjv37W+yTNAyO+wktMzSAASNRoYAgakSqFXAAAAAiX4KqgABqkKdPAAAANyIAEozIK2qIQ0MM6CtqiENDgFQLFMSLpkUKc5IAIQAhgB" + } + ] + ], + "statistics": { + "recordsMatched": 43, + "recordsScanned": 43, + "estimatedRecordsSkipped": 0, + "bytesScanned": 78544, + "estimatedBytesSkipped": 0, + "logGroupsScanned": 1 + }, + "status": "Complete" +} \ No newline at end of file diff --git a/src/handlers/eval/ondemand/__fixtures__/InvokeAgentRuntimeCommand.d7f8ec055ea3add0.json b/src/handlers/eval/ondemand/__fixtures__/InvokeAgentRuntimeCommand.d7f8ec055ea3add0.json new file mode 100644 index 000000000..9f288bec2 --- /dev/null +++ b/src/handlers/eval/ondemand/__fixtures__/InvokeAgentRuntimeCommand.d7f8ec055ea3add0.json @@ -0,0 +1,8 @@ +{ + "contentType": "text/event-stream; charset=utf-8", + "runtimeSessionId": "00000000-0000-4000-8000-000000000001", + "response": { + "$stream": "data: \"Hello! 👋 Welcome\"\n\ndata: \"!\"\n\ndata: \"\\n\\nI'm here to assist\"\n\ndata: \" you today\"\n\ndata: \". I can help you\"\n\ndata: \" with:\"\n\ndata: \"\\n\\n- **Web\"\n\ndata: \" searches** -\"\n\ndata: \" Find\"\n\ndata: \" current information, news, research\"\n\ndata: \", or answers\"\n\ndata: \" to questions on\"\n\ndata: \" any\"\n\ndata: \" topic\\n- **Reading\"\n\ndata: \" webpages** - Extract and\"\n\ndata: \" read content\"\n\ndata: \" from specific URLs or\"\n\ndata: \" articles\"\n\ndata: \"\\n- **Simple\"\n\ndata: \" calculations** - Add numbers together\"\n\ndata: \"\\n\\nWhat would you like help\"\n\ndata: \" with?\"\n\n" + }, + "statusCode": 200 +} \ No newline at end of file diff --git a/src/handlers/eval/ondemand/__fixtures__/StartQueryCommand.3c11aa80c0c5122e.json b/src/handlers/eval/ondemand/__fixtures__/StartQueryCommand.3c11aa80c0c5122e.json new file mode 100644 index 000000000..f2dc06c01 --- /dev/null +++ b/src/handlers/eval/ondemand/__fixtures__/StartQueryCommand.3c11aa80c0c5122e.json @@ -0,0 +1,3 @@ +{ + "queryId": "143d23b0-0eb9-4542-940b-ffd637ebb0a8" +} \ No newline at end of file diff --git a/src/handlers/eval/ondemand/__fixtures__/StartQueryCommand.a6424982fd346c1.json b/src/handlers/eval/ondemand/__fixtures__/StartQueryCommand.a6424982fd346c1.json new file mode 100644 index 000000000..31d074708 --- /dev/null +++ b/src/handlers/eval/ondemand/__fixtures__/StartQueryCommand.a6424982fd346c1.json @@ -0,0 +1,3 @@ +{ + "queryId": "9a682dcb-330e-449b-b7b4-9897021c47bc" +} \ No newline at end of file diff --git a/src/handlers/eval/ondemand/__fixtures__/simulate-ds.jsonl b/src/handlers/eval/ondemand/__fixtures__/simulate-ds.jsonl new file mode 100644 index 000000000..34738ea89 --- /dev/null +++ b/src/handlers/eval/ondemand/__fixtures__/simulate-ds.jsonl @@ -0,0 +1 @@ +{"example_id":"greet","turns":[{"input":"Say hello and offer help"}],"assertions":["is polite"]} diff --git a/src/handlers/eval/ondemand/__fixtures__/simulate.golden.json b/src/handlers/eval/ondemand/__fixtures__/simulate.golden.json new file mode 100644 index 000000000..38b65e80b --- /dev/null +++ b/src/handlers/eval/ondemand/__fixtures__/simulate.golden.json @@ -0,0 +1,125 @@ +{ + "sessionsRequested": 1, + "sessionsEvaluated": 1, + "results": [ + { + "evaluatorArn": "arn:aws:bedrock-agentcore:::evaluator/Builtin.Helpfulness", + "evaluatorId": "Builtin.Helpfulness", + "evaluatorName": "Builtin.Helpfulness", + "context": { + "spanContext": { + "sessionId": "00000000-0000-4000-8000-000000000001", + "traceId": "6a8f5b461983df572e2aca3264d32d4d" + } + }, + "explanation": "The user simply said 'hi', which is a greeting with no specific goal expressed. The assistant responded with a polite greeting and an open invitation to help. This is appropriate conversational behavior that maintains the flow of interaction and invites the user to share their actual needs. Since the user hasn't expressed a specific goal yet, the assistant's response is a standard, appropriate reply that keeps the conversation open. It doesn't advance any specific goal (since none exists yet), but it doesn't hinder progress either. This falls into the 'Neutral/Mixed' category as it's appropriate chit-chat for conversation flow with no specific goal to advance.", + "value": 0.5, + "label": "Neutral/Mixed", + "tokenUsage": { + "inputTokens": 815, + "outputTokens": 155, + "totalTokens": 970 + }, + "ignoredReferenceInputFields": [ + "assertions" + ] + }, + { + "evaluatorArn": "arn:aws:bedrock-agentcore:::evaluator/Builtin.Helpfulness", + "evaluatorId": "Builtin.Helpfulness", + "evaluatorName": "Builtin.Helpfulness", + "context": { + "spanContext": { + "sessionId": "00000000-0000-4000-8000-000000000001", + "traceId": "6a8f5bbc1882c05547b64a9246f17ca4" + } + }, + "explanation": "The user has said 'hi' twice without providing any specific goal or request. The assistant's response 'Hi there! How can I assist you today?' is a standard greeting that keeps the conversation open and invites the user to share their needs. Since the user hasn't expressed a specific goal yet, the assistant can't do much more than respond to the greeting and prompt the user to share what they need. This response is appropriate for the conversational context - it's a polite acknowledgment that maintains the conversation flow and opens the door for the user to state their actual needs. It doesn't advance any specific goal (since none has been stated), but it doesn't hinder progress either. This falls into the 'Neutral/Mixed' category as it's appropriate chit-chat for conversation flow with no specific goal to advance.", + "value": 0.5, + "label": "Neutral/Mixed", + "tokenUsage": { + "inputTokens": 887, + "outputTokens": 194, + "totalTokens": 1081 + }, + "ignoredReferenceInputFields": [ + "assertions" + ] + }, + { + "evaluatorArn": "arn:aws:bedrock-agentcore:::evaluator/Builtin.Helpfulness", + "evaluatorId": "Builtin.Helpfulness", + "evaluatorName": "Builtin.Helpfulness", + "context": { + "spanContext": { + "sessionId": "00000000-0000-4000-8000-000000000001", + "traceId": "6a8f639540ebe6cb6f30d5d80762c2e2" + } + }, + "explanation": "The user has sent 'hi' three times in a row without providing any specific request or goal. The assistant's response 'Hello! How can I help you today?' is a standard greeting that keeps the conversation open and invites the user to share their needs. This is the third identical exchange, and the assistant is simply repeating the same greeting. While the response is appropriate and doesn't obstruct any goal, it also doesn't advance any specific goal since the user hasn't expressed one yet. The response is essentially neutral - it's appropriate chit-chat that maintains conversation flow without moving toward any particular goal (since no goal has been stated). The assistant could potentially note that the user has greeted multiple times and ask if they need help with something specific, which would be slightly more proactive, but the current response is still a reasonable reply to a simple greeting.", + "value": 0.5, + "label": "Neutral/Mixed", + "tokenUsage": { + "inputTokens": 958, + "outputTokens": 207, + "totalTokens": 1165 + }, + "ignoredReferenceInputFields": [ + "assertions" + ] + }, + { + "evaluatorArn": "arn:aws:bedrock-agentcore:::evaluator/Builtin.Helpfulness", + "evaluatorId": "Builtin.Helpfulness", + "evaluatorName": "Builtin.Helpfulness", + "context": { + "spanContext": { + "sessionId": "00000000-0000-4000-8000-000000000001", + "traceId": "6a90a5987755c6e457ada5616a16dddb" + } + }, + "explanation": "The user's final request was 'Say hello and offer help.' The assistant's response directly fulfills this request by:\n1. Saying hello with a greeting and wave emoji\n2. Offering help by listing specific capabilities it can assist with\n\nThe response is friendly, clear, and directly addresses what the user asked for. It goes slightly beyond the minimal requirement by providing specific examples of what it can help with (web search, fetching webpages, calculations), which gives the user actionable information about how to proceed.\n\nHowever, the listed capabilities (web search, fetch webpages, perform calculations) seem oddly specific and somewhat limiting - a general AI assistant can help with many more things like writing, analysis, coding, answering questions, etc. This specificity might actually mislead the user about the assistant's full range of capabilities.\n\nDespite this minor issue, the response does exactly what was requested - says hello and offers help - and does so in a clear, organized manner. The user's goal was simple and the assistant met it directly.", + "value": 0.83, + "label": "Very Helpful", + "tokenUsage": { + "inputTokens": 1112, + "outputTokens": 258, + "totalTokens": 1370 + }, + "ignoredReferenceInputFields": [ + "assertions" + ] + }, + { + "evaluatorArn": "arn:aws:bedrock-agentcore:::evaluator/Builtin.Helpfulness", + "evaluatorId": "Builtin.Helpfulness", + "evaluatorName": "Builtin.Helpfulness", + "context": { + "spanContext": { + "sessionId": "00000000-0000-4000-8000-000000000001", + "traceId": "6a90a74750f33f567076943f7bce7966" + } + }, + "explanation": "The user's request is simple and explicit: 'Say hello and offer help.' The assistant's response directly fulfills this request by greeting the user with 'Hello! 👋 Welcome!' and then offering help with a clear, organized list of capabilities (web searches, reading webpages, and calculations). The response is well-formatted, friendly, and ends with an open invitation for the user to specify what they need. This is essentially the same response as the previous turn (with minor variations like adding 'Welcome!'), which is appropriate since the user repeated the same request. The response fully satisfies the user's stated goal of having the assistant say hello and offer help. It's comprehensive and actionable, clearly communicating what the assistant can do. There's nothing missing or problematic about this response given the user's simple, direct request.", + "value": 0.83, + "label": "Very Helpful", + "tokenUsage": { + "inputTokens": 1267, + "outputTokens": 203, + "totalTokens": 1470 + }, + "ignoredReferenceInputFields": [ + "assertions" + ] + } + ], + "examplesInvoked": 1, + "examplesFailed": 0, + "sessions": [ + { + "exampleId": "greet", + "sessionId": "00000000-0000-4000-8000-000000000001" + } + ], + "failures": [] +} \ No newline at end of file diff --git a/src/handlers/eval/ondemand/index.tsx b/src/handlers/eval/ondemand/index.tsx index d46cc38ea..e731df263 100644 --- a/src/handlers/eval/ondemand/index.tsx +++ b/src/handlers/eval/ondemand/index.tsx @@ -3,11 +3,13 @@ import type { AppIO } from "../../../io"; import type { Core } from "../../types"; import { createHelpDefault } from "../../help"; import { createEvaluateOnDemandHandler } from "./evaluate"; +import { createSimulateOnDemandHandler } from "./simulate"; // ondemand groups the synchronous, client-side evaluation commands. It has no TUI // screen (unlike evaluator/online-eval), so a bare invocation prints help. export function createOnDemandHandler(core: Core, io: AppIO): Router { return new Router("ondemand", "evaluate existing sessions synchronously, client-side") .default(createHelpDefault(io)) - .handler(createEvaluateOnDemandHandler(core, io)); + .handler(createEvaluateOnDemandHandler(core, io)) + .handler(createSimulateOnDemandHandler(core, io)); } diff --git a/src/handlers/eval/ondemand/ondemand.fixture.test.tsx b/src/handlers/eval/ondemand/ondemand.fixture.test.tsx index 39bf5767f..c83263707 100644 --- a/src/handlers/eval/ondemand/ondemand.fixture.test.tsx +++ b/src/handlers/eval/ondemand/ondemand.fixture.test.tsx @@ -4,6 +4,7 @@ import { CoreClient } from "../../../core"; import { createSilentLogger, fixtureFactories, + isRecording, matchGolden, TestGlobalConfigAccessor, testIO, @@ -14,6 +15,7 @@ const REGION = "us-west-2"; const FIXTURES = join(import.meta.dir, "__fixtures__"); const FIXTURE_AGENT = "asdf_MyAgent-3s5axvBC6Q"; +const SIMULATE_DATASET = join(FIXTURES, "simulate-ds.jsonl"); const FIXTURE_SESSION_IDS = [ "67ebf93b-65e3-4127-9e13-483b239f256a", "7f983b9f-9569-4a4d-bdc2-5c997ff346dd", @@ -70,4 +72,47 @@ describe("eval ondemand evaluate (fixture-backed)", () => { // Recording polls live CloudWatch Insights (1s between polls) and calls Evaluate // per session, so it needs well over bun's 5s default; replay is instant. }, 180_000); + + test("simulate replays a dataset, then evaluates the created sessions client-side", async () => { + let n = 0; + const { createControlClient, createDataClient, createIamClient, createLogsClient } = + fixtureFactories(FIXTURES); + const core = new CoreClient({ + createControlClient, + createDataClient, + createIamClient, + createLogsClient, + logger: createSilentLogger(), + newSessionId: () => `00000000-0000-4000-8000-${String(++n).padStart(12, "0")}`, + now: () => Date.parse("2026-08-28T00:00:00Z"), + }); + const io = testIO(); + const root = createRootHandler(core, { + io: io.io, + logger: createSilentLogger(), + globalConfigAccessor: new TestGlobalConfigAccessor(), + }); + + await root.route([ + "node", + "agentcore", + "eval", + "ondemand", + "simulate", + "--runtime-id", + FIXTURE_AGENT, + "--payload-template", + '{"prompt":"{input}"}', + "--dataset", + SIMULATE_DATASET, + "--evaluator", + "Builtin.Helpfulness", + "--ingestion-wait-ms", + isRecording() ? "150000" : "0", + "--region", + REGION, + ]); + + matchGolden(FIXTURES, "simulate.golden.json", io.stdout()); + }, 200_000); }); diff --git a/src/handlers/eval/ondemand/ondemand.test.tsx b/src/handlers/eval/ondemand/ondemand.test.tsx index 903a9140e..64782c855 100644 --- a/src/handlers/eval/ondemand/ondemand.test.tsx +++ b/src/handlers/eval/ondemand/ondemand.test.tsx @@ -180,7 +180,74 @@ describe("eval ondemand command hierarchy", () => { .find((c) => c.name() === "eval") ?.children() .find((c) => c.name() === "ondemand"); - expect(group?.children().map((c) => c.name())).toEqual(["evaluate"]); + expect(group?.children().map((c) => c.name())).toEqual(["evaluate", "simulate"]); + }); +}); + +describe("eval ondemand simulate", () => { + const BASE = [ + "eval", + "ondemand", + "simulate", + "--runtime-id", + "r-1", + "--payload-template", + '{"prompt":"{input}"}', + "--dataset", + "/tmp/ds.jsonl", + "--evaluator", + "Builtin.Helpfulness", + ]; + + test.each<[RegExp, string[]]>([ + [ + /--runtime-id/, + ["--payload-template", "{}", "--dataset", "/tmp/ds.jsonl", "--evaluator", "E"], + ], + [ + /--payload-template/, + ["--runtime-id", "r-1", "--dataset", "/tmp/ds.jsonl", "--evaluator", "E"], + ], + [/--dataset/, ["--runtime-id", "r-1", "--payload-template", "{}", "--evaluator", "E"]], + [ + /--evaluator/, + ["--runtime-id", "r-1", "--payload-template", "{}", "--dataset", "/tmp/ds.jsonl"], + ], + ])("rejects when a required flag is missing (%s)", async (expected, args) => { + await expect(run(["eval", "ondemand", "simulate", ...args])).rejects.toThrow(expected); + }); + + test("refuses to grade when nothing was invoked, naming the first failure", async () => { + await expect( + run(BASE, (c) => + c.eval.setInvokeDatasetResponse({ + sessions: [], + invoked: 0, + failed: 2, + failures: [ + { exampleId: "e1", error: "HTTP 500" }, + { exampleId: "e2", error: "HTTP 500" }, + ], + }), + ), + ).rejects.toThrow(/no examples could be invoked \(2 failed\).*first error: e1 — HTTP 500/); + }); + + test("passes --ingestion-wait-ms through to invokeDataset and renders failures", async () => { + const { core, stdout } = await run([...BASE, "--ingestion-wait-ms", "0"], (c) => + c.eval.setInvokeDatasetResponse({ + sessions: [{ exampleId: "ok1", sessionId: "s1" }], + invoked: 1, + failed: 1, + failures: [{ exampleId: "bad", error: "HTTP 500" }], + }), + ); + const invoke = core.eval.calls.find((c) => c.method === "invokeDataset"); + expect(invoke).toBeDefined(); + expect((invoke!.args[0] as { waitIngestionMs?: number }).waitIngestionMs).toBe(0); + const out = JSON.parse(stdout); + expect(out.failures).toEqual([{ exampleId: "bad", error: "HTTP 500" }]); + expect(out.sessions).toEqual([{ exampleId: "ok1", sessionId: "s1" }]); }); }); diff --git a/src/handlers/eval/ondemand/simulate/index.tsx b/src/handlers/eval/ondemand/simulate/index.tsx new file mode 100644 index 000000000..1b13ac7d9 --- /dev/null +++ b/src/handlers/eval/ondemand/simulate/index.tsx @@ -0,0 +1,124 @@ +import z from "zod"; +import type { EvaluationReferenceInput } from "@aws-sdk/client-bedrock-agentcore"; +import { createHandler, flag } from "../../../../router"; +import { InputValidationError } from "../../../../errors"; +import { JsonRendererKey } from "../../../../tui"; +import type { AppIO } from "../../../../io"; +import type { Core } from "../../../types"; +import type { InvokedSession } from "../../types"; +import { coreOptsFromCtx } from "../../../utils"; +import { parseRuntimeInvokeHeaders } from "../../../runtime/invoke/request"; + +export const createSimulateOnDemandHandler = (core: Core, _io: AppIO) => + createHandler({ + name: "simulate", + description: "replay a dataset against a runtime, then evaluate the sessions client-side", + flags: [ + flag("runtime-id", "runtime id to invoke per scenario", z.string().optional()), + flag("qualifier", "runtime endpoint qualifier (default DEFAULT)", z.string().optional()), + flag( + "payload-template", + 'JSON payload template; {input} is the scenario input, e.g. {"prompt":"{input}"}', + z.string().optional(), + ), + flag("header", "an ordered application header (repeatable)", z.array(z.string()).optional()), + flag( + "bearer-token", + "CUSTOM_JWT bearer token (for JWT-auth runtimes)", + z.string().optional(), + ), + flag("user-id", "runtime user id", z.string().optional()), + flag("dataset", "dataset source: local JSONL path or a dataset id", z.string().optional()), + flag("dataset-version", "dataset version (with a dataset id)", z.string().optional()), + flag("evaluator", "evaluator id(s) to apply", z.array(z.string()).optional()), + flag( + "ingestion-wait-ms", + "ms to wait for span ingestion before grading (default 180000; 0 to skip)", + z.coerce.number().int().nonnegative().optional(), + ), + ], + handle: async (ctx, flags) => { + if (!flags["runtime-id"]) + throw new InputValidationError("required option '--runtime-id' not specified"); + if (!flags["payload-template"]) { + throw new InputValidationError("required option '--payload-template' not specified"); + } + if (!flags["dataset"]) + throw new InputValidationError("required option '--dataset' not specified"); + if (!flags["evaluator"]?.length) { + throw new InputValidationError( + "required option '--evaluator ' not specified", + ); + } + + const controller = new AbortController(); + const interrupt = () => controller.abort(); + process.once("SIGINT", interrupt); + try { + const opts = coreOptsFromCtx(ctx); + + const replay = await core.eval.invokeDataset( + { + runtimeId: flags["runtime-id"], + qualifier: flags["qualifier"], + payloadTemplate: flags["payload-template"], + headers: parseRuntimeInvokeHeaders(flags["header"]), + bearerToken: flags["bearer-token"], + userId: flags["user-id"], + dataset: flags["dataset"], + datasetVersion: flags["dataset-version"], + waitIngestionMs: flags["ingestion-wait-ms"], + }, + opts, + controller.signal, + ); + if (replay.invoked === 0) { + const first = replay.failures[0]; + const detail = first ? `; first error: ${first.exampleId} — ${first.error}` : ""; + throw new InputValidationError( + `no examples could be invoked (${replay.failed} failed) — nothing to evaluate${detail}`, + ); + } + + const traces = await core.eval.getTracesForAgent( + { + agent: flags["runtime-id"], + endpoint: flags["qualifier"], + sessionIds: replay.sessions.map((s) => s.sessionId), + }, + opts, + ); + + const groundTruth = replay.sessions.flatMap(toReferenceInputs); + const result = await core.eval.evaluate( + { traces, evaluatorIds: flags["evaluator"], groundTruth }, + opts, + ); + + ctx.require(JsonRendererKey).renderJson({ + ...result, + examplesInvoked: replay.invoked, + examplesFailed: replay.failed, + sessions: replay.sessions.map((s) => ({ + exampleId: s.exampleId, + sessionId: s.sessionId, + })), + failures: replay.failures, + }); + } finally { + process.off("SIGINT", interrupt); + } + }, + }); + +function toReferenceInputs(s: InvokedSession): EvaluationReferenceInput[] { + const gt = s.groundTruth; + if (!gt?.assertions?.length && !gt?.expectedTrajectory) return []; + return [ + { + context: { spanContext: { sessionId: s.sessionId } }, + ...(gt.assertions?.length && { assertions: gt.assertions }), + ...(gt.expectedTrajectory && { expectedTrajectory: gt.expectedTrajectory }), + }, + ]; +} diff --git a/src/handlers/eval/types.tsx b/src/handlers/eval/types.tsx index b1be77f4b..91a70c705 100644 --- a/src/handlers/eval/types.tsx +++ b/src/handlers/eval/types.tsx @@ -30,6 +30,8 @@ import type { UpdateOnlineEvaluationConfigResponse, } from "@aws-sdk/client-bedrock-agentcore-control"; import type { + CreateABTestResponse, + GatewayFilter, GetABTestResponse, ListABTestsResponse, ABTestExecutionStatus, @@ -231,6 +233,33 @@ export type RoleScopeWarning = { logGroupNames: string[]; }; +export type BundleRef = { configBundle: string; bundleVersion: string }; + +export type CreateConfigBundleABTestInput = { + name: string; + gateway: string; + control: BundleRef; + treatment: BundleRef; + onlineEval: string; + treatmentWeight?: number; + gatewayFilter?: GatewayFilter; + roleArn?: string; + enableOnCreate?: boolean; +}; + +export type TargetVariantRef = { gatewayTarget: string; onlineEval: string }; + +export type CreateTargetBasedABTestInput = { + name: string; + gateway: string; + control: TargetVariantRef; + treatment: TargetVariantRef; + treatmentWeight?: number; + gatewayFilter?: GatewayFilter; + roleArn?: string; + enableOnCreate?: boolean; +}; + export type CreateDatasetInput = CreateDatasetRequest; export type StartRecommendationInput = { name: string; @@ -427,6 +456,14 @@ export interface CoreEvalClient { options: CoreOptions, ): Promise; deleteABTest(id: string, options: CoreOptions): Promise; + createConfigBundleABTest( + input: CreateConfigBundleABTestInput, + options: CoreOptions, + ): Promise; + createTargetBasedABTest( + input: CreateTargetBasedABTestInput, + options: CoreOptions, + ): Promise; // startBatchEvaluation submits an async, service-side evaluation over sessions // the service gathers from the resolved data source. Returns the durable job id // + RUNNING status; poll with getBatchEvaluation. diff --git a/src/handlers/project/add/evaluator/code-based/index.test.ts b/src/handlers/project/add/evaluator/code-based/index.test.ts new file mode 100644 index 000000000..3d67f6e79 --- /dev/null +++ b/src/handlers/project/add/evaluator/code-based/index.test.ts @@ -0,0 +1,353 @@ +import { afterEach, describe, expect, test } from "bun:test"; +import { mkdir, mkdtemp, rm } from "node:fs/promises"; +import { join } from "node:path"; +import { tmpdir } from "node:os"; +import { createRootHandler } from "../../../../index"; +import { + createSilentLogger, + TestCoreClient, + TestGlobalConfigAccessor, + testIO, +} from "../../../../../testing"; +import { InputValidationError } from "../../../../../errors"; + +const originalCwd = process.cwd(); +const tempDirectories: string[] = []; + +async function inTempDirectory(): Promise { + const directory = await mkdtemp(join(tmpdir(), "agentcore-code-eval-")); + tempDirectories.push(directory); + process.chdir(directory); + return process.cwd(); +} + +afterEach(async () => { + process.chdir(originalCwd); + await Promise.all( + tempDirectories.splice(0).map((directory) => rm(directory, { recursive: true, force: true })), + ); +}); + +async function run(args: string[]) { + const io = testIO(); + const root = createRootHandler(new TestCoreClient(), { + io: io.io, + globalConfigAccessor: new TestGlobalConfigAccessor(), + logger: createSilentLogger(), + }); + await root.route(["node", "agentcore", "project", ...args]); + return { io }; +} + +async function inProject(name = "TestProject"): Promise { + const directory = await inTempDirectory(); + await run(["create", "--name", name, "--skip-install", "--skip-git"]); + const projectRoot = join(directory, name); + process.chdir(projectRoot); + return projectRoot; +} + +const spec = (projectRoot: string) => + Bun.file(join(projectRoot, "agentcore", "agentcore.json")).json(); +const evaluator = async (projectRoot: string, name: string) => + ((await spec(projectRoot)).evaluators ?? []).find((e: { name: string }) => e.name === name); + +describe("project add evaluator code-based", () => { + test("3P metric → managed config + scaffolded, rendered Lambda source", async () => { + const projectRoot = await inProject(); + await run([ + "add", + "evaluator", + "code-based", + "--name", + "answer_faithfulness", + "--level", + "SESSION", + "--metric", + "deepeval.FaithfulnessMetric", + "--model", + "bedrock/anthropic.claude-3-5-sonnet-20240620-v1:0", + ]); + + expect(await evaluator(projectRoot, "answer_faithfulness")).toMatchObject({ + name: "answer_faithfulness", + level: "SESSION", + config: { + codeBased: { + managed: { + codeLocation: "app/answer_faithfulness", + entrypoint: "lambda_function.handler", + timeoutSeconds: 300, + additionalPolicies: ["execution-role-policy.json"], + }, + }, + }, + }); + + const appDir = join(projectRoot, "app", "answer_faithfulness"); + const handler = await Bun.file(join(appDir, "lambda_function.py")).text(); + expect(handler).toContain("FaithfulnessMetric"); + expect(handler).toContain("AmazonBedrockModel"); + expect(handler).toContain("anthropic.claude-3-5-sonnet-20240620-v1:0"); + expect(await Bun.file(join(appDir, "execution-role-policy.json")).exists()).toBe(true); + // no unrendered Handlebars left behind + expect(handler).not.toContain("{{"); + }); + + test("autoevals metric with default (non-bedrock) model", async () => { + const projectRoot = await inProject(); + await run([ + "add", + "evaluator", + "code-based", + "--name", + "factuality", + "--level", + "TRACE", + "--metric", + "autoevals.Factuality", + ]); + + expect( + (await evaluator(projectRoot, "factuality")).config.codeBased.managed.timeoutSeconds, + ).toBe(60); + const handler = await Bun.file( + join(projectRoot, "app", "factuality", "lambda_function.py"), + ).text(); + expect(handler).toContain("Factuality"); + expect(handler).not.toContain("{{"); + }); + + test("no metric, no lambda → empty managed stub", async () => { + const projectRoot = await inProject(); + await run(["add", "evaluator", "code-based", "--name", "custom_eval", "--level", "TOOL_CALL"]); + + expect((await evaluator(projectRoot, "custom_eval")).config.codeBased.managed).toMatchObject({ + codeLocation: "app/custom_eval", + timeoutSeconds: 60, + }); + const handler = await Bun.file( + join(projectRoot, "app", "custom_eval", "lambda_function.py"), + ).text(); + expect(handler).toContain("TODO"); + expect(handler).toContain("custom_code_based_evaluator"); + }); + + test("--lambda-arn → external config, no scaffold", async () => { + const projectRoot = await inProject(); + const arn = "arn:aws:lambda:us-west-2:123456789012:function:refund-policy"; + await run([ + "add", + "evaluator", + "code-based", + "--name", + "refund_policy", + "--level", + "SESSION", + "--lambda-arn", + arn, + ]); + + expect((await evaluator(projectRoot, "refund_policy")).config).toEqual({ + codeBased: { external: { lambdaArn: arn } }, + }); + expect( + await Bun.file(join(projectRoot, "app", "refund_policy", "lambda_function.py")).exists(), + ).toBe(false); + }); + + test("persists description, kms key, and tags", async () => { + const projectRoot = await inProject(); + const kms = "arn:aws:kms:us-east-1:123456789012:key/12345678-1234-1234-1234-123456789012"; + await run([ + "add", + "evaluator", + "code-based", + "--name", + "full", + "--level", + "SESSION", + "--lambda-arn", + "arn:aws:lambda:us-west-2:123456789012:function:f", + "--description", + "external scorer", + "--kms-key-arn", + kms, + "--tags", + '{"team":"ml"}', + ]); + + expect(await evaluator(projectRoot, "full")).toMatchObject({ + description: "external scorer", + kmsKeyArn: kms, + tags: { team: "ml" }, + }); + }); + + test.each<[string, string[]]>([ + ["missing --name", ["--level", "SESSION"]], + ["missing --level", ["--name", "x"]], + [ + "--metric and --lambda-arn together", + [ + "--name", + "x", + "--level", + "SESSION", + "--metric", + "deepeval.FaithfulnessMetric", + "--lambda-arn", + "arn:aws:lambda:us-west-2:123456789012:function:f", + ], + ], + [ + "unknown metric library", + ["--name", "x", "--level", "SESSION", "--metric", "ragas.Faithfulness"], + ], + ["metric without a class", ["--name", "x", "--level", "SESSION", "--metric", "deepeval"]], + [ + "namespaced (multi-dot) metric class", + ["--name", "x", "--level", "SESSION", "--metric", "deepeval.metrics.Faithfulness"], + ], + [ + "non-Bedrock --model", + [ + "--name", + "x", + "--level", + "SESSION", + "--metric", + "deepeval.FaithfulnessMetric", + "--model", + "gpt-4o", + ], + ], + [ + "--model bedrock with no slash/id", + [ + "--name", + "x", + "--level", + "SESSION", + "--metric", + "autoevals.Factuality", + "--model", + "bedrock", + ], + ], + ["--model without --metric", ["--name", "x", "--level", "SESSION", "--model", "bedrock/foo"]], + [ + "managed flag with --lambda-arn", + [ + "--name", + "x", + "--level", + "SESSION", + "--lambda-arn", + "arn:aws:lambda:us-west-2:123456789012:function:f", + "--timeout-seconds", + "30", + ], + ], + ["invalid --level", ["--name", "x", "--level", "NOPE"]], + ["invalid --lambda-arn", ["--name", "x", "--level", "SESSION", "--lambda-arn", "not-an-arn"]], + ])("%s", async (_label, flags) => { + await inProject(); + await expect(run(["add", "evaluator", "code-based", ...flags])).rejects.toBeInstanceOf( + InputValidationError, + ); + }); + + test("accepts a bare Bedrock inference-profile model id and renders it into the source", async () => { + const projectRoot = await inProject(); + await run([ + "add", + "evaluator", + "code-based", + "--name", + "prof", + "--level", + "SESSION", + "--metric", + "deepeval.FaithfulnessMetric", + "--model", + "us.anthropic.claude-sonnet-4-5-20250929-v1:0", + ]); + expect(await evaluator(projectRoot, "prof")).toBeDefined(); + const src = await Bun.file(join(projectRoot, "app", "prof", "lambda_function.py")).text(); + expect(src).toContain("us.anthropic.claude-sonnet-4-5-20250929-v1:0"); + }); + + test("rejects a duplicate evaluator name", async () => { + await inProject(); + const flags = [ + "add", + "evaluator", + "code-based", + "--name", + "dup", + "--level", + "SESSION", + "--lambda-arn", + "arn:aws:lambda:us-west-2:123456789012:function:f", + ]; + await run(flags); + await expect(run(flags)).rejects.toBeInstanceOf(InputValidationError); + }); + + test("errors before writing when app/ already exists (cross-resource collision)", async () => { + const projectRoot = await inProject(); + const appDir = join(projectRoot, "app", "collide"); + await mkdir(appDir, { recursive: true }); + await Bun.write(join(appDir, "pyproject.toml"), "# pre-existing\n"); + + await expect( + run(["add", "evaluator", "code-based", "--name", "collide", "--level", "SESSION"]), + ).rejects.toBeInstanceOf(InputValidationError); + + // no spec entry, and the pre-existing dir is left untouched (no mid-write orphans) + expect(await evaluator(projectRoot, "collide")).toBeUndefined(); + expect(await Bun.file(join(appDir, "pyproject.toml")).text()).toBe("# pre-existing\n"); + expect(await Bun.file(join(appDir, "lambda_function.py")).exists()).toBe(false); + }); + + test("empty stub warns it returns Pass until implemented, plus the not-deployed note", async () => { + await inProject(); + const { io } = await run([ + "add", + "evaluator", + "code-based", + "--name", + "stub", + "--level", + "SESSION", + ]); + expect(io.stderr()).toContain("returns Pass for every session"); + expect(io.stderr()).toContain("not yet provisioned"); + }); + + test("external mode prints neither managed note", async () => { + await inProject(); + const { io } = await run([ + "add", + "evaluator", + "code-based", + "--name", + "ext", + "--level", + "SESSION", + "--lambda-arn", + "arn:aws:lambda:us-west-2:123456789012:function:f", + ]); + expect(io.stderr()).not.toContain("returns Pass for every session"); + expect(io.stderr()).not.toContain("not yet provisioned"); + }); + + test("remove evaluator drops it from the spec", async () => { + const projectRoot = await inProject(); + await run(["add", "evaluator", "code-based", "--name", "gone", "--level", "SESSION"]); + expect(await evaluator(projectRoot, "gone")).toBeDefined(); + await run(["remove", "evaluator", "--name", "gone"]); + expect(await evaluator(projectRoot, "gone")).toBeUndefined(); + }); +}); diff --git a/src/handlers/project/add/evaluator/code-based/index.ts b/src/handlers/project/add/evaluator/code-based/index.ts new file mode 100644 index 000000000..60bfe62fc --- /dev/null +++ b/src/handlers/project/add/evaluator/code-based/index.ts @@ -0,0 +1,193 @@ +import z from "zod"; +import { createHandler, flag, ProjectKey } from "../../../../../router"; +import { InputValidationError } from "../../../../../errors"; +import { EvaluatorSchema, isValidBedrockModelId } from "../../../../../projectSchemas/evaluator"; +import { TagsSchema } from "../../../../../projectSchemas/tags"; +import { parseJsonFlagWithSchema } from "../../../../utils"; +import type { AddProjectResourceConfig } from "../../types"; + +// 3P evaluator libraries the CLI can scaffold. The metric class is passed +// through to the library (not allowlisted here) — only the library prefix is +// validated. Default timeouts mirror the old CLI's THIRD_PARTY_EVALUATOR_LIBRARIES. +const LIBRARIES: Record = { + deepeval: { assetDir: "evaluators/deepeval-lambda", defaultTimeoutSeconds: 300 }, + autoevals: { assetDir: "evaluators/autoevals-lambda", defaultTimeoutSeconds: 60 }, +}; + +const EMPTY_ASSET_DIR = "evaluators/python-lambda"; +const EMPTY_DEFAULT_TIMEOUT_SECONDS = 60; + +export const createAddCodeBasedEvaluatorHandler = (config: AddProjectResourceConfig) => + createHandler({ + name: "code-based", + description: + "add a code-based evaluator — a Lambda that scores a session. Pass a 3P metric, an existing Lambda, or neither to scaffold an empty evaluator you fill in", + flags: [ + flag("name", "the name of the evaluator", z.string().optional()), + flag("level", "what to score: SESSION, TRACE, or TOOL_CALL", z.string().optional()), + flag( + "metric", + "3P metric to scaffold as , e.g. deepeval.FaithfulnessMetric or autoevals.Factuality", + z.string().optional(), + ), + flag( + "model", + "judge model for the 3P metric, e.g. bedrock/anthropic.claude-3-5-sonnet-20240620-v1:0", + z.string().optional(), + ), + flag("lambda-arn", "ARN of an existing Lambda that scores a session", z.string().optional()), + flag( + "timeout-seconds", + "Lambda timeout in seconds (1-300)", + z.number().int().min(1).max(300).optional(), + ), + flag("description", "a description of what this evaluator measures", z.string().optional()), + flag( + "kms-key-arn", + "customer-managed KMS key ARN to encrypt the evaluator", + z.string().optional(), + ), + flag("tags", "tags to apply (JSON object of key/value strings)", z.string().optional()), + ], + handle: async (ctx, flags) => { + if (!flags["name"]) + throw new InputValidationError("required option '--name ' not specified"); + if (!flags["level"]) + throw new InputValidationError("required option '--level ' not specified"); + + const hasMetric = flags["metric"] !== undefined; + const hasLambda = flags["lambda-arn"] !== undefined; + if (hasMetric && hasLambda) + throw new InputValidationError( + "provide either --metric (managed) or --lambda-arn (external), not both", + ); + + const tags = parseJsonFlagWithSchema("tags", flags["tags"], TagsSchema); + const base = { + name: flags["name"], + level: flags["level"], + description: flags["description"], + kmsKeyArn: flags["kms-key-arn"], + tags, + }; + + // Kept loose so `--level` stays a plain string for EvaluatorSchema to + // validate (mirrors the llm-as-a-judge handler); safeParse narrows it. + let candidate: Record; + let scaffold: { assetDir: string; context: Record } | undefined; + + if (hasLambda) { + if (flags["metric"] || flags["model"] || flags["timeout-seconds"] !== undefined) + throw new InputValidationError( + "--metric, --model, and --timeout-seconds are managed-only and not valid with --lambda-arn", + ); + candidate = { + ...base, + config: { codeBased: { external: { lambdaArn: flags["lambda-arn"] } } }, + }; + } else { + // managed: 3P metric, or empty stub when no metric is given. + if (flags["model"] && !hasMetric) + throw new InputValidationError("--model requires --metric"); + + let assetDir = EMPTY_ASSET_DIR; + let defaultTimeout = EMPTY_DEFAULT_TIMEOUT_SECONDS; + const context: Record = { Name: toPythonPackageName(flags["name"]) }; + + if (hasMetric) { + const raw = flags["metric"]!; + const dot = raw.indexOf("."); + const library = dot > 0 ? raw.slice(0, dot) : ""; + const metricClass = dot > 0 ? raw.slice(dot + 1) : ""; + const lib = library && metricClass ? LIBRARIES[library] : undefined; + if (!lib) + throw new InputValidationError( + `invalid --metric "${raw}": expected where library is one of ${Object.keys(LIBRARIES).join(", ")} (e.g. deepeval.FaithfulnessMetric)`, + ); + // The class is rendered straight into `from import ` and + // `(...)`, so it must be a single Python identifier. Reject + // dotted/namespaced values (e.g. deepeval.metrics.Faithfulness) that + // would emit invalid Python and only fail at deploy/runtime. + if (!/^[A-Za-z_][A-Za-z0-9_]*$/.test(metricClass)) + throw new InputValidationError( + `invalid metric class "${metricClass}" in --metric "${raw}": expected a single class name like FaithfulnessMetric`, + ); + assetDir = lib.assetDir; + defaultTimeout = lib.defaultTimeoutSeconds; + + const model = resolveBedrockModel(flags["model"]); + context["EvaluatorClass"] = metricClass; + context["Model"] = model ?? ""; + context["ModelProviderBedrock"] = model !== undefined; + context["EvaluatorParams"] = ""; + } + + const timeoutSeconds = flags["timeout-seconds"] ?? defaultTimeout; + candidate = { + ...base, + config: { + codeBased: { + managed: { + codeLocation: `app/${flags["name"]}`, + entrypoint: "lambda_function.handler", + timeoutSeconds, + additionalPolicies: ["execution-role-policy.json"], + }, + }, + }, + }; + scaffold = { assetDir, context }; + } + + const parsed = EvaluatorSchema.safeParse(candidate); + if (!parsed.success) throw new InputValidationError(z.prettifyError(parsed.error)); + + const project = ctx.require(ProjectKey); + for await (const event of config.projectManager.addResource(project, { + resourceType: "evaluator", + resourceConfig: parsed.data, + scaffold, + })) { + config.io.stderr.write(`${event.message}\n`); + } + + config.io.stderr.write(`added evaluator '${flags["name"]}' to '${project.name}'\n`); + // Make the inferred mode and its caveats visible: the empty stub silently + // passes every session, and no managed evaluator is provisioned by deploy + // yet (no CDK/L3 support) — both are silent footguns otherwise. + if (!hasLambda) { + if (!hasMetric) + config.io.stderr.write( + `note: this evaluator returns Pass for every session until you implement app/${flags["name"]}/lambda_function.py\n`, + ); + config.io.stderr.write( + `note: managed code-based evaluators are scaffolded locally but not yet provisioned by 'project deploy' (pending CDK/L3 support)\n`, + ); + } + }, + }); + +// Judge model for a 3P metric. Bedrock-only: accepts a bare Bedrock model id / +// inference-profile-or-foundation-model ARN, optionally prefixed with +// `bedrock/`, and returns the id with the prefix stripped. Anything else errors +// rather than silently degrading — a non-Bedrock value is dropped by the +// deepeval template and passed to the wrong client by autoevals. Returns +// undefined when no --model was given (library default). +function resolveBedrockModel(model: string | undefined): string | undefined { + if (!model) return undefined; + const id = model.startsWith("bedrock/") ? model.slice("bedrock/".length) : model; + if (!isValidBedrockModelId(id)) + throw new InputValidationError( + `invalid --model "${model}": expected a Bedrock model ID (e.g. anthropic.claude-3-5-sonnet-20240620-v1:0) or an inference-profile/foundation-model ARN, optionally prefixed with "bedrock/"`, + ); + return id; +} + +// PEP 508 package name: ASCII letters/numbers/period/underscore/hyphen, must +// start and end alphanumeric. Mirrors the runtime template helper. +function toPythonPackageName(name: string): string { + return name + .replace(/[^a-zA-Z0-9._-]/g, "-") + .replace(/^[^a-zA-Z0-9]+/, "") + .replace(/[^a-zA-Z0-9]+$/, ""); +} diff --git a/src/handlers/project/add/evaluator/index.ts b/src/handlers/project/add/evaluator/index.ts index 5dbc413ab..1c05ea3be 100644 --- a/src/handlers/project/add/evaluator/index.ts +++ b/src/handlers/project/add/evaluator/index.ts @@ -1,9 +1,11 @@ import { Router } from "../../../../router"; import type { AddProjectResourceConfig } from "../types"; import { createAddLlmAsAJudgeEvaluatorHandler } from "./llm-as-a-judge"; +import { createAddCodeBasedEvaluatorHandler } from "./code-based"; export function createAddEvaluatorHandler(config: AddProjectResourceConfig): Router { const evaluator = new Router("evaluator", "add a custom evaluator to the current project"); evaluator.handler(createAddLlmAsAJudgeEvaluatorHandler(config)); + evaluator.handler(createAddCodeBasedEvaluatorHandler(config)); return evaluator; } diff --git a/src/handlers/project/remove/index.ts b/src/handlers/project/remove/index.ts index 8d5973d92..308afe1fb 100644 --- a/src/handlers/project/remove/index.ts +++ b/src/handlers/project/remove/index.ts @@ -26,6 +26,7 @@ export const createRemoveProjectHandler = (config: RemoveProjectResourceConfig) .enum([ "harness", "runtime", + "evaluator", "gateway", "gateway-target", "gateway-connector", diff --git a/src/handlers/project/types.ts b/src/handlers/project/types.ts index 19b3a8ef4..2e6f814dd 100644 --- a/src/handlers/project/types.ts +++ b/src/handlers/project/types.ts @@ -160,6 +160,13 @@ export type AddResourceInput = | { resourceType: "evaluator"; resourceConfig: z.input; + /** + * Present only for managed code-based evaluators, whose code the CLI + * generates. `assetDir` picks the template under src/assets/evaluators; + * `context` holds its Handlebars variables. External and llm-as-a-judge + * evaluators omit this and are spec-only. + */ + scaffold?: { assetDir: string; context: Record }; } | { resourceType: "gateway"; diff --git a/src/testing/TestCoreClient.tsx b/src/testing/TestCoreClient.tsx index 8fcce7a46..686bb6bf0 100644 --- a/src/testing/TestCoreClient.tsx +++ b/src/testing/TestCoreClient.tsx @@ -76,6 +76,7 @@ import type { GetABTestResponse, ListABTestsResponse, ABTestExecutionStatus, + CreateABTestResponse, UpdateABTestResponse, DeleteABTestResponse, DeleteRecommendationResponse, @@ -137,6 +138,8 @@ import type { CodeBasedUpdate, CoreEvalClient, CreateConfigurationBundleInput, + CreateConfigBundleABTestInput, + CreateTargetBasedABTestInput, CreateDatasetInput, CreateOnlineEvalInput, CreateOnlineInsightInput, @@ -281,6 +284,7 @@ const DEFAULT_GET_ABTEST_RESPONSE = {} as GetABTestResponse; const DEFAULT_LIST_ABTESTS_RESPONSE: ListABTestsResponse = { abTests: [] }; const DEFAULT_UPDATE_ABTEST_RESPONSE = {} as UpdateABTestResponse; const DEFAULT_DELETE_ABTEST_RESPONSE = {} as DeleteABTestResponse; +const DEFAULT_CREATE_ABTEST_RESPONSE = {} as CreateABTestResponse; const DEFAULT_START_BATCH_EVAL_RESPONSE = { batchEvaluationId: "batch-eval-test", status: "RUNNING", @@ -1438,6 +1442,7 @@ export class TestEvalClient implements CoreEvalClient { private abTestListResponses = new Map(); private abTestUpdateResponse: UpdateABTestResponse = DEFAULT_UPDATE_ABTEST_RESPONSE; private abTestDeleteResponse: DeleteABTestResponse = DEFAULT_DELETE_ABTEST_RESPONSE; + private abTestCreateResponse: CreateABTestResponse = DEFAULT_CREATE_ABTEST_RESPONSE; private batchEvalResults: BatchEvaluationResultEntry[] = []; private batchEvalResultsError?: unknown; private startBatchEvalResponse: StartBatchEvaluationResponse = DEFAULT_START_BATCH_EVAL_RESPONSE; @@ -1685,6 +1690,11 @@ export class TestEvalClient implements CoreEvalClient { return this; } + setAbTestCreateResponse(response: CreateABTestResponse): this { + this.abTestCreateResponse = response; + return this; + } + // setUpdateDatasetResult sets what updateDatasetExamples resolves to (when not // erroring). setUpdateDatasetResult(result: DatasetUpdateResult): this { @@ -1894,6 +1904,24 @@ export class TestEvalClient implements CoreEvalClient { return this.abTestDeleteResponse; } + async createConfigBundleABTest( + input: CreateConfigBundleABTestInput, + options: CoreOptions, + ): Promise { + this.calls.push({ method: "createConfigBundleABTest", args: [input, options] }); + if (this.error) throw this.error; + return this.abTestCreateResponse; + } + + async createTargetBasedABTest( + input: CreateTargetBasedABTestInput, + options: CoreOptions, + ): Promise { + this.calls.push({ method: "createTargetBasedABTest", args: [input, options] }); + if (this.error) throw this.error; + return this.abTestCreateResponse; + } + async startBatchEvaluation( input: StartBatchEvaluationInput, options: CoreOptions,