diff --git a/src/core/eval.tsx b/src/core/eval.tsx index 1f0d551f1..7da2080e2 100644 --- a/src/core/eval.tsx +++ b/src/core/eval.tsx @@ -225,6 +225,7 @@ export class EvalClient implements CoreEvalClient { // logger for batch-evaluation result-log diagnostics private readonly logger: Logger = noopLogger, private readonly newSessionId: () => string = randomUUID, + private readonly now: () => number = () => Date.now(), ) {} async createEvaluator( @@ -591,7 +592,7 @@ export class EvalClient implements CoreEvalClient { const logGroupName = runtimeLogGroup(runtimeId, qualifier); const serviceName = runtimeServiceName(runtimeName, qualifier); - const endMs = input.window ? +input.window.endTime : Date.now(); + const endMs = input.window ? +input.window.endTime : this.now(); const startMs = input.window ? +input.window.startTime : endMs - SEVEN_DAYS_MS; const startSec = Math.floor(startMs / 1000); const endSec = Math.floor(endMs / 1000); diff --git a/src/core/index.tsx b/src/core/index.tsx index 720e356f2..5a6bab9fb 100644 --- a/src/core/index.tsx +++ b/src/core/index.tsx @@ -45,6 +45,7 @@ type CoreClientConfig = { logger: Logger; fetch?: CoreFetch; newSessionId?: () => string; + now?: () => number; describeBedrockAgent?: DescribeBedrockAgent; }; @@ -93,6 +94,7 @@ export class CoreClient implements AwsClients { fetch, this.logger.child({ module: "eval" }), config.newSessionId, + config.now, ); // Observability resolves a project's deployed runtime from its stack diff --git a/src/handlers/eval/ondemand/__fixtures__/EvaluateCommand.1aea66f982338a76.json b/src/handlers/eval/ondemand/__fixtures__/EvaluateCommand.1aea66f982338a76.json new file mode 100644 index 000000000..8086702a2 --- /dev/null +++ b/src/handlers/eval/ondemand/__fixtures__/EvaluateCommand.1aea66f982338a76.json @@ -0,0 +1,26 @@ +{ + "evaluationResults": [ + { + "evaluatorArn": "arn:aws:bedrock-agentcore:::evaluator/Builtin.Correctness", + "evaluatorId": "Builtin.Correctness", + "evaluatorName": "Builtin.Correctness", + "context": { + "spanContext": { + "sessionId": "00000000-0000-4000-8000-4a4a72800001", + "traceId": "6a91fe944cf38d6b2d8c3c6a66e63ad4" + } + }, + "explanation": "The agent responded with '4' which matches the expected response of '4'. The trailing newline is irrelevant to the correctness of the answer.", + "value": 1, + "label": "Correct", + "tokenUsage": { + "inputTokens": 484, + "outputTokens": 53, + "totalTokens": 537 + }, + "ignoredReferenceInputFields": [ + "assertions" + ] + } + ] +} \ No newline at end of file diff --git a/src/handlers/eval/ondemand/__fixtures__/GetEvaluatorCommand.5ee939ef1fda1ca7.json b/src/handlers/eval/ondemand/__fixtures__/GetEvaluatorCommand.5ee939ef1fda1ca7.json new file mode 100644 index 000000000..d9ff0960e --- /dev/null +++ b/src/handlers/eval/ondemand/__fixtures__/GetEvaluatorCommand.5ee939ef1fda1ca7.json @@ -0,0 +1,41 @@ +{ + "evaluatorArn": "arn:aws:bedrock-agentcore:::evaluator/Builtin.Correctness", + "evaluatorId": "Builtin.Correctness", + "evaluatorName": "Builtin.Correctness", + "evaluatorConfig": { + "llmAsAJudge": { + "ratingScale": { + "numerical": [ + { + "value": { + "string": "0.0", + "type": "bigDecimal" + }, + "label": "Incorrect" + }, + { + "value": 0.5, + "label": "Partially correct" + }, + { + "value": { + "string": "1.0", + "type": "bigDecimal" + }, + "label": "Correct" + } + ] + } + } + }, + "level": "TRACE", + "status": "ACTIVE", + "createdAt": { + "$date": "2024-10-22T00:00:00.000Z" + }, + "updatedAt": { + "$date": "2024-10-22T00:00:00.000Z" + }, + "description": "Response Quality Metric. Evaluates whether the information in the agent's response is factually accurate", + "lockedForModification": true +} \ No newline at end of file diff --git a/src/handlers/eval/ondemand/__fixtures__/GetQueryResultsCommand.548f1c221addde3b.json b/src/handlers/eval/ondemand/__fixtures__/GetQueryResultsCommand.548f1c221addde3b.json new file mode 100644 index 000000000..38cd0642a --- /dev/null +++ b/src/handlers/eval/ondemand/__fixtures__/GetQueryResultsCommand.548f1c221addde3b.json @@ -0,0 +1,146 @@ +{ + "queryLanguage": "CWLI", + "results": [ + [ + { + "field": "@message", + "value": "{\"resource\":{\"attributes\":{\"telemetry.sdk.language\":\"python\",\"telemetry.sdk.name\":\"opentelemetry\",\"telemetry.sdk.version\":\"1.40.0\",\"service.name\":\"asdf_MyAgent.DEFAULT\",\"aws.log.group.names\":\"/aws/bedrock-agentcore/runtimes/asdf_MyAgent-3s5axvBC6Q-DEFAULT\",\"aws.log.stream.names\":\"otel-rt-logs\",\"deployment.environment.name\":\"bedrock-agentcore:default\",\"cloud.resource_id\":\"arn:aws:bedrock-agentcore:us-west-2:685197708687:runtime/asdf_MyAgent-3s5axvBC6Q/runtime-endpoint/DEFAULT:DEFAULT\",\"cloud.platform\":\"aws_bedrock_agentcore\",\"cloud.provider\":\"aws\",\"cloud.region\":\"us-west-2\",\"telemetry.auto.version\":\"0.17.0-aws\",\"aws.local.service\":\"asdf_MyAgent.DEFAULT\",\"aws.service.type\":\"gen_ai_agent\"}},\"scope\":{\"name\":\"strands.telemetry.tracer\"},\"timeUnixNano\":1787952795780790373,\"observedTimeUnixNano\":1787952796735286012,\"severityNumber\":9,\"severityText\":\"\",\"body\":{\"input\":{\"messages\":[{\"role\":\"system\",\"content\":{\"content\":\"[{\\\"text\\\": \\\"\\\\n You are a helpful assistant. Use tools when appropriate.\\\\n \\\"}]\"}},{\"role\":\"user\",\"content\":{\"content\":\"[{\\\"text\\\": \\\"What is 2+2? Reply with just the number.\\\"}]\"}}]},\"output\":{\"messages\":[{\"role\":\"assistant\",\"content\":{\"finish_reason\":\"tool_use\",\"message\":\"[{\\\"toolUse\\\": {\\\"toolUseId\\\": \\\"tooluse_3jtUttUiLMVTz1Qqd4hYGz\\\", \\\"name\\\": \\\"add_numbers\\\", \\\"input\\\": {\\\"a\\\": 2, \\\"b\\\": 2}}}]\"}}]}},\"attributes\":{\"session.id\":\"00000000-0000-4000-8000-4a4a72800001\",\"event.name\":\"strands.telemetry.tracer\"},\"flags\":1,\"traceId\":\"6a91fe944cf38d6b2d8c3c6a66e63ad4\",\"spanId\":\"7b1eeecf45c6e146\"}" + }, + { + "field": "sessionId", + "value": "00000000-0000-4000-8000-4a4a72800001" + }, + { + "field": "traceId", + "value": "6a91fe944cf38d6b2d8c3c6a66e63ad4" + }, + { + "field": "spanId", + "value": "7b1eeecf45c6e146" + }, + { + "field": "@ptr", + "value": "Cs0BCo8BCkw2ODUxOTc3MDg2ODc6L2F3cy9iZWRyb2NrLWFnZW50Y29yZS9ydW50aW1lcy9hc2RmX015QWdlbnQtM3M1YXh2QkM2US1ERUZBVUxUEAAaJGY2ZGY3NDIyLWVlZTQtNDhkNS05NDg3LTE0NTZkYjdkZGE1YiIOCKCUkduCNBCHpMP7hDRA0cHR4dszSAASNxoYAgajvKv7AAAAABTZwGgABqkf6XAAAAKyIAEolYKq0oQ0MJyuqtKENDgqQJe/AkiwkQFQgGYgAhAcGAE=" + } + ], + [ + { + "field": "@message", + "value": "{\"resource\":{\"attributes\":{\"telemetry.sdk.language\":\"python\",\"telemetry.sdk.name\":\"opentelemetry\",\"telemetry.sdk.version\":\"1.40.0\",\"service.name\":\"asdf_MyAgent.DEFAULT\",\"aws.log.group.names\":\"/aws/bedrock-agentcore/runtimes/asdf_MyAgent-3s5axvBC6Q-DEFAULT\",\"aws.log.stream.names\":\"otel-rt-logs\",\"deployment.environment.name\":\"bedrock-agentcore:default\",\"cloud.resource_id\":\"arn:aws:bedrock-agentcore:us-west-2:685197708687:runtime/asdf_MyAgent-3s5axvBC6Q/runtime-endpoint/DEFAULT:DEFAULT\",\"cloud.platform\":\"aws_bedrock_agentcore\",\"cloud.provider\":\"aws\",\"cloud.region\":\"us-west-2\",\"telemetry.auto.version\":\"0.17.0-aws\",\"aws.local.service\":\"asdf_MyAgent.DEFAULT\",\"aws.service.type\":\"gen_ai_agent\"}},\"scope\":{\"name\":\"strands.telemetry.tracer\"},\"timeUnixNano\":1787952795781997134,\"observedTimeUnixNano\":1787952796735435934,\"severityNumber\":9,\"severityText\":\"\",\"body\":{\"input\":{\"messages\":[{\"role\":\"tool\",\"content\":{\"role\":\"tool\",\"content\":\"{\\\"a\\\": 2, \\\"b\\\": 2}\",\"id\":\"tooluse_3jtUttUiLMVTz1Qqd4hYGz\"}}]},\"output\":{\"messages\":[{\"role\":\"assistant\",\"content\":{\"message\":\"[{\\\"text\\\": \\\"4\\\"}]\",\"id\":\"tooluse_3jtUttUiLMVTz1Qqd4hYGz\"}}]}},\"attributes\":{\"session.id\":\"00000000-0000-4000-8000-4a4a72800001\",\"event.name\":\"strands.telemetry.tracer\"},\"flags\":1,\"traceId\":\"6a91fe944cf38d6b2d8c3c6a66e63ad4\",\"spanId\":\"4ad640cbd3587ff3\"}" + }, + { + "field": "sessionId", + "value": "00000000-0000-4000-8000-4a4a72800001" + }, + { + "field": "traceId", + "value": "6a91fe944cf38d6b2d8c3c6a66e63ad4" + }, + { + "field": "spanId", + "value": "4ad640cbd3587ff3" + }, + { + "field": "@ptr", + "value": "Cs0BCo8BCkw2ODUxOTc3MDg2ODc6L2F3cy9iZWRyb2NrLWFnZW50Y29yZS9ydW50aW1lcy9hc2RmX015QWdlbnQtM3M1YXh2QkM2US1ERUZBVUxUEAAaJGY2ZGY3NDIyLWVlZTQtNDhkNS05NDg3LTE0NTZkYjdkZGE1YiIOCKCUkduCNBCHpMP7hDRA0cHR4dszSAASNxoYAgajvKv7AAAAABTZwGgABqkf6XAAAAKyIAEolYKq0oQ0MJyuqtKENDgqQJe/AkiwkQFQgGYgAhAdGAE=" + } + ], + [ + { + "field": "@message", + "value": "{\"resource\":{\"attributes\":{\"telemetry.sdk.language\":\"python\",\"telemetry.sdk.name\":\"opentelemetry\",\"telemetry.sdk.version\":\"1.40.0\",\"service.name\":\"asdf_MyAgent.DEFAULT\",\"aws.log.group.names\":\"/aws/bedrock-agentcore/runtimes/asdf_MyAgent-3s5axvBC6Q-DEFAULT\",\"aws.log.stream.names\":\"otel-rt-logs\",\"deployment.environment.name\":\"bedrock-agentcore:default\",\"cloud.resource_id\":\"arn:aws:bedrock-agentcore:us-west-2:685197708687:runtime/asdf_MyAgent-3s5axvBC6Q/runtime-endpoint/DEFAULT:DEFAULT\",\"cloud.platform\":\"aws_bedrock_agentcore\",\"cloud.provider\":\"aws\",\"cloud.region\":\"us-west-2\",\"telemetry.auto.version\":\"0.17.0-aws\",\"aws.local.service\":\"asdf_MyAgent.DEFAULT\",\"aws.service.type\":\"gen_ai_agent\"}},\"scope\":{\"name\":\"strands.telemetry.tracer\"},\"timeUnixNano\":1787952795782144605,\"observedTimeUnixNano\":1787952796735532818,\"severityNumber\":9,\"severityText\":\"\",\"body\":{\"input\":{\"messages\":[{\"role\":\"user\",\"content\":{\"content\":\"[{\\\"text\\\": \\\"What is 2+2? Reply with just the number.\\\"}]\"}}]},\"output\":{\"messages\":[{\"role\":\"assistant\",\"content\":{\"message\":\"[{\\\"toolUse\\\": {\\\"toolUseId\\\": \\\"tooluse_3jtUttUiLMVTz1Qqd4hYGz\\\", \\\"name\\\": \\\"add_numbers\\\", \\\"input\\\": {\\\"a\\\": 2, \\\"b\\\": 2}}}]\",\"tool.result\":\"[{\\\"toolResult\\\": {\\\"toolUseId\\\": \\\"tooluse_3jtUttUiLMVTz1Qqd4hYGz\\\", \\\"status\\\": \\\"success\\\", \\\"content\\\": [{\\\"text\\\": \\\"4\\\"}]}}]\"}},{\"role\":\"assistant\",\"content\":\"[{\\\"toolResult\\\": {\\\"toolUseId\\\": \\\"tooluse_3jtUttUiLMVTz1Qqd4hYGz\\\", \\\"status\\\": \\\"success\\\", \\\"content\\\": [{\\\"text\\\": \\\"4\\\"}]}}]\"}]}},\"attributes\":{\"session.id\":\"00000000-0000-4000-8000-4a4a72800001\",\"event.name\":\"strands.telemetry.tracer\"},\"flags\":1,\"traceId\":\"6a91fe944cf38d6b2d8c3c6a66e63ad4\",\"spanId\":\"a37ebb08225f5f32\"}" + }, + { + "field": "sessionId", + "value": "00000000-0000-4000-8000-4a4a72800001" + }, + { + "field": "traceId", + "value": "6a91fe944cf38d6b2d8c3c6a66e63ad4" + }, + { + "field": "spanId", + "value": "a37ebb08225f5f32" + }, + { + "field": "@ptr", + "value": "Cs0BCo8BCkw2ODUxOTc3MDg2ODc6L2F3cy9iZWRyb2NrLWFnZW50Y29yZS9ydW50aW1lcy9hc2RmX015QWdlbnQtM3M1YXh2QkM2US1ERUZBVUxUEAAaJGY2ZGY3NDIyLWVlZTQtNDhkNS05NDg3LTE0NTZkYjdkZGE1YiIOCKCUkduCNBCHpMP7hDRA0cHR4dszSAASNxoYAgajvKv7AAAAABTZwGgABqkf6XAAAAKyIAEolYKq0oQ0MJyuqtKENDgqQJe/AkiwkQFQgGYgAhAeGAE=" + } + ], + [ + { + "field": "@message", + "value": "{\"resource\":{\"attributes\":{\"telemetry.sdk.language\":\"python\",\"telemetry.sdk.name\":\"opentelemetry\",\"telemetry.sdk.version\":\"1.40.0\",\"service.name\":\"asdf_MyAgent.DEFAULT\",\"aws.log.group.names\":\"/aws/bedrock-agentcore/runtimes/asdf_MyAgent-3s5axvBC6Q-DEFAULT\",\"aws.log.stream.names\":\"otel-rt-logs\",\"deployment.environment.name\":\"bedrock-agentcore:default\",\"cloud.resource_id\":\"arn:aws:bedrock-agentcore:us-west-2:685197708687:runtime/asdf_MyAgent-3s5axvBC6Q/runtime-endpoint/DEFAULT:DEFAULT\",\"cloud.platform\":\"aws_bedrock_agentcore\",\"cloud.provider\":\"aws\",\"cloud.region\":\"us-west-2\",\"telemetry.auto.version\":\"0.17.0-aws\",\"aws.local.service\":\"asdf_MyAgent.DEFAULT\",\"aws.service.type\":\"gen_ai_agent\"}},\"scope\":{\"name\":\"strands.telemetry.tracer\"},\"timeUnixNano\":1787952797468097953,\"observedTimeUnixNano\":1787952801835110321,\"severityNumber\":9,\"severityText\":\"\",\"body\":{\"input\":{\"messages\":[{\"role\":\"system\",\"content\":{\"content\":\"[{\\\"text\\\": \\\"\\\\n You are a helpful assistant. Use tools when appropriate.\\\\n \\\"}]\"}},{\"role\":\"user\",\"content\":{\"content\":\"[{\\\"text\\\": \\\"What is 2+2? Reply with just the number.\\\"}]\"}},{\"role\":\"tool\",\"content\":{\"content\":\"[{\\\"toolResult\\\": {\\\"toolUseId\\\": \\\"tooluse_3jtUttUiLMVTz1Qqd4hYGz\\\", \\\"status\\\": \\\"success\\\", \\\"content\\\": [{\\\"text\\\": \\\"4\\\"}]}}]\"}}]},\"output\":{\"messages\":[{\"role\":\"assistant\",\"content\":{\"content\":\"[{\\\"toolUse\\\": {\\\"toolUseId\\\": \\\"tooluse_3jtUttUiLMVTz1Qqd4hYGz\\\", \\\"name\\\": \\\"add_numbers\\\", \\\"input\\\": {\\\"a\\\": 2, \\\"b\\\": 2}}}]\"}},{\"role\":\"assistant\",\"content\":{\"finish_reason\":\"end_turn\",\"message\":\"[{\\\"text\\\": \\\"4\\\"}]\"}}]}},\"attributes\":{\"session.id\":\"00000000-0000-4000-8000-4a4a72800001\",\"event.name\":\"strands.telemetry.tracer\"},\"flags\":1,\"traceId\":\"6a91fe944cf38d6b2d8c3c6a66e63ad4\",\"spanId\":\"ad52d36b18057ff1\"}" + }, + { + "field": "sessionId", + "value": "00000000-0000-4000-8000-4a4a72800001" + }, + { + "field": "traceId", + "value": "6a91fe944cf38d6b2d8c3c6a66e63ad4" + }, + { + "field": "spanId", + "value": "ad52d36b18057ff1" + }, + { + "field": "@ptr", + "value": "Cs0BCo8BCkw2ODUxOTc3MDg2ODc6L2F3cy9iZWRyb2NrLWFnZW50Y29yZS9ydW50aW1lcy9hc2RmX015QWdlbnQtM3M1YXh2QkM2US1ERUZBVUxUEAAaJGY2ZGY3NDIyLWVlZTQtNDhkNS05NDg3LTE0NTZkYjdkZGE1YiIOCKCUkduCNBCHpMP7hDRA0cHR4dszSAASNxoYAgajvKv7AAAAABTZwGgABqkf6XAAAAKyIAEolYKq0oQ0MJyuqtKENDgqQJe/AkiwkQFQgGYgAhAnGAE=" + } + ], + [ + { + "field": "@message", + "value": "{\"resource\":{\"attributes\":{\"telemetry.sdk.language\":\"python\",\"telemetry.sdk.name\":\"opentelemetry\",\"telemetry.sdk.version\":\"1.40.0\",\"service.name\":\"asdf_MyAgent.DEFAULT\",\"aws.log.group.names\":\"/aws/bedrock-agentcore/runtimes/asdf_MyAgent-3s5axvBC6Q-DEFAULT\",\"aws.log.stream.names\":\"otel-rt-logs\",\"deployment.environment.name\":\"bedrock-agentcore:default\",\"cloud.resource_id\":\"arn:aws:bedrock-agentcore:us-west-2:685197708687:runtime/asdf_MyAgent-3s5axvBC6Q/runtime-endpoint/DEFAULT:DEFAULT\",\"cloud.platform\":\"aws_bedrock_agentcore\",\"cloud.provider\":\"aws\",\"cloud.region\":\"us-west-2\",\"telemetry.auto.version\":\"0.17.0-aws\",\"aws.local.service\":\"asdf_MyAgent.DEFAULT\",\"aws.service.type\":\"gen_ai_agent\"}},\"scope\":{\"name\":\"strands.telemetry.tracer\"},\"timeUnixNano\":1787952797468400669,\"observedTimeUnixNano\":1787952801835250906,\"severityNumber\":9,\"severityText\":\"\",\"body\":{\"input\":{\"messages\":[{\"role\":\"user\",\"content\":{\"content\":\"[{\\\"text\\\": \\\"What is 2+2? Reply with just the number.\\\"}]\"}},{\"role\":\"tool\",\"content\":{\"content\":\"[{\\\"toolResult\\\": {\\\"toolUseId\\\": \\\"tooluse_3jtUttUiLMVTz1Qqd4hYGz\\\", \\\"status\\\": \\\"success\\\", \\\"content\\\": [{\\\"text\\\": \\\"4\\\"}]}}]\"}}]},\"output\":{\"messages\":[{\"role\":\"assistant\",\"content\":{\"content\":\"[{\\\"toolUse\\\": {\\\"toolUseId\\\": \\\"tooluse_3jtUttUiLMVTz1Qqd4hYGz\\\", \\\"name\\\": \\\"add_numbers\\\", \\\"input\\\": {\\\"a\\\": 2, \\\"b\\\": 2}}}]\"}}]}},\"attributes\":{\"session.id\":\"00000000-0000-4000-8000-4a4a72800001\",\"event.name\":\"strands.telemetry.tracer\"},\"flags\":1,\"traceId\":\"6a91fe944cf38d6b2d8c3c6a66e63ad4\",\"spanId\":\"d7b0a909896809ba\"}" + }, + { + "field": "sessionId", + "value": "00000000-0000-4000-8000-4a4a72800001" + }, + { + "field": "traceId", + "value": "6a91fe944cf38d6b2d8c3c6a66e63ad4" + }, + { + "field": "spanId", + "value": "d7b0a909896809ba" + }, + { + "field": "@ptr", + "value": "Cs0BCo8BCkw2ODUxOTc3MDg2ODc6L2F3cy9iZWRyb2NrLWFnZW50Y29yZS9ydW50aW1lcy9hc2RmX015QWdlbnQtM3M1YXh2QkM2US1ERUZBVUxUEAAaJGY2ZGY3NDIyLWVlZTQtNDhkNS05NDg3LTE0NTZkYjdkZGE1YiIOCKCUkduCNBCHpMP7hDRA0cHR4dszSAASNxoYAgajvKv7AAAAABTZwGgABqkf6XAAAAKyIAEolYKq0oQ0MJyuqtKENDgqQJe/AkiwkQFQgGYgAhAoGAE=" + } + ], + [ + { + "field": "@message", + "value": "{\"resource\":{\"attributes\":{\"telemetry.sdk.language\":\"python\",\"telemetry.sdk.name\":\"opentelemetry\",\"telemetry.sdk.version\":\"1.40.0\",\"service.name\":\"asdf_MyAgent.DEFAULT\",\"aws.log.group.names\":\"/aws/bedrock-agentcore/runtimes/asdf_MyAgent-3s5axvBC6Q-DEFAULT\",\"aws.log.stream.names\":\"otel-rt-logs\",\"deployment.environment.name\":\"bedrock-agentcore:default\",\"cloud.resource_id\":\"arn:aws:bedrock-agentcore:us-west-2:685197708687:runtime/asdf_MyAgent-3s5axvBC6Q/runtime-endpoint/DEFAULT:DEFAULT\",\"cloud.platform\":\"aws_bedrock_agentcore\",\"cloud.provider\":\"aws\",\"cloud.region\":\"us-west-2\",\"telemetry.auto.version\":\"0.17.0-aws\",\"aws.local.service\":\"asdf_MyAgent.DEFAULT\",\"aws.service.type\":\"gen_ai_agent\"}},\"scope\":{\"name\":\"strands.telemetry.tracer\"},\"timeUnixNano\":1787952797468576817,\"observedTimeUnixNano\":1787952801835352043,\"severityNumber\":9,\"severityText\":\"\",\"body\":{\"input\":{\"messages\":[{\"role\":\"system\",\"content\":\"\\n You are a helpful assistant. Use tools when appropriate.\\n \"},{\"role\":\"user\",\"content\":{\"content\":\"[{\\\"text\\\": \\\"What is 2+2? Reply with just the number.\\\"}]\"}}]},\"output\":{\"messages\":[{\"role\":\"assistant\",\"content\":{\"message\":\"4\\n\",\"finish_reason\":\"end_turn\"}}]}},\"attributes\":{\"session.id\":\"00000000-0000-4000-8000-4a4a72800001\",\"event.name\":\"strands.telemetry.tracer\"},\"flags\":1,\"traceId\":\"6a91fe944cf38d6b2d8c3c6a66e63ad4\",\"spanId\":\"5f3fc94f1decac15\"}" + }, + { + "field": "sessionId", + "value": "00000000-0000-4000-8000-4a4a72800001" + }, + { + "field": "traceId", + "value": "6a91fe944cf38d6b2d8c3c6a66e63ad4" + }, + { + "field": "spanId", + "value": "5f3fc94f1decac15" + }, + { + "field": "@ptr", + "value": "Cs0BCo8BCkw2ODUxOTc3MDg2ODc6L2F3cy9iZWRyb2NrLWFnZW50Y29yZS9ydW50aW1lcy9hc2RmX015QWdlbnQtM3M1YXh2QkM2US1ERUZBVUxUEAAaJGY2ZGY3NDIyLWVlZTQtNDhkNS05NDg3LTE0NTZkYjdkZGE1YiIOCKCUkduCNBCHpMP7hDRA0cHR4dszSAASNxoYAgajvKv7AAAAABTZwGgABqkf6XAAAAKyIAEolYKq0oQ0MJyuqtKENDgqQJe/AkiwkQFQgGYgAhApGAE=" + } + ] + ], + "statistics": { + "recordsMatched": 6, + "recordsScanned": 139, + "estimatedRecordsSkipped": 107, + "bytesScanned": 126707, + "estimatedBytesSkipped": 113696, + "logGroupsScanned": 1 + }, + "status": "Complete" +} \ No newline at end of file diff --git a/src/handlers/eval/ondemand/__fixtures__/GetQueryResultsCommand.8bae324b6facaa89.json b/src/handlers/eval/ondemand/__fixtures__/GetQueryResultsCommand.8bae324b6facaa89.json new file mode 100644 index 000000000..adf15b018 --- /dev/null +++ b/src/handlers/eval/ondemand/__fixtures__/GetQueryResultsCommand.8bae324b6facaa89.json @@ -0,0 +1,344 @@ +{ + "queryLanguage": "CWLI", + "results": [ + [ + { + "field": "@message", + "value": "{\"resource\":{\"attributes\":{\"deployment.environment.name\":\"bedrock-agentcore:default\",\"aws.local.service\":\"asdf_MyAgent.DEFAULT\",\"service.name\":\"asdf_MyAgent.DEFAULT\",\"cloud.region\":\"us-west-2\",\"aws.log.stream.names\":\"otel-rt-logs\",\"telemetry.sdk.name\":\"opentelemetry\",\"aws.service.type\":\"gen_ai_agent\",\"telemetry.sdk.language\":\"python\",\"cloud.provider\":\"aws\",\"cloud.resource_id\":\"arn:aws:bedrock-agentcore:us-west-2:685197708687:runtime/asdf_MyAgent-3s5axvBC6Q/runtime-endpoint/DEFAULT:DEFAULT\",\"aws.log.group.names\":\"/aws/bedrock-agentcore/runtimes/asdf_MyAgent-3s5axvBC6Q-DEFAULT\",\"telemetry.sdk.version\":\"1.40.0\",\"cloud.platform\":\"aws_bedrock_agentcore\",\"telemetry.auto.version\":\"0.17.0-aws\"}},\"scope\":{\"name\":\"opentelemetry.instrumentation.urllib3\",\"version\":\"0.61b0\"},\"traceId\":\"6a91fe944cf38d6b2d8c3c6a66e63ad4\",\"spanId\":\"7604363a92848eb2\",\"parentSpanId\":\"ed790aebac4076a8\",\"flags\":256,\"name\":\"PUT\",\"kind\":\"CLIENT\",\"startTimeUnixNano\":1787952793581551046,\"endTimeUnixNano\":1787952793582587769,\"durationNano\":1036723,\"attributes\":{\"aws.local.service\":\"asdf_MyAgent.DEFAULT\",\"telemetry.extended\":\"true\",\"http.url\":\"http://169.254.169.254/latest/api/token\",\"aws.remote.service\":\"169.254.169.254\",\"aws.local.environment\":\"bedrock-agentcore:default\",\"aws.remote.operation\":\"PUT /latest\",\"http.status_code\":200,\"aws.local.operation\":\"UnmappedOperation\",\"aws.span.kind\":\"CLIENT\",\"PlatformType\":\"AWS::BedrockAgentCore\",\"http.method\":\"PUT\",\"http.response.status_code\":200,\"session.id\":\"00000000-0000-4000-8000-4a4a72800001\"},\"status\":{\"code\":\"UNSET\"}}" + }, + { + "field": "sessionId", + "value": "00000000-0000-4000-8000-4a4a72800001" + }, + { + "field": "traceId", + "value": "6a91fe944cf38d6b2d8c3c6a66e63ad4" + }, + { + "field": "spanId", + "value": "7604363a92848eb2" + }, + { + "field": "@ptr", + "value": "CpwBCmAKFjY4NTE5NzcwODY4Nzphd3Mvc3BhbnMQABokZTE0NTI5MzgtZjVjZi00OTc1LWE0N2QtYzk3YzMwZmFlY2ZlIg4IoJSR24I0EIekw/uENDjv37W+yTNAyO+wktMzSAASNhoYAgamDtnsAAAAACb26P0ABqkf6bAAAAHCIAEo7o+q0oQ0MJyuqtKENDgPQJrXAUiue1CBTSACEAAYAQ==" + } + ], + [ + { + "field": "@message", + "value": "{\"resource\":{\"attributes\":{\"deployment.environment.name\":\"bedrock-agentcore:default\",\"aws.local.service\":\"asdf_MyAgent.DEFAULT\",\"service.name\":\"asdf_MyAgent.DEFAULT\",\"cloud.region\":\"us-west-2\",\"aws.log.stream.names\":\"otel-rt-logs\",\"telemetry.sdk.name\":\"opentelemetry\",\"aws.service.type\":\"gen_ai_agent\",\"telemetry.sdk.language\":\"python\",\"cloud.provider\":\"aws\",\"cloud.resource_id\":\"arn:aws:bedrock-agentcore:us-west-2:685197708687:runtime/asdf_MyAgent-3s5axvBC6Q/runtime-endpoint/DEFAULT:DEFAULT\",\"aws.log.group.names\":\"/aws/bedrock-agentcore/runtimes/asdf_MyAgent-3s5axvBC6Q-DEFAULT\",\"telemetry.sdk.version\":\"1.40.0\",\"cloud.platform\":\"aws_bedrock_agentcore\",\"telemetry.auto.version\":\"0.17.0-aws\"}},\"scope\":{\"name\":\"opentelemetry.instrumentation.urllib3\",\"version\":\"0.61b0\"},\"traceId\":\"6a91fe944cf38d6b2d8c3c6a66e63ad4\",\"spanId\":\"16323742446ec885\",\"parentSpanId\":\"ed790aebac4076a8\",\"flags\":256,\"name\":\"GET\",\"kind\":\"CLIENT\",\"startTimeUnixNano\":1787952793582988074,\"endTimeUnixNano\":1787952793583576639,\"durationNano\":588565,\"attributes\":{\"aws.local.service\":\"asdf_MyAgent.DEFAULT\",\"telemetry.extended\":\"true\",\"http.url\":\"http://169.254.169.254/latest/meta-data/iam/security-credentials/\",\"aws.remote.service\":\"169.254.169.254\",\"aws.local.environment\":\"bedrock-agentcore:default\",\"aws.remote.operation\":\"GET /latest\",\"http.status_code\":200,\"aws.local.operation\":\"UnmappedOperation\",\"aws.span.kind\":\"CLIENT\",\"PlatformType\":\"AWS::BedrockAgentCore\",\"http.method\":\"GET\",\"http.response.status_code\":200,\"session.id\":\"00000000-0000-4000-8000-4a4a72800001\"},\"status\":{\"code\":\"UNSET\"}}" + }, + { + "field": "sessionId", + "value": "00000000-0000-4000-8000-4a4a72800001" + }, + { + "field": "traceId", + "value": "6a91fe944cf38d6b2d8c3c6a66e63ad4" + }, + { + "field": "spanId", + "value": "16323742446ec885" + }, + { + "field": "@ptr", + "value": "CpwBCmAKFjY4NTE5NzcwODY4Nzphd3Mvc3BhbnMQABokZTE0NTI5MzgtZjVjZi00OTc1LWE0N2QtYzk3YzMwZmFlY2ZlIg4IoJSR24I0EIekw/uENDjv37W+yTNAyO+wktMzSAASNhoYAgamDtnsAAAAACb26P0ABqkf6bAAAAHCIAEo7o+q0oQ0MJyuqtKENDgPQJrXAUiue1CBTSACEAEYAQ==" + } + ], + [ + { + "field": "@message", + "value": "{\"resource\":{\"attributes\":{\"deployment.environment.name\":\"bedrock-agentcore:default\",\"aws.local.service\":\"asdf_MyAgent.DEFAULT\",\"service.name\":\"asdf_MyAgent.DEFAULT\",\"cloud.region\":\"us-west-2\",\"aws.log.stream.names\":\"otel-rt-logs\",\"telemetry.sdk.name\":\"opentelemetry\",\"aws.service.type\":\"gen_ai_agent\",\"telemetry.sdk.language\":\"python\",\"cloud.provider\":\"aws\",\"cloud.resource_id\":\"arn:aws:bedrock-agentcore:us-west-2:685197708687:runtime/asdf_MyAgent-3s5axvBC6Q/runtime-endpoint/DEFAULT:DEFAULT\",\"aws.log.group.names\":\"/aws/bedrock-agentcore/runtimes/asdf_MyAgent-3s5axvBC6Q-DEFAULT\",\"telemetry.sdk.version\":\"1.40.0\",\"cloud.platform\":\"aws_bedrock_agentcore\",\"telemetry.auto.version\":\"0.17.0-aws\"}},\"scope\":{\"name\":\"opentelemetry.instrumentation.urllib3\",\"version\":\"0.61b0\"},\"traceId\":\"6a91fe944cf38d6b2d8c3c6a66e63ad4\",\"spanId\":\"55ccdd09778b7c13\",\"parentSpanId\":\"ed790aebac4076a8\",\"flags\":256,\"name\":\"GET\",\"kind\":\"CLIENT\",\"startTimeUnixNano\":1787952793583915675,\"endTimeUnixNano\":1787952793584473407,\"durationNano\":557732,\"attributes\":{\"aws.local.service\":\"asdf_MyAgent.DEFAULT\",\"telemetry.extended\":\"true\",\"http.url\":\"http://169.254.169.254/latest/meta-data/iam/security-credentials/execution_role\",\"aws.remote.service\":\"169.254.169.254\",\"aws.local.environment\":\"bedrock-agentcore:default\",\"aws.remote.operation\":\"GET /latest\",\"http.status_code\":200,\"aws.local.operation\":\"UnmappedOperation\",\"aws.span.kind\":\"CLIENT\",\"PlatformType\":\"AWS::BedrockAgentCore\",\"http.method\":\"GET\",\"http.response.status_code\":200,\"session.id\":\"00000000-0000-4000-8000-4a4a72800001\"},\"status\":{\"code\":\"UNSET\"}}" + }, + { + "field": "sessionId", + "value": "00000000-0000-4000-8000-4a4a72800001" + }, + { + "field": "traceId", + "value": "6a91fe944cf38d6b2d8c3c6a66e63ad4" + }, + { + "field": "spanId", + "value": "55ccdd09778b7c13" + }, + { + "field": "@ptr", + "value": "CpwBCmAKFjY4NTE5NzcwODY4Nzphd3Mvc3BhbnMQABokZTE0NTI5MzgtZjVjZi00OTc1LWE0N2QtYzk3YzMwZmFlY2ZlIg4IoJSR24I0EIekw/uENDjv37W+yTNAyO+wktMzSAASNhoYAgamDtnsAAAAACb26P0ABqkf6bAAAAHCIAEo7o+q0oQ0MJyuqtKENDgPQJrXAUiue1CBTSACEAIYAQ==" + } + ], + [ + { + "field": "@message", + "value": "{\"resource\":{\"attributes\":{\"deployment.environment.name\":\"bedrock-agentcore:default\",\"aws.local.service\":\"asdf_MyAgent.DEFAULT\",\"service.name\":\"asdf_MyAgent.DEFAULT\",\"cloud.region\":\"us-west-2\",\"aws.log.stream.names\":\"otel-rt-logs\",\"telemetry.sdk.name\":\"opentelemetry\",\"aws.service.type\":\"gen_ai_agent\",\"telemetry.sdk.language\":\"python\",\"cloud.provider\":\"aws\",\"cloud.resource_id\":\"arn:aws:bedrock-agentcore:us-west-2:685197708687:runtime/asdf_MyAgent-3s5axvBC6Q/runtime-endpoint/DEFAULT:DEFAULT\",\"aws.log.group.names\":\"/aws/bedrock-agentcore/runtimes/asdf_MyAgent-3s5axvBC6Q-DEFAULT\",\"telemetry.sdk.version\":\"1.40.0\",\"cloud.platform\":\"aws_bedrock_agentcore\",\"telemetry.auto.version\":\"0.17.0-aws\"}},\"scope\":{\"name\":\"opentelemetry.instrumentation.httpx\",\"version\":\"0.61b0\"},\"traceId\":\"6a91fe944cf38d6b2d8c3c6a66e63ad4\",\"spanId\":\"516839883f3dccf8\",\"parentSpanId\":\"ed790aebac4076a8\",\"flags\":256,\"name\":\"POST\",\"kind\":\"CLIENT\",\"startTimeUnixNano\":1787952793691038985,\"endTimeUnixNano\":1787952793733308895,\"durationNano\":42269910,\"attributes\":{\"aws.local.service\":\"asdf_MyAgent.DEFAULT\",\"telemetry.extended\":\"true\",\"http.url\":\"https://bugbashgw1776978672-zsy8cbqwts.gateway.bedrock-agentcore.us-west-2.amazonaws.com/mcp\",\"aws.remote.service\":\"bugbashgw1776978672-zsy8cbqwts.gateway.bedrock-agentcore.us-west-2.amazonaws.com\",\"aws.local.environment\":\"bedrock-agentcore:default\",\"aws.remote.operation\":\"POST /mcp\",\"http.status_code\":200,\"aws.local.operation\":\"UnmappedOperation\",\"aws.span.kind\":\"CLIENT\",\"PlatformType\":\"AWS::BedrockAgentCore\",\"http.method\":\"POST\",\"http.response.status_code\":200,\"session.id\":\"00000000-0000-4000-8000-4a4a72800001\"},\"status\":{\"code\":\"UNSET\"}}" + }, + { + "field": "sessionId", + "value": "00000000-0000-4000-8000-4a4a72800001" + }, + { + "field": "traceId", + "value": "6a91fe944cf38d6b2d8c3c6a66e63ad4" + }, + { + "field": "spanId", + "value": "516839883f3dccf8" + }, + { + "field": "@ptr", + "value": "CpwBCmAKFjY4NTE5NzcwODY4Nzphd3Mvc3BhbnMQABokZTE0NTI5MzgtZjVjZi00OTc1LWE0N2QtYzk3YzMwZmFlY2ZlIg4IoJSR24I0EIekw/uENDjv37W+yTNAyO+wktMzSAASNhoYAgamDtnsAAAAACb26P0ABqkf6bAAAAHCIAEo7o+q0oQ0MJyuqtKENDgPQJrXAUiue1CBTSACEAMYAQ==" + } + ], + [ + { + "field": "@message", + "value": "{\"resource\":{\"attributes\":{\"deployment.environment.name\":\"bedrock-agentcore:default\",\"aws.local.service\":\"asdf_MyAgent.DEFAULT\",\"service.name\":\"asdf_MyAgent.DEFAULT\",\"cloud.region\":\"us-west-2\",\"aws.log.stream.names\":\"otel-rt-logs\",\"telemetry.sdk.name\":\"opentelemetry\",\"aws.service.type\":\"gen_ai_agent\",\"telemetry.sdk.language\":\"python\",\"cloud.provider\":\"aws\",\"cloud.resource_id\":\"arn:aws:bedrock-agentcore:us-west-2:685197708687:runtime/asdf_MyAgent-3s5axvBC6Q/runtime-endpoint/DEFAULT:DEFAULT\",\"aws.log.group.names\":\"/aws/bedrock-agentcore/runtimes/asdf_MyAgent-3s5axvBC6Q-DEFAULT\",\"telemetry.sdk.version\":\"1.40.0\",\"cloud.platform\":\"aws_bedrock_agentcore\",\"telemetry.auto.version\":\"0.17.0-aws\"}},\"scope\":{\"name\":\"opentelemetry.instrumentation.httpx\",\"version\":\"0.61b0\"},\"traceId\":\"6a91fe944cf38d6b2d8c3c6a66e63ad4\",\"spanId\":\"4403738c398f6c0c\",\"parentSpanId\":\"ed790aebac4076a8\",\"flags\":256,\"name\":\"POST\",\"kind\":\"CLIENT\",\"startTimeUnixNano\":1787952793735879895,\"endTimeUnixNano\":1787952793763063671,\"durationNano\":27183776,\"attributes\":{\"aws.local.service\":\"asdf_MyAgent.DEFAULT\",\"telemetry.extended\":\"true\",\"http.url\":\"https://bugbashgw1776978672-zsy8cbqwts.gateway.bedrock-agentcore.us-west-2.amazonaws.com/mcp\",\"aws.remote.service\":\"bugbashgw1776978672-zsy8cbqwts.gateway.bedrock-agentcore.us-west-2.amazonaws.com\",\"aws.local.environment\":\"bedrock-agentcore:default\",\"aws.remote.operation\":\"POST /mcp\",\"http.status_code\":202,\"aws.local.operation\":\"UnmappedOperation\",\"aws.span.kind\":\"CLIENT\",\"PlatformType\":\"AWS::BedrockAgentCore\",\"http.method\":\"POST\",\"http.response.status_code\":202,\"session.id\":\"00000000-0000-4000-8000-4a4a72800001\"},\"status\":{\"code\":\"UNSET\"}}" + }, + { + "field": "sessionId", + "value": "00000000-0000-4000-8000-4a4a72800001" + }, + { + "field": "traceId", + "value": "6a91fe944cf38d6b2d8c3c6a66e63ad4" + }, + { + "field": "spanId", + "value": "4403738c398f6c0c" + }, + { + "field": "@ptr", + "value": "CpwBCmAKFjY4NTE5NzcwODY4Nzphd3Mvc3BhbnMQABokZTE0NTI5MzgtZjVjZi00OTc1LWE0N2QtYzk3YzMwZmFlY2ZlIg4IoJSR24I0EIekw/uENDjv37W+yTNAyO+wktMzSAASNhoYAgamDtnsAAAAACb26P0ABqkf6bAAAAHCIAEo7o+q0oQ0MJyuqtKENDgPQJrXAUiue1CBTSACEAQYAQ==" + } + ], + [ + { + "field": "@message", + "value": "{\"resource\":{\"attributes\":{\"deployment.environment.name\":\"bedrock-agentcore:default\",\"aws.local.service\":\"asdf_MyAgent.DEFAULT\",\"service.name\":\"asdf_MyAgent.DEFAULT\",\"cloud.region\":\"us-west-2\",\"aws.log.stream.names\":\"otel-rt-logs\",\"telemetry.sdk.name\":\"opentelemetry\",\"aws.service.type\":\"gen_ai_agent\",\"telemetry.sdk.language\":\"python\",\"cloud.provider\":\"aws\",\"cloud.resource_id\":\"arn:aws:bedrock-agentcore:us-west-2:685197708687:runtime/asdf_MyAgent-3s5axvBC6Q/runtime-endpoint/DEFAULT:DEFAULT\",\"aws.log.group.names\":\"/aws/bedrock-agentcore/runtimes/asdf_MyAgent-3s5axvBC6Q-DEFAULT\",\"telemetry.sdk.version\":\"1.40.0\",\"cloud.platform\":\"aws_bedrock_agentcore\",\"telemetry.auto.version\":\"0.17.0-aws\"}},\"scope\":{\"name\":\"opentelemetry.instrumentation.httpx\",\"version\":\"0.61b0\"},\"traceId\":\"6a91fe944cf38d6b2d8c3c6a66e63ad4\",\"spanId\":\"2aa50f6e0a584472\",\"parentSpanId\":\"ed790aebac4076a8\",\"flags\":256,\"name\":\"POST\",\"kind\":\"CLIENT\",\"startTimeUnixNano\":1787952793764326730,\"endTimeUnixNano\":1787952793832132293,\"durationNano\":67805563,\"attributes\":{\"aws.local.service\":\"asdf_MyAgent.DEFAULT\",\"telemetry.extended\":\"true\",\"http.url\":\"https://bugbashgw1776978672-zsy8cbqwts.gateway.bedrock-agentcore.us-west-2.amazonaws.com/mcp\",\"aws.remote.service\":\"bugbashgw1776978672-zsy8cbqwts.gateway.bedrock-agentcore.us-west-2.amazonaws.com\",\"aws.local.environment\":\"bedrock-agentcore:default\",\"aws.remote.operation\":\"POST /mcp\",\"http.status_code\":200,\"aws.local.operation\":\"UnmappedOperation\",\"aws.span.kind\":\"CLIENT\",\"PlatformType\":\"AWS::BedrockAgentCore\",\"http.method\":\"POST\",\"http.response.status_code\":200,\"session.id\":\"00000000-0000-4000-8000-4a4a72800001\"},\"status\":{\"code\":\"UNSET\"}}" + }, + { + "field": "sessionId", + "value": "00000000-0000-4000-8000-4a4a72800001" + }, + { + "field": "traceId", + "value": "6a91fe944cf38d6b2d8c3c6a66e63ad4" + }, + { + "field": "spanId", + "value": "2aa50f6e0a584472" + }, + { + "field": "@ptr", + "value": "CpwBCmAKFjY4NTE5NzcwODY4Nzphd3Mvc3BhbnMQABokZTE0NTI5MzgtZjVjZi00OTc1LWE0N2QtYzk3YzMwZmFlY2ZlIg4IoJSR24I0EIekw/uENDjv37W+yTNAyO+wktMzSAASNhoYAgamDtnsAAAAACb26P0ABqkf6bAAAAHCIAEo7o+q0oQ0MJyuqtKENDgPQJrXAUiue1CBTSACEAUYAQ==" + } + ], + [ + { + "field": "@message", + "value": "{\"resource\":{\"attributes\":{\"deployment.environment.name\":\"bedrock-agentcore:default\",\"aws.local.service\":\"asdf_MyAgent.DEFAULT\",\"service.name\":\"asdf_MyAgent.DEFAULT\",\"cloud.region\":\"us-west-2\",\"aws.log.stream.names\":\"otel-rt-logs\",\"telemetry.sdk.name\":\"opentelemetry\",\"aws.service.type\":\"gen_ai_agent\",\"telemetry.sdk.language\":\"python\",\"cloud.provider\":\"aws\",\"cloud.resource_id\":\"arn:aws:bedrock-agentcore:us-west-2:685197708687:runtime/asdf_MyAgent-3s5axvBC6Q/runtime-endpoint/DEFAULT:DEFAULT\",\"aws.log.group.names\":\"/aws/bedrock-agentcore/runtimes/asdf_MyAgent-3s5axvBC6Q-DEFAULT\",\"telemetry.sdk.version\":\"1.40.0\",\"cloud.platform\":\"aws_bedrock_agentcore\",\"telemetry.auto.version\":\"0.17.0-aws\"}},\"scope\":{\"name\":\"strands.telemetry.tracer\",\"version\":\"\"},\"traceId\":\"6a91fe944cf38d6b2d8c3c6a66e63ad4\",\"spanId\":\"7b1eeecf45c6e146\",\"parentSpanId\":\"a37ebb08225f5f32\",\"flags\":256,\"name\":\"chat\",\"kind\":\"INTERNAL\",\"startTimeUnixNano\":1787952793835468170,\"endTimeUnixNano\":1787952795780790373,\"durationNano\":1945322203,\"attributes\":{\"aws.local.service\":\"asdf_MyAgent.DEFAULT\",\"gen_ai.usage.prompt_tokens\":1211,\"gen_ai.usage.output_tokens\":70,\"gen_ai.server.request.duration\":1908,\"gen_ai.usage.total_tokens\":1281,\"gen_ai.usage.completion_tokens\":70,\"aws.genai.span_kind\":\"LLM\",\"gen_ai.event.start_time\":\"2026-08-28T21:33:13.835475+00:00\",\"gen_ai.server.time_to_first_token\":1510,\"aws.local.environment\":\"bedrock-agentcore:default\",\"gen_ai.provider.name\":\"strands-agents\",\"gen_ai.operation.name\":\"chat\",\"gen_ai.event.end_time\":\"2026-08-28T21:33:15.780751+00:00\",\"gen_ai.usage.input_tokens\":1211,\"aws.genai.token_count_total\":1281,\"gen_ai.request.model\":\"global.anthropic.claude-sonnet-4-5-20250929-v1:0\",\"PlatformType\":\"AWS::BedrockAgentCore\",\"session.id\":\"00000000-0000-4000-8000-4a4a72800001\",\"gen_ai.system\":\"strands-agents\"},\"status\":{\"code\":\"OK\"}}" + }, + { + "field": "sessionId", + "value": "00000000-0000-4000-8000-4a4a72800001" + }, + { + "field": "traceId", + "value": "6a91fe944cf38d6b2d8c3c6a66e63ad4" + }, + { + "field": "spanId", + "value": "7b1eeecf45c6e146" + }, + { + "field": "@ptr", + "value": "CpwBCmAKFjY4NTE5NzcwODY4Nzphd3Mvc3BhbnMQABokZTE0NTI5MzgtZjVjZi00OTc1LWE0N2QtYzk3YzMwZmFlY2ZlIg4IoJSR24I0EIekw/uENDjv37W+yTNAyO+wktMzSAASNhoYAgamDtnsAAAAACb26P0ABqkf6bAAAAHCIAEo7o+q0oQ0MJyuqtKENDgPQJrXAUiue1CBTSACEAcYAQ==" + } + ], + [ + { + "field": "@message", + "value": "{\"resource\":{\"attributes\":{\"deployment.environment.name\":\"bedrock-agentcore:default\",\"aws.local.service\":\"asdf_MyAgent.DEFAULT\",\"service.name\":\"asdf_MyAgent.DEFAULT\",\"cloud.region\":\"us-west-2\",\"aws.log.stream.names\":\"otel-rt-logs\",\"telemetry.sdk.name\":\"opentelemetry\",\"aws.service.type\":\"gen_ai_agent\",\"telemetry.sdk.language\":\"python\",\"cloud.provider\":\"aws\",\"cloud.resource_id\":\"arn:aws:bedrock-agentcore:us-west-2:685197708687:runtime/asdf_MyAgent-3s5axvBC6Q/runtime-endpoint/DEFAULT:DEFAULT\",\"aws.log.group.names\":\"/aws/bedrock-agentcore/runtimes/asdf_MyAgent-3s5axvBC6Q-DEFAULT\",\"telemetry.sdk.version\":\"1.40.0\",\"cloud.platform\":\"aws_bedrock_agentcore\",\"telemetry.auto.version\":\"0.17.0-aws\"}},\"scope\":{\"name\":\"opentelemetry.instrumentation.botocore.bedrock-runtime\",\"version\":\"0.61b0\"},\"traceId\":\"6a91fe944cf38d6b2d8c3c6a66e63ad4\",\"spanId\":\"9432e4327acddbb4\",\"parentSpanId\":\"7b1eeecf45c6e146\",\"flags\":256,\"name\":\"chat global.anthropic.claude-sonnet-4-5-20250929-v1:0\",\"kind\":\"CLIENT\",\"startTimeUnixNano\":1787952793848383168,\"endTimeUnixNano\":1787952795780207477,\"durationNano\":1931824309,\"attributes\":{\"aws.local.service\":\"asdf_MyAgent.DEFAULT\",\"rpc.service\":\"Bedrock Runtime\",\"aws.remote.resource.identifier\":\"global.anthropic.claude-sonnet-4-5-20250929-v1:0\",\"aws.local.environment\":\"bedrock-agentcore:default\",\"aws.remote.operation\":\"ConverseStream\",\"gen_ai.provider.name\":\"aws.bedrock\",\"server.address\":\"bedrock-runtime.us-west-2.amazonaws.com\",\"aws.request_id\":\"d139b90c-6426-4af8-b100-d99e033a7b14\",\"aws.local.operation\":\"UnmappedOperation\",\"aws.span.kind\":\"CLIENT\",\"aws.auth.region\":\"us-west-2\",\"rpc.method\":\"ConverseStream\",\"gen_ai.response.finish_reasons\":[\"tool_use\"],\"server.port\":443,\"gen_ai.request.model\":\"global.anthropic.claude-sonnet-4-5-20250929-v1:0\",\"http.response.status_code\":200,\"gen_ai.system\":\"aws.bedrock\",\"telemetry.extended\":\"true\",\"gen_ai.usage.output_tokens\":70,\"aws.genai.span_kind\":\"LLM\",\"rpc.system\":\"aws-api\",\"aws.remote.service\":\"AWS::BedrockRuntime\",\"http.status_code\":200,\"aws.region\":\"us-west-2\",\"aws.remote.resource.type\":\"AWS::Bedrock::Model\",\"gen_ai.operation.name\":\"chat\",\"gen_ai.usage.input_tokens\":1211,\"aws.genai.token_count_total\":1281,\"retry_attempts\":0,\"PlatformType\":\"AWS::BedrockAgentCore\",\"aws.auth.account.access_key\":\"ASIAZ7CHXJWH4DWE2LBS\",\"session.id\":\"00000000-0000-4000-8000-4a4a72800001\"},\"status\":{\"code\":\"UNSET\"}}" + }, + { + "field": "sessionId", + "value": "00000000-0000-4000-8000-4a4a72800001" + }, + { + "field": "traceId", + "value": "6a91fe944cf38d6b2d8c3c6a66e63ad4" + }, + { + "field": "spanId", + "value": "9432e4327acddbb4" + }, + { + "field": "@ptr", + "value": "CpwBCmAKFjY4NTE5NzcwODY4Nzphd3Mvc3BhbnMQABokZTE0NTI5MzgtZjVjZi00OTc1LWE0N2QtYzk3YzMwZmFlY2ZlIg4IoJSR24I0EIekw/uENDjv37W+yTNAyO+wktMzSAASNhoYAgamDtnsAAAAACb26P0ABqkf6bAAAAHCIAEo7o+q0oQ0MJyuqtKENDgPQJrXAUiue1CBTSACEAYYAQ==" + } + ], + [ + { + "field": "@message", + "value": "{\"resource\":{\"attributes\":{\"deployment.environment.name\":\"bedrock-agentcore:default\",\"aws.local.service\":\"asdf_MyAgent.DEFAULT\",\"service.name\":\"asdf_MyAgent.DEFAULT\",\"cloud.region\":\"us-west-2\",\"aws.log.stream.names\":\"otel-rt-logs\",\"telemetry.sdk.name\":\"opentelemetry\",\"aws.service.type\":\"gen_ai_agent\",\"telemetry.sdk.language\":\"python\",\"cloud.provider\":\"aws\",\"cloud.resource_id\":\"arn:aws:bedrock-agentcore:us-west-2:685197708687:runtime/asdf_MyAgent-3s5axvBC6Q/runtime-endpoint/DEFAULT:DEFAULT\",\"aws.log.group.names\":\"/aws/bedrock-agentcore/runtimes/asdf_MyAgent-3s5axvBC6Q-DEFAULT\",\"telemetry.sdk.version\":\"1.40.0\",\"cloud.platform\":\"aws_bedrock_agentcore\",\"telemetry.auto.version\":\"0.17.0-aws\"}},\"scope\":{\"name\":\"strands.telemetry.tracer\",\"version\":\"\"},\"traceId\":\"6a91fe944cf38d6b2d8c3c6a66e63ad4\",\"spanId\":\"4ad640cbd3587ff3\",\"parentSpanId\":\"a37ebb08225f5f32\",\"flags\":256,\"name\":\"execute_tool add_numbers\",\"kind\":\"INTERNAL\",\"startTimeUnixNano\":1787952795781281491,\"endTimeUnixNano\":1787952795781997134,\"durationNano\":715643,\"attributes\":{\"aws.local.service\":\"asdf_MyAgent.DEFAULT\",\"gen_ai.tool.status\":\"success\",\"gen_ai.tool.call.id\":\"tooluse_3jtUttUiLMVTz1Qqd4hYGz\",\"gen_ai.tool.description\":\"Return the sum of two numbers\",\"gen_ai.tool.json_schema\":\"{\\\"properties\\\": {\\\"a\\\": {\\\"description\\\": \\\"Parameter a\\\", \\\"type\\\": \\\"integer\\\"}, \\\"b\\\": {\\\"description\\\": \\\"Parameter b\\\", \\\"type\\\": \\\"integer\\\"}}, \\\"required\\\": [\\\"a\\\", \\\"b\\\"], \\\"type\\\": \\\"object\\\"}\",\"aws.genai.span_kind\":\"TOOL\",\"gen_ai.event.start_time\":\"2026-08-28T21:33:15.781295+00:00\",\"aws.local.environment\":\"bedrock-agentcore:default\",\"gen_ai.provider.name\":\"strands-agents\",\"gen_ai.operation.name\":\"execute_tool\",\"gen_ai.event.end_time\":\"2026-08-28T21:33:15.781979+00:00\",\"gen_ai.tool.name\":\"add_numbers\",\"PlatformType\":\"AWS::BedrockAgentCore\",\"session.id\":\"00000000-0000-4000-8000-4a4a72800001\",\"gen_ai.system\":\"strands-agents\"},\"status\":{\"code\":\"OK\"}}" + }, + { + "field": "sessionId", + "value": "00000000-0000-4000-8000-4a4a72800001" + }, + { + "field": "traceId", + "value": "6a91fe944cf38d6b2d8c3c6a66e63ad4" + }, + { + "field": "spanId", + "value": "4ad640cbd3587ff3" + }, + { + "field": "@ptr", + "value": "CpwBCmAKFjY4NTE5NzcwODY4Nzphd3Mvc3BhbnMQABokZTE0NTI5MzgtZjVjZi00OTc1LWE0N2QtYzk3YzMwZmFlY2ZlIg4IoJSR24I0EIekw/uENDjv37W+yTNAyO+wktMzSAASNhoYAgamDtnsAAAAACb26P0ABqkf6bAAAAHCIAEo7o+q0oQ0MJyuqtKENDgPQJrXAUiue1CBTSACEAgYAQ==" + } + ], + [ + { + "field": "@message", + "value": "{\"resource\":{\"attributes\":{\"deployment.environment.name\":\"bedrock-agentcore:default\",\"aws.local.service\":\"asdf_MyAgent.DEFAULT\",\"service.name\":\"asdf_MyAgent.DEFAULT\",\"cloud.region\":\"us-west-2\",\"aws.log.stream.names\":\"otel-rt-logs\",\"telemetry.sdk.name\":\"opentelemetry\",\"aws.service.type\":\"gen_ai_agent\",\"telemetry.sdk.language\":\"python\",\"cloud.provider\":\"aws\",\"cloud.resource_id\":\"arn:aws:bedrock-agentcore:us-west-2:685197708687:runtime/asdf_MyAgent-3s5axvBC6Q/runtime-endpoint/DEFAULT:DEFAULT\",\"aws.log.group.names\":\"/aws/bedrock-agentcore/runtimes/asdf_MyAgent-3s5axvBC6Q-DEFAULT\",\"telemetry.sdk.version\":\"1.40.0\",\"cloud.platform\":\"aws_bedrock_agentcore\",\"telemetry.auto.version\":\"0.17.0-aws\"}},\"scope\":{\"name\":\"strands.telemetry.tracer\",\"version\":\"\"},\"traceId\":\"6a91fe944cf38d6b2d8c3c6a66e63ad4\",\"spanId\":\"a37ebb08225f5f32\",\"parentSpanId\":\"5f3fc94f1decac15\",\"flags\":256,\"name\":\"execute_event_loop_cycle\",\"kind\":\"INTERNAL\",\"startTimeUnixNano\":1787952793835335831,\"endTimeUnixNano\":1787952795782144605,\"durationNano\":1946808774,\"attributes\":{\"aws.local.service\":\"asdf_MyAgent.DEFAULT\",\"gen_ai.operation.name\":\"execute_event_loop_cycle\",\"gen_ai.event.end_time\":\"2026-08-28T21:33:15.782133+00:00\",\"event_loop.cycle_id\":\"0d22d22c-190f-4c18-bf70-c0d3115cb990\",\"gen_ai.event.start_time\":\"2026-08-28T21:33:13.835347+00:00\",\"PlatformType\":\"AWS::BedrockAgentCore\",\"session.id\":\"00000000-0000-4000-8000-4a4a72800001\",\"gen_ai.system\":\"strands-agents\",\"aws.local.environment\":\"bedrock-agentcore:default\",\"gen_ai.provider.name\":\"strands-agents\"},\"status\":{\"code\":\"OK\"}}" + }, + { + "field": "sessionId", + "value": "00000000-0000-4000-8000-4a4a72800001" + }, + { + "field": "traceId", + "value": "6a91fe944cf38d6b2d8c3c6a66e63ad4" + }, + { + "field": "spanId", + "value": "a37ebb08225f5f32" + }, + { + "field": "@ptr", + "value": "CpwBCmAKFjY4NTE5NzcwODY4Nzphd3Mvc3BhbnMQABokZTE0NTI5MzgtZjVjZi00OTc1LWE0N2QtYzk3YzMwZmFlY2ZlIg4IoJSR24I0EIekw/uENDjv37W+yTNAyO+wktMzSAASNhoYAgamDtnsAAAAACb26P0ABqkf6bAAAAHCIAEo7o+q0oQ0MJyuqtKENDgPQJrXAUiue1CBTSACEAkYAQ==" + } + ], + [ + { + "field": "@message", + "value": "{\"resource\":{\"attributes\":{\"deployment.environment.name\":\"bedrock-agentcore:default\",\"aws.local.service\":\"asdf_MyAgent.DEFAULT\",\"service.name\":\"asdf_MyAgent.DEFAULT\",\"cloud.region\":\"us-west-2\",\"aws.log.stream.names\":\"otel-rt-logs\",\"telemetry.sdk.name\":\"opentelemetry\",\"aws.service.type\":\"gen_ai_agent\",\"telemetry.sdk.language\":\"python\",\"cloud.provider\":\"aws\",\"cloud.resource_id\":\"arn:aws:bedrock-agentcore:us-west-2:685197708687:runtime/asdf_MyAgent-3s5axvBC6Q/runtime-endpoint/DEFAULT:DEFAULT\",\"aws.log.group.names\":\"/aws/bedrock-agentcore/runtimes/asdf_MyAgent-3s5axvBC6Q-DEFAULT\",\"telemetry.sdk.version\":\"1.40.0\",\"cloud.platform\":\"aws_bedrock_agentcore\",\"telemetry.auto.version\":\"0.17.0-aws\"}},\"scope\":{\"name\":\"opentelemetry.instrumentation.botocore.bedrock-runtime\",\"version\":\"0.61b0\"},\"traceId\":\"6a91fe944cf38d6b2d8c3c6a66e63ad4\",\"spanId\":\"072a6b01f30dcf74\",\"parentSpanId\":\"ad52d36b18057ff1\",\"flags\":256,\"name\":\"chat global.anthropic.claude-sonnet-4-5-20250929-v1:0\",\"kind\":\"CLIENT\",\"startTimeUnixNano\":1787952795783348917,\"endTimeUnixNano\":1787952797467653794,\"durationNano\":1684304877,\"attributes\":{\"aws.local.service\":\"asdf_MyAgent.DEFAULT\",\"rpc.service\":\"Bedrock Runtime\",\"aws.remote.resource.identifier\":\"global.anthropic.claude-sonnet-4-5-20250929-v1:0\",\"aws.local.environment\":\"bedrock-agentcore:default\",\"aws.remote.operation\":\"ConverseStream\",\"gen_ai.provider.name\":\"aws.bedrock\",\"server.address\":\"bedrock-runtime.us-west-2.amazonaws.com\",\"aws.request_id\":\"f988e74d-ba06-4055-9e7a-17d48cff7413\",\"aws.local.operation\":\"UnmappedOperation\",\"aws.span.kind\":\"CLIENT\",\"aws.auth.region\":\"us-west-2\",\"rpc.method\":\"ConverseStream\",\"gen_ai.response.finish_reasons\":[\"end_turn\"],\"server.port\":443,\"gen_ai.request.model\":\"global.anthropic.claude-sonnet-4-5-20250929-v1:0\",\"http.response.status_code\":200,\"gen_ai.system\":\"aws.bedrock\",\"telemetry.extended\":\"true\",\"gen_ai.usage.output_tokens\":4,\"aws.genai.span_kind\":\"LLM\",\"rpc.system\":\"aws-api\",\"aws.remote.service\":\"AWS::BedrockRuntime\",\"http.status_code\":200,\"aws.region\":\"us-west-2\",\"aws.remote.resource.type\":\"AWS::Bedrock::Model\",\"gen_ai.operation.name\":\"chat\",\"gen_ai.usage.input_tokens\":1295,\"aws.genai.token_count_total\":1299,\"retry_attempts\":0,\"PlatformType\":\"AWS::BedrockAgentCore\",\"aws.auth.account.access_key\":\"ASIAZ7CHXJWH4DWE2LBS\",\"session.id\":\"00000000-0000-4000-8000-4a4a72800001\"},\"status\":{\"code\":\"UNSET\"}}" + }, + { + "field": "sessionId", + "value": "00000000-0000-4000-8000-4a4a72800001" + }, + { + "field": "traceId", + "value": "6a91fe944cf38d6b2d8c3c6a66e63ad4" + }, + { + "field": "spanId", + "value": "072a6b01f30dcf74" + }, + { + "field": "@ptr", + "value": "CpwBCmAKFjY4NTE5NzcwODY4Nzphd3Mvc3BhbnMQABokZTE0NTI5MzgtZjVjZi00OTc1LWE0N2QtYzk3YzMwZmFlY2ZlIg4IoJSR24I0EIekw/uENDjv37W+yTNAyO+wktMzSAASNhoYAgamDtnsAAAAACb26P0ABqkf6bAAAAHCIAEo7o+q0oQ0MJyuqtKENDgPQJrXAUiue1CBTSACEAoYAQ==" + } + ], + [ + { + "field": "@message", + "value": "{\"resource\":{\"attributes\":{\"deployment.environment.name\":\"bedrock-agentcore:default\",\"aws.local.service\":\"asdf_MyAgent.DEFAULT\",\"service.name\":\"asdf_MyAgent.DEFAULT\",\"cloud.region\":\"us-west-2\",\"aws.log.stream.names\":\"otel-rt-logs\",\"telemetry.sdk.name\":\"opentelemetry\",\"aws.service.type\":\"gen_ai_agent\",\"telemetry.sdk.language\":\"python\",\"cloud.provider\":\"aws\",\"cloud.resource_id\":\"arn:aws:bedrock-agentcore:us-west-2:685197708687:runtime/asdf_MyAgent-3s5axvBC6Q/runtime-endpoint/DEFAULT:DEFAULT\",\"aws.log.group.names\":\"/aws/bedrock-agentcore/runtimes/asdf_MyAgent-3s5axvBC6Q-DEFAULT\",\"telemetry.sdk.version\":\"1.40.0\",\"cloud.platform\":\"aws_bedrock_agentcore\",\"telemetry.auto.version\":\"0.17.0-aws\"}},\"scope\":{\"name\":\"strands.telemetry.tracer\",\"version\":\"\"},\"traceId\":\"6a91fe944cf38d6b2d8c3c6a66e63ad4\",\"spanId\":\"5f3fc94f1decac15\",\"parentSpanId\":\"ed790aebac4076a8\",\"flags\":256,\"name\":\"invoke_agent Strands Agents\",\"kind\":\"INTERNAL\",\"startTimeUnixNano\":1787952793834752450,\"endTimeUnixNano\":1787952797468576817,\"durationNano\":3633824367,\"attributes\":{\"aws.local.service\":\"asdf_MyAgent.DEFAULT\",\"gen_ai.usage.prompt_tokens\":2506,\"gen_ai.usage.output_tokens\":74,\"gen_ai.usage.cache_write_input_tokens\":0,\"gen_ai.agent.name\":\"Strands Agents\",\"gen_ai.usage.total_tokens\":2580,\"gen_ai.usage.completion_tokens\":74,\"aws.genai.span_kind\":\"AGENT\",\"gen_ai.event.start_time\":\"2026-08-28T21:33:13.834769+00:00\",\"aws.local.environment\":\"bedrock-agentcore:default\",\"gen_ai.provider.name\":\"strands-agents\",\"gen_ai.operation.name\":\"invoke_agent\",\"gen_ai.event.end_time\":\"2026-08-28T21:33:17.468556+00:00\",\"gen_ai.usage.input_tokens\":2506,\"aws.genai.token_count_total\":2580,\"gen_ai.request.model\":\"global.anthropic.claude-sonnet-4-5-20250929-v1:0\",\"gen_ai.usage.cache_read_input_tokens\":0,\"gen_ai.agent.tools\":\"[\\\"add_numbers\\\", \\\"x_amz_bedrock_agentcore_search\\\", \\\"mcpTarget___web_fetch_exa\\\", \\\"mcpTarget___web_search_exa\\\"]\",\"PlatformType\":\"AWS::BedrockAgentCore\",\"session.id\":\"00000000-0000-4000-8000-4a4a72800001\",\"gen_ai.system\":\"strands-agents\"},\"status\":{\"code\":\"OK\"}}" + }, + { + "field": "sessionId", + "value": "00000000-0000-4000-8000-4a4a72800001" + }, + { + "field": "traceId", + "value": "6a91fe944cf38d6b2d8c3c6a66e63ad4" + }, + { + "field": "spanId", + "value": "5f3fc94f1decac15" + }, + { + "field": "@ptr", + "value": "CpwBCmAKFjY4NTE5NzcwODY4Nzphd3Mvc3BhbnMQABokZTE0NTI5MzgtZjVjZi00OTc1LWE0N2QtYzk3YzMwZmFlY2ZlIg4IoJSR24I0EIekw/uENDjv37W+yTNAyO+wktMzSAASNhoYAgamDtnsAAAAACb26P0ABqkf6bAAAAHCIAEo7o+q0oQ0MJyuqtKENDgPQJrXAUiue1CBTSACEA0YAQ==" + } + ], + [ + { + "field": "@message", + "value": "{\"resource\":{\"attributes\":{\"deployment.environment.name\":\"bedrock-agentcore:default\",\"aws.local.service\":\"asdf_MyAgent.DEFAULT\",\"service.name\":\"asdf_MyAgent.DEFAULT\",\"cloud.region\":\"us-west-2\",\"aws.log.stream.names\":\"otel-rt-logs\",\"telemetry.sdk.name\":\"opentelemetry\",\"aws.service.type\":\"gen_ai_agent\",\"telemetry.sdk.language\":\"python\",\"cloud.provider\":\"aws\",\"cloud.resource_id\":\"arn:aws:bedrock-agentcore:us-west-2:685197708687:runtime/asdf_MyAgent-3s5axvBC6Q/runtime-endpoint/DEFAULT:DEFAULT\",\"aws.log.group.names\":\"/aws/bedrock-agentcore/runtimes/asdf_MyAgent-3s5axvBC6Q-DEFAULT\",\"telemetry.sdk.version\":\"1.40.0\",\"cloud.platform\":\"aws_bedrock_agentcore\",\"telemetry.auto.version\":\"0.17.0-aws\"}},\"scope\":{\"name\":\"opentelemetry.instrumentation.starlette\",\"version\":\"0.61b0\"},\"traceId\":\"6a91fe944cf38d6b2d8c3c6a66e63ad4\",\"spanId\":\"ed790aebac4076a8\",\"parentSpanId\":\"33f5dbcf78a5fa01\",\"flags\":768,\"name\":\"POST /invocations\",\"kind\":\"SERVER\",\"startTimeUnixNano\":1787952793540648784,\"endTimeUnixNano\":1787952797468963069,\"durationNano\":3928314285,\"attributes\":{\"aws.local.service\":\"asdf_MyAgent.DEFAULT\",\"net.peer.port\":48820,\"telemetry.extended\":\"true\",\"http.target\":\"/invocations\",\"http.flavor\":\"1.1\",\"http.url\":\"http://cell01.us-west-2.prod.arp.kepler-analytics.aws.dev/invocations\",\"net.peer.ip\":\"127.0.0.1\",\"http.host\":\"127.0.0.1:8080\",\"aws.local.environment\":\"bedrock-agentcore:default\",\"http.status_code\":200,\"aws.local.operation\":\"POST /invocations\",\"aws.span.kind\":\"SERVER\",\"http.server_name\":\"cell01.us-west-2.prod.arp.kepler-analytics.aws.dev\",\"net.host.port\":8080,\"http.route\":\"/invocations\",\"PlatformType\":\"AWS::BedrockAgentCore\",\"http.method\":\"POST\",\"http.response.status_code\":200,\"session.id\":\"00000000-0000-4000-8000-4a4a72800001\",\"http.scheme\":\"http\"},\"status\":{\"code\":\"UNSET\"}}" + }, + { + "field": "sessionId", + "value": "00000000-0000-4000-8000-4a4a72800001" + }, + { + "field": "traceId", + "value": "6a91fe944cf38d6b2d8c3c6a66e63ad4" + }, + { + "field": "spanId", + "value": "ed790aebac4076a8" + }, + { + "field": "@ptr", + "value": "CpwBCmAKFjY4NTE5NzcwODY4Nzphd3Mvc3BhbnMQABokZTE0NTI5MzgtZjVjZi00OTc1LWE0N2QtYzk3YzMwZmFlY2ZlIg4IoJSR24I0EIekw/uENDjv37W+yTNAyO+wktMzSAASNhoYAgamDtnsAAAAACb26P0ABqkf6bAAAAHCIAEo7o+q0oQ0MJyuqtKENDgPQJrXAUiue1CBTSACEA4YAQ==" + } + ], + [ + { + "field": "@message", + "value": "{\"resource\":{\"attributes\":{\"deployment.environment.name\":\"bedrock-agentcore:default\",\"aws.local.service\":\"asdf_MyAgent.DEFAULT\",\"service.name\":\"asdf_MyAgent.DEFAULT\",\"cloud.region\":\"us-west-2\",\"aws.log.stream.names\":\"otel-rt-logs\",\"telemetry.sdk.name\":\"opentelemetry\",\"aws.service.type\":\"gen_ai_agent\",\"telemetry.sdk.language\":\"python\",\"cloud.provider\":\"aws\",\"cloud.resource_id\":\"arn:aws:bedrock-agentcore:us-west-2:685197708687:runtime/asdf_MyAgent-3s5axvBC6Q/runtime-endpoint/DEFAULT:DEFAULT\",\"aws.log.group.names\":\"/aws/bedrock-agentcore/runtimes/asdf_MyAgent-3s5axvBC6Q-DEFAULT\",\"telemetry.sdk.version\":\"1.40.0\",\"cloud.platform\":\"aws_bedrock_agentcore\",\"telemetry.auto.version\":\"0.17.0-aws\"}},\"scope\":{\"name\":\"strands.telemetry.tracer\",\"version\":\"\"},\"traceId\":\"6a91fe944cf38d6b2d8c3c6a66e63ad4\",\"spanId\":\"d7b0a909896809ba\",\"parentSpanId\":\"5f3fc94f1decac15\",\"flags\":256,\"name\":\"execute_event_loop_cycle\",\"kind\":\"INTERNAL\",\"startTimeUnixNano\":1787952795782385331,\"endTimeUnixNano\":1787952797468400669,\"durationNano\":1686015338,\"attributes\":{\"aws.local.service\":\"asdf_MyAgent.DEFAULT\",\"gen_ai.operation.name\":\"execute_event_loop_cycle\",\"gen_ai.event.end_time\":\"2026-08-28T21:33:17.468386+00:00\",\"event_loop.cycle_id\":\"5d873f75-9a9e-4e1a-b494-8e7857bbf933\",\"gen_ai.event.start_time\":\"2026-08-28T21:33:15.782395+00:00\",\"event_loop.parent_cycle_id\":\"0d22d22c-190f-4c18-bf70-c0d3115cb990\",\"PlatformType\":\"AWS::BedrockAgentCore\",\"session.id\":\"00000000-0000-4000-8000-4a4a72800001\",\"gen_ai.system\":\"strands-agents\",\"aws.local.environment\":\"bedrock-agentcore:default\",\"gen_ai.provider.name\":\"strands-agents\"},\"status\":{\"code\":\"OK\"}}" + }, + { + "field": "sessionId", + "value": "00000000-0000-4000-8000-4a4a72800001" + }, + { + "field": "traceId", + "value": "6a91fe944cf38d6b2d8c3c6a66e63ad4" + }, + { + "field": "spanId", + "value": "d7b0a909896809ba" + }, + { + "field": "@ptr", + "value": "CpwBCmAKFjY4NTE5NzcwODY4Nzphd3Mvc3BhbnMQABokZTE0NTI5MzgtZjVjZi00OTc1LWE0N2QtYzk3YzMwZmFlY2ZlIg4IoJSR24I0EIekw/uENDjv37W+yTNAyO+wktMzSAASNhoYAgamDtnsAAAAACb26P0ABqkf6bAAAAHCIAEo7o+q0oQ0MJyuqtKENDgPQJrXAUiue1CBTSACEAwYAQ==" + } + ], + [ + { + "field": "@message", + "value": "{\"resource\":{\"attributes\":{\"deployment.environment.name\":\"bedrock-agentcore:default\",\"aws.local.service\":\"asdf_MyAgent.DEFAULT\",\"service.name\":\"asdf_MyAgent.DEFAULT\",\"cloud.region\":\"us-west-2\",\"aws.log.stream.names\":\"otel-rt-logs\",\"telemetry.sdk.name\":\"opentelemetry\",\"aws.service.type\":\"gen_ai_agent\",\"telemetry.sdk.language\":\"python\",\"cloud.provider\":\"aws\",\"cloud.resource_id\":\"arn:aws:bedrock-agentcore:us-west-2:685197708687:runtime/asdf_MyAgent-3s5axvBC6Q/runtime-endpoint/DEFAULT:DEFAULT\",\"aws.log.group.names\":\"/aws/bedrock-agentcore/runtimes/asdf_MyAgent-3s5axvBC6Q-DEFAULT\",\"telemetry.sdk.version\":\"1.40.0\",\"cloud.platform\":\"aws_bedrock_agentcore\",\"telemetry.auto.version\":\"0.17.0-aws\"}},\"scope\":{\"name\":\"strands.telemetry.tracer\",\"version\":\"\"},\"traceId\":\"6a91fe944cf38d6b2d8c3c6a66e63ad4\",\"spanId\":\"ad52d36b18057ff1\",\"parentSpanId\":\"d7b0a909896809ba\",\"flags\":256,\"name\":\"chat\",\"kind\":\"INTERNAL\",\"startTimeUnixNano\":1787952795782789184,\"endTimeUnixNano\":1787952797468097953,\"durationNano\":1685308769,\"attributes\":{\"aws.local.service\":\"asdf_MyAgent.DEFAULT\",\"gen_ai.usage.prompt_tokens\":1295,\"gen_ai.usage.output_tokens\":4,\"gen_ai.server.request.duration\":1679,\"gen_ai.usage.total_tokens\":1299,\"gen_ai.usage.completion_tokens\":4,\"aws.genai.span_kind\":\"LLM\",\"gen_ai.event.start_time\":\"2026-08-28T21:33:15.782800+00:00\",\"gen_ai.server.time_to_first_token\":1490,\"aws.local.environment\":\"bedrock-agentcore:default\",\"gen_ai.provider.name\":\"strands-agents\",\"gen_ai.operation.name\":\"chat\",\"gen_ai.event.end_time\":\"2026-08-28T21:33:17.468062+00:00\",\"gen_ai.usage.input_tokens\":1295,\"aws.genai.token_count_total\":1299,\"gen_ai.request.model\":\"global.anthropic.claude-sonnet-4-5-20250929-v1:0\",\"PlatformType\":\"AWS::BedrockAgentCore\",\"session.id\":\"00000000-0000-4000-8000-4a4a72800001\",\"gen_ai.system\":\"strands-agents\"},\"status\":{\"code\":\"OK\"}}" + }, + { + "field": "sessionId", + "value": "00000000-0000-4000-8000-4a4a72800001" + }, + { + "field": "traceId", + "value": "6a91fe944cf38d6b2d8c3c6a66e63ad4" + }, + { + "field": "spanId", + "value": "ad52d36b18057ff1" + }, + { + "field": "@ptr", + "value": "CpwBCmAKFjY4NTE5NzcwODY4Nzphd3Mvc3BhbnMQABokZTE0NTI5MzgtZjVjZi00OTc1LWE0N2QtYzk3YzMwZmFlY2ZlIg4IoJSR24I0EIekw/uENDjv37W+yTNAyO+wktMzSAASNhoYAgamDtnsAAAAACb26P0ABqkf6bAAAAHCIAEo7o+q0oQ0MJyuqtKENDgPQJrXAUiue1CBTSACEAsYAQ==" + } + ] + ], + "statistics": { + "recordsMatched": 15, + "recordsScanned": 30, + "estimatedRecordsSkipped": 43, + "bytesScanned": 55094, + "estimatedBytesSkipped": 78544, + "logGroupsScanned": 1 + }, + "status": "Complete" +} \ No newline at end of file diff --git a/src/handlers/eval/ondemand/__fixtures__/InvokeAgentRuntimeCommand.11e09d7b5858fc78.json b/src/handlers/eval/ondemand/__fixtures__/InvokeAgentRuntimeCommand.11e09d7b5858fc78.json new file mode 100644 index 000000000..fedcc64d6 --- /dev/null +++ b/src/handlers/eval/ondemand/__fixtures__/InvokeAgentRuntimeCommand.11e09d7b5858fc78.json @@ -0,0 +1,8 @@ +{ + "contentType": "text/event-stream; charset=utf-8", + "runtimeSessionId": "00000000-0000-4000-8000-4a4a72800001", + "response": { + "$stream": "data: \"4\"\n\n" + }, + "statusCode": 200 +} \ No newline at end of file diff --git a/src/handlers/eval/ondemand/__fixtures__/StartQueryCommand.6610e857ee073463.json b/src/handlers/eval/ondemand/__fixtures__/StartQueryCommand.6610e857ee073463.json new file mode 100644 index 000000000..2cc2d8499 --- /dev/null +++ b/src/handlers/eval/ondemand/__fixtures__/StartQueryCommand.6610e857ee073463.json @@ -0,0 +1,3 @@ +{ + "queryId": "0295bf88-1c2b-40d3-9cf0-7051da35fa27" +} \ No newline at end of file diff --git a/src/handlers/eval/ondemand/__fixtures__/StartQueryCommand.f4f9750bf334f026.json b/src/handlers/eval/ondemand/__fixtures__/StartQueryCommand.f4f9750bf334f026.json new file mode 100644 index 000000000..131fdad22 --- /dev/null +++ b/src/handlers/eval/ondemand/__fixtures__/StartQueryCommand.f4f9750bf334f026.json @@ -0,0 +1,3 @@ +{ + "queryId": "50df6ae9-e322-4d47-bb01-af32563ce3e1" +} \ No newline at end of file diff --git a/src/handlers/eval/ondemand/__fixtures__/session-nonce.json b/src/handlers/eval/ondemand/__fixtures__/session-nonce.json new file mode 100644 index 000000000..8a80220e1 --- /dev/null +++ b/src/handlers/eval/ondemand/__fixtures__/session-nonce.json @@ -0,0 +1 @@ +"1a04a4a7280" \ No newline at end of file diff --git a/src/handlers/eval/ondemand/__fixtures__/simulate-ds.jsonl b/src/handlers/eval/ondemand/__fixtures__/simulate-ds.jsonl new file mode 100644 index 000000000..565ae49c1 --- /dev/null +++ b/src/handlers/eval/ondemand/__fixtures__/simulate-ds.jsonl @@ -0,0 +1 @@ +{"example_id":"greet","turns":[{"input":"What is 2+2? Reply with just the number.","expected_response":"4"}],"assertions":["is concise"]} diff --git a/src/handlers/eval/ondemand/__fixtures__/simulate.golden.json b/src/handlers/eval/ondemand/__fixtures__/simulate.golden.json new file mode 100644 index 000000000..0e5a89e91 --- /dev/null +++ b/src/handlers/eval/ondemand/__fixtures__/simulate.golden.json @@ -0,0 +1,37 @@ +{ + "sessionsRequested": 1, + "sessionsEvaluated": 1, + "results": [ + { + "evaluatorArn": "arn:aws:bedrock-agentcore:::evaluator/Builtin.Correctness", + "evaluatorId": "Builtin.Correctness", + "evaluatorName": "Builtin.Correctness", + "context": { + "spanContext": { + "sessionId": "00000000-0000-4000-8000-4a4a72800001", + "traceId": "6a91fe944cf38d6b2d8c3c6a66e63ad4" + } + }, + "explanation": "The agent responded with '4' which matches the expected response of '4'. The trailing newline is irrelevant to the correctness of the answer.", + "value": 1, + "label": "Correct", + "tokenUsage": { + "inputTokens": 484, + "outputTokens": 53, + "totalTokens": 537 + }, + "ignoredReferenceInputFields": [ + "assertions" + ] + } + ], + "examplesInvoked": 1, + "examplesFailed": 0, + "sessions": [ + { + "exampleId": "greet", + "sessionId": "00000000-0000-4000-8000-4a4a72800001" + } + ], + "failures": [] +} \ No newline at end of file diff --git a/src/handlers/eval/ondemand/__fixtures__/trace-window-now.json b/src/handlers/eval/ondemand/__fixtures__/trace-window-now.json new file mode 100644 index 000000000..73edd27c6 --- /dev/null +++ b/src/handlers/eval/ondemand/__fixtures__/trace-window-now.json @@ -0,0 +1 @@ +1788039188097 \ No newline at end of file diff --git a/src/handlers/eval/ondemand/index.tsx b/src/handlers/eval/ondemand/index.tsx index d46cc38ea..e731df263 100644 --- a/src/handlers/eval/ondemand/index.tsx +++ b/src/handlers/eval/ondemand/index.tsx @@ -3,11 +3,13 @@ import type { AppIO } from "../../../io"; import type { Core } from "../../types"; import { createHelpDefault } from "../../help"; import { createEvaluateOnDemandHandler } from "./evaluate"; +import { createSimulateOnDemandHandler } from "./simulate"; // ondemand groups the synchronous, client-side evaluation commands. It has no TUI // screen (unlike evaluator/online-eval), so a bare invocation prints help. export function createOnDemandHandler(core: Core, io: AppIO): Router { return new Router("ondemand", "evaluate existing sessions synchronously, client-side") .default(createHelpDefault(io)) - .handler(createEvaluateOnDemandHandler(core, io)); + .handler(createEvaluateOnDemandHandler(core, io)) + .handler(createSimulateOnDemandHandler(core, io)); } diff --git a/src/handlers/eval/ondemand/ondemand.fixture.test.tsx b/src/handlers/eval/ondemand/ondemand.fixture.test.tsx index 39bf5767f..b1aefddc1 100644 --- a/src/handlers/eval/ondemand/ondemand.fixture.test.tsx +++ b/src/handlers/eval/ondemand/ondemand.fixture.test.tsx @@ -4,9 +4,11 @@ import { CoreClient } from "../../../core"; import { createSilentLogger, fixtureFactories, + isRecording, matchGolden, TestGlobalConfigAccessor, testIO, + uniquePerRecording, } from "../../../testing"; import { createRootHandler } from "../../index"; @@ -14,6 +16,7 @@ const REGION = "us-west-2"; const FIXTURES = join(import.meta.dir, "__fixtures__"); const FIXTURE_AGENT = "asdf_MyAgent-3s5axvBC6Q"; +const SIMULATE_DATASET = join(FIXTURES, "simulate-ds.jsonl"); const FIXTURE_SESSION_IDS = [ "67ebf93b-65e3-4127-9e13-483b239f256a", "7f983b9f-9569-4a4d-bdc2-5c997ff346dd", @@ -70,4 +73,54 @@ describe("eval ondemand evaluate (fixture-backed)", () => { // Recording polls live CloudWatch Insights (1s between polls) and calls Evaluate // per session, so it needs well over bun's 5s default; replay is instant. }, 180_000); + + test("simulate replays a dataset, then evaluates the created sessions client-side", async () => { + const nonce = uniquePerRecording(FIXTURES, "session-nonce", () => Date.now().toString(16)); + const windowNow = uniquePerRecording( + FIXTURES, + "trace-window-now", + () => Date.now() + 24 * 60 * 60 * 1000, + ); + let n = 0; + const { createControlClient, createDataClient, createIamClient, createLogsClient } = + fixtureFactories(FIXTURES); + const core = new CoreClient({ + createControlClient, + createDataClient, + createIamClient, + createLogsClient, + logger: createSilentLogger(), + newSessionId: () => + `00000000-0000-4000-8000-${nonce.slice(-8).padStart(8, "0")}${String(++n).padStart(4, "0")}`, + now: () => windowNow, + }); + const io = testIO(); + const root = createRootHandler(core, { + io: io.io, + logger: createSilentLogger(), + globalConfigAccessor: new TestGlobalConfigAccessor(), + }); + + await root.route([ + "node", + "agentcore", + "eval", + "ondemand", + "simulate", + "--runtime-id", + FIXTURE_AGENT, + "--payload-template", + '{"prompt":"{input}"}', + "--dataset", + SIMULATE_DATASET, + "--evaluator", + "Builtin.Correctness", + "--ingestion-wait-ms", + isRecording() ? "150000" : "0", + "--region", + REGION, + ]); + + matchGolden(FIXTURES, "simulate.golden.json", io.stdout()); + }, 200_000); }); diff --git a/src/handlers/eval/ondemand/ondemand.test.tsx b/src/handlers/eval/ondemand/ondemand.test.tsx index 903a9140e..58991327d 100644 --- a/src/handlers/eval/ondemand/ondemand.test.tsx +++ b/src/handlers/eval/ondemand/ondemand.test.tsx @@ -180,7 +180,99 @@ describe("eval ondemand command hierarchy", () => { .find((c) => c.name() === "eval") ?.children() .find((c) => c.name() === "ondemand"); - expect(group?.children().map((c) => c.name())).toEqual(["evaluate"]); + expect(group?.children().map((c) => c.name())).toEqual(["evaluate", "simulate"]); + }); +}); + +describe("eval ondemand simulate", () => { + const BASE = [ + "eval", + "ondemand", + "simulate", + "--runtime-id", + "r-1", + "--payload-template", + '{"prompt":"{input}"}', + "--dataset", + "/tmp/ds.jsonl", + "--evaluator", + "Builtin.Helpfulness", + ]; + + test.each<[RegExp, string[]]>([ + [ + /--runtime-id/, + ["--payload-template", "{}", "--dataset", "/tmp/ds.jsonl", "--evaluator", "E"], + ], + [ + /--payload-template/, + ["--runtime-id", "r-1", "--dataset", "/tmp/ds.jsonl", "--evaluator", "E"], + ], + [/--dataset/, ["--runtime-id", "r-1", "--payload-template", "{}", "--evaluator", "E"]], + [ + /--evaluator/, + ["--runtime-id", "r-1", "--payload-template", "{}", "--dataset", "/tmp/ds.jsonl"], + ], + ])("rejects when a required flag is missing (%s)", async (expected, args) => { + await expect(run(["eval", "ondemand", "simulate", ...args])).rejects.toThrow(expected); + }); + + test("refuses to grade when nothing was invoked, naming the first failure", async () => { + await expect( + run(BASE, (c) => + c.eval.setInvokeDatasetResponse({ + sessions: [], + invoked: 0, + failed: 2, + failures: [ + { exampleId: "e1", error: "HTTP 500" }, + { exampleId: "e2", error: "HTTP 500" }, + ], + }), + ), + ).rejects.toThrow(/no examples could be invoked \(2 failed\).*first error: e1 — HTTP 500/); + }); + + test("passes --ingestion-wait-ms through to invokeDataset and renders failures", async () => { + const { core, stdout } = await run([...BASE, "--ingestion-wait-ms", "0"], (c) => + c.eval.setInvokeDatasetResponse({ + sessions: [{ exampleId: "ok1", sessionId: "s1" }], + invoked: 1, + failed: 1, + failures: [{ exampleId: "bad", error: "HTTP 500" }], + }), + ); + const invoke = core.eval.calls.find((c) => c.method === "invokeDataset"); + expect(invoke).toBeDefined(); + expect((invoke!.args[0] as { waitIngestionMs?: number }).waitIngestionMs).toBe(0); + const out = JSON.parse(stdout); + expect(out.failures).toEqual([{ exampleId: "bad", error: "HTTP 500" }]); + expect(out.sessions).toEqual([{ exampleId: "ok1", sessionId: "s1" }]); + }); + + test("forwards a turn's expectedResponse as a reference input", async () => { + const { core } = await run(BASE, (c) => + c.eval.setInvokeDatasetResponse({ + sessions: [ + { + exampleId: "e1", + sessionId: "s1", + groundTruth: { turns: [{ input: { prompt: "q" }, expectedResponse: { text: "42" } }] }, + }, + ], + invoked: 1, + failed: 0, + failures: [], + }), + ); + // expectedResponse is trace-level: correlated to the session's first trace (TRACE.traceIds[0]). + const evaluate = core.eval.calls.find((c) => c.method === "evaluate"); + expect((evaluate!.args[0] as { groundTruth: unknown[] }).groundTruth).toEqual([ + { + context: { spanContext: { sessionId: "s1", traceId: "t1" } }, + expectedResponse: { text: "42" }, + }, + ]); }); }); diff --git a/src/handlers/eval/ondemand/simulate/index.tsx b/src/handlers/eval/ondemand/simulate/index.tsx new file mode 100644 index 000000000..37c0b41db --- /dev/null +++ b/src/handlers/eval/ondemand/simulate/index.tsx @@ -0,0 +1,139 @@ +import z from "zod"; +import type { EvaluationReferenceInput } from "@aws-sdk/client-bedrock-agentcore"; +import { createHandler, flag } from "../../../../router"; +import { InputValidationError } from "../../../../errors"; +import { JsonRendererKey } from "../../../../tui"; +import type { AppIO } from "../../../../io"; +import type { Core } from "../../../types"; +import type { InvokedSession } from "../../types"; +import { coreOptsFromCtx } from "../../../utils"; +import { parseRuntimeInvokeHeaders } from "../../../runtime/invoke/request"; +import { withUserCancellation } from "../../../../runnable"; + +export const createSimulateOnDemandHandler = (core: Core, _io: AppIO) => + createHandler({ + name: "simulate", + description: "replay a dataset against a runtime, then evaluate the sessions client-side", + flags: [ + flag("runtime-id", "runtime id to invoke per scenario", z.string().optional()), + flag("qualifier", "runtime endpoint qualifier (default DEFAULT)", z.string().optional()), + flag( + "payload-template", + 'JSON payload template; {input} is the scenario input, e.g. {"prompt":"{input}"}', + z.string().optional(), + ), + flag("header", "an ordered application header (repeatable)", z.array(z.string()).optional()), + flag( + "bearer-token", + "CUSTOM_JWT bearer token (for JWT-auth runtimes)", + z.string().optional(), + ), + flag("user-id", "runtime user id", z.string().optional()), + flag("dataset", "dataset source: local JSONL path or a dataset id", z.string().optional()), + flag("dataset-version", "dataset version (with a dataset id)", z.string().optional()), + flag("evaluator", "evaluator id(s) to apply", z.array(z.string()).optional()), + flag( + "ingestion-wait-ms", + "ms to wait for span ingestion before grading (default 180000; 0 to skip)", + z.coerce.number().int().nonnegative().optional(), + ), + ], + handle: async (ctx, flags) => { + if (!flags["runtime-id"]) + throw new InputValidationError("required option '--runtime-id' not specified"); + if (!flags["payload-template"]) { + throw new InputValidationError("required option '--payload-template' not specified"); + } + if (!flags["dataset"]) + throw new InputValidationError("required option '--dataset' not specified"); + if (!flags["evaluator"]?.length) { + throw new InputValidationError( + "required option '--evaluator ' not specified", + ); + } + + const runtimeId = flags["runtime-id"]; + const payloadTemplate = flags["payload-template"]; + const dataset = flags["dataset"]; + const evaluatorIds = flags["evaluator"]; + + await withUserCancellation(async (signal) => { + const opts = coreOptsFromCtx(ctx); + + const replay = await core.eval.invokeDataset( + { + runtimeId, + qualifier: flags["qualifier"], + payloadTemplate, + headers: parseRuntimeInvokeHeaders(flags["header"]), + bearerToken: flags["bearer-token"], + userId: flags["user-id"], + dataset, + datasetVersion: flags["dataset-version"], + waitIngestionMs: flags["ingestion-wait-ms"], + }, + opts, + signal, + ); + if (replay.invoked === 0) { + const first = replay.failures[0]; + const detail = first ? `; first error: ${first.exampleId} — ${first.error}` : ""; + throw new InputValidationError( + `no examples could be invoked (${replay.failed} failed) — nothing to evaluate${detail}`, + ); + } + + const traces = await core.eval.getTracesForAgent( + { + agent: runtimeId, + endpoint: flags["qualifier"], + sessionIds: replay.sessions.map((s) => s.sessionId), + }, + opts, + ); + + const traceIdsBySession = new Map(traces.map((t) => [t.sessionId, t.traceIds])); + const groundTruth = replay.sessions.flatMap((s) => + toReferenceInputs(s, traceIdsBySession.get(s.sessionId) ?? []), + ); + const result = await core.eval.evaluate({ traces, evaluatorIds, groundTruth }, opts); + + ctx.require(JsonRendererKey).renderJson({ + ...result, + examplesInvoked: replay.invoked, + examplesFailed: replay.failed, + sessions: replay.sessions.map((s) => ({ + exampleId: s.exampleId, + sessionId: s.sessionId, + })), + failures: replay.failures, + }); + }); + }, + }); + +// assertions + expectedTrajectory are session-level; expectedResponse is trace-level and the +// Evaluate API rejects it under a session-only context, so each turn's expectedResponse is +// correlated to its trace by position (turn i → the session's i-th trace). +function toReferenceInputs(s: InvokedSession, traceIds: string[]): EvaluationReferenceInput[] { + const gt = s.groundTruth; + if (!gt) return []; + const refs: EvaluationReferenceInput[] = []; + if (gt.assertions?.length || gt.expectedTrajectory) { + refs.push({ + context: { spanContext: { sessionId: s.sessionId } }, + ...(gt.assertions?.length && { assertions: gt.assertions }), + ...(gt.expectedTrajectory && { expectedTrajectory: gt.expectedTrajectory }), + }); + } + (gt.turns ?? []).forEach((turn, i) => { + const traceId = traceIds[i]; + if (turn.expectedResponse && traceId) { + refs.push({ + context: { spanContext: { sessionId: s.sessionId, traceId } }, + expectedResponse: turn.expectedResponse, + }); + } + }); + return refs; +} diff --git a/src/testing/fixtures.tsx b/src/testing/fixtures.tsx index 60271ee56..2ea230002 100644 --- a/src/testing/fixtures.tsx +++ b/src/testing/fixtures.tsx @@ -53,6 +53,20 @@ export async function settle(ms = 5_000): Promise { await new Promise((resolve) => setTimeout(resolve, ms)); } +// Use this when a test needs a value that is unique for every recording but identical on every +// replay. For example, when we run `agentcore eval ondemand simulate`, each recording needs a +// different session id so we don't contaminate the run by reading back traces from old sessions. +export function uniquePerRecording(dir: string, name: string, generate: () => T): T { + const path = join(dir, `${name}.json`); + if (isRecording()) { + mkdirSync(dir, { recursive: true }); + const value = generate(); + writeFileSync(path, stringify(value)); + return value; + } + return parse(readFileSync(path, "utf8")); +} + // An AWS SDK command as seen at the `.send()` boundary: its class carries the // operation name and it holds the request `input`. We only read these. interface SdkCommand { diff --git a/src/testing/index.tsx b/src/testing/index.tsx index 7597dd900..06ede485c 100644 --- a/src/testing/index.tsx +++ b/src/testing/index.tsx @@ -1,5 +1,12 @@ export { parse, stringify } from "./serialization"; -export { fixtureFactories, fixtureFetch, isRecording, matchGolden, settle } from "./fixtures"; +export { + fixtureFactories, + fixtureFetch, + isRecording, + matchGolden, + settle, + uniquePerRecording, +} from "./fixtures"; export { testIO, ttyTestIO, type TestIO, type TestIOOptions, type TtyInput } from "./testIO"; export { tick, waitFor, WaitForTimeoutError } from "./timing"; export {