From d4bffdfd7aa29b6160cd74bc37cd84d98b82aa2a Mon Sep 17 00:00:00 2001 From: silanhe Date: Mon, 27 Jul 2026 21:09:21 +0000 Subject: [PATCH 1/6] Add workflow-rooted ExecutionOtelPlugin and fix OtelPlugin status parity Port the Workflow-rooted ExecutionOtelPlugin from the Python/JS SDKs into the otel-plugin module, alongside the existing invocation-rooted OtelPlugin: - ExecutionOtelPlugin: deterministic Workflow root span (exported once per execution, terminal-only), invocation span as its child, operations parented to the Workflow span and linked to the invocation span; deterministic span IDs stitch suspended/resumed operations into a single logical span. - OtelPlugin parity fix: invocation-span status now maps SUCCEEDED/PENDING -> OK and RETRYING/FAILED -> ERROR (was FAILED-only). - DeterministicIdGenerator: add setNextSpanId(raw) and generateWorkflowSpanId(). - SpanAttributes: add durable.execution.status for the Workflow span. - Tests: new 16-test ExecutionOtelPluginTest; update OtelPluginTest SUCCEEDED assertion to OK. otel-plugin 105/105, core SDK 1098/1098. --- .../otel/DeterministicIdGenerator.java | 35 ++ .../durable/otel/ExecutionOtelPlugin.java | 497 ++++++++++++++++++ .../lambda/durable/otel/OtelPlugin.java | 17 +- .../lambda/durable/otel/SpanAttributes.java | 1 + .../durable/otel/ExecutionOtelPluginTest.java | 389 ++++++++++++++ .../lambda/durable/otel/OtelPluginTest.java | 2 +- 6 files changed, 936 insertions(+), 5 deletions(-) create mode 100644 otel-plugin/src/main/java/software/amazon/lambda/durable/otel/ExecutionOtelPlugin.java create mode 100644 otel-plugin/src/test/java/software/amazon/lambda/durable/otel/ExecutionOtelPluginTest.java diff --git a/otel-plugin/src/main/java/software/amazon/lambda/durable/otel/DeterministicIdGenerator.java b/otel-plugin/src/main/java/software/amazon/lambda/durable/otel/DeterministicIdGenerator.java index 255af2d5e..b74ef845b 100644 --- a/otel-plugin/src/main/java/software/amazon/lambda/durable/otel/DeterministicIdGenerator.java +++ b/otel-plugin/src/main/java/software/amazon/lambda/durable/otel/DeterministicIdGenerator.java @@ -34,6 +34,7 @@ public class DeterministicIdGenerator implements IdGenerator { private final AtomicReference extractedTraceId = new AtomicReference<>(null); private final AtomicReference arnDerivedTraceId = new AtomicReference<>(null); private final ThreadLocal pendingSpanOperationId = new ThreadLocal<>(); + private final ThreadLocal pendingRawSpanId = new ThreadLocal<>(); private final AtomicReference durableExecutionArn = new AtomicReference<>(null); /** @@ -66,6 +67,17 @@ public void setNextSpanOperationId(String operationId) { this.pendingSpanOperationId.set(operationId); } + /** + * Queues the next span to use the given pre-computed span ID verbatim. Unlike {@link #setNextSpanOperationId}, the + * supplied value is used directly rather than derived from an operation ID. Used for the Workflow root span, whose + * ID is derived once from the execution ARN via {@link #generateWorkflowSpanId()}. + * + * @param spanId a 16-char lowercase hex span ID + */ + public void setNextSpanId(String spanId) { + this.pendingRawSpanId.set(spanId); + } + /** * Generates a deterministic span ID for a given operation ID without consuming the ThreadLocal state. * @@ -76,6 +88,24 @@ public String generateSpanIdForOperation(String operationId) { return generateSpanIdFromOperation(operationId); } + /** + * Generates the deterministic span ID for the Workflow root span from the current execution ARN, using the seed + * {@code "workflow:" + arn} (SHA-256, truncated to 16 hex chars). Stable across all invocations of the same + * execution so the Workflow span is exported once as a single logical span. Guarded to never be all-zero (an + * invalid OTel span ID). + * + * @return a deterministic 16-char hex span ID + */ + public String generateWorkflowSpanId() { + var arn = durableExecutionArn.get(); + var seed = "workflow:" + (arn != null ? arn : ""); + var spanId = sha256(seed).substring(0, 16); + if (spanId.equals("0000000000000000")) { + spanId = "0000000000000001"; + } + return spanId; + } + @Override public String generateTraceId() { // Priority 1: extracted from X-Ray header (backend propagates same Root across invocations) @@ -94,6 +124,11 @@ public String generateTraceId() { @Override public String generateSpanId() { + var raw = pendingRawSpanId.get(); + if (raw != null) { + pendingRawSpanId.remove(); + return raw; + } var operationId = pendingSpanOperationId.get(); if (operationId != null) { pendingSpanOperationId.remove(); diff --git a/otel-plugin/src/main/java/software/amazon/lambda/durable/otel/ExecutionOtelPlugin.java b/otel-plugin/src/main/java/software/amazon/lambda/durable/otel/ExecutionOtelPlugin.java new file mode 100644 index 000000000..c039f9040 --- /dev/null +++ b/otel-plugin/src/main/java/software/amazon/lambda/durable/otel/ExecutionOtelPlugin.java @@ -0,0 +1,497 @@ +// Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +// SPDX-License-Identifier: Apache-2.0 +package software.amazon.lambda.durable.otel; + +import static software.amazon.lambda.durable.otel.SpanAttributes.*; + +import io.opentelemetry.api.common.AttributeKey; +import io.opentelemetry.api.trace.Span; +import io.opentelemetry.api.trace.SpanBuilder; +import io.opentelemetry.api.trace.SpanContext; +import io.opentelemetry.api.trace.SpanKind; +import io.opentelemetry.api.trace.StatusCode; +import io.opentelemetry.api.trace.TraceFlags; +import io.opentelemetry.api.trace.TraceState; +import io.opentelemetry.api.trace.Tracer; +import io.opentelemetry.context.Context; +import io.opentelemetry.context.Scope; +import io.opentelemetry.sdk.trace.SdkTracerProvider; +import io.opentelemetry.sdk.trace.SdkTracerProviderBuilder; +import java.time.Instant; +import java.util.concurrent.ConcurrentHashMap; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import software.amazon.lambda.durable.plugin.DurableExecutionPlugin; +import software.amazon.lambda.durable.plugin.InvocationEndInfo; +import software.amazon.lambda.durable.plugin.InvocationInfo; +import software.amazon.lambda.durable.plugin.OperationEndInfo; +import software.amazon.lambda.durable.plugin.OperationInfo; +import software.amazon.lambda.durable.plugin.UserFunctionEndInfo; +import software.amazon.lambda.durable.plugin.UserFunctionStartInfo; + +/** + * Workflow-rooted OpenTelemetry plugin for the AWS Lambda Durable Execution SDK. + * + *

This is the Java port of the {@code ExecutionOtelPlugin} in the Python and JavaScript SDKs. It renders the full + * durable-execution hierarchy: + * + *

    + *
  • Workflow span — one logical root span per durable execution. Its span ID is derived + * deterministically from the execution ARN, so every invocation of the same execution produces the same ID. It is + * ended (and therefore exported) exactly once, on the terminal invocation. + *
  • Invocation span — one per Lambda invocation, a child of the Workflow span. Created and ended every + * invocation. + *
  • Operation span — parented to its parent operation span (or the Workflow span) and carrying a + * link to the current Invocation span for correlation. Deterministic ID keyed by operation ID, so a + * suspended-then-resumed operation stitches into a single logical span across invocations. + *
  • Attempt span — one per user-function execution (step attempt, child-context run), child of the operation + * span, linked to the current Invocation span. + *
+ * + *

Contrast with {@link OtelPlugin} (the invocation-rooted variant, equivalent to the reference + * {@code InvocationOtelPlugin}): there the invocation span is the root and there is no Workflow span. Here the Workflow + * span is the root, operations hang off the Workflow span, and each operation/attempt links to the invocation that ran + * it. Both plugins share {@link DeterministicIdGenerator}, {@link ContextExtractor}, {@link SpanAttributes}, and + * {@link MdcSpanEnricher}. + * + *

Trace ID resolution matches {@link OtelPlugin}: the X-Ray trace ID from {@code _X_AMZN_TRACE_ID} when available + * (the backend propagates the same Root to all invocations, unifying the trace), else a deterministic trace ID derived + * from the execution ARN. + * + *

Status mapping (parity with the Python/JS references): + * + *

    + *
  • Invocation span: {@code SUCCEEDED}/{@code PENDING} → {@link StatusCode#OK}; {@code RETRYING}/{@code FAILED} → + * {@link StatusCode#ERROR}. + *
  • Workflow span (terminal only): {@code SUCCEEDED} → {@link StatusCode#OK}; {@code FAILED} → + * {@link StatusCode#ERROR}. Non-terminal statuses never end the Workflow span, so it is not exported this + * invocation. + *
+ * + *

Thread-safe: uses {@link ConcurrentHashMap} for span/scope storage since the SDK runs user code on multiple + * threads. + * + * @deprecated This is a preview API that is experimental and may be changed or removed in future releases. + */ +@Deprecated +public class ExecutionOtelPlugin implements DurableExecutionPlugin { + + private static final Logger logger = LoggerFactory.getLogger(ExecutionOtelPlugin.class); + private static final String INSTRUMENTATION_NAME = "aws-durable-execution-sdk-java"; + private static final String DEFAULT_WORKFLOW_SPAN_NAME = "Workflow"; + + private final SdkTracerProvider tracerProvider; + private final Tracer tracer; + private final DeterministicIdGenerator idGenerator; + private final ContextExtractor contextExtractor; + private final boolean enableMdc; + private final String workflowSpanName; + + // Per-invocation state + private volatile Span workflowSpan; + private volatile Span invocationSpan; + private volatile String durableExecutionArn; + + // Thread-safe storage for operation spans (keyed by operationId) — open spans that need ending + private final ConcurrentHashMap operationSpans = new ConcurrentHashMap<>(); + + // Thread-safe storage for attempt spans/scopes (keyed by operationId + "-" + attempt) + private final ConcurrentHashMap attemptSpans = new ConcurrentHashMap<>(); + private final ConcurrentHashMap attemptScopes = new ConcurrentHashMap<>(); + + // Store operation span contexts for parent resolution (keyed by operationId) + private final ConcurrentHashMap operationContexts = new ConcurrentHashMap<>(); + + /** + * Creates a workflow-rooted OTel plugin with default settings: X-Ray context extraction, MDC enabled, root span + * named {@code "Workflow"}. + * + * @param tracerProviderBuilder the tracer provider builder (ID generator will be overridden) + */ + public ExecutionOtelPlugin(SdkTracerProviderBuilder tracerProviderBuilder) { + this(tracerProviderBuilder, new XRayContextExtractor(), true, DEFAULT_WORKFLOW_SPAN_NAME); + } + + /** + * Creates a workflow-rooted OTel plugin with a custom context extractor, MDC enabled, root span named + * {@code "Workflow"}. + * + * @param tracerProviderBuilder the tracer provider builder (ID generator will be overridden) + * @param contextExtractor extracts parent trace context from the Lambda environment + */ + public ExecutionOtelPlugin(SdkTracerProviderBuilder tracerProviderBuilder, ContextExtractor contextExtractor) { + this(tracerProviderBuilder, contextExtractor, true, DEFAULT_WORKFLOW_SPAN_NAME); + } + + /** + * Creates a workflow-rooted OTel plugin with full configuration. + * + * @param tracerProviderBuilder the tracer provider builder (ID generator will be overridden) + * @param contextExtractor extracts parent trace context from the Lambda environment + * @param enableMdc if true, injects traceId/spanId/traceSampled into SLF4J MDC for log correlation + * @param workflowSpanName the name for the Workflow root span + */ + public ExecutionOtelPlugin( + SdkTracerProviderBuilder tracerProviderBuilder, + ContextExtractor contextExtractor, + boolean enableMdc, + String workflowSpanName) { + this.idGenerator = new DeterministicIdGenerator(); + this.tracerProvider = tracerProviderBuilder.setIdGenerator(idGenerator).build(); + this.tracer = tracerProvider.get(INSTRUMENTATION_NAME); + this.contextExtractor = contextExtractor; + this.enableMdc = enableMdc; + this.workflowSpanName = workflowSpanName != null ? workflowSpanName : DEFAULT_WORKFLOW_SPAN_NAME; + } + + // ─── Invocation hooks ──────────────────────────────────────────────── + + @Override + public void onInvocationStart(InvocationInfo info) { + this.durableExecutionArn = info.durableExecutionArn(); + + // Set execution ARN for deterministic span/trace ID generation + idGenerator.setDurableExecutionArn(info.durableExecutionArn()); + + // Extract trace context from environment (X-Ray header). Only the trace ID is used — the Workflow span is a + // true root, so the X-Ray parent span ID is intentionally not used for parenting (unlike OtelPlugin). + var extractedContext = contextExtractor.extract(); + if (extractedContext != null) { + idGenerator.setExtractedTraceId(extractedContext.traceId()); + } + + // Workflow root span — deterministic span ID from the ARN, no parent. Recreated every invocation with the + // same ID so it is exported once as a single logical span (on the terminal invocation only). + idGenerator.setNextSpanId(idGenerator.generateWorkflowSpanId()); + workflowSpan = tracer.spanBuilder(workflowSpanName) + .setNoParent() + .setAttribute(DURABLE_EXECUTION_ARN, info.durableExecutionArn()) + .startSpan(); + + // Invocation span — child of the Workflow span, INTERNAL kind, random span ID (new every invocation). + var spanBuilder = tracer.spanBuilder("invocation") + .setSpanKind(SpanKind.INTERNAL) + .setParent(Context.root().with(workflowSpan)) + .setAttribute(DURABLE_EXECUTION_ARN, info.durableExecutionArn()) + .setAttribute(DURABLE_FIRST_INVOCATION, info.isFirstInvocation()); + + if (info.requestId() != null) { + spanBuilder.setAttribute(AttributeKey.stringKey("faas.invocation_id"), info.requestId()); + } + + invocationSpan = spanBuilder.startSpan(); + } + + @Override + public void onInvocationEnd(InvocationEndInfo info) { + // End any operation spans that are still open (operations that didn't complete in this invocation) + for (var entry : operationSpans.entrySet()) { + var span = entry.getValue(); + span.setAttribute(DURABLE_OPERATION_STATUS, "PENDING"); + span.end(); + } + operationSpans.clear(); + operationContexts.clear(); + + // End any attempt spans that are still open (e.g., crash before onUserFunctionEnd) + for (var entry : attemptScopes.entrySet()) { + entry.getValue().close(); + } + attemptScopes.clear(); + for (var entry : attemptSpans.entrySet()) { + entry.getValue().end(); + } + attemptSpans.clear(); + + // End the invocation span every invocation. + if (invocationSpan != null) { + invocationSpan.setAttribute( + DURABLE_INVOCATION_STATUS, info.invocationStatus().name()); + applyInvocationStatus(invocationSpan, info); + invocationSpan.end(); + invocationSpan = null; + } + + // End the Workflow span only on a terminal status, so it is exported exactly once per execution. + if (workflowSpan != null) { + if (isTerminal(info)) { + workflowSpan.setAttribute( + DURABLE_EXECUTION_STATUS, info.invocationStatus().name()); + switch (info.invocationStatus()) { + case FAILED -> { + var message = info.executionError() != null + ? info.executionError().getMessage() + : null; + workflowSpan.setStatus(StatusCode.ERROR, message); + if (info.executionError() != null) { + workflowSpan.recordException(info.executionError()); + } + } + default -> workflowSpan.setStatus(StatusCode.OK); // SUCCEEDED + } + workflowSpan.end(); + } + // Non-terminal (PENDING/RETRYING): leave the Workflow span un-ended (not exported this invocation). + workflowSpan = null; + } + + // Flush spans before Lambda freezes + var flushResult = tracerProvider.forceFlush().join(5, java.util.concurrent.TimeUnit.SECONDS); + if (!flushResult.isSuccess()) { + logger.warn("OTel span flush failed or timed out — some spans may be lost"); + } + } + + // ─── Operation hooks ───────────────────────────────────────────────── + + @Override + public void onOperationStart(OperationInfo info) { + if (info.id() == null) return; + + var parentContext = resolveParentContext(info.parentId()); + + // Always use a deterministic span ID keyed by operation ID (regardless of replay) so a suspended-then-resumed + // operation stitches into a single logical span across invocations. + idGenerator.setNextSpanOperationId(info.id()); + + var spanBuilder = tracer.spanBuilder(spanName(info.type(), info.subType(), info.name())) + .setParent(parentContext) + .setAttribute(DURABLE_EXECUTION_ARN, durableExecutionArn) + .setAttribute(DURABLE_OPERATION_ID, info.id()) + .setAttribute(DURABLE_OPERATION_TYPE, info.type()); + addInvocationLink(spanBuilder); + + if (info.name() != null) { + spanBuilder.setAttribute(DURABLE_OPERATION_NAME, info.name()); + } + if (info.subType() != null) { + spanBuilder.setAttribute(DURABLE_OPERATION_SUBTYPE, info.subType()); + } + + var span = spanBuilder.startSpan(); + + // Store the open span — will be ended in onOperationEnd or onInvocationEnd + operationSpans.put(info.id(), span); + operationContexts.put(info.id(), span.getSpanContext()); + } + + @Override + public void onOperationEnd(OperationEndInfo info) { + if (info.id() == null) return; + + var span = operationSpans.remove(info.id()); + + if (span != null) { + // Operation was started in this invocation — end normally + if (info.status() != null) { + span.setAttribute(DURABLE_OPERATION_STATUS, info.status()); + } + if (info.error() != null) { + span.setStatus(StatusCode.ERROR, info.error().getMessage()); + span.recordException(info.error()); + } + endSpan(span, info.endTimestamp()); + } else { + // Operation completed between invocations (started in a prior invocation). Recreate it with the same + // deterministic span ID so it stitches to the original, plus a link to the current invocation. + operationContexts.remove(info.id()); + idGenerator.setNextSpanOperationId(info.id()); + + var parentContext = resolveParentContext(info.parentId()); + + var spanBuilder = tracer.spanBuilder(spanName(info.type(), info.subType(), info.name())) + .setParent(parentContext) + .setAttribute(DURABLE_EXECUTION_ARN, durableExecutionArn) + .setAttribute(DURABLE_OPERATION_ID, info.id()) + .setAttribute(DURABLE_OPERATION_TYPE, info.type()); + addInvocationLink(spanBuilder); + + if (info.startTimestamp() != null) { + spanBuilder.setStartTimestamp(info.startTimestamp()); + } + if (info.name() != null) { + spanBuilder.setAttribute(DURABLE_OPERATION_NAME, info.name()); + } + if (info.subType() != null) { + spanBuilder.setAttribute(DURABLE_OPERATION_SUBTYPE, info.subType()); + } + + var continuationSpan = spanBuilder.startSpan(); + + if (info.status() != null) { + continuationSpan.setAttribute(DURABLE_OPERATION_STATUS, info.status()); + } + if (info.error() != null) { + continuationSpan.setStatus(StatusCode.ERROR, info.error().getMessage()); + continuationSpan.recordException(info.error()); + } + + endSpan(continuationSpan, info.endTimestamp()); + } + } + + // ─── User function hooks ───────────────────────────────────────────── + + @Override + public void onUserFunctionStart(UserFunctionStartInfo info) { + // Skip attempt spans for CONTEXT operations — they are a scoping construct, not a retriable unit of work. Still + // make the operation span current so auto-instrumented calls become children. + if ("CONTEXT".equals(info.type())) { + var operationSpan = operationSpans.get(info.id()); + if (operationSpan != null) { + var scope = operationSpan.makeCurrent(); + var key = attemptKey(info.id(), info.attempt()); + attemptScopes.put(key, scope); + } + if (enableMdc) { + MdcSpanEnricher.inject(); + } + return; + } + + var key = attemptKey(info.id(), info.attempt()); + + // Parent the attempt span to its operation span. + var parentContext = resolveParentContext(info.id()); + + var spanBuilder = tracer.spanBuilder(attemptSpanName(info.type(), info.subType(), info.name(), info.attempt())) + .setParent(parentContext) + .setStartTimestamp(info.startTimestamp() != null ? info.startTimestamp() : Instant.now()); + addInvocationLink(spanBuilder); + + spanBuilder.setAttribute(DURABLE_EXECUTION_ARN, durableExecutionArn); + spanBuilder.setAttribute(DURABLE_OPERATION_ID, info.id()); + + if (info.type() != null) { + spanBuilder.setAttribute(DURABLE_OPERATION_TYPE, info.type()); + } + if (info.name() != null) { + spanBuilder.setAttribute(DURABLE_OPERATION_NAME, info.name()); + } + if (info.attempt() != null) { + spanBuilder.setAttribute(DURABLE_ATTEMPT_NUMBER, info.attempt().longValue()); + } + + var span = spanBuilder.startSpan(); + attemptSpans.put(key, span); + + // Make span current on this thread so auto-instrumented calls become children + var scope = span.makeCurrent(); + attemptScopes.put(key, scope); + + if (enableMdc) { + MdcSpanEnricher.inject(); + } + } + + @Override + public void onUserFunctionEnd(UserFunctionEndInfo info) { + var key = attemptKey(info.id(), info.attempt()); + + // Close scope first (must happen on same thread as makeCurrent) + var scope = attemptScopes.remove(key); + if (scope != null) { + scope.close(); + } + + if (enableMdc) { + MdcSpanEnricher.clear(); + } + + // CONTEXT operations don't have attempt spans — scope cleanup is all we need + if ("CONTEXT".equals(info.type())) { + return; + } + + var span = attemptSpans.remove(key); + if (span == null) return; + + var outcome = info.succeeded() ? "SUCCEEDED" : "FAILED"; + span.setAttribute(DURABLE_ATTEMPT_OUTCOME, outcome); + + if (!info.succeeded() && info.error() != null) { + span.setStatus(StatusCode.ERROR, info.error().getMessage()); + span.recordException(info.error()); + } + + endSpan(span, info.endTimestamp()); + } + + // ─── Helpers ───────────────────────────────────────────────────────── + + private void applyInvocationStatus(Span span, InvocationEndInfo info) { + switch (info.invocationStatus()) { + case SUCCEEDED, PENDING -> span.setStatus(StatusCode.OK); + case RETRYING, FAILED -> { + var message = + info.executionError() != null ? info.executionError().getMessage() : null; + span.setStatus(StatusCode.ERROR, message); + if (info.executionError() != null) { + span.recordException(info.executionError()); + } + } + } + } + + private static boolean isTerminal(InvocationEndInfo info) { + return switch (info.invocationStatus()) { + case SUCCEEDED, FAILED -> true; + case PENDING, RETRYING -> false; + }; + } + + /** Adds a link to the current invocation span, if one exists, for correlation. */ + private void addInvocationLink(SpanBuilder spanBuilder) { + var currentInvocationSpan = invocationSpan; + if (currentInvocationSpan != null) { + spanBuilder.addLink(currentInvocationSpan.getSpanContext()); + } + } + + private Context resolveParentContext(String parentId) { + if (parentId != null) { + var parentSpanContext = operationContexts.get(parentId); + if (parentSpanContext != null) { + return Context.current().with(Span.wrap(parentSpanContext)); + } + // Parent operation from a prior invocation — create a non-recording placeholder with its deterministic ID. + var deterministicParentSpanId = idGenerator.generateSpanIdForOperation(parentId); + var traceId = idGenerator.generateTraceId(); + var placeholderContext = SpanContext.create( + traceId, deterministicParentSpanId, TraceFlags.getSampled(), TraceState.getDefault()); + return Context.current().with(Span.wrap(placeholderContext)); + } + // No parent operation — hang off the Workflow root span. + if (workflowSpan != null) { + return Context.current().with(workflowSpan); + } + return Context.current(); + } + + private static void endSpan(Span span, Instant endTimestamp) { + if (endTimestamp != null) { + span.end(endTimestamp); + } else { + span.end(); + } + } + + private static String spanName(String type, String subType, String name) { + if (name != null) { + return name; + } + return subType != null ? subType.toLowerCase() : type.toLowerCase(); + } + + private static String attemptSpanName(String type, String subType, String name, Integer attempt) { + var base = spanName(type, subType, name); + if (attempt != null) { + return base + " attempt " + attempt; + } + return base; + } + + private static String attemptKey(String operationId, Integer attempt) { + return operationId + "-" + (attempt != null ? attempt : "ctx"); + } +} diff --git a/otel-plugin/src/main/java/software/amazon/lambda/durable/otel/OtelPlugin.java b/otel-plugin/src/main/java/software/amazon/lambda/durable/otel/OtelPlugin.java index 25fde5cde..7f7636f88 100644 --- a/otel-plugin/src/main/java/software/amazon/lambda/durable/otel/OtelPlugin.java +++ b/otel-plugin/src/main/java/software/amazon/lambda/durable/otel/OtelPlugin.java @@ -26,7 +26,6 @@ import software.amazon.lambda.durable.plugin.DurableExecutionPlugin; import software.amazon.lambda.durable.plugin.InvocationEndInfo; import software.amazon.lambda.durable.plugin.InvocationInfo; -import software.amazon.lambda.durable.plugin.InvocationStatus; import software.amazon.lambda.durable.plugin.OperationEndInfo; import software.amazon.lambda.durable.plugin.OperationInfo; import software.amazon.lambda.durable.plugin.UserFunctionEndInfo; @@ -220,9 +219,19 @@ public void onInvocationEnd(InvocationEndInfo info) { invocationSpan.setAttribute( DURABLE_INVOCATION_STATUS, info.invocationStatus().name()); - if (info.invocationStatus() == InvocationStatus.FAILED && info.executionError() != null) { - invocationSpan.setStatus(StatusCode.ERROR, info.executionError().getMessage()); - invocationSpan.recordException(info.executionError()); + // Status mapping (parity with Python/JS references): + // SUCCEEDED, PENDING -> OK + // RETRYING, FAILED -> ERROR (records the execution error when present) + switch (info.invocationStatus()) { + case SUCCEEDED, PENDING -> invocationSpan.setStatus(StatusCode.OK); + case RETRYING, FAILED -> { + var message = + info.executionError() != null ? info.executionError().getMessage() : null; + invocationSpan.setStatus(StatusCode.ERROR, message); + if (info.executionError() != null) { + invocationSpan.recordException(info.executionError()); + } + } } invocationSpan.end(); diff --git a/otel-plugin/src/main/java/software/amazon/lambda/durable/otel/SpanAttributes.java b/otel-plugin/src/main/java/software/amazon/lambda/durable/otel/SpanAttributes.java index de048acc0..a6f8c2873 100644 --- a/otel-plugin/src/main/java/software/amazon/lambda/durable/otel/SpanAttributes.java +++ b/otel-plugin/src/main/java/software/amazon/lambda/durable/otel/SpanAttributes.java @@ -15,6 +15,7 @@ final class SpanAttributes { private SpanAttributes() {} static final AttributeKey DURABLE_EXECUTION_ARN = AttributeKey.stringKey("durable.execution.arn"); + static final AttributeKey DURABLE_EXECUTION_STATUS = AttributeKey.stringKey("durable.execution.status"); static final AttributeKey DURABLE_OPERATION_ID = AttributeKey.stringKey("durable.operation.id"); static final AttributeKey DURABLE_OPERATION_TYPE = AttributeKey.stringKey("durable.operation.type"); static final AttributeKey DURABLE_OPERATION_NAME = AttributeKey.stringKey("durable.operation.name"); diff --git a/otel-plugin/src/test/java/software/amazon/lambda/durable/otel/ExecutionOtelPluginTest.java b/otel-plugin/src/test/java/software/amazon/lambda/durable/otel/ExecutionOtelPluginTest.java new file mode 100644 index 000000000..4283064e1 --- /dev/null +++ b/otel-plugin/src/test/java/software/amazon/lambda/durable/otel/ExecutionOtelPluginTest.java @@ -0,0 +1,389 @@ +// Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +// SPDX-License-Identifier: Apache-2.0 +package software.amazon.lambda.durable.otel; + +import static org.junit.jupiter.api.Assertions.*; + +import io.opentelemetry.api.trace.SpanKind; +import io.opentelemetry.api.trace.StatusCode; +import io.opentelemetry.sdk.testing.exporter.InMemorySpanExporter; +import io.opentelemetry.sdk.trace.SdkTracerProvider; +import io.opentelemetry.sdk.trace.export.SimpleSpanProcessor; +import java.time.Instant; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import software.amazon.lambda.durable.plugin.*; + +class ExecutionOtelPluginTest { + + private static final String ARN = "arn:aws:lambda:us-east-1:123:function:test:$LATEST/durable/exec1"; + + private InMemorySpanExporter spanExporter; + private ExecutionOtelPlugin plugin; + + @BeforeEach + void setUp() { + spanExporter = InMemorySpanExporter.create(); + plugin = new ExecutionOtelPlugin( + SdkTracerProvider.builder().addSpanProcessor(SimpleSpanProcessor.create(spanExporter)), + () -> null, + false, + "Workflow"); + } + + // ─── Workflow root span lifecycle ──────────────────────────────────── + + @Test + void terminalInvocation_exportsWorkflowAndInvocationSpans() { + plugin.onInvocationStart(new InvocationInfo("req-1", ARN, true)); + plugin.onInvocationEnd(new InvocationEndInfo("req-1", ARN, true, InvocationStatus.SUCCEEDED, null)); + + var spans = spanExporter.getFinishedSpanItems(); + assertEquals(2, spans.size(), "Terminal invocation should export the Workflow span and the invocation span"); + + var workflowSpan = spanByName(spans, "Workflow"); + var invocationSpan = spanByName(spans, "invocation"); + + assertEquals(StatusCode.OK, workflowSpan.getStatus().getStatusCode()); + assertEquals(StatusCode.OK, invocationSpan.getStatus().getStatusCode()); + } + + @Test + void workflowSpan_isRoot_invocationSpanIsChild() { + plugin.onInvocationStart(new InvocationInfo("req-1", ARN, true)); + plugin.onInvocationEnd(new InvocationEndInfo("req-1", ARN, true, InvocationStatus.SUCCEEDED, null)); + + var spans = spanExporter.getFinishedSpanItems(); + var workflowSpan = spanByName(spans, "Workflow"); + var invocationSpan = spanByName(spans, "invocation"); + + assertFalse( + invocationSpan.getParentSpanContext().getSpanId().equals("0000000000000000"), + "Invocation span must have a parent"); + assertEquals( + workflowSpan.getSpanId(), + invocationSpan.getParentSpanId(), + "Invocation span must be a child of the Workflow root span"); + assertEquals(SpanKind.INTERNAL, invocationSpan.getKind()); + } + + @Test + void nonTerminalInvocation_doesNotExportWorkflowSpan() { + plugin.onInvocationStart(new InvocationInfo("req-1", ARN, true)); + plugin.onInvocationEnd(new InvocationEndInfo("req-1", ARN, true, InvocationStatus.PENDING, null)); + + var spans = spanExporter.getFinishedSpanItems(); + // Only the invocation span is exported; the Workflow span is not ended on non-terminal status. + assertEquals(1, spans.size()); + assertEquals("invocation", spans.get(0).getName()); + assertEquals(StatusCode.OK, spans.get(0).getStatus().getStatusCode(), "PENDING invocation span maps to OK"); + } + + @Test + void workflowSpan_exportedOnceAcrossInvocations_sameSpanId() { + // Invocation 1: non-terminal → no Workflow span exported + plugin.onInvocationStart(new InvocationInfo("req-1", ARN, true)); + plugin.onInvocationEnd(new InvocationEndInfo("req-1", ARN, true, InvocationStatus.PENDING, null)); + assertTrue( + spanExporter.getFinishedSpanItems().stream() + .noneMatch(s -> s.getName().equals("Workflow")), + "Workflow span must not be exported on a non-terminal invocation"); + spanExporter.reset(); + + // Invocation 2: terminal → Workflow span exported with the deterministic ID + plugin.onInvocationStart(new InvocationInfo("req-2", ARN, false)); + plugin.onInvocationEnd(new InvocationEndInfo("req-2", ARN, false, InvocationStatus.SUCCEEDED, null)); + + var workflowSpan = spanByName(spanExporter.getFinishedSpanItems(), "Workflow"); + assertEquals(StatusCode.OK, workflowSpan.getStatus().getStatusCode()); + assertTrue(workflowSpan.getSpanId().matches("[0-9a-f]{16}")); + } + + // ─── Status mapping ────────────────────────────────────────────────── + + @Test + void failedInvocation_setsErrorOnBothWorkflowAndInvocationSpans() { + plugin.onInvocationStart(new InvocationInfo("req-1", ARN, true)); + plugin.onInvocationEnd( + new InvocationEndInfo("req-1", ARN, true, InvocationStatus.FAILED, new RuntimeException("boom"))); + + var spans = spanExporter.getFinishedSpanItems(); + assertEquals(StatusCode.ERROR, spanByName(spans, "Workflow").getStatus().getStatusCode()); + assertEquals( + StatusCode.ERROR, spanByName(spans, "invocation").getStatus().getStatusCode()); + } + + @Test + void retryingInvocation_invocationSpanError_workflowNotExported() { + plugin.onInvocationStart(new InvocationInfo("req-1", ARN, true)); + plugin.onInvocationEnd(new InvocationEndInfo( + "req-1", ARN, true, InvocationStatus.RETRYING, new RuntimeException("transient"))); + + var spans = spanExporter.getFinishedSpanItems(); + assertEquals(1, spans.size(), "RETRYING is non-terminal — Workflow span not exported"); + var invocationSpan = spans.get(0); + assertEquals("invocation", invocationSpan.getName()); + assertEquals(StatusCode.ERROR, invocationSpan.getStatus().getStatusCode()); + } + + // ─── Operation topology: parented to Workflow, linked to invocation ── + + @Test + void operationSpan_parentedToWorkflow_linkedToInvocation() { + plugin.onInvocationStart(new InvocationInfo("req-1", ARN, true)); + plugin.onOperationStart(new OperationInfo("op-1", "step-a", "STEP", "Step", null, Instant.now(), null, false)); + plugin.onOperationEnd(new OperationEndInfo( + "op-1", "step-a", "STEP", "Step", null, Instant.now(), Instant.now(), "SUCCEEDED", false, null)); + plugin.onInvocationEnd(new InvocationEndInfo("req-1", ARN, true, InvocationStatus.SUCCEEDED, null)); + + var spans = spanExporter.getFinishedSpanItems(); + var workflowSpan = spanByName(spans, "Workflow"); + var invocationSpan = spanByName(spans, "invocation"); + var operationSpan = spanByName(spans, "step-a"); + + assertEquals( + workflowSpan.getSpanId(), + operationSpan.getParentSpanId(), + "Operation span must be parented to the Workflow span, not the invocation span"); + assertTrue( + operationSpan.getLinks().stream() + .anyMatch(l -> l.getSpanContext().getSpanId().equals(invocationSpan.getSpanId())), + "Operation span must carry a link to the invocation span"); + } + + @Test + void attemptSpan_childOfOperation_linkedToInvocation() { + plugin.onInvocationStart(new InvocationInfo("req-1", ARN, true)); + plugin.onOperationStart(new OperationInfo("op-1", "compute", "STEP", "Step", null, Instant.now(), null, false)); + plugin.onUserFunctionStart( + new UserFunctionStartInfo("op-1", "compute", "STEP", "Step", null, Instant.now(), false, 1)); + plugin.onUserFunctionEnd(new UserFunctionEndInfo( + "op-1", "compute", "STEP", "Step", null, Instant.now(), Instant.now(), false, 1, true, null)); + plugin.onOperationEnd(new OperationEndInfo( + "op-1", "compute", "STEP", "Step", null, Instant.now(), Instant.now(), "SUCCEEDED", false, null)); + plugin.onInvocationEnd(new InvocationEndInfo("req-1", ARN, true, InvocationStatus.SUCCEEDED, null)); + + var spans = spanExporter.getFinishedSpanItems(); + var operationSpan = spanByName(spans, "compute"); + var invocationSpan = spanByName(spans, "invocation"); + var attemptSpan = spans.stream() + .filter(s -> s.getName().contains("attempt")) + .findFirst() + .orElseThrow(); + + assertEquals("compute attempt 1", attemptSpan.getName()); + assertEquals( + operationSpan.getSpanId(), + attemptSpan.getParentSpanId(), + "Attempt span must be a child of its operation span"); + assertTrue( + attemptSpan.getLinks().stream() + .anyMatch(l -> l.getSpanContext().getSpanId().equals(invocationSpan.getSpanId())), + "Attempt span must carry a link to the invocation span"); + } + + @Test + void childOperation_parentedToParentOperationSpan() { + plugin.onInvocationStart(new InvocationInfo("req-1", ARN, true)); + plugin.onOperationStart(new OperationInfo( + "op-parent", "my-context", "CONTEXT", "RunInChildContext", null, Instant.now(), null, false)); + plugin.onOperationStart( + new OperationInfo("op-child", "inner-step", "STEP", "Step", "op-parent", Instant.now(), null, false)); + plugin.onOperationEnd(new OperationEndInfo( + "op-child", + "inner-step", + "STEP", + "Step", + "op-parent", + Instant.now(), + Instant.now(), + "SUCCEEDED", + false, + null)); + plugin.onOperationEnd(new OperationEndInfo( + "op-parent", + "my-context", + "CONTEXT", + "RunInChildContext", + null, + Instant.now(), + Instant.now(), + "SUCCEEDED", + false, + null)); + plugin.onInvocationEnd(new InvocationEndInfo("req-1", ARN, true, InvocationStatus.SUCCEEDED, null)); + + var spans = spanExporter.getFinishedSpanItems(); + var parentSpan = spanByName(spans, "my-context"); + var childSpan = spanByName(spans, "inner-step"); + assertEquals( + parentSpan.getSpanId(), + childSpan.getParentSpanId(), + "Child operation should be parented to its parent operation span"); + } + + // ─── Failure propagation ───────────────────────────────────────────── + + @Test + void userFunctionFailure_setsErrorOnAttemptSpan() { + plugin.onInvocationStart(new InvocationInfo("req-1", ARN, true)); + plugin.onUserFunctionStart( + new UserFunctionStartInfo("op-1", "failing", "STEP", "Step", null, Instant.now(), false, 1)); + plugin.onUserFunctionEnd(new UserFunctionEndInfo( + "op-1", + "failing", + "STEP", + "Step", + null, + Instant.now(), + Instant.now(), + false, + 1, + false, + new RuntimeException("step failed"))); + plugin.onInvocationEnd(new InvocationEndInfo("req-1", ARN, true, InvocationStatus.FAILED, null)); + + var attemptSpan = spanExporter.getFinishedSpanItems().stream() + .filter(s -> s.getName().contains("attempt")) + .findFirst() + .orElseThrow(); + assertEquals(StatusCode.ERROR, attemptSpan.getStatus().getStatusCode()); + } + + @Test + void operationNotCompleted_endedPendingAtInvocationEnd() { + plugin.onInvocationStart(new InvocationInfo("req-1", ARN, true)); + plugin.onOperationStart(new OperationInfo("op-1", "my-wait", "WAIT", "Wait", null, Instant.now(), null, false)); + plugin.onInvocationEnd(new InvocationEndInfo("req-1", ARN, true, InvocationStatus.PENDING, null)); + + var spans = spanExporter.getFinishedSpanItems(); + // invocation span + PENDING operation span (Workflow not exported — non-terminal) + assertEquals(2, spans.size()); + var operationSpan = spanByName(spans, "my-wait"); + assertEquals( + "PENDING", + operationSpan + .getAttributes() + .get(io.opentelemetry.api.common.AttributeKey.stringKey("durable.operation.status"))); + } + + // ─── Cross-invocation stitching ────────────────────────────────────── + + @Test + void allSpansShareTraceId_acrossInvocations() { + plugin.onInvocationStart(new InvocationInfo("req-1", ARN, true)); + plugin.onOperationStart(new OperationInfo("op-1", "step-1", "STEP", "Step", null, Instant.now(), null, false)); + plugin.onOperationEnd(new OperationEndInfo( + "op-1", "step-1", "STEP", "Step", null, Instant.now(), Instant.now(), "SUCCEEDED", false, null)); + plugin.onInvocationEnd(new InvocationEndInfo("req-1", ARN, true, InvocationStatus.PENDING, null)); + var firstTraceId = spanExporter.getFinishedSpanItems().get(0).getTraceId(); + spanExporter.reset(); + + plugin.onInvocationStart(new InvocationInfo("req-2", ARN, false)); + plugin.onInvocationEnd(new InvocationEndInfo("req-2", ARN, false, InvocationStatus.SUCCEEDED, null)); + var secondSpans = spanExporter.getFinishedSpanItems(); + + assertTrue( + secondSpans.stream().allMatch(s -> s.getTraceId().equals(firstTraceId)), + "All spans of one execution must share the same trace ID"); + } + + @Test + void operationEnd_withoutStart_createsContinuationSpanWithLink() { + plugin.onInvocationStart(new InvocationInfo("req-2", ARN, false)); + // Operation completed between invocations — no matching onOperationStart in this invocation. + plugin.onOperationEnd(new OperationEndInfo( + "op-wait-1", "my-wait", "WAIT", "Wait", null, Instant.now(), Instant.now(), "SUCCEEDED", false, null)); + plugin.onInvocationEnd(new InvocationEndInfo("req-2", ARN, false, InvocationStatus.SUCCEEDED, null)); + + var spans = spanExporter.getFinishedSpanItems(); + var continuationSpan = spanByName(spans, "my-wait"); + var invocationSpan = spanByName(spans, "invocation"); + assertFalse(continuationSpan.getLinks().isEmpty(), "Continuation span should have a link"); + assertTrue( + continuationSpan.getLinks().stream() + .anyMatch(l -> l.getSpanContext().getSpanId().equals(invocationSpan.getSpanId())), + "Continuation span should link to the current invocation span"); + } + + @Test + void deterministicWorkflowSpanId_stableAcrossInvocations() { + plugin.onInvocationStart(new InvocationInfo("req-1", ARN, true)); + plugin.onInvocationEnd(new InvocationEndInfo("req-1", ARN, true, InvocationStatus.SUCCEEDED, null)); + var firstWorkflowSpanId = + spanByName(spanExporter.getFinishedSpanItems(), "Workflow").getSpanId(); + spanExporter.reset(); + + // A second (independent) plugin for the same execution ARN must derive the same Workflow span ID. + var exporter2 = InMemorySpanExporter.create(); + var plugin2 = new ExecutionOtelPlugin( + SdkTracerProvider.builder().addSpanProcessor(SimpleSpanProcessor.create(exporter2)), + () -> null, + false, + "Workflow"); + plugin2.onInvocationStart(new InvocationInfo("req-9", ARN, true)); + plugin2.onInvocationEnd(new InvocationEndInfo("req-9", ARN, true, InvocationStatus.SUCCEEDED, null)); + var secondWorkflowSpanId = + spanByName(exporter2.getFinishedSpanItems(), "Workflow").getSpanId(); + + assertEquals( + firstWorkflowSpanId, + secondWorkflowSpanId, + "Workflow span ID must be deterministic for a given execution ARN"); + } + + // ─── Sampling ──────────────────────────────────────────────────────── + + @Test + void sampling_disabled_producesNoSpans() { + var exporter = InMemorySpanExporter.create(); + var sampledPlugin = new ExecutionOtelPlugin( + SdkTracerProvider.builder() + .setSampler(io.opentelemetry.sdk.trace.samplers.Sampler.alwaysOff()) + .addSpanProcessor(SimpleSpanProcessor.create(exporter)), + () -> null, + false, + "Workflow"); + sampledPlugin.onInvocationStart(new InvocationInfo("req-1", ARN, true)); + sampledPlugin.onInvocationEnd(new InvocationEndInfo("req-1", ARN, true, InvocationStatus.SUCCEEDED, null)); + assertTrue(exporter.getFinishedSpanItems().isEmpty(), "No spans should be exported with 0% sampling"); + } + + // ─── X-Ray trace ID ────────────────────────────────────────────────── + + @Test + void xrayExtraction_allSpansShareExtractedTraceId() { + var xrayTraceId = "aabbccddee112233445566778899aabb"; + var exporter = InMemorySpanExporter.create(); + var xrayPlugin = new ExecutionOtelPlugin( + SdkTracerProvider.builder().addSpanProcessor(SimpleSpanProcessor.create(exporter)), + () -> new ExtractedContext(xrayTraceId, null), + false, + "Workflow"); + xrayPlugin.onInvocationStart(new InvocationInfo("req-1", ARN, true)); + xrayPlugin.onOperationStart( + new OperationInfo("op-1", "step-a", "STEP", "Step", null, Instant.now(), null, false)); + xrayPlugin.onOperationEnd(new OperationEndInfo( + "op-1", "step-a", "STEP", "Step", null, Instant.now(), Instant.now(), "SUCCEEDED", false, null)); + xrayPlugin.onInvocationEnd(new InvocationEndInfo("req-1", ARN, true, InvocationStatus.SUCCEEDED, null)); + + var spans = exporter.getFinishedSpanItems(); + assertTrue(spans.size() >= 3, "Workflow + invocation + operation spans expected"); + assertTrue( + spans.stream().allMatch(s -> s.getTraceId().equals(xrayTraceId)), + "All spans must share the extracted X-Ray trace ID"); + } + + // ─── Helpers ───────────────────────────────────────────────────────── + + private static io.opentelemetry.sdk.trace.data.SpanData spanByName( + java.util.List spans, String name) { + return spans.stream() + .filter(s -> s.getName().equals(name)) + .findFirst() + .orElseThrow(() -> new AssertionError("No span named '" + name + "' in " + + spans.stream() + .map(io.opentelemetry.sdk.trace.data.SpanData::getName) + .toList())); + } +} diff --git a/otel-plugin/src/test/java/software/amazon/lambda/durable/otel/OtelPluginTest.java b/otel-plugin/src/test/java/software/amazon/lambda/durable/otel/OtelPluginTest.java index 4945c9a34..17c2b4820 100644 --- a/otel-plugin/src/test/java/software/amazon/lambda/durable/otel/OtelPluginTest.java +++ b/otel-plugin/src/test/java/software/amazon/lambda/durable/otel/OtelPluginTest.java @@ -46,7 +46,7 @@ void invocationStart_and_end_createsSpan() { var span = spans.get(0); assertEquals("invocation", span.getName()); - assertEquals(StatusCode.UNSET, span.getStatus().getStatusCode()); + assertEquals(StatusCode.OK, span.getStatus().getStatusCode()); } @Test From 56d15f929491d45dccbf8962fd9345c684e94c1f Mon Sep 17 00:00:00 2001 From: silanhe Date: Mon, 27 Jul 2026 22:07:50 +0000 Subject: [PATCH 2/6] Rename OtelPlugin to InvocationOtelPlugin Rename the invocation-rooted plugin to InvocationOtelPlugin to match the Python/JS SDK naming, now that ExecutionOtelPlugin (workflow-rooted) exists alongside it. Prerelease, so no backwards-compat alias is kept. - git mv OtelPlugin{,Test,IntegrationTest}.java -> InvocationOtelPlugin*.java - Update all references: MdcSpanEnricher(+Test), ExecutionOtelPlugin javadoc, 4 example handlers, otel-plugin/README.md, AGENTS.md. - ExecutionOtelPlugin and all other classes unchanged. Build: otel-plugin + examples green. --- AGENTS.md | 4 ++-- .../durable/examples/general/OtelExample.java | 4 ++-- .../examples/otel/OtelXRayExamples.java | 6 ++--- .../examples/otel/OtelXRayStepExample.java | 6 ++--- .../examples/otel/OtelXRayWaitExample.java | 6 ++--- otel-plugin/README.md | 14 ++++++------ .../durable/otel/ExecutionOtelPlugin.java | 18 +++++++-------- ...lPlugin.java => InvocationOtelPlugin.java} | 12 +++++----- .../lambda/durable/otel/MdcSpanEnricher.java | 4 ++-- ... InvocationOtelPluginIntegrationTest.java} | 6 ++--- ...est.java => InvocationOtelPluginTest.java} | 22 +++++++++---------- .../durable/otel/MdcSpanEnricherTest.java | 2 +- 12 files changed, 52 insertions(+), 52 deletions(-) rename otel-plugin/src/main/java/software/amazon/lambda/durable/otel/{OtelPlugin.java => InvocationOtelPlugin.java} (98%) rename otel-plugin/src/test/java/software/amazon/lambda/durable/otel/{OtelPluginIntegrationTest.java => InvocationOtelPluginIntegrationTest.java} (99%) rename otel-plugin/src/test/java/software/amazon/lambda/durable/otel/{OtelPluginTest.java => InvocationOtelPluginTest.java} (98%) diff --git a/AGENTS.md b/AGENTS.md index ed00b2862..643f53dbc 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -160,7 +160,7 @@ sdk-integration-tests/src/test/ # SDK components working together via LocalDura ├── MapIntegrationTest ├── ParallelIntegrationTest ├── WaitForConditionIntegrationTest -└── OtelPluginIntegrationTest +└── InvocationOtelPluginIntegrationTest otel-plugin/src/test/ # OpenTelemetry plugin unit tests @@ -272,7 +272,7 @@ void testAgainstRealLambda() { | `ParallelOperation` | Runs named child-context branches concurrently | | `ConcurrencyOperation` | Shared base for map/parallel concurrency limiting and completion evaluation | | `DurableExecutionPlugin` | Preview lifecycle hook interface | -| `OtelPlugin` | OpenTelemetry plugin implementation in `otel-plugin` | +| `InvocationOtelPlugin` | OpenTelemetry plugin implementation in `otel-plugin` | ### Serialization diff --git a/examples/src/main/java/software/amazon/lambda/durable/examples/general/OtelExample.java b/examples/src/main/java/software/amazon/lambda/durable/examples/general/OtelExample.java index 94b2be751..d8135975b 100644 --- a/examples/src/main/java/software/amazon/lambda/durable/examples/general/OtelExample.java +++ b/examples/src/main/java/software/amazon/lambda/durable/examples/general/OtelExample.java @@ -9,7 +9,7 @@ import software.amazon.lambda.durable.DurableContext; import software.amazon.lambda.durable.DurableHandler; import software.amazon.lambda.durable.examples.types.GreetingRequest; -import software.amazon.lambda.durable.otel.OtelPlugin; +import software.amazon.lambda.durable.otel.InvocationOtelPlugin; /** * Example demonstrating OpenTelemetry instrumentation with the Durable Execution SDK. @@ -39,7 +39,7 @@ public class OtelExample extends DurableHandler { @Override protected DurableConfig createConfiguration() { - var otelPlugin = new OtelPlugin( + var otelPlugin = new InvocationOtelPlugin( SdkTracerProvider.builder().addSpanProcessor(SimpleSpanProcessor.create(LoggingSpanExporter.create()))); return DurableConfig.builder().withPlugins(otelPlugin).build(); diff --git a/examples/src/main/java/software/amazon/lambda/durable/examples/otel/OtelXRayExamples.java b/examples/src/main/java/software/amazon/lambda/durable/examples/otel/OtelXRayExamples.java index e1e58b431..dcd6a875b 100644 --- a/examples/src/main/java/software/amazon/lambda/durable/examples/otel/OtelXRayExamples.java +++ b/examples/src/main/java/software/amazon/lambda/durable/examples/otel/OtelXRayExamples.java @@ -10,7 +10,7 @@ import software.amazon.lambda.durable.DurableContext; import software.amazon.lambda.durable.DurableHandler; import software.amazon.lambda.durable.examples.types.GreetingRequest; -import software.amazon.lambda.durable.otel.OtelPlugin; +import software.amazon.lambda.durable.otel.InvocationOtelPlugin; /** * OTel examples for map, parallel, and nested context operations. These are local-only examples (not deployed to @@ -22,8 +22,8 @@ private OtelXRayExamples() {} private static DurableConfig otelConfig() { var otlpExporter = OtlpGrpcSpanExporter.getDefault(); - var otelPlugin = - new OtelPlugin(SdkTracerProvider.builder().addSpanProcessor(SimpleSpanProcessor.create(otlpExporter))); + var otelPlugin = new InvocationOtelPlugin( + SdkTracerProvider.builder().addSpanProcessor(SimpleSpanProcessor.create(otlpExporter))); return DurableConfig.builder().withPlugins(otelPlugin).build(); } diff --git a/examples/src/main/java/software/amazon/lambda/durable/examples/otel/OtelXRayStepExample.java b/examples/src/main/java/software/amazon/lambda/durable/examples/otel/OtelXRayStepExample.java index f78b0fda7..14ef4814b 100644 --- a/examples/src/main/java/software/amazon/lambda/durable/examples/otel/OtelXRayStepExample.java +++ b/examples/src/main/java/software/amazon/lambda/durable/examples/otel/OtelXRayStepExample.java @@ -10,7 +10,7 @@ import software.amazon.lambda.durable.DurableHandler; import software.amazon.lambda.durable.examples.ExampleTemplate; import software.amazon.lambda.durable.examples.types.GreetingRequest; -import software.amazon.lambda.durable.otel.OtelPlugin; +import software.amazon.lambda.durable.otel.InvocationOtelPlugin; /** * OTel + X-Ray example: simple steps in a single invocation. @@ -40,8 +40,8 @@ protected DurableConfig createConfiguration() { // OTLP exporter sends spans to the ADOT collector (localhost:4317 by default) var otlpExporter = OtlpGrpcSpanExporter.getDefault(); - var otelPlugin = - new OtelPlugin(SdkTracerProvider.builder().addSpanProcessor(SimpleSpanProcessor.create(otlpExporter))); + var otelPlugin = new InvocationOtelPlugin( + SdkTracerProvider.builder().addSpanProcessor(SimpleSpanProcessor.create(otlpExporter))); return DurableConfig.builder().withPlugins(otelPlugin).build(); } diff --git a/examples/src/main/java/software/amazon/lambda/durable/examples/otel/OtelXRayWaitExample.java b/examples/src/main/java/software/amazon/lambda/durable/examples/otel/OtelXRayWaitExample.java index 4ee8e6551..f58fea837 100644 --- a/examples/src/main/java/software/amazon/lambda/durable/examples/otel/OtelXRayWaitExample.java +++ b/examples/src/main/java/software/amazon/lambda/durable/examples/otel/OtelXRayWaitExample.java @@ -11,7 +11,7 @@ import software.amazon.lambda.durable.DurableHandler; import software.amazon.lambda.durable.examples.ExampleTemplate; import software.amazon.lambda.durable.examples.types.GreetingRequest; -import software.amazon.lambda.durable.otel.OtelPlugin; +import software.amazon.lambda.durable.otel.InvocationOtelPlugin; /** * OTel + X-Ray example: step → wait → step pattern that forces multiple Lambda invocations. @@ -52,8 +52,8 @@ protected DurableConfig createConfiguration() { // OTLP exporter sends spans to the ADOT collector (localhost:4317 by default) var otlpExporter = OtlpGrpcSpanExporter.getDefault(); - var otelPlugin = - new OtelPlugin(SdkTracerProvider.builder().addSpanProcessor(SimpleSpanProcessor.create(otlpExporter))); + var otelPlugin = new InvocationOtelPlugin( + SdkTracerProvider.builder().addSpanProcessor(SimpleSpanProcessor.create(otlpExporter))); return DurableConfig.builder().withPlugins(otelPlugin).build(); } diff --git a/otel-plugin/README.md b/otel-plugin/README.md index fb77f9a12..8b2a06ea5 100644 --- a/otel-plugin/README.md +++ b/otel-plugin/README.md @@ -41,7 +41,7 @@ You also need the OpenTelemetry SDK and an exporter: 1. Add the ADOT Lambda Layer to your function 2. Enable X-Ray Active Tracing on the function -3. Register `OtelPlugin` in your handler's `DurableConfig` +3. Register `InvocationOtelPlugin` in your handler's `DurableConfig` 4. Grant X-Ray write permissions ### 1. ADOT Lambda Layer @@ -110,7 +110,7 @@ import io.opentelemetry.sdk.trace.export.SimpleSpanProcessor; import software.amazon.lambda.durable.DurableConfig; import software.amazon.lambda.durable.DurableContext; import software.amazon.lambda.durable.DurableHandler; -import software.amazon.lambda.durable.otel.OtelPlugin; +import software.amazon.lambda.durable.otel.InvocationOtelPlugin; public class MyHandler extends DurableHandler { @@ -118,7 +118,7 @@ public class MyHandler extends DurableHandler { protected DurableConfig createConfiguration() { var otlpExporter = OtlpGrpcSpanExporter.getDefault(); - var otelPlugin = new OtelPlugin( + var otelPlugin = new InvocationOtelPlugin( SdkTracerProvider.builder() .addSpanProcessor(SimpleSpanProcessor.create(otlpExporter))); @@ -215,13 +215,13 @@ These appear automatically in structured log output (Log4j2 JSON, Logback JSON) ```java // Default: X-Ray context extraction, MDC enabled -new OtelPlugin(tracerProviderBuilder); +new InvocationOtelPlugin(tracerProviderBuilder); // Custom context extractor, MDC enabled -new OtelPlugin(tracerProviderBuilder, contextExtractor); +new InvocationOtelPlugin(tracerProviderBuilder, contextExtractor); // Full configuration -new OtelPlugin(tracerProviderBuilder, contextExtractor, enableMdc); +new InvocationOtelPlugin(tracerProviderBuilder, contextExtractor, enableMdc); ``` | Parameter | Description | Default | @@ -262,7 +262,7 @@ For local testing, use a logging exporter to print spans to stdout: ```java import io.opentelemetry.exporter.logging.LoggingSpanExporter; -var otelPlugin = new OtelPlugin( +var otelPlugin = new InvocationOtelPlugin( SdkTracerProvider.builder() .addSpanProcessor(SimpleSpanProcessor.create(LoggingSpanExporter.create()))); ``` diff --git a/otel-plugin/src/main/java/software/amazon/lambda/durable/otel/ExecutionOtelPlugin.java b/otel-plugin/src/main/java/software/amazon/lambda/durable/otel/ExecutionOtelPlugin.java index c039f9040..814ff5830 100644 --- a/otel-plugin/src/main/java/software/amazon/lambda/durable/otel/ExecutionOtelPlugin.java +++ b/otel-plugin/src/main/java/software/amazon/lambda/durable/otel/ExecutionOtelPlugin.java @@ -48,15 +48,15 @@ * span, linked to the current Invocation span. * * - *

Contrast with {@link OtelPlugin} (the invocation-rooted variant, equivalent to the reference - * {@code InvocationOtelPlugin}): there the invocation span is the root and there is no Workflow span. Here the Workflow - * span is the root, operations hang off the Workflow span, and each operation/attempt links to the invocation that ran - * it. Both plugins share {@link DeterministicIdGenerator}, {@link ContextExtractor}, {@link SpanAttributes}, and - * {@link MdcSpanEnricher}. + *

Contrast with {@link InvocationOtelPlugin} (the invocation-rooted variant, equivalent to the reference + * {@code InvocationInvocationOtelPlugin}): there the invocation span is the root and there is no Workflow span. Here + * the Workflow span is the root, operations hang off the Workflow span, and each operation/attempt links to the + * invocation that ran it. Both plugins share {@link DeterministicIdGenerator}, {@link ContextExtractor}, + * {@link SpanAttributes}, and {@link MdcSpanEnricher}. * - *

Trace ID resolution matches {@link OtelPlugin}: the X-Ray trace ID from {@code _X_AMZN_TRACE_ID} when available - * (the backend propagates the same Root to all invocations, unifying the trace), else a deterministic trace ID derived - * from the execution ARN. + *

Trace ID resolution matches {@link InvocationOtelPlugin}: the X-Ray trace ID from {@code _X_AMZN_TRACE_ID} when + * available (the backend propagates the same Root to all invocations, unifying the trace), else a deterministic trace + * ID derived from the execution ARN. * *

Status mapping (parity with the Python/JS references): * @@ -154,7 +154,7 @@ public void onInvocationStart(InvocationInfo info) { idGenerator.setDurableExecutionArn(info.durableExecutionArn()); // Extract trace context from environment (X-Ray header). Only the trace ID is used — the Workflow span is a - // true root, so the X-Ray parent span ID is intentionally not used for parenting (unlike OtelPlugin). + // true root, so the X-Ray parent span ID is intentionally not used for parenting (unlike InvocationOtelPlugin). var extractedContext = contextExtractor.extract(); if (extractedContext != null) { idGenerator.setExtractedTraceId(extractedContext.traceId()); diff --git a/otel-plugin/src/main/java/software/amazon/lambda/durable/otel/OtelPlugin.java b/otel-plugin/src/main/java/software/amazon/lambda/durable/otel/InvocationOtelPlugin.java similarity index 98% rename from otel-plugin/src/main/java/software/amazon/lambda/durable/otel/OtelPlugin.java rename to otel-plugin/src/main/java/software/amazon/lambda/durable/otel/InvocationOtelPlugin.java index 7f7636f88..f0ff03db2 100644 --- a/otel-plugin/src/main/java/software/amazon/lambda/durable/otel/OtelPlugin.java +++ b/otel-plugin/src/main/java/software/amazon/lambda/durable/otel/InvocationOtelPlugin.java @@ -67,9 +67,9 @@ * @deprecated This is a preview API that is experimental and may be changed or removed in future releases. */ @Deprecated -public class OtelPlugin implements DurableExecutionPlugin { +public class InvocationOtelPlugin implements DurableExecutionPlugin { - private static final Logger logger = LoggerFactory.getLogger(OtelPlugin.class); + private static final Logger logger = LoggerFactory.getLogger(InvocationOtelPlugin.class); private static final String INSTRUMENTATION_NAME = "aws-durable-execution-sdk-java"; private final SdkTracerProvider tracerProvider; @@ -102,13 +102,13 @@ public class OtelPlugin implements DurableExecutionPlugin { * * 

{@code
      * var otlpExporter = OtlpGrpcSpanExporter.getDefault(); // sends to localhost:4317
-     * var plugin = new OtelPlugin(
+     * var plugin = new InvocationOtelPlugin(
      *     SdkTracerProvider.builder().addSpanProcessor(SimpleSpanProcessor.create(otlpExporter)));
      * }
* * @param tracerProviderBuilder the tracer provider builder (ID generator will be overridden) */ - public OtelPlugin(SdkTracerProviderBuilder tracerProviderBuilder) { + public InvocationOtelPlugin(SdkTracerProviderBuilder tracerProviderBuilder) { this(tracerProviderBuilder, new XRayContextExtractor(), true); } @@ -118,7 +118,7 @@ public OtelPlugin(SdkTracerProviderBuilder tracerProviderBuilder) { * @param tracerProviderBuilder the tracer provider builder (ID generator will be overridden) * @param contextExtractor extracts parent trace context from the Lambda environment */ - public OtelPlugin(SdkTracerProviderBuilder tracerProviderBuilder, ContextExtractor contextExtractor) { + public InvocationOtelPlugin(SdkTracerProviderBuilder tracerProviderBuilder, ContextExtractor contextExtractor) { this(tracerProviderBuilder, contextExtractor, true); } @@ -129,7 +129,7 @@ public OtelPlugin(SdkTracerProviderBuilder tracerProviderBuilder, ContextExtract * @param contextExtractor extracts parent trace context from the Lambda environment * @param enableMdc if true, injects traceId/spanId/traceSampled into SLF4J MDC for log correlation */ - public OtelPlugin( + public InvocationOtelPlugin( SdkTracerProviderBuilder tracerProviderBuilder, ContextExtractor contextExtractor, boolean enableMdc) { this.idGenerator = new DeterministicIdGenerator(); diff --git a/otel-plugin/src/main/java/software/amazon/lambda/durable/otel/MdcSpanEnricher.java b/otel-plugin/src/main/java/software/amazon/lambda/durable/otel/MdcSpanEnricher.java index 8a8571cc3..2b769e308 100644 --- a/otel-plugin/src/main/java/software/amazon/lambda/durable/otel/MdcSpanEnricher.java +++ b/otel-plugin/src/main/java/software/amazon/lambda/durable/otel/MdcSpanEnricher.java @@ -20,8 +20,8 @@ * * *

Usage: Call {@link #inject()} in {@code onUserFunctionStart} (after span is active) and {@link #clear()} in - * {@code onUserFunctionEnd}. Or use the convenience plugin {@link OtelPlugin} which handles this automatically when MDC - * enrichment is enabled. + * {@code onUserFunctionEnd}. Or use the convenience plugin {@link InvocationOtelPlugin} which handles this + * automatically when MDC enrichment is enabled. * * @deprecated This is a preview API that is experimental and may be changed or removed in future releases. */ diff --git a/otel-plugin/src/test/java/software/amazon/lambda/durable/otel/OtelPluginIntegrationTest.java b/otel-plugin/src/test/java/software/amazon/lambda/durable/otel/InvocationOtelPluginIntegrationTest.java similarity index 99% rename from otel-plugin/src/test/java/software/amazon/lambda/durable/otel/OtelPluginIntegrationTest.java rename to otel-plugin/src/test/java/software/amazon/lambda/durable/otel/InvocationOtelPluginIntegrationTest.java index d30555ecd..e52519a3d 100644 --- a/otel-plugin/src/test/java/software/amazon/lambda/durable/otel/OtelPluginIntegrationTest.java +++ b/otel-plugin/src/test/java/software/amazon/lambda/durable/otel/InvocationOtelPluginIntegrationTest.java @@ -27,7 +27,7 @@ * Integration tests verifying the OTel plugin produces correct spans when running with the real SDK execution engine * (LocalDurableTestRunner). */ -class OtelPluginIntegrationTest { +class InvocationOtelPluginIntegrationTest { private InMemorySpanExporter spanExporter; private DurableConfig otelConfig; @@ -36,7 +36,7 @@ class OtelPluginIntegrationTest { void setUp() { spanExporter = InMemorySpanExporter.create(); - var plugin = new OtelPlugin( + var plugin = new InvocationOtelPlugin( SdkTracerProvider.builder().addSpanProcessor(SimpleSpanProcessor.create(spanExporter)), () -> null, false); @@ -307,7 +307,7 @@ void failedStep_producesErrorSpan() { void sampling_off_producesNoSpans() { var sampledExporter = InMemorySpanExporter.create(); - var noSamplePlugin = new OtelPlugin( + var noSamplePlugin = new InvocationOtelPlugin( SdkTracerProvider.builder() .setSampler(Sampler.alwaysOff()) .addSpanProcessor(SimpleSpanProcessor.create(sampledExporter)), diff --git a/otel-plugin/src/test/java/software/amazon/lambda/durable/otel/OtelPluginTest.java b/otel-plugin/src/test/java/software/amazon/lambda/durable/otel/InvocationOtelPluginTest.java similarity index 98% rename from otel-plugin/src/test/java/software/amazon/lambda/durable/otel/OtelPluginTest.java rename to otel-plugin/src/test/java/software/amazon/lambda/durable/otel/InvocationOtelPluginTest.java index 17c2b4820..179661de2 100644 --- a/otel-plugin/src/test/java/software/amazon/lambda/durable/otel/OtelPluginTest.java +++ b/otel-plugin/src/test/java/software/amazon/lambda/durable/otel/InvocationOtelPluginTest.java @@ -15,16 +15,16 @@ import software.amazon.lambda.durable.execution.SuspendExecutionException; import software.amazon.lambda.durable.plugin.*; -class OtelPluginTest { +class InvocationOtelPluginTest { private InMemorySpanExporter spanExporter; - private OtelPlugin plugin; + private InvocationOtelPlugin plugin; @BeforeEach void setUp() { spanExporter = InMemorySpanExporter.create(); - plugin = new OtelPlugin( + plugin = new InvocationOtelPlugin( SdkTracerProvider.builder().addSpanProcessor(SimpleSpanProcessor.create(spanExporter)), () -> null, false); @@ -274,7 +274,7 @@ void operationNotCompleted_spanEndedAtInvocationEnd() { @Test void sampling_disabled_producesNoSpans() { spanExporter = InMemorySpanExporter.create(); - var sampledPlugin = new OtelPlugin( + var sampledPlugin = new InvocationOtelPlugin( SdkTracerProvider.builder() .setSampler(io.opentelemetry.sdk.trace.samplers.Sampler.alwaysOff()) .addSpanProcessor(SimpleSpanProcessor.create(spanExporter)), @@ -302,7 +302,7 @@ void xrayExtraction_usesExtractedTraceId_overArnDerived() { var extractedContext = new ExtractedContext(xrayTraceId, null); spanExporter = InMemorySpanExporter.create(); - var xrayPlugin = new OtelPlugin( + var xrayPlugin = new InvocationOtelPlugin( SdkTracerProvider.builder().addSpanProcessor(SimpleSpanProcessor.create(spanExporter)), () -> extractedContext, false); @@ -321,7 +321,7 @@ void xrayExtraction_allSpansShareExtractedTraceId() { var extractedContext = new ExtractedContext(xrayTraceId, null); spanExporter = InMemorySpanExporter.create(); - var xrayPlugin = new OtelPlugin( + var xrayPlugin = new InvocationOtelPlugin( SdkTracerProvider.builder().addSpanProcessor(SimpleSpanProcessor.create(spanExporter)), () -> extractedContext, false); @@ -351,7 +351,7 @@ void xrayExtraction_withParentSpanId_invocationSpanHasCorrectParent() { var extractedContext = new ExtractedContext(xrayTraceId, parentSpanId); spanExporter = InMemorySpanExporter.create(); - var xrayPlugin = new OtelPlugin( + var xrayPlugin = new InvocationOtelPlugin( SdkTracerProvider.builder().addSpanProcessor(SimpleSpanProcessor.create(spanExporter)), () -> extractedContext, false); @@ -376,7 +376,7 @@ void xrayExtraction_withoutParentSpanId_invocationSpanIsRoot() { var extractedContext = new ExtractedContext(xrayTraceId, null); spanExporter = InMemorySpanExporter.create(); - var xrayPlugin = new OtelPlugin( + var xrayPlugin = new InvocationOtelPlugin( SdkTracerProvider.builder().addSpanProcessor(SimpleSpanProcessor.create(spanExporter)), () -> extractedContext, false); @@ -406,7 +406,7 @@ void xrayExtraction_multipleInvocations_sameTraceId_unifiedTrace() { var extractedContext = new ExtractedContext(xrayTraceId, "53995c3f42cd8ad8"); spanExporter = InMemorySpanExporter.create(); - var xrayPlugin = new OtelPlugin( + var xrayPlugin = new InvocationOtelPlugin( SdkTracerProvider.builder().addSpanProcessor(SimpleSpanProcessor.create(spanExporter)), () -> extractedContext, false); @@ -440,7 +440,7 @@ void xrayExtraction_multipleInvocations_sameTraceId_unifiedTrace() { @Test void xrayExtraction_nullExtractor_fallsBackToArnDerived() { spanExporter = InMemorySpanExporter.create(); - var noXrayPlugin = new OtelPlugin( + var noXrayPlugin = new InvocationOtelPlugin( SdkTracerProvider.builder().addSpanProcessor(SimpleSpanProcessor.create(spanExporter)), () -> null, false); @@ -471,7 +471,7 @@ void xrayExtraction_extractedTraceIdMatchesXrayConversion() { // Now feed it through the plugin var extractedContext = new ExtractedContext(convertedId, null); spanExporter = InMemorySpanExporter.create(); - var xrayPlugin = new OtelPlugin( + var xrayPlugin = new InvocationOtelPlugin( SdkTracerProvider.builder().addSpanProcessor(SimpleSpanProcessor.create(spanExporter)), () -> extractedContext, false); diff --git a/otel-plugin/src/test/java/software/amazon/lambda/durable/otel/MdcSpanEnricherTest.java b/otel-plugin/src/test/java/software/amazon/lambda/durable/otel/MdcSpanEnricherTest.java index 3aaa627b0..e2fb17c60 100644 --- a/otel-plugin/src/test/java/software/amazon/lambda/durable/otel/MdcSpanEnricherTest.java +++ b/otel-plugin/src/test/java/software/amazon/lambda/durable/otel/MdcSpanEnricherTest.java @@ -46,7 +46,7 @@ void inject_withNoActiveSpan_doesNotSetMdcFields() { void plugin_withMdcEnabled_setsFieldsInMdc() { var spanExporter = InMemorySpanExporter.create(); - var plugin = new OtelPlugin( + var plugin = new InvocationOtelPlugin( SdkTracerProvider.builder().addSpanProcessor(SimpleSpanProcessor.create(spanExporter)), () -> null, true); From 8e5e84928f4a66ac515d78528146b58009eab5e6 Mon Sep 17 00:00:00 2001 From: silanhe Date: Mon, 27 Jul 2026 22:59:23 +0000 Subject: [PATCH 3/6] Set Workflow span kind to INTERNAL explicitly The workflow root span previously relied on OTel's implicit INTERNAL default; set it explicitly via setSpanKind(SpanKind.INTERNAL) and add a test asserting it. Workflow and invocation span status mappings are unchanged (already: Workflow SUCCEEDED->OK/FAILED->ERROR, non-terminal->UNSET/not-exported; Invocation SUCCEEDED+PENDING->OK, RETRYING+FAILED->ERROR). --- .../lambda/durable/otel/ExecutionOtelPlugin.java | 1 + .../lambda/durable/otel/ExecutionOtelPluginTest.java | 11 +++++++++++ 2 files changed, 12 insertions(+) diff --git a/otel-plugin/src/main/java/software/amazon/lambda/durable/otel/ExecutionOtelPlugin.java b/otel-plugin/src/main/java/software/amazon/lambda/durable/otel/ExecutionOtelPlugin.java index 814ff5830..202c6f8c1 100644 --- a/otel-plugin/src/main/java/software/amazon/lambda/durable/otel/ExecutionOtelPlugin.java +++ b/otel-plugin/src/main/java/software/amazon/lambda/durable/otel/ExecutionOtelPlugin.java @@ -164,6 +164,7 @@ public void onInvocationStart(InvocationInfo info) { // same ID so it is exported once as a single logical span (on the terminal invocation only). idGenerator.setNextSpanId(idGenerator.generateWorkflowSpanId()); workflowSpan = tracer.spanBuilder(workflowSpanName) + .setSpanKind(SpanKind.INTERNAL) .setNoParent() .setAttribute(DURABLE_EXECUTION_ARN, info.durableExecutionArn()) .startSpan(); diff --git a/otel-plugin/src/test/java/software/amazon/lambda/durable/otel/ExecutionOtelPluginTest.java b/otel-plugin/src/test/java/software/amazon/lambda/durable/otel/ExecutionOtelPluginTest.java index 4283064e1..d5d5907f8 100644 --- a/otel-plugin/src/test/java/software/amazon/lambda/durable/otel/ExecutionOtelPluginTest.java +++ b/otel-plugin/src/test/java/software/amazon/lambda/durable/otel/ExecutionOtelPluginTest.java @@ -48,6 +48,17 @@ void terminalInvocation_exportsWorkflowAndInvocationSpans() { assertEquals(StatusCode.OK, invocationSpan.getStatus().getStatusCode()); } + @Test + void workflowSpan_hasInternalKind() { + plugin.onInvocationStart(new InvocationInfo("req-1", ARN, true)); + plugin.onInvocationEnd(new InvocationEndInfo("req-1", ARN, true, InvocationStatus.SUCCEEDED, null)); + + assertEquals( + SpanKind.INTERNAL, + spanByName(spanExporter.getFinishedSpanItems(), "Workflow").getKind(), + "Workflow span must be INTERNAL kind"); + } + @Test void workflowSpan_isRoot_invocationSpanIsChild() { plugin.onInvocationStart(new InvocationInfo("req-1", ARN, true)); From 319bbc961fb78c2f469211cf06122ba60217b04f Mon Sep 17 00:00:00 2001 From: silanhe Date: Mon, 27 Jul 2026 23:25:23 +0000 Subject: [PATCH 4/6] ExecutionOtelPlugin: don't end open operation spans at invocation end Match the JS/Python ExecutionOtelPlugin export-once semantics: onInvocationEnd no longer force-ends still-open operation spans as PENDING. An operation left open when the invocation suspends is materialized exactly once (deterministic span ID + link to the completing invocation) when onOperationEnd fires later. Per-invocation state is still reset and lingering attempt scopes are closed defensively, but their spans are no longer force-ended. InvocationOtelPlugin (legacy, invocation-rooted) keeps its PENDING-at-end behavior unchanged. Tests: replace operationNotCompleted_endedPendingAtInvocationEnd with operationNotCompleted_notEndedAtInvocationEnd and add operationOpenedThenCompletedNextInvocation_exportedOnceOnOperationEnd. --- .../durable/otel/ExecutionOtelPlugin.java | 21 ++++----- .../durable/otel/ExecutionOtelPluginTest.java | 46 +++++++++++++++---- 2 files changed, 46 insertions(+), 21 deletions(-) diff --git a/otel-plugin/src/main/java/software/amazon/lambda/durable/otel/ExecutionOtelPlugin.java b/otel-plugin/src/main/java/software/amazon/lambda/durable/otel/ExecutionOtelPlugin.java index 202c6f8c1..4547510b9 100644 --- a/otel-plugin/src/main/java/software/amazon/lambda/durable/otel/ExecutionOtelPlugin.java +++ b/otel-plugin/src/main/java/software/amazon/lambda/durable/otel/ExecutionOtelPlugin.java @@ -185,23 +185,20 @@ public void onInvocationStart(InvocationInfo info) { @Override public void onInvocationEnd(InvocationEndInfo info) { - // End any operation spans that are still open (operations that didn't complete in this invocation) - for (var entry : operationSpans.entrySet()) { - var span = entry.getValue(); - span.setAttribute(DURABLE_OPERATION_STATUS, "PENDING"); - span.end(); - } + // Reset per-invocation operation state WITHOUT ending open operation spans. Matching the JS/Python + // ExecutionOtelPlugin, an operation span is only ended in onOperationEnd. An operation still open when the + // invocation suspends is left un-exported here and is re-materialized once (with its deterministic span ID, + // plus a link to the invocation that completes it) when onOperationEnd fires in a later invocation. operationSpans.clear(); operationContexts.clear(); - // End any attempt spans that are still open (e.g., crash before onUserFunctionEnd) - for (var entry : attemptScopes.entrySet()) { - entry.getValue().close(); + // Defensively close any lingering attempt scopes so OTel context is not leaked on worker threads (normally + // every onUserFunctionStart is paired with onUserFunctionEnd within the invocation). The attempt spans + // themselves are left un-ended rather than force-ended, consistent with not ending open spans here. + for (var scope : attemptScopes.values()) { + scope.close(); } attemptScopes.clear(); - for (var entry : attemptSpans.entrySet()) { - entry.getValue().end(); - } attemptSpans.clear(); // End the invocation span every invocation. diff --git a/otel-plugin/src/test/java/software/amazon/lambda/durable/otel/ExecutionOtelPluginTest.java b/otel-plugin/src/test/java/software/amazon/lambda/durable/otel/ExecutionOtelPluginTest.java index d5d5907f8..3c02432e5 100644 --- a/otel-plugin/src/test/java/software/amazon/lambda/durable/otel/ExecutionOtelPluginTest.java +++ b/otel-plugin/src/test/java/software/amazon/lambda/durable/otel/ExecutionOtelPluginTest.java @@ -262,20 +262,48 @@ void userFunctionFailure_setsErrorOnAttemptSpan() { } @Test - void operationNotCompleted_endedPendingAtInvocationEnd() { + void operationNotCompleted_notEndedAtInvocationEnd() { plugin.onInvocationStart(new InvocationInfo("req-1", ARN, true)); plugin.onOperationStart(new OperationInfo("op-1", "my-wait", "WAIT", "Wait", null, Instant.now(), null, false)); plugin.onInvocationEnd(new InvocationEndInfo("req-1", ARN, true, InvocationStatus.PENDING, null)); var spans = spanExporter.getFinishedSpanItems(); - // invocation span + PENDING operation span (Workflow not exported — non-terminal) - assertEquals(2, spans.size()); - var operationSpan = spanByName(spans, "my-wait"); - assertEquals( - "PENDING", - operationSpan - .getAttributes() - .get(io.opentelemetry.api.common.AttributeKey.stringKey("durable.operation.status"))); + // Only the invocation span is exported. The still-open operation span is NOT force-ended (no PENDING + // span here), and the Workflow span is not exported on a non-terminal invocation. + assertEquals(1, spans.size()); + assertEquals("invocation", spans.get(0).getName()); + assertTrue( + spans.stream().noneMatch(s -> s.getName().equals("my-wait")), + "An operation still open at invocation end must not be ended/exported in onInvocationEnd"); + } + + @Test + void operationOpenedThenCompletedNextInvocation_exportedOnceOnOperationEnd() { + // Invocation 1: operation opens but does not complete. + plugin.onInvocationStart(new InvocationInfo("req-1", ARN, true)); + plugin.onOperationStart(new OperationInfo("op-1", "my-wait", "WAIT", "Wait", null, Instant.now(), null, false)); + plugin.onInvocationEnd(new InvocationEndInfo("req-1", ARN, true, InvocationStatus.PENDING, null)); + assertTrue( + spanExporter.getFinishedSpanItems().stream() + .noneMatch(s -> s.getName().equals("my-wait")), + "Operation span must not be exported by the suspending invocation"); + spanExporter.reset(); + + // Invocation 2: the operation completes → materialized once via onOperationEnd, linked to this invocation. + plugin.onInvocationStart(new InvocationInfo("req-2", ARN, false)); + plugin.onOperationEnd(new OperationEndInfo( + "op-1", "my-wait", "WAIT", "Wait", null, Instant.now(), Instant.now(), "SUCCEEDED", false, null)); + plugin.onInvocationEnd(new InvocationEndInfo("req-2", ARN, false, InvocationStatus.SUCCEEDED, null)); + + var spans = spanExporter.getFinishedSpanItems(); + var waitSpans = + spans.stream().filter(s -> s.getName().equals("my-wait")).toList(); + assertEquals(1, waitSpans.size(), "The operation must be exported exactly once, when it completes"); + var invocationSpan = spanByName(spans, "invocation"); + assertTrue( + waitSpans.get(0).getLinks().stream() + .anyMatch(l -> l.getSpanContext().getSpanId().equals(invocationSpan.getSpanId())), + "Completed operation span should link to the invocation that completed it"); } // ─── Cross-invocation stitching ────────────────────────────────────── From 0f023619abbd750dab5b90a0c1c8b05a578e93b8 Mon Sep 17 00:00:00 2001 From: silanhe Date: Tue, 28 Jul 2026 00:03:39 +0000 Subject: [PATCH 5/6] Map RETRYING invocation-span status to UNSET, document rationale Invocation span status now maps SUCCEEDED/PENDING -> OK, FAILED -> ERROR, and RETRYING -> UNSET (previously ERROR), in both ExecutionOtelPlugin and InvocationOtelPlugin. RETRYING is left UNSET because the plugin interface does not expose whether the invocation/workflow was STOPPED or TIMED_OUT versus retried for a transient error, so ERROR cannot be asserted reliably. Workflow/execution span mapping is unchanged (FAILED->ERROR, SUCCEEDED->OK, RETRY/PENDING->UNSET via terminal-only export) and the workflow span kind remains explicitly INTERNAL. Class javadoc updated; tests updated/added for the RETRYING->UNSET behavior in both plugins. --- .../durable/otel/ExecutionOtelPlugin.java | 22 ++++++++++++++----- .../durable/otel/InvocationOtelPlugin.java | 15 +++++++++---- .../durable/otel/ExecutionOtelPluginTest.java | 7 ++++-- .../otel/InvocationOtelPluginTest.java | 14 ++++++++++++ 4 files changed, 47 insertions(+), 11 deletions(-) diff --git a/otel-plugin/src/main/java/software/amazon/lambda/durable/otel/ExecutionOtelPlugin.java b/otel-plugin/src/main/java/software/amazon/lambda/durable/otel/ExecutionOtelPlugin.java index 4547510b9..01dc1cdbc 100644 --- a/otel-plugin/src/main/java/software/amazon/lambda/durable/otel/ExecutionOtelPlugin.java +++ b/otel-plugin/src/main/java/software/amazon/lambda/durable/otel/ExecutionOtelPlugin.java @@ -61,11 +61,13 @@ *

Status mapping (parity with the Python/JS references): * *

    - *
  • Invocation span: {@code SUCCEEDED}/{@code PENDING} → {@link StatusCode#OK}; {@code RETRYING}/{@code FAILED} → - * {@link StatusCode#ERROR}. + *
  • Invocation span: {@code SUCCEEDED}/{@code PENDING} → {@link StatusCode#OK}; {@code FAILED} → + * {@link StatusCode#ERROR}; {@code RETRYING} → {@link StatusCode#UNSET}. {@code RETRYING} is left {@code UNSET} + * because the plugin interface does not expose whether the invocation/workflow was STOPPED or TIMED_OUT versus + * retried for a transient error, so an ERROR status cannot be asserted reliably. *
  • Workflow span (terminal only): {@code SUCCEEDED} → {@link StatusCode#OK}; {@code FAILED} → - * {@link StatusCode#ERROR}. Non-terminal statuses never end the Workflow span, so it is not exported this - * invocation. + * {@link StatusCode#ERROR}. Non-terminal statuses ({@code PENDING}/{@code RETRYING}) never end the Workflow span, + * so it is not exported this invocation (effectively {@link StatusCode#UNSET}). *
* *

Thread-safe: uses {@link ConcurrentHashMap} for span/scope storage since the SDK runs user code on multiple @@ -418,9 +420,16 @@ public void onUserFunctionEnd(UserFunctionEndInfo info) { // ─── Helpers ───────────────────────────────────────────────────────── private void applyInvocationStatus(Span span, InvocationEndInfo info) { + // Invocation span status mapping: + // SUCCEEDED, PENDING -> OK + // FAILED -> ERROR (records the execution error when present) + // RETRYING -> UNSET (left unset) + // RETRYING is left UNSET on purpose: the plugin interface does not expose whether the invocation/workflow + // was STOPPED or TIMED_OUT versus retried for a transient error, so an ERROR status cannot be asserted + // reliably for a retrying invocation. switch (info.invocationStatus()) { case SUCCEEDED, PENDING -> span.setStatus(StatusCode.OK); - case RETRYING, FAILED -> { + case FAILED -> { var message = info.executionError() != null ? info.executionError().getMessage() : null; span.setStatus(StatusCode.ERROR, message); @@ -428,6 +437,9 @@ private void applyInvocationStatus(Span span, InvocationEndInfo info) { span.recordException(info.executionError()); } } + case RETRYING -> { + // UNSET — see note above. + } } } diff --git a/otel-plugin/src/main/java/software/amazon/lambda/durable/otel/InvocationOtelPlugin.java b/otel-plugin/src/main/java/software/amazon/lambda/durable/otel/InvocationOtelPlugin.java index f0ff03db2..9543cca92 100644 --- a/otel-plugin/src/main/java/software/amazon/lambda/durable/otel/InvocationOtelPlugin.java +++ b/otel-plugin/src/main/java/software/amazon/lambda/durable/otel/InvocationOtelPlugin.java @@ -219,12 +219,16 @@ public void onInvocationEnd(InvocationEndInfo info) { invocationSpan.setAttribute( DURABLE_INVOCATION_STATUS, info.invocationStatus().name()); - // Status mapping (parity with Python/JS references): - // SUCCEEDED, PENDING -> OK - // RETRYING, FAILED -> ERROR (records the execution error when present) + // Invocation span status mapping: + // SUCCEEDED, PENDING -> OK + // FAILED -> ERROR (records the execution error when present) + // RETRYING -> UNSET + // RETRYING is left UNSET on purpose: the plugin interface does not expose whether the invocation/workflow + // was STOPPED or TIMED_OUT versus retried for a transient error, so an ERROR status cannot be asserted + // reliably for a retrying invocation. switch (info.invocationStatus()) { case SUCCEEDED, PENDING -> invocationSpan.setStatus(StatusCode.OK); - case RETRYING, FAILED -> { + case FAILED -> { var message = info.executionError() != null ? info.executionError().getMessage() : null; invocationSpan.setStatus(StatusCode.ERROR, message); @@ -232,6 +236,9 @@ public void onInvocationEnd(InvocationEndInfo info) { invocationSpan.recordException(info.executionError()); } } + case RETRYING -> { + // UNSET — see note above. + } } invocationSpan.end(); diff --git a/otel-plugin/src/test/java/software/amazon/lambda/durable/otel/ExecutionOtelPluginTest.java b/otel-plugin/src/test/java/software/amazon/lambda/durable/otel/ExecutionOtelPluginTest.java index 3c02432e5..5d6fa90f1 100644 --- a/otel-plugin/src/test/java/software/amazon/lambda/durable/otel/ExecutionOtelPluginTest.java +++ b/otel-plugin/src/test/java/software/amazon/lambda/durable/otel/ExecutionOtelPluginTest.java @@ -125,7 +125,7 @@ void failedInvocation_setsErrorOnBothWorkflowAndInvocationSpans() { } @Test - void retryingInvocation_invocationSpanError_workflowNotExported() { + void retryingInvocation_invocationSpanUnset_workflowNotExported() { plugin.onInvocationStart(new InvocationInfo("req-1", ARN, true)); plugin.onInvocationEnd(new InvocationEndInfo( "req-1", ARN, true, InvocationStatus.RETRYING, new RuntimeException("transient"))); @@ -134,7 +134,10 @@ void retryingInvocation_invocationSpanError_workflowNotExported() { assertEquals(1, spans.size(), "RETRYING is non-terminal — Workflow span not exported"); var invocationSpan = spans.get(0); assertEquals("invocation", invocationSpan.getName()); - assertEquals(StatusCode.ERROR, invocationSpan.getStatus().getStatusCode()); + assertEquals( + StatusCode.UNSET, + invocationSpan.getStatus().getStatusCode(), + "RETRYING invocation span is UNSET (interface cannot distinguish STOPPED/TIMED_OUT)"); } // ─── Operation topology: parented to Workflow, linked to invocation ── diff --git a/otel-plugin/src/test/java/software/amazon/lambda/durable/otel/InvocationOtelPluginTest.java b/otel-plugin/src/test/java/software/amazon/lambda/durable/otel/InvocationOtelPluginTest.java index 179661de2..aaadbdad0 100644 --- a/otel-plugin/src/test/java/software/amazon/lambda/durable/otel/InvocationOtelPluginTest.java +++ b/otel-plugin/src/test/java/software/amazon/lambda/durable/otel/InvocationOtelPluginTest.java @@ -119,6 +119,20 @@ void invocationEnd_withFailure_setsErrorStatus() { assertEquals(StatusCode.ERROR, spans.get(0).getStatus().getStatusCode()); } + @Test + void invocationEnd_withRetrying_leavesStatusUnset() { + plugin.onInvocationStart(new InvocationInfo("req-123", "arn:exec1", true)); + plugin.onInvocationEnd(new InvocationEndInfo( + "req-123", "arn:exec1", true, InvocationStatus.RETRYING, new RuntimeException("transient"))); + + var spans = spanExporter.getFinishedSpanItems(); + assertEquals(1, spans.size()); + assertEquals( + StatusCode.UNSET, + spans.get(0).getStatus().getStatusCode(), + "RETRYING invocation span is UNSET (interface cannot distinguish STOPPED/TIMED_OUT)"); + } + @Test void operationStart_createsSpan_operationEnd_endsIt() { plugin.onInvocationStart(new InvocationInfo("req-1", "arn:exec1", true)); From d5643d4ac71c36d807d7634ba7a668457ec694c3 Mon Sep 17 00:00:00 2001 From: silanhe Date: Tue, 28 Jul 2026 00:26:30 +0000 Subject: [PATCH 6/6] Fix inaccurate comment in ExecutionOtelPlugin continuation branch The prior comment said the continuation span 'stitches to the original' with the same deterministic ID. That is misleading: OTel spans cannot be edited after export, and the prior invocation drops its in-memory span un-exported. This branch is simply the single export of the operation's span, using its deterministic ID. Comment-only change. --- .../amazon/lambda/durable/otel/ExecutionOtelPlugin.java | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/otel-plugin/src/main/java/software/amazon/lambda/durable/otel/ExecutionOtelPlugin.java b/otel-plugin/src/main/java/software/amazon/lambda/durable/otel/ExecutionOtelPlugin.java index 01dc1cdbc..253e7a94c 100644 --- a/otel-plugin/src/main/java/software/amazon/lambda/durable/otel/ExecutionOtelPlugin.java +++ b/otel-plugin/src/main/java/software/amazon/lambda/durable/otel/ExecutionOtelPlugin.java @@ -292,8 +292,10 @@ public void onOperationEnd(OperationEndInfo info) { } endSpan(span, info.endTimestamp()); } else { - // Operation completed between invocations (started in a prior invocation). Recreate it with the same - // deterministic span ID so it stitches to the original, plus a link to the current invocation. + // Operation completed between invocations: its onOperationStart ran in a prior invocation, whose + // in-memory span was dropped un-exported at that invocation's end. Emit the operation's single span + // now, using its deterministic span ID (stable across the execution), plus a link to the invocation + // that completed it. operationContexts.remove(info.id()); idGenerator.setNextSpanOperationId(info.id());