In production, replace {@code LoggingSpanExporter} with {@code OtlpGrpcSpanExporter} to send spans to an OTLP - * collector (X-Ray, Datadog, etc.). + *
In production, replace {@code LoggingSpanExporter} with the exporter for your observability backend. * *
Expected trace structure: * *
- * durable.invocation - * ├── durable.step:create-greeting [attempt 1] - * ├── durable.step:create-greeting (operation, backfilled) - * ├── durable.step:transform [attempt 1] - * └── durable.step:transform (operation, backfilled) + * invocation + * ├── create-greeting + * │ └── create-greeting attempt 1 + * └── transform + * └── transform attempt 1 **/ public class OtelExample extends DurableHandler
{@link OtelPlugin#OtelPlugin()} uses the global provider initialized by the ADOT Java agent with deterministic
+ * span ID generation installed by the plugin's OpenTelemetry autoconfigure SPI.
+ */
+@ExampleTemplate(tracing = true, javaAgent = true)
+public class OtelXRayDefaultConstructorExample extends DurableHandler Exports spans through the ADOT Java agent global OpenTelemetry provider. Requires:
*
* Expected trace structure in X-Ray:
@@ -32,18 +30,12 @@
* └── transform attempt 1
*
*/
-@ExampleTemplate(tracing = true)
+@ExampleTemplate(tracing = true, javaAgent = true)
public class OtelXRayStepExample extends DurableHandler Exports spans via OTLP to the ADOT Lambda Layer collector. Requires:
+ * Exports spans through the ADOT Java agent global OpenTelemetry provider. Requires:
*
* Expected trace structure in X-Ray (all under one trace ID — backend propagates same Root):
*
* These tests deploy Lambda functions configured with:
*
*
*
*
*
*
*
*
* Trace (single trace ID across both invocations)
- * ├── durable.invocation (invocation 1)
- * │ ├── durable.step:before-wait
- * │ │ └── durable.step:before-wait [attempt 1]
- * │ └── durable.wait:pause (ended as PENDING)
- * └── durable.invocation (invocation 2)
- * ├── durable.wait:pause (completed)
- * └── durable.step:after-wait
- * └── durable.step:after-wait [attempt 1]
+ * ├── invocation (invocation 1)
+ * │ ├── before-wait
+ * │ │ └── before-wait attempt 1
+ * │ └── pause (ended as PENDING)
+ * └── invocation (invocation 2)
+ * ├── pause (completed)
+ * └── after-wait
+ * └── after-wait attempt 1
*
*/
-@ExampleTemplate(tracing = true)
+@ExampleTemplate(tracing = true, javaAgent = true)
public class OtelXRayWaitExample extends DurableHandler
- *
*
@@ -121,7 +126,7 @@ void simpleSteps_producesUnifiedTraceInXRay() throws Exception {
var uniqueInput = "XRay-" + System.currentTimeMillis();
var result = runner.run(new GreetingRequest(uniqueInput));
- assertEquals(ExecutionStatus.SUCCEEDED, result.getStatus(), "Execution failed: " + result);
+ assertExecutionSucceeded(result);
assertEquals("HELLO, " + uniqueInput.toUpperCase() + "!", result.getResult());
// 2. Wait for X-Ray ingestion
@@ -190,7 +195,7 @@ void waitAndResume_producesUnifiedTraceAcrossInvocations() throws Exception {
var uniqueInput = "Wait-" + System.currentTimeMillis();
var result = runner.run(new GreetingRequest(uniqueInput));
- assertEquals(ExecutionStatus.SUCCEEDED, result.getStatus(), "Execution failed: " + result);
+ assertExecutionSucceeded(result);
assertTrue(
result.getResult().contains("Resumed and completed"),
"Expected result to contain 'Resumed and completed', got: " + result.getResult());
@@ -251,8 +256,136 @@ void waitAndResume_producesUnifiedTraceAcrossInvocations() throws Exception {
+ invocationCount + " invocations in trace " + durableTrace.id());
}
+ @Test
+ void defaultConstructorWithAdotJavaAgent_producesDurableSpansInXRay() throws Exception {
+ var startTime = Instant.now();
+
+ var runner = CloudDurableTestRunner.create(
+ arn("otel-xray-default-constructor-example"), GreetingRequest.class, String.class, lambdaClient);
+ var uniqueInput = "Default-" + System.currentTimeMillis();
+ var execution = runner.startAsync(new GreetingRequest(uniqueInput));
+ var result = execution.pollUntilComplete();
+
+ assertExecutionSucceeded(result);
+ assertEquals("HELLO, " + uniqueInput.toUpperCase() + "!", result.getResult());
+
+ Thread.sleep(XRAY_INGESTION_DELAY.toMillis());
+
+ var traces = queryTracesWithRetry(startTime, Instant.now(), "otel-xray-default-constructor-example");
+ assertFalse(traces.isEmpty(), "Expected at least one trace in X-Ray after execution");
+
+ var allTraces = getFullTraces(traces);
+ var durableTrace = allTraces.stream()
+ .filter(trace ->
+ trace.segments().stream().anyMatch(seg -> segmentContains(seg, "default-create-greeting")))
+ .findFirst()
+ .orElse(null);
+
+ assertNotNull(
+ durableTrace,
+ "Expected to find a trace with default-create-greeting segment. Segment names: "
+ + allTraces.stream()
+ .flatMap(t -> t.segments().stream())
+ .map(CloudBasedOtelIntegrationTest::getSegmentName)
+ .collect(Collectors.joining(", ")));
+
+ var segmentDocuments =
+ durableTrace.segments().stream().map(Segment::document).toList();
+ var allSegmentText = String.join("\n", segmentDocuments);
+
+ assertTrue(allSegmentText.contains("invocation"), "Expected invocation span in trace");
+ assertTrue(
+ allSegmentText.contains("default-create-greeting"), "Expected default-create-greeting span in trace");
+ assertTrue(allSegmentText.contains("default-transform"), "Expected default-transform span in trace");
+ }
+
// ─── Helpers ─────────────────────────────────────────────────────────
+ private static void assertExecutionSucceeded(TestResult result) {
+ assertTrue(
+ result.isSucceeded() && result.getError().isEmpty(),
+ () -> "Execution failed:\n" + summarizeExecutionHistory(result));
+ }
+
+ private static String summarizeExecutionHistory(TestResult result) {
+ var failureSummary = result.getHistoryEvents().stream()
+ .map(CloudBasedOtelIntegrationTest::formatFailureEvent)
+ .filter(summary -> !summary.isBlank())
+ .collect(Collectors.joining("\n"));
+ if (!failureSummary.isBlank()) {
+ return failureSummary;
+ }
+
+ return result.getError()
+ .map(error -> "Execution error: " + formatError(error))
+ .orElseGet(() -> "Events: "
+ + result.getHistoryEvents().stream()
+ .map(event -> event.eventId() + ":" + event.eventType())
+ .collect(Collectors.joining(", ")));
+ }
+
+ private static String formatFailureEvent(Event event) {
+ if (EventType.INVOCATION_COMPLETED.equals(event.eventType())) {
+ var details = event.invocationCompletedDetails();
+ if (details != null && details.error() != null && details.error().payload() != null) {
+ return formatEventError(event, details.error().payload());
+ }
+ }
+
+ if (EventType.EXECUTION_FAILED.equals(event.eventType())) {
+ var details = event.executionFailedDetails();
+ if (details != null && details.error() != null && details.error().payload() != null) {
+ return formatEventError(event, details.error().payload());
+ }
+ }
+
+ if (EventType.EXECUTION_TIMED_OUT.equals(event.eventType())) {
+ var details = event.executionTimedOutDetails();
+ if (details != null && details.error() != null && details.error().payload() != null) {
+ return formatEventError(event, details.error().payload());
+ }
+ }
+
+ if (EventType.EXECUTION_STOPPED.equals(event.eventType())) {
+ var details = event.executionStoppedDetails();
+ if (details != null && details.error() != null && details.error().payload() != null) {
+ return formatEventError(event, details.error().payload());
+ }
+ }
+
+ return "";
+ }
+
+ private static String formatEventError(Event event, ErrorObject error) {
+ return event.eventType() + " event " + event.eventId() + ": " + formatError(error);
+ }
+
+ private static String formatError(ErrorObject error) {
+ var parts = new ArrayList
Requires the ADOT Lambda Layer for trace export. Configure with: * *
Note: Do NOT set {@code AWS_LAMBDA_EXEC_WRAPPER}. The collector extension runs independently. The wrapper would - * attach the auto-instrumentation agent which creates a competing TracerProvider. + *
When using {@link #OtelPlugin()}, the plugin requires {@code OtelPluginAutoConfigurationCustomizerProvider} to + * have been installed by the OpenTelemetry Java agent and uses the global provider directly. * *
Thread-safe: uses {@link ConcurrentHashMap} for span/scope storage since the SDK runs user code on multiple * threads. @@ -73,7 +78,7 @@ public class OtelPlugin implements DurableExecutionPlugin { private static final Logger logger = LoggerFactory.getLogger(OtelPlugin.class); private static final String INSTRUMENTATION_NAME = "aws-durable-execution-sdk-java"; - private final SdkTracerProvider tracerProvider; + private final SdkTracerProvider sdkTracerProvider; private final Tracer tracer; private final DeterministicIdGenerator idGenerator; private final ContextExtractor contextExtractor; @@ -99,12 +104,13 @@ public class OtelPlugin implements DurableExecutionPlugin { *
Uses the provided tracer provider builder. Customers configure exporters and span processors on the builder — * the plugin handles ID generation. * - *
For ADOT layer usage, configure with an OTLP exporter: + *
For ADOT Java agent usage, prefer {@link #OtelPlugin()} with the plugin jar configured through + * {@code OTEL_JAVAAGENT_EXTENSIONS}. Use this builder constructor when you want to own the exporter pipeline: * *
{@code
- * var otlpExporter = OtlpGrpcSpanExporter.getDefault(); // sends to localhost:4317
+ * var exporter = LoggingSpanExporter.create();
* var plugin = new OtelPlugin(
- * SdkTracerProvider.builder().addSpanProcessor(SimpleSpanProcessor.create(otlpExporter)));
+ * SdkTracerProvider.builder().addSpanProcessor(SimpleSpanProcessor.create(exporter)));
* }
*
* @param tracerProviderBuilder the tracer provider builder (ID generator will be overridden)
@@ -113,6 +119,16 @@ public OtelPlugin(SdkTracerProviderBuilder tracerProviderBuilder) {
this(tracerProviderBuilder, new XRayContextExtractor(), true);
}
+ /**
+ * Creates an OTel plugin with default settings: X-Ray context extraction and MDC enabled.
+ *
+ * Uses {@code GlobalOpenTelemetry} directly and assumes deterministic ID generation was installed by
+ * {@code OtelPluginAutoConfigurationCustomizerProvider}.
+ */
+ public OtelPlugin() {
+ this(getDefaultTracerProvider(), createDefaultIdGenerator());
+ }
+
/**
* Creates an OTel plugin with a custom context extractor, MDC enabled.
*
@@ -139,12 +155,22 @@ public OtelPlugin(
var resource = Resource.create(Attributes.of(ServiceAttributes.SERVICE_NAME, "invocation"));
tracerProviderBuilder.addResource(resource);
- this.tracerProvider = tracerProviderBuilder.setIdGenerator(idGenerator).build();
- this.tracer = tracerProvider.get(INSTRUMENTATION_NAME);
+ this.sdkTracerProvider =
+ tracerProviderBuilder.setIdGenerator(idGenerator).build();
+ this.tracer = sdkTracerProvider.get(INSTRUMENTATION_NAME);
this.contextExtractor = contextExtractor;
this.enableMdc = enableMdc;
}
+ private OtelPlugin(TracerProvider tracerProvider, DeterministicIdGenerator idGenerator) {
+ this.idGenerator = idGenerator;
+ this.sdkTracerProvider = getSdkTracerProviderForFlush(tracerProvider);
+ this.tracer = tracerProvider.get(INSTRUMENTATION_NAME);
+
+ this.contextExtractor = new XRayContextExtractor();
+ this.enableMdc = true;
+ }
+
// ─── Invocation hooks ────────────────────────────────────────────────
@Override
@@ -156,10 +182,15 @@ public void onInvocationStart(InvocationInfo info) {
// Extract trace context from environment (X-Ray header)
var extractedContext = contextExtractor.extract();
+ if (extractedContext == null) {
+ extractedContext = extractCurrentSpanContext();
+ }
if (extractedContext != null) {
// Use the X-Ray trace ID — backend propagates same Root across all invocations
idGenerator.setExtractedTraceId(extractedContext.traceId());
+ } else {
+ idGenerator.setExtractedTraceId(null);
}
// If no extracted context, idGenerator falls back to ARN-derived trace ID
@@ -167,7 +198,7 @@ public void onInvocationStart(InvocationInfo info) {
Context parentContext;
if (extractedContext != null && extractedContext.parentSpanId() != null) {
// X-Ray header has parent — create the invocation span as a child of that segment.
- // This connects our OTLP-exported spans to the Lambda service's X-Ray segments.
+ // This connects plugin spans to the Lambda service's X-Ray segments.
var parentSpanContext = SpanContext.createFromRemoteParent(
extractedContext.traceId(),
extractedContext.parentSpanId(),
@@ -228,10 +259,12 @@ public void onInvocationEnd(InvocationEndInfo info) {
invocationSpan.end();
invocationSpan = null;
- // Flush spans before Lambda freezes
- var flushResult = tracerProvider.forceFlush().join(5, java.util.concurrent.TimeUnit.SECONDS);
- if (!flushResult.isSuccess()) {
- logger.warn("OTel span flush failed or timed out — some spans may be lost");
+ if (sdkTracerProvider != null) {
+ // Flush spans before Lambda freezes
+ var flushResult = sdkTracerProvider.forceFlush().join(5, TimeUnit.SECONDS);
+ if (!flushResult.isSuccess()) {
+ logger.warn("OTel span flush failed or timed out — some spans may be lost");
+ }
}
}
@@ -465,4 +498,82 @@ private static String attemptSpanName(String type, String subType, String name,
private static String attemptKey(String operationId, Integer attempt) {
return operationId + "-" + (attempt != null ? attempt : "ctx");
}
+
+ private static ExtractedContext extractCurrentSpanContext() {
+ var spanContext = Span.current().getSpanContext();
+ if (!spanContext.isValid()) {
+ return null;
+ }
+ return new ExtractedContext(spanContext.getTraceId(), spanContext.getSpanId());
+ }
+
+ private static TracerProvider getDefaultTracerProvider() {
+ validateAutoConfigurationCustomizerProviderInstalled();
+
+ var globalTracerProvider = GlobalOpenTelemetry.getTracerProvider();
+ if (globalTracerProvider == TracerProvider.noop()) {
+ throw new IllegalStateException("OtelPlugin() requires GlobalOpenTelemetry to be initialized by "
+ + "OtelPluginAutoConfigurationCustomizerProvider through the OpenTelemetry Java agent.");
+ }
+ logger.info(
+ "OtelPlugin initialized from existing GlobalOpenTelemetry tracer provider {}; assuming deterministic "
+ + "span IDs were installed through AutoConfigurationCustomizerProvider",
+ globalTracerProvider.getClass().getName());
+ return globalTracerProvider;
+ }
+
+ private static DeterministicIdGenerator createDefaultIdGenerator() {
+ // This is intentionally a separate instance from the SPI provider's generator. The Java agent extension and
+ // application may load this plugin in different class loaders, so DeterministicIdGenerator bridges invocation
+ // state through system properties that the SPI-installed generator can read when spans are started.
+ return new DeterministicIdGenerator();
+ }
+
+ private static void validateAutoConfigurationCustomizerProviderInstalled() {
+ if (OtelPluginAutoConfigurationState.isInstalled()) {
+ return;
+ }
+ throw new IllegalStateException(
+ "OtelPlugin() requires OtelPluginAutoConfigurationCustomizerProvider to be installed by the "
+ + "OpenTelemetry Java agent. Package this plugin jar as an agent extension and set "
+ + "OTEL_JAVAAGENT_EXTENSIONS or -Dotel.javaagent.extensions to that jar before constructing "
+ + "OtelPlugin(). "
+ + javaAgentExtensionsDiagnostic());
+ }
+
+ private static String javaAgentExtensionsDiagnostic() {
+ var propertyValue = System.getProperty("otel.javaagent.extensions");
+ var environmentValue = System.getenv("OTEL_JAVAAGENT_EXTENSIONS");
+ var configuredPath = propertyValue != null ? propertyValue : environmentValue;
+ return "otel.javaagent.extensions="
+ + valueOrUnset(propertyValue)
+ + ", OTEL_JAVAAGENT_EXTENSIONS="
+ + valueOrUnset(environmentValue)
+ + ", configured extension path exists="
+ + extensionPathExists(configuredPath);
+ }
+
+ private static String valueOrUnset(String value) {
+ return value != null ? value : "