From 55509cb057213814e23e8796e5cb7f8fcd4fb302 Mon Sep 17 00:00:00 2001 From: Frank Chen Date: Mon, 13 Jul 2026 17:21:05 -0700 Subject: [PATCH 01/36] Add default constructor support for OTel plugin --- .../lambda/durable/otel/OtelPlugin.java | 82 ++++++++++++++++++- .../otel/OtelPluginIntegrationTest.java | 37 +++++++++ .../lambda/durable/otel/OtelPluginTest.java | 34 ++++++++ 3 files changed, 152 insertions(+), 1 deletion(-) diff --git a/otel-plugin/src/main/java/software/amazon/lambda/durable/otel/OtelPlugin.java b/otel-plugin/src/main/java/software/amazon/lambda/durable/otel/OtelPlugin.java index 3b89aa195..30dbeede4 100644 --- a/otel-plugin/src/main/java/software/amazon/lambda/durable/otel/OtelPlugin.java +++ b/otel-plugin/src/main/java/software/amazon/lambda/durable/otel/OtelPlugin.java @@ -4,6 +4,7 @@ import static software.amazon.lambda.durable.otel.SpanAttributes.*; +import io.opentelemetry.api.GlobalOpenTelemetry; import io.opentelemetry.api.common.AttributeKey; import io.opentelemetry.api.common.Attributes; import io.opentelemetry.api.trace.Span; @@ -13,14 +14,23 @@ import io.opentelemetry.api.trace.TraceFlags; import io.opentelemetry.api.trace.TraceState; import io.opentelemetry.api.trace.Tracer; +import io.opentelemetry.api.trace.TracerProvider; import io.opentelemetry.context.Context; import io.opentelemetry.context.Scope; +import io.opentelemetry.sdk.common.Clock; import io.opentelemetry.sdk.resources.Resource; import io.opentelemetry.sdk.trace.SdkTracerProvider; import io.opentelemetry.sdk.trace.SdkTracerProviderBuilder; +import io.opentelemetry.sdk.trace.SpanLimits; +import io.opentelemetry.sdk.trace.SpanProcessor; +import io.opentelemetry.sdk.trace.samplers.Sampler; import io.opentelemetry.semconv.ServiceAttributes; +import java.lang.reflect.Field; +import java.lang.reflect.InvocationTargetException; import java.time.Instant; import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.TimeUnit; +import java.util.function.Supplier; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import software.amazon.lambda.durable.plugin.DurableExecutionPlugin; @@ -113,6 +123,10 @@ public OtelPlugin(SdkTracerProviderBuilder tracerProviderBuilder) { this(tracerProviderBuilder, new XRayContextExtractor(), true); } + public OtelPlugin() { + this(getDefaultTracerProviderBuilder()); + } + /** * Creates an OTel plugin with a custom context extractor, MDC enabled. * @@ -229,7 +243,7 @@ public void onInvocationEnd(InvocationEndInfo info) { invocationSpan = null; // Flush spans before Lambda freezes - var flushResult = tracerProvider.forceFlush().join(5, java.util.concurrent.TimeUnit.SECONDS); + var flushResult = tracerProvider.forceFlush().join(5, TimeUnit.SECONDS); if (!flushResult.isSuccess()) { logger.warn("OTel span flush failed or timed out — some spans may be lost"); } @@ -461,4 +475,70 @@ private static String attemptSpanName(String type, String subType, String name, private static String attemptKey(String operationId, Integer attempt) { return operationId + "-" + (attempt != null ? attempt : "ctx"); } + + private static SdkTracerProviderBuilder getDefaultTracerProviderBuilder() { + var sdkTracerProvider = getGlobalSdkTracerProvider(); + var sharedState = getField(sdkTracerProvider, "sharedState", Object.class); + + return SdkTracerProvider.builder() + .setClock(getField(sharedState, "clock", Clock.class)) + .setResource(getField(sharedState, "resource", Resource.class)) + .setSpanLimits(getSpanLimitsSupplier(sharedState)) + .setSampler(getField(sharedState, "sampler", Sampler.class)) + .addSpanProcessor(getField(sharedState, "activeSpanProcessor", SpanProcessor.class)); + } + + private static SdkTracerProvider getGlobalSdkTracerProvider() { + var tracerProvider = GlobalOpenTelemetry.getTracerProvider(); + if (tracerProvider instanceof SdkTracerProvider sdkTracerProvider) { + return sdkTracerProvider; + } + return unobfuscateSdkTracerProvider(tracerProvider); + } + + private static SdkTracerProvider unobfuscateSdkTracerProvider(TracerProvider tracerProvider) { + try { + var unobfuscate = tracerProvider.getClass().getDeclaredMethod("unobfuscate"); + unobfuscate.setAccessible(true); + var sdkTracerProvider = unobfuscate.invoke(tracerProvider); + if (sdkTracerProvider instanceof SdkTracerProvider result) { + return result; + } + throw new IllegalStateException("GlobalOpenTelemetry is not backed by an SdkTracerProvider"); + } catch (NoSuchMethodException e) { + throw new IllegalStateException( + "OtelPlugin() requires GlobalOpenTelemetry to be backed by the OpenTelemetry SDK. " + + "Use OtelPlugin(SdkTracerProviderBuilder) when the global provider cannot be copied.", + e); + } catch (IllegalAccessException | InvocationTargetException e) { + throw new IllegalStateException("Unable to copy the global OpenTelemetry tracer provider", e); + } + } + + @SuppressWarnings("unchecked") + private static Supplier getSpanLimitsSupplier(Object sharedState) { + return (Supplier) getField(sharedState, "spanLimitsSupplier", Supplier.class); + } + + private static T getField(Object target, String fieldName, Class fieldType) { + try { + var field = findField(target.getClass(), fieldName); + field.setAccessible(true); + return fieldType.cast(field.get(target)); + } catch (NoSuchFieldException | IllegalAccessException e) { + throw new IllegalStateException("Unable to copy OpenTelemetry SDK field: " + fieldName, e); + } + } + + private static Field findField(Class type, String fieldName) throws NoSuchFieldException { + var current = type; + while (current != null) { + try { + return current.getDeclaredField(fieldName); + } catch (NoSuchFieldException e) { + current = current.getSuperclass(); + } + } + throw new NoSuchFieldException(fieldName); + } } diff --git a/otel-plugin/src/test/java/software/amazon/lambda/durable/otel/OtelPluginIntegrationTest.java b/otel-plugin/src/test/java/software/amazon/lambda/durable/otel/OtelPluginIntegrationTest.java index d30555ecd..3c6078355 100644 --- a/otel-plugin/src/test/java/software/amazon/lambda/durable/otel/OtelPluginIntegrationTest.java +++ b/otel-plugin/src/test/java/software/amazon/lambda/durable/otel/OtelPluginIntegrationTest.java @@ -4,7 +4,9 @@ import static org.junit.jupiter.api.Assertions.*; +import io.opentelemetry.api.GlobalOpenTelemetry; import io.opentelemetry.api.trace.StatusCode; +import io.opentelemetry.sdk.OpenTelemetrySdk; import io.opentelemetry.sdk.testing.exporter.InMemorySpanExporter; import io.opentelemetry.sdk.trace.SdkTracerProvider; import io.opentelemetry.sdk.trace.data.SpanData; @@ -13,6 +15,7 @@ import java.time.Duration; import java.util.List; import java.util.concurrent.atomic.AtomicInteger; +import org.junit.jupiter.api.AfterEach; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; import software.amazon.awssdk.services.lambda.model.ErrorObject; @@ -44,6 +47,11 @@ void setUp() { otelConfig = DurableConfig.builder().withPlugins(plugin).build(); } + @AfterEach + void tearDown() { + GlobalOpenTelemetry.resetForTest(); + } + @Test void simpleStep_producesInvocationAndOperationAndAttemptSpans() { var runner = LocalDurableTestRunner.create( @@ -67,6 +75,35 @@ void simpleStep_producesInvocationAndOperationAndAttemptSpans() { assertTrue(spans.stream().allMatch(s -> s.getTraceId().equals(traceId))); } + @Test + void defaultConstructor_copiesGlobalSdkTracerProviderPipeline() { + GlobalOpenTelemetry.resetForTest(); + var globalExporter = InMemorySpanExporter.create(); + var globalTracerProvider = SdkTracerProvider.builder() + .addSpanProcessor(SimpleSpanProcessor.create(globalExporter)) + .build(); + OpenTelemetrySdk.builder().setTracerProvider(globalTracerProvider).buildAndRegisterGlobal(); + + var defaultConfig = + DurableConfig.builder().withPlugins(new OtelPlugin()).build(); + var runner = LocalDurableTestRunner.create( + String.class, + (input, ctx) -> ctx.step("global-step", String.class, stepCtx -> "Hello " + input), + defaultConfig); + + var result = runner.runUntilComplete("World"); + assertEquals(ExecutionStatus.SUCCEEDED, result.getStatus()); + + var spans = globalExporter.getFinishedSpanItems(); + assertTrue(spans.size() >= 3, "Expected at least 3 spans, got " + spans.size()); + assertSpanExists(spans, "invocation"); + assertSpanExists(spans, "global-step"); + assertSpanExists(spans, "global-step attempt 1"); + + var traceId = spans.get(0).getTraceId(); + assertTrue(spans.stream().allMatch(span -> span.getTraceId().equals(traceId))); + } + @Test void multipleSteps_producesSpansForEach() { var runner = LocalDurableTestRunner.create( diff --git a/otel-plugin/src/test/java/software/amazon/lambda/durable/otel/OtelPluginTest.java b/otel-plugin/src/test/java/software/amazon/lambda/durable/otel/OtelPluginTest.java index de88ea195..af5e46c00 100644 --- a/otel-plugin/src/test/java/software/amazon/lambda/durable/otel/OtelPluginTest.java +++ b/otel-plugin/src/test/java/software/amazon/lambda/durable/otel/OtelPluginTest.java @@ -4,12 +4,15 @@ import static org.junit.jupiter.api.Assertions.*; +import io.opentelemetry.api.GlobalOpenTelemetry; import io.opentelemetry.api.trace.SpanKind; import io.opentelemetry.api.trace.StatusCode; +import io.opentelemetry.sdk.OpenTelemetrySdk; import io.opentelemetry.sdk.testing.exporter.InMemorySpanExporter; import io.opentelemetry.sdk.trace.SdkTracerProvider; import io.opentelemetry.sdk.trace.export.SimpleSpanProcessor; import java.time.Instant; +import org.junit.jupiter.api.AfterEach; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; import software.amazon.lambda.durable.execution.SuspendExecutionException; @@ -30,6 +33,37 @@ void setUp() { false); } + @AfterEach + void tearDown() { + GlobalOpenTelemetry.resetForTest(); + } + + @Test + void defaultConstructor_copiesGlobalSdkTracerProviderPipeline() { + GlobalOpenTelemetry.resetForTest(); + var globalExporter = InMemorySpanExporter.create(); + var globalTracerProvider = SdkTracerProvider.builder() + .addSpanProcessor(SimpleSpanProcessor.create(globalExporter)) + .build(); + OpenTelemetrySdk.builder().setTracerProvider(globalTracerProvider).buildAndRegisterGlobal(); + + var defaultPlugin = new OtelPlugin(); + defaultPlugin.onInvocationStart(new InvocationInfo("req-1", "arn:exec1", true)); + defaultPlugin.onOperationStart( + new OperationInfo("op-1", "step", "STEP", "Step", null, Instant.now(), null, false)); + defaultPlugin.onOperationEnd(new OperationEndInfo( + "op-1", "step", "STEP", "Step", null, Instant.now(), Instant.now(), "SUCCEEDED", false, null)); + defaultPlugin.onInvocationEnd( + new InvocationEndInfo("req-1", "arn:exec1", true, InvocationStatus.SUCCEEDED, null)); + + var spans = globalExporter.getFinishedSpanItems(); + assertEquals(2, spans.size()); + assertTrue(spans.stream().anyMatch(span -> span.getName().equals("invocation"))); + assertTrue(spans.stream().anyMatch(span -> span.getName().equals("step"))); + var traceId = spans.get(0).getTraceId(); + assertTrue(spans.stream().allMatch(span -> span.getTraceId().equals(traceId))); + } + @Test void invocationStart_and_end_createsSpan() { plugin.onInvocationStart(new InvocationInfo( From 7677f17439d51c7f86a44a5951b1d867f877b7a3 Mon Sep 17 00:00:00 2001 From: Zhongke Chen Date: Tue, 14 Jul 2026 00:51:52 +0000 Subject: [PATCH 02/36] Add cloud coverage for default OTel plugin constructor --- examples/README.md | 1 + examples/generate-template.py | 24 ++++++-- .../durable/examples/ExampleTemplate.java | 2 + .../OtelXRayDefaultConstructorExample.java | 37 ++++++++++++ .../CloudBasedOtelIntegrationTest.java | 56 +++++++++++++++++++ 5 files changed, 115 insertions(+), 5 deletions(-) create mode 100644 examples/src/main/java/software/amazon/lambda/durable/examples/otel/OtelXRayDefaultConstructorExample.java diff --git a/examples/README.md b/examples/README.md index f10eac11d..f660d0d2d 100644 --- a/examples/README.md +++ b/examples/README.md @@ -96,6 +96,7 @@ mvn test -Dtest=CloudBasedIntegrationTest \ | [CustomShouldCompleteMapExample](src/main/java/software/amazon/lambda/durable/examples/map/CustomShouldCompleteMapExample.java) | Custom map completion with `shouldComplete` decisions | | [WaitForConditionExample](src/main/java/software/amazon/lambda/durable/examples/wait/WaitForConditionExample.java) | Poll a condition until met with `waitForCondition()` | | [OtelExample](src/main/java/software/amazon/lambda/durable/examples/general/OtelExample.java) | OpenTelemetry instrumentation with logging span export | +| [OtelXRayDefaultConstructorExample](src/main/java/software/amazon/lambda/durable/examples/otel/OtelXRayDefaultConstructorExample.java) | Export spans to X-Ray with the ADOT Java agent and `new OtelPlugin()` | | [OtelXRayStepExample](src/main/java/software/amazon/lambda/durable/examples/otel/OtelXRayStepExample.java) | Export step spans to X-Ray through the ADOT Lambda Layer | | [OtelXRayWaitExample](src/main/java/software/amazon/lambda/durable/examples/otel/OtelXRayWaitExample.java) | Trace a step-wait-step workflow across Lambda invocations | | [OtelXRayMapExample](src/main/java/software/amazon/lambda/durable/examples/otel/OtelXRayMapExample.java) | Trace concurrent map operations and item steps in X-Ray | diff --git a/examples/generate-template.py b/examples/generate-template.py index 75e6cce09..332a02570 100755 --- a/examples/generate-template.py +++ b/examples/generate-template.py @@ -22,6 +22,7 @@ class ExampleFunction: suffix: str condition: str | None tracing: bool + adot_java_agent: bool @property def logical_id(self) -> str: @@ -58,24 +59,26 @@ def is_top_level_durable_handler(source: str, class_name: str) -> bool: return bool(match and "extends DurableHandler" in match.group("header")) -def read_template_annotation(source: str, class_name: str) -> tuple[str | None, bool]: +def read_template_annotation(source: str, class_name: str) -> tuple[str | None, bool, bool]: class_match = re.search(rf"public\s+(?:final\s+)?class\s+{class_name}\b", source) if not class_match: - return None, False + return None, False, False prefix = source[: class_match.start()] matches = list( re.finditer(rf"@(?:[A-Za-z_][\w.]*\.)?{TEMPLATE_ANNOTATION}\s*(?:\((?P.*?)\))?", prefix, re.DOTALL) ) if not matches: - return None, False + return None, False, False body = matches[-1].group("body") or "" condition_match = re.search(r'condition\s*=\s*"([^"]+)"', body) tracing_match = re.search(r"tracing\s*=\s*(true|false)", body) condition = condition_match.group(1) if condition_match else None tracing = tracing_match.group(1) == "true" if tracing_match else False - return condition, tracing + adot_java_agent_match = re.search(r"adotJavaAgent\s*=\s*(true|false)", body) + adot_java_agent = adot_java_agent_match.group(1) == "true" if adot_java_agent_match else False + return condition, tracing, adot_java_agent def discover_examples() -> list[ExampleFunction]: @@ -86,7 +89,7 @@ def discover_examples() -> list[ExampleFunction]: if not is_top_level_durable_handler(source, class_name): continue - condition, tracing = read_template_annotation(source, class_name) + condition, tracing, adot_java_agent = read_template_annotation(source, class_name) package_name = read_package(source, path) examples.append( ExampleFunction( @@ -95,6 +98,7 @@ def discover_examples() -> list[ExampleFunction]: suffix=kebab_case(class_name), condition=condition, tracing=tracing, + adot_java_agent=adot_java_agent, ) ) return examples @@ -128,6 +132,16 @@ def emit_function(lines: list[str], example: ExampleFunction) -> None: " - AdotArch: amd64", ] ) + if example.adot_java_agent: + lines.extend( + [ + " Environment:", + " Variables:", + " AWS_LAMBDA_EXEC_WRAPPER: /opt/otel-handler", + " OTEL_METRICS_EXPORTER: none", + " OTEL_LOGS_EXPORTER: none", + ] + ) lines.append("") diff --git a/examples/src/main/java/software/amazon/lambda/durable/examples/ExampleTemplate.java b/examples/src/main/java/software/amazon/lambda/durable/examples/ExampleTemplate.java index 898634884..dcb8417c3 100644 --- a/examples/src/main/java/software/amazon/lambda/durable/examples/ExampleTemplate.java +++ b/examples/src/main/java/software/amazon/lambda/durable/examples/ExampleTemplate.java @@ -14,4 +14,6 @@ String condition() default ""; boolean tracing() default false; + + boolean adotJavaAgent() default false; } diff --git a/examples/src/main/java/software/amazon/lambda/durable/examples/otel/OtelXRayDefaultConstructorExample.java b/examples/src/main/java/software/amazon/lambda/durable/examples/otel/OtelXRayDefaultConstructorExample.java new file mode 100644 index 000000000..ded1fb50a --- /dev/null +++ b/examples/src/main/java/software/amazon/lambda/durable/examples/otel/OtelXRayDefaultConstructorExample.java @@ -0,0 +1,37 @@ +// Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +// SPDX-License-Identifier: Apache-2.0 +package software.amazon.lambda.durable.examples.otel; + +import software.amazon.lambda.durable.DurableConfig; +import software.amazon.lambda.durable.DurableContext; +import software.amazon.lambda.durable.DurableHandler; +import software.amazon.lambda.durable.examples.ExampleTemplate; +import software.amazon.lambda.durable.examples.types.GreetingRequest; +import software.amazon.lambda.durable.otel.OtelPlugin; + +/** + * OTel + X-Ray example that uses the no-arg plugin constructor with the ADOT Java agent. + * + *

The ADOT Java agent registers the global OpenTelemetry SDK provider. {@link OtelPlugin#OtelPlugin()} copies that + * provider's export pipeline, then installs the durable deterministic ID generator. + */ +@ExampleTemplate(tracing = true, adotJavaAgent = true) +public class OtelXRayDefaultConstructorExample extends DurableHandler { + + @Override + protected DurableConfig createConfiguration() { + return DurableConfig.builder().withPlugins(new OtelPlugin()).build(); + } + + @Override + public String handleRequest(GreetingRequest input, DurableContext context) { + context.getLogger().info("Starting OTel X-Ray default constructor example for {}", input.getName()); + + var greeting = context.step("default-create-greeting", String.class, stepCtx -> "Hello, " + input.getName()); + + var result = context.step("default-transform", String.class, stepCtx -> greeting.toUpperCase() + "!"); + + context.getLogger().info("OTel X-Ray default constructor example complete: {}", result); + return result; + } +} diff --git a/examples/src/test/java/software/amazon/lambda/durable/examples/CloudBasedOtelIntegrationTest.java b/examples/src/test/java/software/amazon/lambda/durable/examples/CloudBasedOtelIntegrationTest.java index 7c0c47564..b9aa339fd 100644 --- a/examples/src/test/java/software/amazon/lambda/durable/examples/CloudBasedOtelIntegrationTest.java +++ b/examples/src/test/java/software/amazon/lambda/durable/examples/CloudBasedOtelIntegrationTest.java @@ -7,6 +7,7 @@ import com.fasterxml.jackson.databind.ObjectMapper; import java.time.Duration; import java.time.Instant; +import java.util.ArrayList; import java.util.List; import java.util.Set; import java.util.stream.Collectors; @@ -22,6 +23,7 @@ import software.amazon.awssdk.services.xray.model.GetTraceSummariesRequest; import software.amazon.awssdk.services.xray.model.Segment; import software.amazon.awssdk.services.xray.model.TimeRangeType; +import software.amazon.awssdk.services.xray.model.Trace; import software.amazon.awssdk.services.xray.model.TraceSummary; import software.amazon.lambda.durable.examples.types.GreetingRequest; import software.amazon.lambda.durable.model.ExecutionStatus; @@ -251,8 +253,62 @@ void waitAndResume_producesUnifiedTraceAcrossInvocations() throws Exception { + invocationCount + " invocations in trace " + durableTrace.id()); } + @Test + void defaultConstructorWithAdotJavaAgent_producesDurableSpansInXRay() throws Exception { + var startTime = Instant.now(); + + var runner = CloudDurableTestRunner.create( + arn("otel-xray-default-constructor-example"), GreetingRequest.class, String.class, lambdaClient); + var uniqueInput = "Default-" + System.currentTimeMillis(); + var result = runner.run(new GreetingRequest(uniqueInput)); + + assertEquals(ExecutionStatus.SUCCEEDED, result.getStatus(), "Execution failed: " + result); + assertEquals("HELLO, " + uniqueInput.toUpperCase() + "!", result.getResult()); + + Thread.sleep(XRAY_INGESTION_DELAY.toMillis()); + + var traces = queryTracesWithRetry(startTime, Instant.now(), "otel-xray-default-constructor-example"); + assertFalse(traces.isEmpty(), "Expected at least one trace in X-Ray after execution"); + + var allTraces = getFullTraces(traces); + var durableTrace = allTraces.stream() + .filter(trace -> trace.segments().stream() + .anyMatch(seg -> segmentContains(seg, "default-create-greeting"))) + .findFirst() + .orElse(null); + + assertNotNull( + durableTrace, + "Expected to find a trace with default-create-greeting segment. Segment names: " + + allTraces.stream() + .flatMap(t -> t.segments().stream()) + .map(CloudBasedOtelIntegrationTest::getSegmentName) + .collect(Collectors.joining(", "))); + + var segmentDocuments = + durableTrace.segments().stream().map(Segment::document).toList(); + var allSegmentText = String.join("\n", segmentDocuments); + + assertTrue(allSegmentText.contains("invocation"), "Expected invocation span in trace"); + assertTrue( + allSegmentText.contains("default-create-greeting"), "Expected default-create-greeting span in trace"); + assertTrue(allSegmentText.contains("default-transform"), "Expected default-transform span in trace"); + } + // ─── Helpers ───────────────────────────────────────────────────────── + private List getFullTraces(List traces) { + var traceIds = traces.stream().map(TraceSummary::id).toList(); + var allTraces = new ArrayList(); + for (int i = 0; i < traceIds.size(); i += 5) { + var batch = traceIds.subList(i, Math.min(i + 5, traceIds.size())); + var batchResult = xrayClient.batchGetTraces( + BatchGetTracesRequest.builder().traceIds(batch).build()); + allTraces.addAll(batchResult.traces()); + } + return allTraces; + } + /** Queries X-Ray for traces with retry logic to handle eventual consistency. */ private List queryTracesWithRetry(Instant startTime, Instant endTime, String functionName) throws InterruptedException { From 24c8830733c2783b404a76d4027553efee188a19 Mon Sep 17 00:00:00 2001 From: Zhongke Chen Date: Tue, 14 Jul 2026 00:55:09 +0000 Subject: [PATCH 03/36] Fix Spotless formatting --- .../durable/examples/CloudBasedOtelIntegrationTest.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/examples/src/test/java/software/amazon/lambda/durable/examples/CloudBasedOtelIntegrationTest.java b/examples/src/test/java/software/amazon/lambda/durable/examples/CloudBasedOtelIntegrationTest.java index b9aa339fd..c73323077 100644 --- a/examples/src/test/java/software/amazon/lambda/durable/examples/CloudBasedOtelIntegrationTest.java +++ b/examples/src/test/java/software/amazon/lambda/durable/examples/CloudBasedOtelIntegrationTest.java @@ -272,8 +272,8 @@ void defaultConstructorWithAdotJavaAgent_producesDurableSpansInXRay() throws Exc var allTraces = getFullTraces(traces); var durableTrace = allTraces.stream() - .filter(trace -> trace.segments().stream() - .anyMatch(seg -> segmentContains(seg, "default-create-greeting"))) + .filter(trace -> + trace.segments().stream().anyMatch(seg -> segmentContains(seg, "default-create-greeting"))) .findFirst() .orElse(null); From 1bf20f7e51abe2800f9b71ef3efe3a6c62e953d4 Mon Sep 17 00:00:00 2001 From: Zhongke Chen Date: Tue, 14 Jul 2026 01:09:22 +0000 Subject: [PATCH 04/36] Use async start for ADOT default constructor e2e --- .../lambda/durable/examples/CloudBasedOtelIntegrationTest.java | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/examples/src/test/java/software/amazon/lambda/durable/examples/CloudBasedOtelIntegrationTest.java b/examples/src/test/java/software/amazon/lambda/durable/examples/CloudBasedOtelIntegrationTest.java index c73323077..30f290116 100644 --- a/examples/src/test/java/software/amazon/lambda/durable/examples/CloudBasedOtelIntegrationTest.java +++ b/examples/src/test/java/software/amazon/lambda/durable/examples/CloudBasedOtelIntegrationTest.java @@ -260,7 +260,8 @@ void defaultConstructorWithAdotJavaAgent_producesDurableSpansInXRay() throws Exc var runner = CloudDurableTestRunner.create( arn("otel-xray-default-constructor-example"), GreetingRequest.class, String.class, lambdaClient); var uniqueInput = "Default-" + System.currentTimeMillis(); - var result = runner.run(new GreetingRequest(uniqueInput)); + var execution = runner.startAsync(new GreetingRequest(uniqueInput)); + var result = execution.pollUntilComplete(); assertEquals(ExecutionStatus.SUCCEEDED, result.getStatus(), "Execution failed: " + result); assertEquals("HELLO, " + uniqueInput.toUpperCase() + "!", result.getResult()); From d17784d13dfc9aa32646ed0ab4c5284404cc9bbe Mon Sep 17 00:00:00 2001 From: Zhongke Chen Date: Tue, 14 Jul 2026 04:01:13 +0000 Subject: [PATCH 05/36] Use global SDK provider for default OTel e2e --- examples/README.md | 2 +- examples/generate-template.py | 24 ++++--------------- .../durable/examples/ExampleTemplate.java | 2 -- .../OtelXRayDefaultConstructorExample.java | 20 ++++++++++++---- .../CloudBasedOtelIntegrationTest.java | 2 +- 5 files changed, 23 insertions(+), 27 deletions(-) diff --git a/examples/README.md b/examples/README.md index f660d0d2d..1fd29b152 100644 --- a/examples/README.md +++ b/examples/README.md @@ -96,7 +96,7 @@ mvn test -Dtest=CloudBasedIntegrationTest \ | [CustomShouldCompleteMapExample](src/main/java/software/amazon/lambda/durable/examples/map/CustomShouldCompleteMapExample.java) | Custom map completion with `shouldComplete` decisions | | [WaitForConditionExample](src/main/java/software/amazon/lambda/durable/examples/wait/WaitForConditionExample.java) | Poll a condition until met with `waitForCondition()` | | [OtelExample](src/main/java/software/amazon/lambda/durable/examples/general/OtelExample.java) | OpenTelemetry instrumentation with logging span export | -| [OtelXRayDefaultConstructorExample](src/main/java/software/amazon/lambda/durable/examples/otel/OtelXRayDefaultConstructorExample.java) | Export spans to X-Ray with the ADOT Java agent and `new OtelPlugin()` | +| [OtelXRayDefaultConstructorExample](src/main/java/software/amazon/lambda/durable/examples/otel/OtelXRayDefaultConstructorExample.java) | Export spans to X-Ray with a global SDK provider and `new OtelPlugin()` | | [OtelXRayStepExample](src/main/java/software/amazon/lambda/durable/examples/otel/OtelXRayStepExample.java) | Export step spans to X-Ray through the ADOT Lambda Layer | | [OtelXRayWaitExample](src/main/java/software/amazon/lambda/durable/examples/otel/OtelXRayWaitExample.java) | Trace a step-wait-step workflow across Lambda invocations | | [OtelXRayMapExample](src/main/java/software/amazon/lambda/durable/examples/otel/OtelXRayMapExample.java) | Trace concurrent map operations and item steps in X-Ray | diff --git a/examples/generate-template.py b/examples/generate-template.py index 332a02570..75e6cce09 100755 --- a/examples/generate-template.py +++ b/examples/generate-template.py @@ -22,7 +22,6 @@ class ExampleFunction: suffix: str condition: str | None tracing: bool - adot_java_agent: bool @property def logical_id(self) -> str: @@ -59,26 +58,24 @@ def is_top_level_durable_handler(source: str, class_name: str) -> bool: return bool(match and "extends DurableHandler" in match.group("header")) -def read_template_annotation(source: str, class_name: str) -> tuple[str | None, bool, bool]: +def read_template_annotation(source: str, class_name: str) -> tuple[str | None, bool]: class_match = re.search(rf"public\s+(?:final\s+)?class\s+{class_name}\b", source) if not class_match: - return None, False, False + return None, False prefix = source[: class_match.start()] matches = list( re.finditer(rf"@(?:[A-Za-z_][\w.]*\.)?{TEMPLATE_ANNOTATION}\s*(?:\((?P.*?)\))?", prefix, re.DOTALL) ) if not matches: - return None, False, False + return None, False body = matches[-1].group("body") or "" condition_match = re.search(r'condition\s*=\s*"([^"]+)"', body) tracing_match = re.search(r"tracing\s*=\s*(true|false)", body) condition = condition_match.group(1) if condition_match else None tracing = tracing_match.group(1) == "true" if tracing_match else False - adot_java_agent_match = re.search(r"adotJavaAgent\s*=\s*(true|false)", body) - adot_java_agent = adot_java_agent_match.group(1) == "true" if adot_java_agent_match else False - return condition, tracing, adot_java_agent + return condition, tracing def discover_examples() -> list[ExampleFunction]: @@ -89,7 +86,7 @@ def discover_examples() -> list[ExampleFunction]: if not is_top_level_durable_handler(source, class_name): continue - condition, tracing, adot_java_agent = read_template_annotation(source, class_name) + condition, tracing = read_template_annotation(source, class_name) package_name = read_package(source, path) examples.append( ExampleFunction( @@ -98,7 +95,6 @@ def discover_examples() -> list[ExampleFunction]: suffix=kebab_case(class_name), condition=condition, tracing=tracing, - adot_java_agent=adot_java_agent, ) ) return examples @@ -132,16 +128,6 @@ def emit_function(lines: list[str], example: ExampleFunction) -> None: " - AdotArch: amd64", ] ) - if example.adot_java_agent: - lines.extend( - [ - " Environment:", - " Variables:", - " AWS_LAMBDA_EXEC_WRAPPER: /opt/otel-handler", - " OTEL_METRICS_EXPORTER: none", - " OTEL_LOGS_EXPORTER: none", - ] - ) lines.append("") diff --git a/examples/src/main/java/software/amazon/lambda/durable/examples/ExampleTemplate.java b/examples/src/main/java/software/amazon/lambda/durable/examples/ExampleTemplate.java index dcb8417c3..898634884 100644 --- a/examples/src/main/java/software/amazon/lambda/durable/examples/ExampleTemplate.java +++ b/examples/src/main/java/software/amazon/lambda/durable/examples/ExampleTemplate.java @@ -14,6 +14,4 @@ String condition() default ""; boolean tracing() default false; - - boolean adotJavaAgent() default false; } diff --git a/examples/src/main/java/software/amazon/lambda/durable/examples/otel/OtelXRayDefaultConstructorExample.java b/examples/src/main/java/software/amazon/lambda/durable/examples/otel/OtelXRayDefaultConstructorExample.java index ded1fb50a..747781c19 100644 --- a/examples/src/main/java/software/amazon/lambda/durable/examples/otel/OtelXRayDefaultConstructorExample.java +++ b/examples/src/main/java/software/amazon/lambda/durable/examples/otel/OtelXRayDefaultConstructorExample.java @@ -2,6 +2,10 @@ // SPDX-License-Identifier: Apache-2.0 package software.amazon.lambda.durable.examples.otel; +import io.opentelemetry.exporter.otlp.trace.OtlpGrpcSpanExporter; +import io.opentelemetry.sdk.OpenTelemetrySdk; +import io.opentelemetry.sdk.trace.SdkTracerProvider; +import io.opentelemetry.sdk.trace.export.SimpleSpanProcessor; import software.amazon.lambda.durable.DurableConfig; import software.amazon.lambda.durable.DurableContext; import software.amazon.lambda.durable.DurableHandler; @@ -10,14 +14,22 @@ import software.amazon.lambda.durable.otel.OtelPlugin; /** - * OTel + X-Ray example that uses the no-arg plugin constructor with the ADOT Java agent. + * OTel + X-Ray example that uses the no-arg plugin constructor with a global SDK provider. * - *

The ADOT Java agent registers the global OpenTelemetry SDK provider. {@link OtelPlugin#OtelPlugin()} copies that - * provider's export pipeline, then installs the durable deterministic ID generator. + *

The global provider exports through the ADOT collector layer. {@link OtelPlugin#OtelPlugin()} copies that + * provider's export pipeline, then installs the durable deterministic ID generator for the plugin's provider. */ -@ExampleTemplate(tracing = true, adotJavaAgent = true) +@ExampleTemplate(tracing = true) public class OtelXRayDefaultConstructorExample extends DurableHandler { + static { + var otlpExporter = OtlpGrpcSpanExporter.getDefault(); + var tracerProvider = SdkTracerProvider.builder() + .addSpanProcessor(SimpleSpanProcessor.create(otlpExporter)) + .build(); + OpenTelemetrySdk.builder().setTracerProvider(tracerProvider).buildAndRegisterGlobal(); + } + @Override protected DurableConfig createConfiguration() { return DurableConfig.builder().withPlugins(new OtelPlugin()).build(); diff --git a/examples/src/test/java/software/amazon/lambda/durable/examples/CloudBasedOtelIntegrationTest.java b/examples/src/test/java/software/amazon/lambda/durable/examples/CloudBasedOtelIntegrationTest.java index 30f290116..1dfe759df 100644 --- a/examples/src/test/java/software/amazon/lambda/durable/examples/CloudBasedOtelIntegrationTest.java +++ b/examples/src/test/java/software/amazon/lambda/durable/examples/CloudBasedOtelIntegrationTest.java @@ -254,7 +254,7 @@ void waitAndResume_producesUnifiedTraceAcrossInvocations() throws Exception { } @Test - void defaultConstructorWithAdotJavaAgent_producesDurableSpansInXRay() throws Exception { + void defaultConstructorWithGlobalProvider_producesDurableSpansInXRay() throws Exception { var startTime = Instant.now(); var runner = CloudDurableTestRunner.create( From 85cd24c4d46dcff969485b79391c0c6f6723b0e8 Mon Sep 17 00:00:00 2001 From: Zhongke Chen Date: Tue, 14 Jul 2026 04:08:48 +0000 Subject: [PATCH 06/36] Improve cloud OTel history diagnostics --- .../CloudBasedOtelIntegrationTest.java | 82 ++++++++++++++++++- 1 file changed, 78 insertions(+), 4 deletions(-) diff --git a/examples/src/test/java/software/amazon/lambda/durable/examples/CloudBasedOtelIntegrationTest.java b/examples/src/test/java/software/amazon/lambda/durable/examples/CloudBasedOtelIntegrationTest.java index 1dfe759df..68b0292da 100644 --- a/examples/src/test/java/software/amazon/lambda/durable/examples/CloudBasedOtelIntegrationTest.java +++ b/examples/src/test/java/software/amazon/lambda/durable/examples/CloudBasedOtelIntegrationTest.java @@ -17,6 +17,9 @@ import software.amazon.awssdk.auth.credentials.DefaultCredentialsProvider; import software.amazon.awssdk.regions.Region; import software.amazon.awssdk.services.lambda.LambdaClient; +import software.amazon.awssdk.services.lambda.model.ErrorObject; +import software.amazon.awssdk.services.lambda.model.Event; +import software.amazon.awssdk.services.lambda.model.EventType; import software.amazon.awssdk.services.sts.StsClient; import software.amazon.awssdk.services.xray.XRayClient; import software.amazon.awssdk.services.xray.model.BatchGetTracesRequest; @@ -26,8 +29,8 @@ import software.amazon.awssdk.services.xray.model.Trace; import software.amazon.awssdk.services.xray.model.TraceSummary; import software.amazon.lambda.durable.examples.types.GreetingRequest; -import software.amazon.lambda.durable.model.ExecutionStatus; import software.amazon.lambda.durable.testing.CloudDurableTestRunner; +import software.amazon.lambda.durable.testing.TestResult; /** * Integration tests that verify OTel spans exported via ADOT appear correctly in AWS X-Ray. @@ -123,7 +126,7 @@ void simpleSteps_producesUnifiedTraceInXRay() throws Exception { var uniqueInput = "XRay-" + System.currentTimeMillis(); var result = runner.run(new GreetingRequest(uniqueInput)); - assertEquals(ExecutionStatus.SUCCEEDED, result.getStatus(), "Execution failed: " + result); + assertExecutionSucceeded(result); assertEquals("HELLO, " + uniqueInput.toUpperCase() + "!", result.getResult()); // 2. Wait for X-Ray ingestion @@ -192,7 +195,7 @@ void waitAndResume_producesUnifiedTraceAcrossInvocations() throws Exception { var uniqueInput = "Wait-" + System.currentTimeMillis(); var result = runner.run(new GreetingRequest(uniqueInput)); - assertEquals(ExecutionStatus.SUCCEEDED, result.getStatus(), "Execution failed: " + result); + assertExecutionSucceeded(result); assertTrue( result.getResult().contains("Resumed and completed"), "Expected result to contain 'Resumed and completed', got: " + result.getResult()); @@ -263,7 +266,7 @@ void defaultConstructorWithGlobalProvider_producesDurableSpansInXRay() throws Ex var execution = runner.startAsync(new GreetingRequest(uniqueInput)); var result = execution.pollUntilComplete(); - assertEquals(ExecutionStatus.SUCCEEDED, result.getStatus(), "Execution failed: " + result); + assertExecutionSucceeded(result); assertEquals("HELLO, " + uniqueInput.toUpperCase() + "!", result.getResult()); Thread.sleep(XRAY_INGESTION_DELAY.toMillis()); @@ -298,6 +301,77 @@ void defaultConstructorWithGlobalProvider_producesDurableSpansInXRay() throws Ex // ─── Helpers ───────────────────────────────────────────────────────── + private static void assertExecutionSucceeded(TestResult result) { + assertTrue(result.isSucceeded() && result.getError().isEmpty(), () -> "Execution failed:\n" + + summarizeExecutionHistory(result)); + } + + private static String summarizeExecutionHistory(TestResult result) { + var failureSummary = result.getHistoryEvents().stream() + .map(CloudBasedOtelIntegrationTest::formatFailureEvent) + .filter(summary -> !summary.isBlank()) + .collect(Collectors.joining("\n")); + if (!failureSummary.isBlank()) { + return failureSummary; + } + + return result.getError() + .map(error -> "Execution error: " + formatError(error)) + .orElseGet(() -> "Events: " + result.getHistoryEvents().stream() + .map(event -> event.eventId() + ":" + event.eventType()) + .collect(Collectors.joining(", "))); + } + + private static String formatFailureEvent(Event event) { + if (EventType.INVOCATION_COMPLETED.equals(event.eventType())) { + var details = event.invocationCompletedDetails(); + if (details != null && details.error() != null && details.error().payload() != null) { + return formatEventError(event, details.error().payload()); + } + } + + if (EventType.EXECUTION_FAILED.equals(event.eventType())) { + var details = event.executionFailedDetails(); + if (details != null && details.error() != null && details.error().payload() != null) { + return formatEventError(event, details.error().payload()); + } + } + + if (EventType.EXECUTION_TIMED_OUT.equals(event.eventType())) { + var details = event.executionTimedOutDetails(); + if (details != null && details.error() != null && details.error().payload() != null) { + return formatEventError(event, details.error().payload()); + } + } + + if (EventType.EXECUTION_STOPPED.equals(event.eventType())) { + var details = event.executionStoppedDetails(); + if (details != null && details.error() != null && details.error().payload() != null) { + return formatEventError(event, details.error().payload()); + } + } + + return ""; + } + + private static String formatEventError(Event event, ErrorObject error) { + return event.eventType() + " event " + event.eventId() + ": " + formatError(error); + } + + private static String formatError(ErrorObject error) { + var parts = new ArrayList(); + if (error.errorType() != null) { + parts.add("type=" + error.errorType()); + } + if (error.errorMessage() != null) { + parts.add("message=" + error.errorMessage()); + } + if (error.errorData() != null) { + parts.add("data=" + error.errorData()); + } + return String.join(", ", parts); + } + private List getFullTraces(List traces) { var traceIds = traces.stream().map(TraceSummary::id).toList(); var allTraces = new ArrayList(); From 704c01a1abbcd654f9fdf67074d00751e742bd73 Mon Sep 17 00:00:00 2001 From: Zhongke Chen Date: Tue, 14 Jul 2026 04:10:44 +0000 Subject: [PATCH 07/36] Fix OTel cloud test formatting --- .../examples/CloudBasedOtelIntegrationTest.java | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/examples/src/test/java/software/amazon/lambda/durable/examples/CloudBasedOtelIntegrationTest.java b/examples/src/test/java/software/amazon/lambda/durable/examples/CloudBasedOtelIntegrationTest.java index 68b0292da..eb8a706c5 100644 --- a/examples/src/test/java/software/amazon/lambda/durable/examples/CloudBasedOtelIntegrationTest.java +++ b/examples/src/test/java/software/amazon/lambda/durable/examples/CloudBasedOtelIntegrationTest.java @@ -302,8 +302,9 @@ void defaultConstructorWithGlobalProvider_producesDurableSpansInXRay() throws Ex // ─── Helpers ───────────────────────────────────────────────────────── private static void assertExecutionSucceeded(TestResult result) { - assertTrue(result.isSucceeded() && result.getError().isEmpty(), () -> "Execution failed:\n" - + summarizeExecutionHistory(result)); + assertTrue( + result.isSucceeded() && result.getError().isEmpty(), + () -> "Execution failed:\n" + summarizeExecutionHistory(result)); } private static String summarizeExecutionHistory(TestResult result) { @@ -317,9 +318,10 @@ private static String summarizeExecutionHistory(TestResult result) { return result.getError() .map(error -> "Execution error: " + formatError(error)) - .orElseGet(() -> "Events: " + result.getHistoryEvents().stream() - .map(event -> event.eventId() + ":" + event.eventType()) - .collect(Collectors.joining(", "))); + .orElseGet(() -> "Events: " + + result.getHistoryEvents().stream() + .map(event -> event.eventId() + ":" + event.eventType()) + .collect(Collectors.joining(", "))); } private static String formatFailureEvent(Event event) { From 3f51759cbdbfe26283f93682f7df14a17c0acb0d Mon Sep 17 00:00:00 2001 From: Frank Chen Date: Mon, 13 Jul 2026 21:25:18 -0700 Subject: [PATCH 08/36] Use ADOT distro Java layer for OTel examples --- examples/generate-template.py | 4 +--- otel-plugin/README.md | 8 +++----- .../software/amazon/lambda/durable/otel/OtelPlugin.java | 2 +- 3 files changed, 5 insertions(+), 9 deletions(-) diff --git a/examples/generate-template.py b/examples/generate-template.py index 75e6cce09..8d8e3bdca 100755 --- a/examples/generate-template.py +++ b/examples/generate-template.py @@ -123,9 +123,7 @@ def emit_function(lines: list[str], example: ExampleFunction) -> None: [ " Tracing: Active", " Layers:", - " - !Sub", - " - arn:aws:lambda:${AWS::Region}:901920570463:layer:aws-otel-java-agent-${AdotArch}-ver-1-32-0:6", - " - AdotArch: amd64", + " - !Sub arn:aws:lambda:${AWS::Region}:615299751070:layer:AWSOpenTelemetryDistroJava:15", ] ) lines.append("") diff --git a/otel-plugin/README.md b/otel-plugin/README.md index fb77f9a12..5c4516036 100644 --- a/otel-plugin/README.md +++ b/otel-plugin/README.md @@ -53,7 +53,7 @@ This plugin uses the [AWS Distro for OpenTelemetry (ADOT) Lambda layer](https:// The layer ARN follows the format: ``` -arn:aws:lambda::901920570463:layer:aws-otel-java-agent--ver-1-32-0:6 +arn:aws:lambda::615299751070:layer:AWSOpenTelemetryDistroJava:15 ``` **CloudFormation / SAM:** @@ -64,9 +64,7 @@ MyFunction: Properties: Tracing: Active Layers: - - !Sub - - arn:aws:lambda:${AWS::Region}:901920570463:layer:aws-otel-java-agent-${Arch}-ver-1-32-0:6 - - Arch: amd64 + - !Sub arn:aws:lambda:${AWS::Region}:615299751070:layer:AWSOpenTelemetryDistroJava:15 ``` **AWS CLI:** @@ -74,7 +72,7 @@ MyFunction: ```bash aws lambda update-function-configuration \ --function-name your-function-name \ - --layers "arn:aws:lambda::901920570463:layer:aws-otel-java-agent-amd64-ver-1-32-0:6" + --layers "arn:aws:lambda::615299751070:layer:AWSOpenTelemetryDistroJava:15" ``` ### 2. AWS X-Ray Active Tracing diff --git a/otel-plugin/src/main/java/software/amazon/lambda/durable/otel/OtelPlugin.java b/otel-plugin/src/main/java/software/amazon/lambda/durable/otel/OtelPlugin.java index 30dbeede4..29651a8f2 100644 --- a/otel-plugin/src/main/java/software/amazon/lambda/durable/otel/OtelPlugin.java +++ b/otel-plugin/src/main/java/software/amazon/lambda/durable/otel/OtelPlugin.java @@ -65,7 +65,7 @@ *

Requires the ADOT Lambda Layer for trace export. Configure with: * *

    - *
  • Lambda Layer: {@code aws-otel-java-agent} (provides the OTLP collector extension) + *
  • Lambda Layer: {@code AWSOpenTelemetryDistroJava} (provides the OTLP collector extension) *
  • Tracing: Active (to populate {@code _X_AMZN_TRACE_ID}) *
* From e6ebf21ee93bdf4c20493b041db150afe2daef7f Mon Sep 17 00:00:00 2001 From: Frank Chen Date: Mon, 13 Jul 2026 21:42:17 -0700 Subject: [PATCH 09/36] Join ADOT current context for default OTel plugin --- examples/generate-template.py | 3 +++ .../OtelXRayDefaultConstructorExample.java | 19 ++++----------- otel-plugin/README.md | 14 ++++++----- .../lambda/durable/otel/OtelPlugin.java | 16 +++++++++++-- .../lambda/durable/otel/OtelPluginTest.java | 24 +++++++++++++++++++ 5 files changed, 53 insertions(+), 23 deletions(-) diff --git a/examples/generate-template.py b/examples/generate-template.py index 8d8e3bdca..c97d58f2f 100755 --- a/examples/generate-template.py +++ b/examples/generate-template.py @@ -124,6 +124,9 @@ def emit_function(lines: list[str], example: ExampleFunction) -> None: " Tracing: Active", " Layers:", " - !Sub arn:aws:lambda:${AWS::Region}:615299751070:layer:AWSOpenTelemetryDistroJava:15", + " Environment:", + " Variables:", + " AWS_LAMBDA_EXEC_WRAPPER: /opt/otel-handler", ] ) lines.append("") diff --git a/examples/src/main/java/software/amazon/lambda/durable/examples/otel/OtelXRayDefaultConstructorExample.java b/examples/src/main/java/software/amazon/lambda/durable/examples/otel/OtelXRayDefaultConstructorExample.java index 747781c19..8c53fc09b 100644 --- a/examples/src/main/java/software/amazon/lambda/durable/examples/otel/OtelXRayDefaultConstructorExample.java +++ b/examples/src/main/java/software/amazon/lambda/durable/examples/otel/OtelXRayDefaultConstructorExample.java @@ -2,10 +2,6 @@ // SPDX-License-Identifier: Apache-2.0 package software.amazon.lambda.durable.examples.otel; -import io.opentelemetry.exporter.otlp.trace.OtlpGrpcSpanExporter; -import io.opentelemetry.sdk.OpenTelemetrySdk; -import io.opentelemetry.sdk.trace.SdkTracerProvider; -import io.opentelemetry.sdk.trace.export.SimpleSpanProcessor; import software.amazon.lambda.durable.DurableConfig; import software.amazon.lambda.durable.DurableContext; import software.amazon.lambda.durable.DurableHandler; @@ -14,22 +10,15 @@ import software.amazon.lambda.durable.otel.OtelPlugin; /** - * OTel + X-Ray example that uses the no-arg plugin constructor with a global SDK provider. + * OTel + X-Ray example that uses the no-arg plugin constructor with the ADOT-managed global SDK provider. * - *

The global provider exports through the ADOT collector layer. {@link OtelPlugin#OtelPlugin()} copies that - * provider's export pipeline, then installs the durable deterministic ID generator for the plugin's provider. + *

ADOT auto-instrumentation initializes {@code GlobalOpenTelemetry}. {@link OtelPlugin#OtelPlugin()} copies that + * provider's export pipeline and joins the current Lambda span context, then installs the durable deterministic ID + * generator for the plugin's provider. */ @ExampleTemplate(tracing = true) public class OtelXRayDefaultConstructorExample extends DurableHandler { - static { - var otlpExporter = OtlpGrpcSpanExporter.getDefault(); - var tracerProvider = SdkTracerProvider.builder() - .addSpanProcessor(SimpleSpanProcessor.create(otlpExporter)) - .build(); - OpenTelemetrySdk.builder().setTracerProvider(tracerProvider).buildAndRegisterGlobal(); - } - @Override protected DurableConfig createConfiguration() { return DurableConfig.builder().withPlugins(new OtelPlugin()).build(); diff --git a/otel-plugin/README.md b/otel-plugin/README.md index 5c4516036..8c5db228a 100644 --- a/otel-plugin/README.md +++ b/otel-plugin/README.md @@ -46,9 +46,7 @@ You also need the OpenTelemetry SDK and an exporter: ### 1. ADOT Lambda Layer -This plugin uses the [AWS Distro for OpenTelemetry (ADOT) Lambda layer](https://aws-otel.github.io/docs/getting-started/lambda) for trace export. The layer provides an OTLP collector that receives spans from the plugin and exports them to X-Ray. - -> **Note:** Do NOT set `AWS_LAMBDA_EXEC_WRAPPER`. The ADOT layer's collector extension runs independently as a Lambda External Extension. The wrapper would attach the auto-instrumentation agent which creates a competing TracerProvider, causing disconnected service nodes in the X-Ray trace map. +This plugin uses the [AWS Distro for OpenTelemetry (ADOT) Lambda layer](https://aws-otel.github.io/docs/getting-started/lambda) for trace export. When using the no-arg `OtelPlugin()` constructor, enable ADOT auto-instrumentation so the layer initializes `GlobalOpenTelemetry`; the plugin copies that provider's export pipeline and joins the current Lambda span context. The layer ARN follows the format: @@ -65,6 +63,9 @@ MyFunction: Tracing: Active Layers: - !Sub arn:aws:lambda:${AWS::Region}:615299751070:layer:AWSOpenTelemetryDistroJava:15 + Environment: + Variables: + AWS_LAMBDA_EXEC_WRAPPER: /opt/otel-handler ``` **AWS CLI:** @@ -72,7 +73,8 @@ MyFunction: ```bash aws lambda update-function-configuration \ --function-name your-function-name \ - --layers "arn:aws:lambda::615299751070:layer:AWSOpenTelemetryDistroJava:15" + --layers "arn:aws:lambda::615299751070:layer:AWSOpenTelemetryDistroJava:15" \ + --environment "Variables={AWS_LAMBDA_EXEC_WRAPPER=/opt/otel-handler}" ``` ### 2. AWS X-Ray Active Tracing @@ -249,9 +251,9 @@ After deploying your function with the plugin configured: | Traces appear but are fragmented | X-Ray active tracing not enabled on the Lambda function | | Missing spans for some operations | Sampling is configured below 1.0 | | `_X_AMZN_TRACE_ID` not populated | X-Ray active tracing not enabled | -| Two service nodes in trace map | `AWS_LAMBDA_EXEC_WRAPPER` is set — remove it (see note above) | +| Plugin spans missing but Lambda/runtime spans appear | `OtelPlugin()` is not joining the current ADOT span context, or `GlobalOpenTelemetry` was manually initialized instead of using the ADOT-managed provider | -> **Note on ADOT wrapper:** Do not set `AWS_LAMBDA_EXEC_WRAPPER`. The wrapper attaches the auto-instrumentation agent which creates a separate TracerProvider. Since the plugin needs its own TracerProvider (for deterministic ID generation), having two providers causes X-Ray to render disconnected service nodes. +> **Note on ADOT wrapper:** When `AWS_LAMBDA_EXEC_WRAPPER=/opt/otel-handler` is set, use the no-arg `OtelPlugin()` constructor and do not manually call `GlobalOpenTelemetry.set` / `buildAndRegisterGlobal`. ADOT owns the global provider in this mode; the plugin copies its export pipeline and uses a plugin-owned provider for deterministic durable span IDs. ## Local Development diff --git a/otel-plugin/src/main/java/software/amazon/lambda/durable/otel/OtelPlugin.java b/otel-plugin/src/main/java/software/amazon/lambda/durable/otel/OtelPlugin.java index 29651a8f2..083656c9a 100644 --- a/otel-plugin/src/main/java/software/amazon/lambda/durable/otel/OtelPlugin.java +++ b/otel-plugin/src/main/java/software/amazon/lambda/durable/otel/OtelPlugin.java @@ -67,10 +67,11 @@ *

    *
  • Lambda Layer: {@code AWSOpenTelemetryDistroJava} (provides the OTLP collector extension) *
  • Tracing: Active (to populate {@code _X_AMZN_TRACE_ID}) + *
  • Wrapper: {@code AWS_LAMBDA_EXEC_WRAPPER=/opt/otel-handler} when using {@link #OtelPlugin()} *
* - *

Note: Do NOT set {@code AWS_LAMBDA_EXEC_WRAPPER}. The collector extension runs independently. The wrapper would - * attach the auto-instrumentation agent which creates a competing TracerProvider. + *

When using {@link #OtelPlugin()}, ADOT auto-instrumentation owns {@code GlobalOpenTelemetry}; do not manually + * register another global provider. * *

Thread-safe: uses {@link ConcurrentHashMap} for span/scope storage since the SDK runs user code on multiple * threads. @@ -170,6 +171,9 @@ public void onInvocationStart(InvocationInfo info) { // Extract trace context from environment (X-Ray header) var extractedContext = contextExtractor.extract(); + if (extractedContext == null) { + extractedContext = extractCurrentSpanContext(); + } if (extractedContext != null) { // Use the X-Ray trace ID — backend propagates same Root across all invocations @@ -476,6 +480,14 @@ private static String attemptKey(String operationId, Integer attempt) { return operationId + "-" + (attempt != null ? attempt : "ctx"); } + private static ExtractedContext extractCurrentSpanContext() { + var spanContext = Span.current().getSpanContext(); + if (!spanContext.isValid()) { + return null; + } + return new ExtractedContext(spanContext.getTraceId(), spanContext.getSpanId()); + } + private static SdkTracerProviderBuilder getDefaultTracerProviderBuilder() { var sdkTracerProvider = getGlobalSdkTracerProvider(); var sharedState = getField(sdkTracerProvider, "sharedState", Object.class); diff --git a/otel-plugin/src/test/java/software/amazon/lambda/durable/otel/OtelPluginTest.java b/otel-plugin/src/test/java/software/amazon/lambda/durable/otel/OtelPluginTest.java index af5e46c00..8361589b1 100644 --- a/otel-plugin/src/test/java/software/amazon/lambda/durable/otel/OtelPluginTest.java +++ b/otel-plugin/src/test/java/software/amazon/lambda/durable/otel/OtelPluginTest.java @@ -5,8 +5,12 @@ import static org.junit.jupiter.api.Assertions.*; import io.opentelemetry.api.GlobalOpenTelemetry; +import io.opentelemetry.api.trace.Span; +import io.opentelemetry.api.trace.SpanContext; import io.opentelemetry.api.trace.SpanKind; import io.opentelemetry.api.trace.StatusCode; +import io.opentelemetry.api.trace.TraceFlags; +import io.opentelemetry.api.trace.TraceState; import io.opentelemetry.sdk.OpenTelemetrySdk; import io.opentelemetry.sdk.testing.exporter.InMemorySpanExporter; import io.opentelemetry.sdk.trace.SdkTracerProvider; @@ -64,6 +68,26 @@ void defaultConstructor_copiesGlobalSdkTracerProviderPipeline() { assertTrue(spans.stream().allMatch(span -> span.getTraceId().equals(traceId))); } + @Test + void invocationStart_usesCurrentSpanContext_whenExtractorReturnsNull() { + var traceId = "5759e988bd862e3fe1be46a994272793"; + var parentSpanId = "53995c3f42cd8ad8"; + var parentSpanContext = + SpanContext.create(traceId, parentSpanId, TraceFlags.getSampled(), TraceState.getDefault()); + + try (var ignored = Span.wrap(parentSpanContext).makeCurrent()) { + plugin.onInvocationStart(new InvocationInfo("req-1", "arn:exec1", true)); + } + plugin.onInvocationEnd(new InvocationEndInfo("req-1", "arn:exec1", true, InvocationStatus.SUCCEEDED, null)); + + var invocationSpan = spanExporter.getFinishedSpanItems().stream() + .filter(span -> span.getName().equals("invocation")) + .findFirst() + .orElseThrow(); + assertEquals(traceId, invocationSpan.getTraceId()); + assertEquals(parentSpanId, invocationSpan.getParentSpanId()); + } + @Test void invocationStart_and_end_createsSpan() { plugin.onInvocationStart(new InvocationInfo( From 473c82116ae11b38cf0a37256453182075b13c1f Mon Sep 17 00:00:00 2001 From: Frank Chen Date: Mon, 13 Jul 2026 21:50:20 -0700 Subject: [PATCH 10/36] Remove ADOT wrapper from OTel examples --- examples/generate-template.py | 3 --- .../OtelXRayDefaultConstructorExample.java | 19 +++++++++++++++---- otel-plugin/README.md | 12 ++++-------- .../lambda/durable/otel/OtelPlugin.java | 5 ++--- 4 files changed, 21 insertions(+), 18 deletions(-) diff --git a/examples/generate-template.py b/examples/generate-template.py index c97d58f2f..8d8e3bdca 100755 --- a/examples/generate-template.py +++ b/examples/generate-template.py @@ -124,9 +124,6 @@ def emit_function(lines: list[str], example: ExampleFunction) -> None: " Tracing: Active", " Layers:", " - !Sub arn:aws:lambda:${AWS::Region}:615299751070:layer:AWSOpenTelemetryDistroJava:15", - " Environment:", - " Variables:", - " AWS_LAMBDA_EXEC_WRAPPER: /opt/otel-handler", ] ) lines.append("") diff --git a/examples/src/main/java/software/amazon/lambda/durable/examples/otel/OtelXRayDefaultConstructorExample.java b/examples/src/main/java/software/amazon/lambda/durable/examples/otel/OtelXRayDefaultConstructorExample.java index 8c53fc09b..747781c19 100644 --- a/examples/src/main/java/software/amazon/lambda/durable/examples/otel/OtelXRayDefaultConstructorExample.java +++ b/examples/src/main/java/software/amazon/lambda/durable/examples/otel/OtelXRayDefaultConstructorExample.java @@ -2,6 +2,10 @@ // SPDX-License-Identifier: Apache-2.0 package software.amazon.lambda.durable.examples.otel; +import io.opentelemetry.exporter.otlp.trace.OtlpGrpcSpanExporter; +import io.opentelemetry.sdk.OpenTelemetrySdk; +import io.opentelemetry.sdk.trace.SdkTracerProvider; +import io.opentelemetry.sdk.trace.export.SimpleSpanProcessor; import software.amazon.lambda.durable.DurableConfig; import software.amazon.lambda.durable.DurableContext; import software.amazon.lambda.durable.DurableHandler; @@ -10,15 +14,22 @@ import software.amazon.lambda.durable.otel.OtelPlugin; /** - * OTel + X-Ray example that uses the no-arg plugin constructor with the ADOT-managed global SDK provider. + * OTel + X-Ray example that uses the no-arg plugin constructor with a global SDK provider. * - *

ADOT auto-instrumentation initializes {@code GlobalOpenTelemetry}. {@link OtelPlugin#OtelPlugin()} copies that - * provider's export pipeline and joins the current Lambda span context, then installs the durable deterministic ID - * generator for the plugin's provider. + *

The global provider exports through the ADOT collector layer. {@link OtelPlugin#OtelPlugin()} copies that + * provider's export pipeline, then installs the durable deterministic ID generator for the plugin's provider. */ @ExampleTemplate(tracing = true) public class OtelXRayDefaultConstructorExample extends DurableHandler { + static { + var otlpExporter = OtlpGrpcSpanExporter.getDefault(); + var tracerProvider = SdkTracerProvider.builder() + .addSpanProcessor(SimpleSpanProcessor.create(otlpExporter)) + .build(); + OpenTelemetrySdk.builder().setTracerProvider(tracerProvider).buildAndRegisterGlobal(); + } + @Override protected DurableConfig createConfiguration() { return DurableConfig.builder().withPlugins(new OtelPlugin()).build(); diff --git a/otel-plugin/README.md b/otel-plugin/README.md index 8c5db228a..9027275fa 100644 --- a/otel-plugin/README.md +++ b/otel-plugin/README.md @@ -46,7 +46,7 @@ You also need the OpenTelemetry SDK and an exporter: ### 1. ADOT Lambda Layer -This plugin uses the [AWS Distro for OpenTelemetry (ADOT) Lambda layer](https://aws-otel.github.io/docs/getting-started/lambda) for trace export. When using the no-arg `OtelPlugin()` constructor, enable ADOT auto-instrumentation so the layer initializes `GlobalOpenTelemetry`; the plugin copies that provider's export pipeline and joins the current Lambda span context. +This plugin uses the [AWS Distro for OpenTelemetry (ADOT) Lambda layer](https://aws-otel.github.io/docs/getting-started/lambda) for trace export. Configure an OpenTelemetry SDK provider with an OTLP exporter; `OtelPlugin()` copies that provider's export pipeline from `GlobalOpenTelemetry`. The layer ARN follows the format: @@ -63,9 +63,6 @@ MyFunction: Tracing: Active Layers: - !Sub arn:aws:lambda:${AWS::Region}:615299751070:layer:AWSOpenTelemetryDistroJava:15 - Environment: - Variables: - AWS_LAMBDA_EXEC_WRAPPER: /opt/otel-handler ``` **AWS CLI:** @@ -73,8 +70,7 @@ MyFunction: ```bash aws lambda update-function-configuration \ --function-name your-function-name \ - --layers "arn:aws:lambda::615299751070:layer:AWSOpenTelemetryDistroJava:15" \ - --environment "Variables={AWS_LAMBDA_EXEC_WRAPPER=/opt/otel-handler}" + --layers "arn:aws:lambda::615299751070:layer:AWSOpenTelemetryDistroJava:15" ``` ### 2. AWS X-Ray Active Tracing @@ -251,9 +247,9 @@ After deploying your function with the plugin configured: | Traces appear but are fragmented | X-Ray active tracing not enabled on the Lambda function | | Missing spans for some operations | Sampling is configured below 1.0 | | `_X_AMZN_TRACE_ID` not populated | X-Ray active tracing not enabled | -| Plugin spans missing but Lambda/runtime spans appear | `OtelPlugin()` is not joining the current ADOT span context, or `GlobalOpenTelemetry` was manually initialized instead of using the ADOT-managed provider | +| Plugin spans missing but Lambda/runtime spans appear | `GlobalOpenTelemetry` was not initialized with an SDK provider and OTLP exporter, or the exporter cannot reach the ADOT collector | -> **Note on ADOT wrapper:** When `AWS_LAMBDA_EXEC_WRAPPER=/opt/otel-handler` is set, use the no-arg `OtelPlugin()` constructor and do not manually call `GlobalOpenTelemetry.set` / `buildAndRegisterGlobal`. ADOT owns the global provider in this mode; the plugin copies its export pipeline and uses a plugin-owned provider for deterministic durable span IDs. +> **Note on ADOT wrapper:** Do not set `AWS_LAMBDA_EXEC_WRAPPER` for these examples. The plugin owns a provider with a deterministic durable ID generator and exports through the SDK provider configured by your application. ## Local Development diff --git a/otel-plugin/src/main/java/software/amazon/lambda/durable/otel/OtelPlugin.java b/otel-plugin/src/main/java/software/amazon/lambda/durable/otel/OtelPlugin.java index 083656c9a..947be5a2b 100644 --- a/otel-plugin/src/main/java/software/amazon/lambda/durable/otel/OtelPlugin.java +++ b/otel-plugin/src/main/java/software/amazon/lambda/durable/otel/OtelPlugin.java @@ -67,11 +67,10 @@ *

    *
  • Lambda Layer: {@code AWSOpenTelemetryDistroJava} (provides the OTLP collector extension) *
  • Tracing: Active (to populate {@code _X_AMZN_TRACE_ID}) - *
  • Wrapper: {@code AWS_LAMBDA_EXEC_WRAPPER=/opt/otel-handler} when using {@link #OtelPlugin()} *
* - *

When using {@link #OtelPlugin()}, ADOT auto-instrumentation owns {@code GlobalOpenTelemetry}; do not manually - * register another global provider. + *

When using {@link #OtelPlugin()}, initialize {@code GlobalOpenTelemetry} with an SDK provider and OTLP exporter + * before creating the plugin. * *

Thread-safe: uses {@link ConcurrentHashMap} for span/scope storage since the SDK runs user code on multiple * threads. From d0e470904b37deeeb0c5b4756b24ea4959b3339d Mon Sep 17 00:00:00 2001 From: Frank Chen Date: Mon, 13 Jul 2026 21:53:53 -0700 Subject: [PATCH 11/36] Make default OTel plugin use OTLP exporter fallback --- examples/README.md | 2 +- .../OtelXRayDefaultConstructorExample.java | 18 +----- otel-plugin/README.md | 26 ++++---- .../lambda/durable/otel/OtelPlugin.java | 59 ++++++++++++++++++- .../otlp/trace/OtlpGrpcSpanExporter.java | 44 ++++++++++++++ .../otel/OtelPluginIntegrationTest.java | 24 ++++++++ .../lambda/durable/otel/OtelPluginTest.java | 22 +++++++ 7 files changed, 161 insertions(+), 34 deletions(-) create mode 100644 otel-plugin/src/test/java/io/opentelemetry/exporter/otlp/trace/OtlpGrpcSpanExporter.java diff --git a/examples/README.md b/examples/README.md index 1fd29b152..600484a67 100644 --- a/examples/README.md +++ b/examples/README.md @@ -96,7 +96,7 @@ mvn test -Dtest=CloudBasedIntegrationTest \ | [CustomShouldCompleteMapExample](src/main/java/software/amazon/lambda/durable/examples/map/CustomShouldCompleteMapExample.java) | Custom map completion with `shouldComplete` decisions | | [WaitForConditionExample](src/main/java/software/amazon/lambda/durable/examples/wait/WaitForConditionExample.java) | Poll a condition until met with `waitForCondition()` | | [OtelExample](src/main/java/software/amazon/lambda/durable/examples/general/OtelExample.java) | OpenTelemetry instrumentation with logging span export | -| [OtelXRayDefaultConstructorExample](src/main/java/software/amazon/lambda/durable/examples/otel/OtelXRayDefaultConstructorExample.java) | Export spans to X-Ray with a global SDK provider and `new OtelPlugin()` | +| [OtelXRayDefaultConstructorExample](src/main/java/software/amazon/lambda/durable/examples/otel/OtelXRayDefaultConstructorExample.java) | Export spans to X-Ray with `new OtelPlugin()` and no handler-side OpenTelemetry initialization | | [OtelXRayStepExample](src/main/java/software/amazon/lambda/durable/examples/otel/OtelXRayStepExample.java) | Export step spans to X-Ray through the ADOT Lambda Layer | | [OtelXRayWaitExample](src/main/java/software/amazon/lambda/durable/examples/otel/OtelXRayWaitExample.java) | Trace a step-wait-step workflow across Lambda invocations | | [OtelXRayMapExample](src/main/java/software/amazon/lambda/durable/examples/otel/OtelXRayMapExample.java) | Trace concurrent map operations and item steps in X-Ray | diff --git a/examples/src/main/java/software/amazon/lambda/durable/examples/otel/OtelXRayDefaultConstructorExample.java b/examples/src/main/java/software/amazon/lambda/durable/examples/otel/OtelXRayDefaultConstructorExample.java index 747781c19..bee0fd497 100644 --- a/examples/src/main/java/software/amazon/lambda/durable/examples/otel/OtelXRayDefaultConstructorExample.java +++ b/examples/src/main/java/software/amazon/lambda/durable/examples/otel/OtelXRayDefaultConstructorExample.java @@ -2,10 +2,6 @@ // SPDX-License-Identifier: Apache-2.0 package software.amazon.lambda.durable.examples.otel; -import io.opentelemetry.exporter.otlp.trace.OtlpGrpcSpanExporter; -import io.opentelemetry.sdk.OpenTelemetrySdk; -import io.opentelemetry.sdk.trace.SdkTracerProvider; -import io.opentelemetry.sdk.trace.export.SimpleSpanProcessor; import software.amazon.lambda.durable.DurableConfig; import software.amazon.lambda.durable.DurableContext; import software.amazon.lambda.durable.DurableHandler; @@ -14,22 +10,14 @@ import software.amazon.lambda.durable.otel.OtelPlugin; /** - * OTel + X-Ray example that uses the no-arg plugin constructor with a global SDK provider. + * OTel + X-Ray example that uses the no-arg plugin constructor. * - *

The global provider exports through the ADOT collector layer. {@link OtelPlugin#OtelPlugin()} copies that - * provider's export pipeline, then installs the durable deterministic ID generator for the plugin's provider. + *

{@link OtelPlugin#OtelPlugin()} exports through the ADOT collector layer by creating the default OTLP exporter + * from the application classpath. */ @ExampleTemplate(tracing = true) public class OtelXRayDefaultConstructorExample extends DurableHandler { - static { - var otlpExporter = OtlpGrpcSpanExporter.getDefault(); - var tracerProvider = SdkTracerProvider.builder() - .addSpanProcessor(SimpleSpanProcessor.create(otlpExporter)) - .build(); - OpenTelemetrySdk.builder().setTracerProvider(tracerProvider).buildAndRegisterGlobal(); - } - @Override protected DurableConfig createConfiguration() { return DurableConfig.builder().withPlugins(new OtelPlugin()).build(); diff --git a/otel-plugin/README.md b/otel-plugin/README.md index 9027275fa..7218a3c7d 100644 --- a/otel-plugin/README.md +++ b/otel-plugin/README.md @@ -10,7 +10,7 @@ OpenTelemetry instrumentation plugin for the AWS Lambda Durable Execution SDK fo - **Span-per-Operation**: Each durable operation (step, wait, map, etc.) gets its own span with accurate timing - **Attempt Spans**: Each user function execution (step attempt, child context run) gets a span, including retries - **Log Correlation**: Injects `traceId`, `spanId`, and `traceSampled` into SLF4J MDC for end-to-end observability -- **Self-Contained Setup**: No manual TracerProvider configuration required beyond the exporter +- **Self-Contained Setup**: `new OtelPlugin()` can export to the ADOT collector without handler-side OpenTelemetry initialization ## Installation @@ -46,7 +46,7 @@ You also need the OpenTelemetry SDK and an exporter: ### 1. ADOT Lambda Layer -This plugin uses the [AWS Distro for OpenTelemetry (ADOT) Lambda layer](https://aws-otel.github.io/docs/getting-started/lambda) for trace export. Configure an OpenTelemetry SDK provider with an OTLP exporter; `OtelPlugin()` copies that provider's export pipeline from `GlobalOpenTelemetry`. +This plugin uses the [AWS Distro for OpenTelemetry (ADOT) Lambda layer](https://aws-otel.github.io/docs/getting-started/lambda) for trace export. `OtelPlugin()` uses a global SDK provider when one exists; otherwise it creates the default OTLP gRPC exporter and sends spans to the ADOT collector layer. The layer ARN follows the format: @@ -100,9 +100,6 @@ MyFunction: ### 3. In Your Lambda Handler ```java -import io.opentelemetry.exporter.otlp.trace.OtlpGrpcSpanExporter; -import io.opentelemetry.sdk.trace.SdkTracerProvider; -import io.opentelemetry.sdk.trace.export.SimpleSpanProcessor; import software.amazon.lambda.durable.DurableConfig; import software.amazon.lambda.durable.DurableContext; import software.amazon.lambda.durable.DurableHandler; @@ -112,13 +109,7 @@ public class MyHandler extends DurableHandler { @Override protected DurableConfig createConfiguration() { - var otlpExporter = OtlpGrpcSpanExporter.getDefault(); - - var otelPlugin = new OtelPlugin( - SdkTracerProvider.builder() - .addSpanProcessor(SimpleSpanProcessor.create(otlpExporter))); - - return DurableConfig.builder().withPlugins(otelPlugin).build(); + return DurableConfig.builder().withPlugins(new OtelPlugin()).build(); } @Override @@ -210,7 +201,10 @@ These appear automatically in structured log output (Log4j2 JSON, Logback JSON) ### Constructor Options ```java -// Default: X-Ray context extraction, MDC enabled +// Default: X-Ray context extraction, MDC enabled, default OTLP export +new OtelPlugin(); + +// Custom tracer provider pipeline new OtelPlugin(tracerProviderBuilder); // Custom context extractor, MDC enabled @@ -222,7 +216,7 @@ new OtelPlugin(tracerProviderBuilder, contextExtractor, enableMdc); | Parameter | Description | Default | |-----------|-------------|---------| -| `tracerProviderBuilder` | `SdkTracerProviderBuilder` with your exporter/processor configured | Required | +| `tracerProviderBuilder` | `SdkTracerProviderBuilder` with your exporter/processor configured | Copies global SDK provider when present, otherwise uses default OTLP exporter | | `contextExtractor` | Extracts parent trace context from the Lambda environment | `XRayContextExtractor` | | `enableMdc` | If true, injects `traceId`/`spanId`/`traceSampled` into SLF4J MDC | `true` | @@ -247,9 +241,9 @@ After deploying your function with the plugin configured: | Traces appear but are fragmented | X-Ray active tracing not enabled on the Lambda function | | Missing spans for some operations | Sampling is configured below 1.0 | | `_X_AMZN_TRACE_ID` not populated | X-Ray active tracing not enabled | -| Plugin spans missing but Lambda/runtime spans appear | `GlobalOpenTelemetry` was not initialized with an SDK provider and OTLP exporter, or the exporter cannot reach the ADOT collector | +| Plugin spans missing but Lambda/runtime spans appear | `opentelemetry-exporter-otlp` is missing from the application, or the exporter cannot reach the ADOT collector | -> **Note on ADOT wrapper:** Do not set `AWS_LAMBDA_EXEC_WRAPPER` for these examples. The plugin owns a provider with a deterministic durable ID generator and exports through the SDK provider configured by your application. +> **Note on ADOT wrapper:** Do not set `AWS_LAMBDA_EXEC_WRAPPER` for these examples. The plugin owns a provider with a deterministic durable ID generator and exports through the default OTLP exporter when no global SDK provider exists. ## Local Development diff --git a/otel-plugin/src/main/java/software/amazon/lambda/durable/otel/OtelPlugin.java b/otel-plugin/src/main/java/software/amazon/lambda/durable/otel/OtelPlugin.java index 947be5a2b..60dcc1db1 100644 --- a/otel-plugin/src/main/java/software/amazon/lambda/durable/otel/OtelPlugin.java +++ b/otel-plugin/src/main/java/software/amazon/lambda/durable/otel/OtelPlugin.java @@ -23,10 +23,13 @@ import io.opentelemetry.sdk.trace.SdkTracerProviderBuilder; import io.opentelemetry.sdk.trace.SpanLimits; import io.opentelemetry.sdk.trace.SpanProcessor; +import io.opentelemetry.sdk.trace.export.SimpleSpanProcessor; +import io.opentelemetry.sdk.trace.export.SpanExporter; import io.opentelemetry.sdk.trace.samplers.Sampler; import io.opentelemetry.semconv.ServiceAttributes; import java.lang.reflect.Field; import java.lang.reflect.InvocationTargetException; +import java.lang.reflect.Method; import java.time.Instant; import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.TimeUnit; @@ -69,8 +72,8 @@ *

  • Tracing: Active (to populate {@code _X_AMZN_TRACE_ID}) * * - *

    When using {@link #OtelPlugin()}, initialize {@code GlobalOpenTelemetry} with an SDK provider and OTLP exporter - * before creating the plugin. + *

    When using {@link #OtelPlugin()}, the plugin copies a global SDK provider when one exists; otherwise it creates a + * default OTLP exporter from the application classpath. * *

    Thread-safe: uses {@link ConcurrentHashMap} for span/scope storage since the SDK runs user code on multiple * threads. @@ -123,6 +126,13 @@ public OtelPlugin(SdkTracerProviderBuilder tracerProviderBuilder) { this(tracerProviderBuilder, new XRayContextExtractor(), true); } + /** + * Creates an OTel plugin with default settings: X-Ray context extraction, MDC enabled, and default OTLP export. + * + *

    If {@code GlobalOpenTelemetry} is backed by the SDK, the plugin copies that provider's export pipeline. + * Otherwise, the plugin creates {@code OtlpGrpcSpanExporter.getDefault()} from the application classpath, which + * sends to the ADOT collector layer's default OTLP endpoint. + */ public OtelPlugin() { this(getDefaultTracerProviderBuilder()); } @@ -488,6 +498,24 @@ private static ExtractedContext extractCurrentSpanContext() { } private static SdkTracerProviderBuilder getDefaultTracerProviderBuilder() { + var globalBuilder = copyGlobalTracerProviderBuilder(); + if (globalBuilder != null) { + return globalBuilder; + } + + return SdkTracerProvider.builder().addSpanProcessor(SimpleSpanProcessor.create(createDefaultOtlpExporter())); + } + + private static SdkTracerProviderBuilder copyGlobalTracerProviderBuilder() { + try { + return getGlobalTracerProviderBuilder(); + } catch (IllegalStateException e) { + logger.debug("GlobalOpenTelemetry is not backed by an SDK tracer provider; using default OTLP exporter", e); + return null; + } + } + + private static SdkTracerProviderBuilder getGlobalTracerProviderBuilder() { var sdkTracerProvider = getGlobalSdkTracerProvider(); var sharedState = getField(sdkTracerProvider, "sharedState", Object.class); @@ -499,6 +527,33 @@ private static SdkTracerProviderBuilder getDefaultTracerProviderBuilder() { .addSpanProcessor(getField(sharedState, "activeSpanProcessor", SpanProcessor.class)); } + private static SpanExporter createDefaultOtlpExporter() { + try { + var exporterClass = Class.forName("io.opentelemetry.exporter.otlp.trace.OtlpGrpcSpanExporter"); + var getDefault = exporterClass.getMethod("getDefault"); + return asSpanExporter(getDefault); + } catch (ClassNotFoundException e) { + throw new IllegalStateException( + "OtelPlugin() requires either GlobalOpenTelemetry backed by the OpenTelemetry SDK or " + + "opentelemetry-exporter-otlp on the application classpath.", + e); + } catch (NoSuchMethodException e) { + throw new IllegalStateException("Unable to create default OTLP exporter", e); + } + } + + private static SpanExporter asSpanExporter(Method getDefault) { + try { + var exporter = getDefault.invoke(null); + if (exporter instanceof SpanExporter spanExporter) { + return spanExporter; + } + throw new IllegalStateException("Default OTLP exporter is not an OpenTelemetry SpanExporter"); + } catch (IllegalAccessException | InvocationTargetException e) { + throw new IllegalStateException("Unable to create default OTLP exporter", e); + } + } + private static SdkTracerProvider getGlobalSdkTracerProvider() { var tracerProvider = GlobalOpenTelemetry.getTracerProvider(); if (tracerProvider instanceof SdkTracerProvider sdkTracerProvider) { diff --git a/otel-plugin/src/test/java/io/opentelemetry/exporter/otlp/trace/OtlpGrpcSpanExporter.java b/otel-plugin/src/test/java/io/opentelemetry/exporter/otlp/trace/OtlpGrpcSpanExporter.java new file mode 100644 index 000000000..c0c3cdb10 --- /dev/null +++ b/otel-plugin/src/test/java/io/opentelemetry/exporter/otlp/trace/OtlpGrpcSpanExporter.java @@ -0,0 +1,44 @@ +// Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +// SPDX-License-Identifier: Apache-2.0 +package io.opentelemetry.exporter.otlp.trace; + +import io.opentelemetry.sdk.common.CompletableResultCode; +import io.opentelemetry.sdk.trace.data.SpanData; +import io.opentelemetry.sdk.trace.export.SpanExporter; +import java.util.ArrayList; +import java.util.Collection; +import java.util.List; +import java.util.concurrent.CopyOnWriteArrayList; + +public final class OtlpGrpcSpanExporter implements SpanExporter { + + private static final List FINISHED_SPANS = new CopyOnWriteArrayList<>(); + + public static SpanExporter getDefault() { + return new OtlpGrpcSpanExporter(); + } + + public static List getFinishedSpanItems() { + return new ArrayList<>(FINISHED_SPANS); + } + + public static void reset() { + FINISHED_SPANS.clear(); + } + + @Override + public CompletableResultCode export(Collection spans) { + FINISHED_SPANS.addAll(spans); + return CompletableResultCode.ofSuccess(); + } + + @Override + public CompletableResultCode flush() { + return CompletableResultCode.ofSuccess(); + } + + @Override + public CompletableResultCode shutdown() { + return CompletableResultCode.ofSuccess(); + } +} diff --git a/otel-plugin/src/test/java/software/amazon/lambda/durable/otel/OtelPluginIntegrationTest.java b/otel-plugin/src/test/java/software/amazon/lambda/durable/otel/OtelPluginIntegrationTest.java index 3c6078355..fd8ac24f6 100644 --- a/otel-plugin/src/test/java/software/amazon/lambda/durable/otel/OtelPluginIntegrationTest.java +++ b/otel-plugin/src/test/java/software/amazon/lambda/durable/otel/OtelPluginIntegrationTest.java @@ -6,6 +6,7 @@ import io.opentelemetry.api.GlobalOpenTelemetry; import io.opentelemetry.api.trace.StatusCode; +import io.opentelemetry.exporter.otlp.trace.OtlpGrpcSpanExporter; import io.opentelemetry.sdk.OpenTelemetrySdk; import io.opentelemetry.sdk.testing.exporter.InMemorySpanExporter; import io.opentelemetry.sdk.trace.SdkTracerProvider; @@ -50,6 +51,7 @@ void setUp() { @AfterEach void tearDown() { GlobalOpenTelemetry.resetForTest(); + OtlpGrpcSpanExporter.reset(); } @Test @@ -104,6 +106,28 @@ void defaultConstructor_copiesGlobalSdkTracerProviderPipeline() { assertTrue(spans.stream().allMatch(span -> span.getTraceId().equals(traceId))); } + @Test + void defaultConstructor_usesDefaultOtlpExporter_whenGlobalProviderIsNotSdk() { + GlobalOpenTelemetry.resetForTest(); + OtlpGrpcSpanExporter.reset(); + + var defaultConfig = + DurableConfig.builder().withPlugins(new OtelPlugin()).build(); + var runner = LocalDurableTestRunner.create( + String.class, + (input, ctx) -> ctx.step("default-otlp-step", String.class, stepCtx -> "Hello " + input), + defaultConfig); + + var result = runner.runUntilComplete("World"); + assertEquals(ExecutionStatus.SUCCEEDED, result.getStatus()); + + var spans = OtlpGrpcSpanExporter.getFinishedSpanItems(); + assertTrue(spans.size() >= 3, "Expected at least 3 spans, got " + spans.size()); + assertSpanExists(spans, "invocation"); + assertSpanExists(spans, "default-otlp-step"); + assertSpanExists(spans, "default-otlp-step attempt 1"); + } + @Test void multipleSteps_producesSpansForEach() { var runner = LocalDurableTestRunner.create( diff --git a/otel-plugin/src/test/java/software/amazon/lambda/durable/otel/OtelPluginTest.java b/otel-plugin/src/test/java/software/amazon/lambda/durable/otel/OtelPluginTest.java index 8361589b1..5ab4e2b16 100644 --- a/otel-plugin/src/test/java/software/amazon/lambda/durable/otel/OtelPluginTest.java +++ b/otel-plugin/src/test/java/software/amazon/lambda/durable/otel/OtelPluginTest.java @@ -11,6 +11,7 @@ import io.opentelemetry.api.trace.StatusCode; import io.opentelemetry.api.trace.TraceFlags; import io.opentelemetry.api.trace.TraceState; +import io.opentelemetry.exporter.otlp.trace.OtlpGrpcSpanExporter; import io.opentelemetry.sdk.OpenTelemetrySdk; import io.opentelemetry.sdk.testing.exporter.InMemorySpanExporter; import io.opentelemetry.sdk.trace.SdkTracerProvider; @@ -40,6 +41,7 @@ void setUp() { @AfterEach void tearDown() { GlobalOpenTelemetry.resetForTest(); + OtlpGrpcSpanExporter.reset(); } @Test @@ -68,6 +70,26 @@ void defaultConstructor_copiesGlobalSdkTracerProviderPipeline() { assertTrue(spans.stream().allMatch(span -> span.getTraceId().equals(traceId))); } + @Test + void defaultConstructor_usesDefaultOtlpExporter_whenGlobalProviderIsNotSdk() { + GlobalOpenTelemetry.resetForTest(); + OtlpGrpcSpanExporter.reset(); + + var defaultPlugin = new OtelPlugin(); + defaultPlugin.onInvocationStart(new InvocationInfo("req-1", "arn:exec1", true)); + defaultPlugin.onOperationStart( + new OperationInfo("op-1", "step", "STEP", "Step", null, Instant.now(), null, false)); + defaultPlugin.onOperationEnd(new OperationEndInfo( + "op-1", "step", "STEP", "Step", null, Instant.now(), Instant.now(), "SUCCEEDED", false, null)); + defaultPlugin.onInvocationEnd( + new InvocationEndInfo("req-1", "arn:exec1", true, InvocationStatus.SUCCEEDED, null)); + + var spans = OtlpGrpcSpanExporter.getFinishedSpanItems(); + assertEquals(2, spans.size()); + assertTrue(spans.stream().anyMatch(span -> span.getName().equals("invocation"))); + assertTrue(spans.stream().anyMatch(span -> span.getName().equals("step"))); + } + @Test void invocationStart_usesCurrentSpanContext_whenExtractorReturnsNull() { var traceId = "5759e988bd862e3fe1be46a994272793"; From 0dbae73a85a9b33f7d6d3514bb3d39c2602442b4 Mon Sep 17 00:00:00 2001 From: Frank Chen Date: Mon, 13 Jul 2026 21:59:20 -0700 Subject: [PATCH 12/36] Log default OTel plugin provider source --- .../java/software/amazon/lambda/durable/otel/OtelPlugin.java | 2 ++ 1 file changed, 2 insertions(+) diff --git a/otel-plugin/src/main/java/software/amazon/lambda/durable/otel/OtelPlugin.java b/otel-plugin/src/main/java/software/amazon/lambda/durable/otel/OtelPlugin.java index 60dcc1db1..e1e20ebcc 100644 --- a/otel-plugin/src/main/java/software/amazon/lambda/durable/otel/OtelPlugin.java +++ b/otel-plugin/src/main/java/software/amazon/lambda/durable/otel/OtelPlugin.java @@ -500,9 +500,11 @@ private static ExtractedContext extractCurrentSpanContext() { private static SdkTracerProviderBuilder getDefaultTracerProviderBuilder() { var globalBuilder = copyGlobalTracerProviderBuilder(); if (globalBuilder != null) { + logger.info("OtelPlugin initialized from existing GlobalOpenTelemetry SDK tracer provider"); return globalBuilder; } + logger.info("OtelPlugin initialized with a new default OTLP tracer provider"); return SdkTracerProvider.builder().addSpanProcessor(SimpleSpanProcessor.create(createDefaultOtlpExporter())); } From 7a502d538b99d58b948b06034e5e33d8df8a510f Mon Sep 17 00:00:00 2001 From: Frank Chen Date: Mon, 13 Jul 2026 22:18:56 -0700 Subject: [PATCH 13/36] Use ADOT Java instrumentation wrapper --- examples/generate-template.py | 3 +++ .../otel/OtelXRayDefaultConstructorExample.java | 4 ++-- otel-plugin/README.md | 12 ++++++++---- .../amazon/lambda/durable/otel/OtelPlugin.java | 3 ++- 4 files changed, 15 insertions(+), 7 deletions(-) diff --git a/examples/generate-template.py b/examples/generate-template.py index 8d8e3bdca..087c90013 100755 --- a/examples/generate-template.py +++ b/examples/generate-template.py @@ -124,6 +124,9 @@ def emit_function(lines: list[str], example: ExampleFunction) -> None: " Tracing: Active", " Layers:", " - !Sub arn:aws:lambda:${AWS::Region}:615299751070:layer:AWSOpenTelemetryDistroJava:15", + " Environment:", + " Variables:", + " AWS_LAMBDA_EXEC_WRAPPER: /opt/otel-instrument", ] ) lines.append("") diff --git a/examples/src/main/java/software/amazon/lambda/durable/examples/otel/OtelXRayDefaultConstructorExample.java b/examples/src/main/java/software/amazon/lambda/durable/examples/otel/OtelXRayDefaultConstructorExample.java index bee0fd497..a0864265a 100644 --- a/examples/src/main/java/software/amazon/lambda/durable/examples/otel/OtelXRayDefaultConstructorExample.java +++ b/examples/src/main/java/software/amazon/lambda/durable/examples/otel/OtelXRayDefaultConstructorExample.java @@ -12,8 +12,8 @@ /** * OTel + X-Ray example that uses the no-arg plugin constructor. * - *

    {@link OtelPlugin#OtelPlugin()} exports through the ADOT collector layer by creating the default OTLP exporter - * from the application classpath. + *

    {@link OtelPlugin#OtelPlugin()} copies the SDK provider initialized by the ADOT Java wrapper, then installs the + * durable deterministic ID generator for the plugin's provider. */ @ExampleTemplate(tracing = true) public class OtelXRayDefaultConstructorExample extends DurableHandler { diff --git a/otel-plugin/README.md b/otel-plugin/README.md index 7218a3c7d..4e271f223 100644 --- a/otel-plugin/README.md +++ b/otel-plugin/README.md @@ -46,7 +46,7 @@ You also need the OpenTelemetry SDK and an exporter: ### 1. ADOT Lambda Layer -This plugin uses the [AWS Distro for OpenTelemetry (ADOT) Lambda layer](https://aws-otel.github.io/docs/getting-started/lambda) for trace export. `OtelPlugin()` uses a global SDK provider when one exists; otherwise it creates the default OTLP gRPC exporter and sends spans to the ADOT collector layer. +This plugin uses the [AWS Distro for OpenTelemetry (ADOT) Lambda layer](https://aws-otel.github.io/docs/getting-started/lambda) for trace export. `OtelPlugin()` uses the global SDK provider initialized by the ADOT Java wrapper when one exists; otherwise it creates the default OTLP gRPC exporter and sends spans to a reachable OTLP collector. The layer ARN follows the format: @@ -63,6 +63,9 @@ MyFunction: Tracing: Active Layers: - !Sub arn:aws:lambda:${AWS::Region}:615299751070:layer:AWSOpenTelemetryDistroJava:15 + Environment: + Variables: + AWS_LAMBDA_EXEC_WRAPPER: /opt/otel-instrument ``` **AWS CLI:** @@ -70,7 +73,8 @@ MyFunction: ```bash aws lambda update-function-configuration \ --function-name your-function-name \ - --layers "arn:aws:lambda::615299751070:layer:AWSOpenTelemetryDistroJava:15" + --layers "arn:aws:lambda::615299751070:layer:AWSOpenTelemetryDistroJava:15" \ + --environment "Variables={AWS_LAMBDA_EXEC_WRAPPER=/opt/otel-instrument}" ``` ### 2. AWS X-Ray Active Tracing @@ -241,9 +245,9 @@ After deploying your function with the plugin configured: | Traces appear but are fragmented | X-Ray active tracing not enabled on the Lambda function | | Missing spans for some operations | Sampling is configured below 1.0 | | `_X_AMZN_TRACE_ID` not populated | X-Ray active tracing not enabled | -| Plugin spans missing but Lambda/runtime spans appear | `opentelemetry-exporter-otlp` is missing from the application, or the exporter cannot reach the ADOT collector | +| Plugin spans missing but Lambda/runtime spans appear | The ADOT wrapper did not initialize the global SDK provider, or the fallback OTLP exporter cannot reach a collector | -> **Note on ADOT wrapper:** Do not set `AWS_LAMBDA_EXEC_WRAPPER` for these examples. The plugin owns a provider with a deterministic durable ID generator and exports through the default OTLP exporter when no global SDK provider exists. +> **Note on ADOT wrapper:** Use `AWS_LAMBDA_EXEC_WRAPPER=/opt/otel-instrument` with the `AWSOpenTelemetryDistroJava` layer. The older `/opt/otel-handler` path is not valid for this layer and can fail before the Java handler starts. ## Local Development diff --git a/otel-plugin/src/main/java/software/amazon/lambda/durable/otel/OtelPlugin.java b/otel-plugin/src/main/java/software/amazon/lambda/durable/otel/OtelPlugin.java index e1e20ebcc..9d0b31f0d 100644 --- a/otel-plugin/src/main/java/software/amazon/lambda/durable/otel/OtelPlugin.java +++ b/otel-plugin/src/main/java/software/amazon/lambda/durable/otel/OtelPlugin.java @@ -504,7 +504,8 @@ private static SdkTracerProviderBuilder getDefaultTracerProviderBuilder() { return globalBuilder; } - logger.info("OtelPlugin initialized with a new default OTLP tracer provider"); + logger.info( + "OtelPlugin initialized with a new default OTLP tracer provider; ensure an OTLP collector is reachable"); return SdkTracerProvider.builder().addSpanProcessor(SimpleSpanProcessor.create(createDefaultOtlpExporter())); } From 7d7b0a68c5ebfdcdfe06f834fcbb0865933055fa Mon Sep 17 00:00:00 2001 From: Frank Chen Date: Mon, 13 Jul 2026 22:36:40 -0700 Subject: [PATCH 14/36] Log global OTel provider copy failures --- .../amazon/lambda/durable/otel/OtelPlugin.java | 16 +++++++++++----- 1 file changed, 11 insertions(+), 5 deletions(-) diff --git a/otel-plugin/src/main/java/software/amazon/lambda/durable/otel/OtelPlugin.java b/otel-plugin/src/main/java/software/amazon/lambda/durable/otel/OtelPlugin.java index 9d0b31f0d..62f203b1f 100644 --- a/otel-plugin/src/main/java/software/amazon/lambda/durable/otel/OtelPlugin.java +++ b/otel-plugin/src/main/java/software/amazon/lambda/durable/otel/OtelPlugin.java @@ -513,7 +513,10 @@ private static SdkTracerProviderBuilder copyGlobalTracerProviderBuilder() { try { return getGlobalTracerProviderBuilder(); } catch (IllegalStateException e) { - logger.debug("GlobalOpenTelemetry is not backed by an SDK tracer provider; using default OTLP exporter", e); + logger.warn( + "OtelPlugin could not initialize from GlobalOpenTelemetry; using default OTLP exporter. Cause: {}", + e.getMessage(), + e); return null; } } @@ -566,6 +569,7 @@ private static SdkTracerProvider getGlobalSdkTracerProvider() { } private static SdkTracerProvider unobfuscateSdkTracerProvider(TracerProvider tracerProvider) { + var providerClassName = tracerProvider.getClass().getName(); try { var unobfuscate = tracerProvider.getClass().getDeclaredMethod("unobfuscate"); unobfuscate.setAccessible(true); @@ -573,14 +577,16 @@ private static SdkTracerProvider unobfuscateSdkTracerProvider(TracerProvider tra if (sdkTracerProvider instanceof SdkTracerProvider result) { return result; } - throw new IllegalStateException("GlobalOpenTelemetry is not backed by an SdkTracerProvider"); + throw new IllegalStateException( + "GlobalOpenTelemetry provider " + providerClassName + " is not backed by an SdkTracerProvider"); } catch (NoSuchMethodException e) { throw new IllegalStateException( - "OtelPlugin() requires GlobalOpenTelemetry to be backed by the OpenTelemetry SDK. " - + "Use OtelPlugin(SdkTracerProviderBuilder) when the global provider cannot be copied.", + "GlobalOpenTelemetry provider " + providerClassName + " does not expose unobfuscate(); " + + "ADOT Java instrumentation may not have initialized an SDK provider.", e); } catch (IllegalAccessException | InvocationTargetException e) { - throw new IllegalStateException("Unable to copy the global OpenTelemetry tracer provider", e); + throw new IllegalStateException( + "Unable to invoke unobfuscate() on GlobalOpenTelemetry provider " + providerClassName, e); } } From 2eacd2caab6d02d4f7518697e57755d4c0a0bc0b Mon Sep 17 00:00:00 2001 From: Frank Chen Date: Mon, 13 Jul 2026 22:45:36 -0700 Subject: [PATCH 15/36] Use ADOT Java agent global tracer provider --- .../OtelXRayDefaultConstructorExample.java | 3 +- otel-plugin/README.md | 6 +- .../lambda/durable/otel/OtelPlugin.java | 106 ++++++++++++------ .../testing/FakeJavaAgentTracerProvider.java | 31 +++++ .../otel/OtelPluginIntegrationTest.java | 42 +++++++ .../lambda/durable/otel/OtelPluginTest.java | 40 +++++++ 6 files changed, 191 insertions(+), 37 deletions(-) create mode 100644 otel-plugin/src/test/java/io/opentelemetry/javaagent/testing/FakeJavaAgentTracerProvider.java diff --git a/examples/src/main/java/software/amazon/lambda/durable/examples/otel/OtelXRayDefaultConstructorExample.java b/examples/src/main/java/software/amazon/lambda/durable/examples/otel/OtelXRayDefaultConstructorExample.java index a0864265a..732222b17 100644 --- a/examples/src/main/java/software/amazon/lambda/durable/examples/otel/OtelXRayDefaultConstructorExample.java +++ b/examples/src/main/java/software/amazon/lambda/durable/examples/otel/OtelXRayDefaultConstructorExample.java @@ -12,8 +12,7 @@ /** * OTel + X-Ray example that uses the no-arg plugin constructor. * - *

    {@link OtelPlugin#OtelPlugin()} copies the SDK provider initialized by the ADOT Java wrapper, then installs the - * durable deterministic ID generator for the plugin's provider. + *

    {@link OtelPlugin#OtelPlugin()} uses the global provider initialized by the ADOT Java wrapper. */ @ExampleTemplate(tracing = true) public class OtelXRayDefaultConstructorExample extends DurableHandler { diff --git a/otel-plugin/README.md b/otel-plugin/README.md index 4e271f223..db6a05de0 100644 --- a/otel-plugin/README.md +++ b/otel-plugin/README.md @@ -46,7 +46,7 @@ You also need the OpenTelemetry SDK and an exporter: ### 1. ADOT Lambda Layer -This plugin uses the [AWS Distro for OpenTelemetry (ADOT) Lambda layer](https://aws-otel.github.io/docs/getting-started/lambda) for trace export. `OtelPlugin()` uses the global SDK provider initialized by the ADOT Java wrapper when one exists; otherwise it creates the default OTLP gRPC exporter and sends spans to a reachable OTLP collector. +This plugin uses the [AWS Distro for OpenTelemetry (ADOT) Lambda layer](https://aws-otel.github.io/docs/getting-started/lambda) for trace export. `OtelPlugin()` uses the global provider initialized by the ADOT Java wrapper when one exists; otherwise it creates the default OTLP gRPC exporter and sends spans to a reachable OTLP collector. The layer ARN follows the format: @@ -220,7 +220,7 @@ new OtelPlugin(tracerProviderBuilder, contextExtractor, enableMdc); | Parameter | Description | Default | |-----------|-------------|---------| -| `tracerProviderBuilder` | `SdkTracerProviderBuilder` with your exporter/processor configured | Copies global SDK provider when present, otherwise uses default OTLP exporter | +| `tracerProviderBuilder` | `SdkTracerProviderBuilder` with your exporter/processor configured | Uses global provider when present, otherwise uses default OTLP exporter | | `contextExtractor` | Extracts parent trace context from the Lambda environment | `XRayContextExtractor` | | `enableMdc` | If true, injects `traceId`/`spanId`/`traceSampled` into SLF4J MDC | `true` | @@ -245,7 +245,7 @@ After deploying your function with the plugin configured: | Traces appear but are fragmented | X-Ray active tracing not enabled on the Lambda function | | Missing spans for some operations | Sampling is configured below 1.0 | | `_X_AMZN_TRACE_ID` not populated | X-Ray active tracing not enabled | -| Plugin spans missing but Lambda/runtime spans appear | The ADOT wrapper did not initialize the global SDK provider, or the fallback OTLP exporter cannot reach a collector | +| Plugin spans missing but Lambda/runtime spans appear | The ADOT wrapper did not initialize `GlobalOpenTelemetry`, or the fallback OTLP exporter cannot reach a collector | > **Note on ADOT wrapper:** Use `AWS_LAMBDA_EXEC_WRAPPER=/opt/otel-instrument` with the `AWSOpenTelemetryDistroJava` layer. The older `/opt/otel-handler` path is not valid for this layer and can fail before the Java handler starts. diff --git a/otel-plugin/src/main/java/software/amazon/lambda/durable/otel/OtelPlugin.java b/otel-plugin/src/main/java/software/amazon/lambda/durable/otel/OtelPlugin.java index 62f203b1f..1566a82f7 100644 --- a/otel-plugin/src/main/java/software/amazon/lambda/durable/otel/OtelPlugin.java +++ b/otel-plugin/src/main/java/software/amazon/lambda/durable/otel/OtelPlugin.java @@ -72,8 +72,9 @@ *

  • Tracing: Active (to populate {@code _X_AMZN_TRACE_ID}) * * - *

    When using {@link #OtelPlugin()}, the plugin copies a global SDK provider when one exists; otherwise it creates a - * default OTLP exporter from the application classpath. + *

    When using {@link #OtelPlugin()}, the plugin copies a global SDK provider when one exists. If ADOT exposes a Java + * agent bridge provider, the plugin uses that provider directly. Otherwise it creates a default OTLP exporter from the + * application classpath. * *

    Thread-safe: uses {@link ConcurrentHashMap} for span/scope storage since the SDK runs user code on multiple * threads. @@ -86,7 +87,7 @@ public class OtelPlugin implements DurableExecutionPlugin { private static final Logger logger = LoggerFactory.getLogger(OtelPlugin.class); private static final String INSTRUMENTATION_NAME = "aws-durable-execution-sdk-java"; - private final SdkTracerProvider tracerProvider; + private final SdkTracerProvider sdkTracerProvider; private final Tracer tracer; private final DeterministicIdGenerator idGenerator; private final ContextExtractor contextExtractor; @@ -129,12 +130,12 @@ public OtelPlugin(SdkTracerProviderBuilder tracerProviderBuilder) { /** * Creates an OTel plugin with default settings: X-Ray context extraction, MDC enabled, and default OTLP export. * - *

    If {@code GlobalOpenTelemetry} is backed by the SDK, the plugin copies that provider's export pipeline. - * Otherwise, the plugin creates {@code OtlpGrpcSpanExporter.getDefault()} from the application classpath, which - * sends to the ADOT collector layer's default OTLP endpoint. + *

    If {@code GlobalOpenTelemetry} is backed by the SDK, the plugin copies that provider's export pipeline. If + * ADOT exposes a Java agent bridge provider, the plugin uses that provider directly. Otherwise, the plugin creates + * {@code OtlpGrpcSpanExporter.getDefault()} from the application classpath. */ public OtelPlugin() { - this(getDefaultTracerProviderBuilder()); + this(getDefaultTracerProvider()); } /** @@ -156,15 +157,32 @@ public OtelPlugin(SdkTracerProviderBuilder tracerProviderBuilder, ContextExtract */ public OtelPlugin( SdkTracerProviderBuilder tracerProviderBuilder, ContextExtractor contextExtractor, boolean enableMdc) { + this(DefaultTracerProvider.fromSdkBuilder(tracerProviderBuilder), contextExtractor, enableMdc); + } + + private OtelPlugin(DefaultTracerProvider defaultTracerProvider) { + this(defaultTracerProvider, new XRayContextExtractor(), true); + } + + private OtelPlugin( + DefaultTracerProvider defaultTracerProvider, ContextExtractor contextExtractor, boolean enableMdc) { this.idGenerator = new DeterministicIdGenerator(); - // Set service.name to "invocation" — X-Ray uses this as the display name for SERVER spans, - // creating a separate service node in the trace map labeled "invocation". - var resource = Resource.create(Attributes.of(ServiceAttributes.SERVICE_NAME, "invocation")); - tracerProviderBuilder.addResource(resource); + if (defaultTracerProvider.sdkBuilder() != null) { + // Set service.name to "invocation" — X-Ray uses this as the display name for SERVER spans, + // creating a separate service node in the trace map labeled "invocation". + var resource = Resource.create(Attributes.of(ServiceAttributes.SERVICE_NAME, "invocation")); + defaultTracerProvider.sdkBuilder().addResource(resource); + this.sdkTracerProvider = defaultTracerProvider + .sdkBuilder() + .setIdGenerator(idGenerator) + .build(); + this.tracer = sdkTracerProvider.get(INSTRUMENTATION_NAME); + } else { + this.sdkTracerProvider = null; + this.tracer = defaultTracerProvider.tracerProvider().get(INSTRUMENTATION_NAME); + } - this.tracerProvider = tracerProviderBuilder.setIdGenerator(idGenerator).build(); - this.tracer = tracerProvider.get(INSTRUMENTATION_NAME); this.contextExtractor = contextExtractor; this.enableMdc = enableMdc; } @@ -255,10 +273,12 @@ public void onInvocationEnd(InvocationEndInfo info) { invocationSpan.end(); invocationSpan = null; - // Flush spans before Lambda freezes - var flushResult = tracerProvider.forceFlush().join(5, TimeUnit.SECONDS); - if (!flushResult.isSuccess()) { - logger.warn("OTel span flush failed or timed out — some spans may be lost"); + if (sdkTracerProvider != null) { + // Flush spans before Lambda freezes + var flushResult = sdkTracerProvider.forceFlush().join(5, TimeUnit.SECONDS); + if (!flushResult.isSuccess()) { + logger.warn("OTel span flush failed or timed out — some spans may be lost"); + } } } @@ -497,27 +517,34 @@ private static ExtractedContext extractCurrentSpanContext() { return new ExtractedContext(spanContext.getTraceId(), spanContext.getSpanId()); } - private static SdkTracerProviderBuilder getDefaultTracerProviderBuilder() { - var globalBuilder = copyGlobalTracerProviderBuilder(); - if (globalBuilder != null) { - logger.info("OtelPlugin initialized from existing GlobalOpenTelemetry SDK tracer provider"); - return globalBuilder; - } - - logger.info( - "OtelPlugin initialized with a new default OTLP tracer provider; ensure an OTLP collector is reachable"); - return SdkTracerProvider.builder().addSpanProcessor(SimpleSpanProcessor.create(createDefaultOtlpExporter())); - } - - private static SdkTracerProviderBuilder copyGlobalTracerProviderBuilder() { + private static DefaultTracerProvider getDefaultTracerProvider() { try { - return getGlobalTracerProviderBuilder(); + var globalBuilder = getGlobalTracerProviderBuilder(); + logger.info("OtelPlugin initialized from existing GlobalOpenTelemetry SDK tracer provider"); + return DefaultTracerProvider.fromSdkBuilder(globalBuilder); } catch (IllegalStateException e) { + var globalTracerProvider = GlobalOpenTelemetry.getTracerProvider(); + if (isJavaAgentTracerProvider(globalTracerProvider)) { + logger.info( + "OtelPlugin initialized from existing GlobalOpenTelemetry Java agent tracer provider {}; " + + "SDK provider could not be copied, so deterministic span IDs cannot be installed. " + + "Cause: {}", + globalTracerProvider.getClass().getName(), + e.getMessage(), + e); + return DefaultTracerProvider.fromTracerProvider(globalTracerProvider); + } + logger.warn( "OtelPlugin could not initialize from GlobalOpenTelemetry; using default OTLP exporter. Cause: {}", e.getMessage(), e); - return null; + + var fallbackBuilder = SdkTracerProvider.builder() + .addSpanProcessor(SimpleSpanProcessor.create(createDefaultOtlpExporter())); + logger.info( + "OtelPlugin initialized with a new default OTLP tracer provider; ensure an OTLP collector is reachable"); + return DefaultTracerProvider.fromSdkBuilder(fallbackBuilder); } } @@ -568,6 +595,10 @@ private static SdkTracerProvider getGlobalSdkTracerProvider() { return unobfuscateSdkTracerProvider(tracerProvider); } + private static boolean isJavaAgentTracerProvider(TracerProvider tracerProvider) { + return tracerProvider.getClass().getName().startsWith("io.opentelemetry.javaagent."); + } + private static SdkTracerProvider unobfuscateSdkTracerProvider(TracerProvider tracerProvider) { var providerClassName = tracerProvider.getClass().getName(); try { @@ -616,4 +647,15 @@ private static Field findField(Class type, String fieldName) throws NoSuchFie } throw new NoSuchFieldException(fieldName); } + + private record DefaultTracerProvider(SdkTracerProviderBuilder sdkBuilder, TracerProvider tracerProvider) { + + private static DefaultTracerProvider fromSdkBuilder(SdkTracerProviderBuilder sdkBuilder) { + return new DefaultTracerProvider(sdkBuilder, null); + } + + private static DefaultTracerProvider fromTracerProvider(TracerProvider tracerProvider) { + return new DefaultTracerProvider(null, tracerProvider); + } + } } diff --git a/otel-plugin/src/test/java/io/opentelemetry/javaagent/testing/FakeJavaAgentTracerProvider.java b/otel-plugin/src/test/java/io/opentelemetry/javaagent/testing/FakeJavaAgentTracerProvider.java new file mode 100644 index 000000000..f6ca91d22 --- /dev/null +++ b/otel-plugin/src/test/java/io/opentelemetry/javaagent/testing/FakeJavaAgentTracerProvider.java @@ -0,0 +1,31 @@ +// Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +// SPDX-License-Identifier: Apache-2.0 +package io.opentelemetry.javaagent.testing; + +import io.opentelemetry.api.trace.Tracer; +import io.opentelemetry.api.trace.TracerBuilder; +import io.opentelemetry.api.trace.TracerProvider; + +public final class FakeJavaAgentTracerProvider implements TracerProvider { + + private final TracerProvider delegate; + + public FakeJavaAgentTracerProvider(TracerProvider delegate) { + this.delegate = delegate; + } + + @Override + public Tracer get(String instrumentationName) { + return delegate.get(instrumentationName); + } + + @Override + public Tracer get(String instrumentationName, String instrumentationVersion) { + return delegate.get(instrumentationName, instrumentationVersion); + } + + @Override + public TracerBuilder tracerBuilder(String instrumentationScopeName) { + return delegate.tracerBuilder(instrumentationScopeName); + } +} diff --git a/otel-plugin/src/test/java/software/amazon/lambda/durable/otel/OtelPluginIntegrationTest.java b/otel-plugin/src/test/java/software/amazon/lambda/durable/otel/OtelPluginIntegrationTest.java index fd8ac24f6..6c76cf278 100644 --- a/otel-plugin/src/test/java/software/amazon/lambda/durable/otel/OtelPluginIntegrationTest.java +++ b/otel-plugin/src/test/java/software/amazon/lambda/durable/otel/OtelPluginIntegrationTest.java @@ -5,8 +5,11 @@ import static org.junit.jupiter.api.Assertions.*; import io.opentelemetry.api.GlobalOpenTelemetry; +import io.opentelemetry.api.OpenTelemetry; import io.opentelemetry.api.trace.StatusCode; +import io.opentelemetry.context.propagation.ContextPropagators; import io.opentelemetry.exporter.otlp.trace.OtlpGrpcSpanExporter; +import io.opentelemetry.javaagent.testing.FakeJavaAgentTracerProvider; import io.opentelemetry.sdk.OpenTelemetrySdk; import io.opentelemetry.sdk.testing.exporter.InMemorySpanExporter; import io.opentelemetry.sdk.trace.SdkTracerProvider; @@ -128,6 +131,45 @@ void defaultConstructor_usesDefaultOtlpExporter_whenGlobalProviderIsNotSdk() { assertSpanExists(spans, "default-otlp-step attempt 1"); } + @Test + void defaultConstructor_usesJavaAgentGlobalTracerProviderDirectly_whenSdkCannotBeCopied() { + GlobalOpenTelemetry.resetForTest(); + OtlpGrpcSpanExporter.reset(); + var globalExporter = InMemorySpanExporter.create(); + var sdkTracerProvider = SdkTracerProvider.builder() + .addSpanProcessor(SimpleSpanProcessor.create(globalExporter)) + .build(); + var javaAgentTracerProvider = new FakeJavaAgentTracerProvider(sdkTracerProvider); + GlobalOpenTelemetry.set(new OpenTelemetry() { + @Override + public io.opentelemetry.api.trace.TracerProvider getTracerProvider() { + return javaAgentTracerProvider; + } + + @Override + public ContextPropagators getPropagators() { + return ContextPropagators.noop(); + } + }); + + var defaultConfig = + DurableConfig.builder().withPlugins(new OtelPlugin()).build(); + var runner = LocalDurableTestRunner.create( + String.class, + (input, ctx) -> ctx.step("javaagent-step", String.class, stepCtx -> "Hello " + input), + defaultConfig); + + var result = runner.runUntilComplete("World"); + assertEquals(ExecutionStatus.SUCCEEDED, result.getStatus()); + + var spans = globalExporter.getFinishedSpanItems(); + assertTrue(spans.size() >= 3, "Expected at least 3 spans, got " + spans.size()); + assertSpanExists(spans, "invocation"); + assertSpanExists(spans, "javaagent-step"); + assertSpanExists(spans, "javaagent-step attempt 1"); + assertTrue(OtlpGrpcSpanExporter.getFinishedSpanItems().isEmpty()); + } + @Test void multipleSteps_producesSpansForEach() { var runner = LocalDurableTestRunner.create( diff --git a/otel-plugin/src/test/java/software/amazon/lambda/durable/otel/OtelPluginTest.java b/otel-plugin/src/test/java/software/amazon/lambda/durable/otel/OtelPluginTest.java index 5ab4e2b16..97a038dba 100644 --- a/otel-plugin/src/test/java/software/amazon/lambda/durable/otel/OtelPluginTest.java +++ b/otel-plugin/src/test/java/software/amazon/lambda/durable/otel/OtelPluginTest.java @@ -5,13 +5,16 @@ import static org.junit.jupiter.api.Assertions.*; import io.opentelemetry.api.GlobalOpenTelemetry; +import io.opentelemetry.api.OpenTelemetry; import io.opentelemetry.api.trace.Span; import io.opentelemetry.api.trace.SpanContext; import io.opentelemetry.api.trace.SpanKind; import io.opentelemetry.api.trace.StatusCode; import io.opentelemetry.api.trace.TraceFlags; import io.opentelemetry.api.trace.TraceState; +import io.opentelemetry.context.propagation.ContextPropagators; import io.opentelemetry.exporter.otlp.trace.OtlpGrpcSpanExporter; +import io.opentelemetry.javaagent.testing.FakeJavaAgentTracerProvider; import io.opentelemetry.sdk.OpenTelemetrySdk; import io.opentelemetry.sdk.testing.exporter.InMemorySpanExporter; import io.opentelemetry.sdk.trace.SdkTracerProvider; @@ -90,6 +93,43 @@ void defaultConstructor_usesDefaultOtlpExporter_whenGlobalProviderIsNotSdk() { assertTrue(spans.stream().anyMatch(span -> span.getName().equals("step"))); } + @Test + void defaultConstructor_usesJavaAgentGlobalTracerProviderDirectly_whenSdkCannotBeCopied() { + GlobalOpenTelemetry.resetForTest(); + OtlpGrpcSpanExporter.reset(); + var globalExporter = InMemorySpanExporter.create(); + var sdkTracerProvider = SdkTracerProvider.builder() + .addSpanProcessor(SimpleSpanProcessor.create(globalExporter)) + .build(); + var javaAgentTracerProvider = new FakeJavaAgentTracerProvider(sdkTracerProvider); + GlobalOpenTelemetry.set(new OpenTelemetry() { + @Override + public io.opentelemetry.api.trace.TracerProvider getTracerProvider() { + return javaAgentTracerProvider; + } + + @Override + public ContextPropagators getPropagators() { + return ContextPropagators.noop(); + } + }); + + var defaultPlugin = new OtelPlugin(); + defaultPlugin.onInvocationStart(new InvocationInfo("req-1", "arn:exec1", true)); + defaultPlugin.onOperationStart( + new OperationInfo("op-1", "step", "STEP", "Step", null, Instant.now(), null, false)); + defaultPlugin.onOperationEnd(new OperationEndInfo( + "op-1", "step", "STEP", "Step", null, Instant.now(), Instant.now(), "SUCCEEDED", false, null)); + defaultPlugin.onInvocationEnd( + new InvocationEndInfo("req-1", "arn:exec1", true, InvocationStatus.SUCCEEDED, null)); + + var spans = globalExporter.getFinishedSpanItems(); + assertEquals(2, spans.size()); + assertTrue(spans.stream().anyMatch(span -> span.getName().equals("invocation"))); + assertTrue(spans.stream().anyMatch(span -> span.getName().equals("step"))); + assertTrue(OtlpGrpcSpanExporter.getFinishedSpanItems().isEmpty()); + } + @Test void invocationStart_usesCurrentSpanContext_whenExtractorReturnsNull() { var traceId = "5759e988bd862e3fe1be46a994272793"; From eab94b53e33e7ab57fd311bb51638b8e957a129a Mon Sep 17 00:00:00 2001 From: Frank Chen Date: Mon, 13 Jul 2026 22:52:56 -0700 Subject: [PATCH 16/36] Install deterministic IDs in ADOT Java agent provider --- .../OtelXRayDefaultConstructorExample.java | 3 +- otel-plugin/README.md | 4 +- .../lambda/durable/otel/OtelPlugin.java | 85 ++++++++++++++----- .../testing/FakeJavaAgentTracerProvider.java | 12 +-- .../otel/OtelPluginIntegrationTest.java | 2 +- .../lambda/durable/otel/OtelPluginTest.java | 9 +- 6 files changed, 83 insertions(+), 32 deletions(-) diff --git a/examples/src/main/java/software/amazon/lambda/durable/examples/otel/OtelXRayDefaultConstructorExample.java b/examples/src/main/java/software/amazon/lambda/durable/examples/otel/OtelXRayDefaultConstructorExample.java index 732222b17..fed15671a 100644 --- a/examples/src/main/java/software/amazon/lambda/durable/examples/otel/OtelXRayDefaultConstructorExample.java +++ b/examples/src/main/java/software/amazon/lambda/durable/examples/otel/OtelXRayDefaultConstructorExample.java @@ -12,7 +12,8 @@ /** * OTel + X-Ray example that uses the no-arg plugin constructor. * - *

    {@link OtelPlugin#OtelPlugin()} uses the global provider initialized by the ADOT Java wrapper. + *

    {@link OtelPlugin#OtelPlugin()} copies a global SDK provider when possible, or uses the ADOT Java agent provider + * directly after installing deterministic span ID generation. */ @ExampleTemplate(tracing = true) public class OtelXRayDefaultConstructorExample extends DurableHandler { diff --git a/otel-plugin/README.md b/otel-plugin/README.md index db6a05de0..f3990575b 100644 --- a/otel-plugin/README.md +++ b/otel-plugin/README.md @@ -46,7 +46,7 @@ You also need the OpenTelemetry SDK and an exporter: ### 1. ADOT Lambda Layer -This plugin uses the [AWS Distro for OpenTelemetry (ADOT) Lambda layer](https://aws-otel.github.io/docs/getting-started/lambda) for trace export. `OtelPlugin()` uses the global provider initialized by the ADOT Java wrapper when one exists; otherwise it creates the default OTLP gRPC exporter and sends spans to a reachable OTLP collector. +This plugin uses the [AWS Distro for OpenTelemetry (ADOT) Lambda layer](https://aws-otel.github.io/docs/getting-started/lambda) for trace export. `OtelPlugin()` copies a global SDK provider when possible, or uses the ADOT Java agent provider directly after installing deterministic span ID generation. If no usable global provider exists, it creates the default OTLP gRPC exporter and sends spans to a reachable OTLP collector. The layer ARN follows the format: @@ -220,7 +220,7 @@ new OtelPlugin(tracerProviderBuilder, contextExtractor, enableMdc); | Parameter | Description | Default | |-----------|-------------|---------| -| `tracerProviderBuilder` | `SdkTracerProviderBuilder` with your exporter/processor configured | Uses global provider when present, otherwise uses default OTLP exporter | +| `tracerProviderBuilder` | `SdkTracerProviderBuilder` with your exporter/processor configured | Copies global SDK provider when possible, or uses the ADOT Java agent provider directly | | `contextExtractor` | Extracts parent trace context from the Lambda environment | `XRayContextExtractor` | | `enableMdc` | If true, injects `traceId`/`spanId`/`traceSampled` into SLF4J MDC | `true` | diff --git a/otel-plugin/src/main/java/software/amazon/lambda/durable/otel/OtelPlugin.java b/otel-plugin/src/main/java/software/amazon/lambda/durable/otel/OtelPlugin.java index 1566a82f7..a96794ecb 100644 --- a/otel-plugin/src/main/java/software/amazon/lambda/durable/otel/OtelPlugin.java +++ b/otel-plugin/src/main/java/software/amazon/lambda/durable/otel/OtelPlugin.java @@ -19,6 +19,7 @@ import io.opentelemetry.context.Scope; import io.opentelemetry.sdk.common.Clock; import io.opentelemetry.sdk.resources.Resource; +import io.opentelemetry.sdk.trace.IdGenerator; import io.opentelemetry.sdk.trace.SdkTracerProvider; import io.opentelemetry.sdk.trace.SdkTracerProviderBuilder; import io.opentelemetry.sdk.trace.SpanLimits; @@ -73,8 +74,8 @@ * * *

    When using {@link #OtelPlugin()}, the plugin copies a global SDK provider when one exists. If ADOT exposes a Java - * agent bridge provider, the plugin uses that provider directly. Otherwise it creates a default OTLP exporter from the - * application classpath. + * agent bridge provider, the plugin uses that provider directly after installing its deterministic ID generator through + * reflection. Otherwise it creates a default OTLP exporter from the application classpath. * *

    Thread-safe: uses {@link ConcurrentHashMap} for span/scope storage since the SDK runs user code on multiple * threads. @@ -131,7 +132,8 @@ public OtelPlugin(SdkTracerProviderBuilder tracerProviderBuilder) { * Creates an OTel plugin with default settings: X-Ray context extraction, MDC enabled, and default OTLP export. * *

    If {@code GlobalOpenTelemetry} is backed by the SDK, the plugin copies that provider's export pipeline. If - * ADOT exposes a Java agent bridge provider, the plugin uses that provider directly. Otherwise, the plugin creates + * ADOT exposes a Java agent bridge provider, the plugin uses that provider directly after installing deterministic + * ID generation into the underlying SDK provider. Otherwise, the plugin creates * {@code OtlpGrpcSpanExporter.getDefault()} from the application classpath. */ public OtelPlugin() { @@ -157,7 +159,10 @@ public OtelPlugin(SdkTracerProviderBuilder tracerProviderBuilder, ContextExtract */ public OtelPlugin( SdkTracerProviderBuilder tracerProviderBuilder, ContextExtractor contextExtractor, boolean enableMdc) { - this(DefaultTracerProvider.fromSdkBuilder(tracerProviderBuilder), contextExtractor, enableMdc); + this( + DefaultTracerProvider.fromSdkBuilder(tracerProviderBuilder, new DeterministicIdGenerator()), + contextExtractor, + enableMdc); } private OtelPlugin(DefaultTracerProvider defaultTracerProvider) { @@ -166,20 +171,21 @@ private OtelPlugin(DefaultTracerProvider defaultTracerProvider) { private OtelPlugin( DefaultTracerProvider defaultTracerProvider, ContextExtractor contextExtractor, boolean enableMdc) { - this.idGenerator = new DeterministicIdGenerator(); + this.idGenerator = defaultTracerProvider.idGenerator(); if (defaultTracerProvider.sdkBuilder() != null) { // Set service.name to "invocation" — X-Ray uses this as the display name for SERVER spans, // creating a separate service node in the trace map labeled "invocation". var resource = Resource.create(Attributes.of(ServiceAttributes.SERVICE_NAME, "invocation")); defaultTracerProvider.sdkBuilder().addResource(resource); + this.sdkTracerProvider = defaultTracerProvider .sdkBuilder() .setIdGenerator(idGenerator) .build(); this.tracer = sdkTracerProvider.get(INSTRUMENTATION_NAME); } else { - this.sdkTracerProvider = null; + this.sdkTracerProvider = defaultTracerProvider.sdkTracerProvider(); this.tracer = defaultTracerProvider.tracerProvider().get(INSTRUMENTATION_NAME); } @@ -518,33 +524,35 @@ private static ExtractedContext extractCurrentSpanContext() { } private static DefaultTracerProvider getDefaultTracerProvider() { + var idGenerator = new DeterministicIdGenerator(); try { var globalBuilder = getGlobalTracerProviderBuilder(); logger.info("OtelPlugin initialized from existing GlobalOpenTelemetry SDK tracer provider"); - return DefaultTracerProvider.fromSdkBuilder(globalBuilder); + return DefaultTracerProvider.fromSdkBuilder(globalBuilder, idGenerator); } catch (IllegalStateException e) { var globalTracerProvider = GlobalOpenTelemetry.getTracerProvider(); if (isJavaAgentTracerProvider(globalTracerProvider)) { + var sdkTracerProvider = getJavaAgentSdkTracerProvider(globalTracerProvider); + overrideIdGenerator(sdkTracerProvider, idGenerator); logger.info( "OtelPlugin initialized from existing GlobalOpenTelemetry Java agent tracer provider {}; " - + "SDK provider could not be copied, so deterministic span IDs cannot be installed. " - + "Cause: {}", + + "deterministic span IDs installed through reflection. Cause: {}", globalTracerProvider.getClass().getName(), + e.getMessage()); + return DefaultTracerProvider.fromTracerProvider(globalTracerProvider, sdkTracerProvider, idGenerator); + } else { + logger.warn( + "OtelPlugin could not initialize from GlobalOpenTelemetry; using default OTLP exporter. " + + "Cause: {}", e.getMessage(), e); - return DefaultTracerProvider.fromTracerProvider(globalTracerProvider); } - logger.warn( - "OtelPlugin could not initialize from GlobalOpenTelemetry; using default OTLP exporter. Cause: {}", - e.getMessage(), - e); - var fallbackBuilder = SdkTracerProvider.builder() .addSpanProcessor(SimpleSpanProcessor.create(createDefaultOtlpExporter())); logger.info( "OtelPlugin initialized with a new default OTLP tracer provider; ensure an OTLP collector is reachable"); - return DefaultTracerProvider.fromSdkBuilder(fallbackBuilder); + return DefaultTracerProvider.fromSdkBuilder(fallbackBuilder, idGenerator); } } @@ -599,6 +607,23 @@ private static boolean isJavaAgentTracerProvider(TracerProvider tracerProvider) return tracerProvider.getClass().getName().startsWith("io.opentelemetry.javaagent."); } + private static SdkTracerProvider getJavaAgentSdkTracerProvider(TracerProvider tracerProvider) { + var agentTracerProvider = getField(tracerProvider, "agentTracerProvider", Object.class); + if (agentTracerProvider instanceof SdkTracerProvider sdkTracerProvider) { + return sdkTracerProvider; + } + if (agentTracerProvider instanceof TracerProvider nestedTracerProvider) { + return unobfuscateSdkTracerProvider(nestedTracerProvider); + } + throw new IllegalStateException("Java agent tracer provider field agentTracerProvider is not a TracerProvider"); + } + + private static void overrideIdGenerator(SdkTracerProvider sdkTracerProvider, IdGenerator idGenerator) { + var sharedState = getField(sdkTracerProvider, "sharedState", Object.class); + setField(sharedState, "idGenerator", idGenerator); + setField(sharedState, "idGeneratorSafeToSkipIdValidation", false); + } + private static SdkTracerProvider unobfuscateSdkTracerProvider(TracerProvider tracerProvider) { var providerClassName = tracerProvider.getClass().getName(); try { @@ -636,6 +661,16 @@ private static T getField(Object target, String fieldName, Class fieldTyp } } + private static void setField(Object target, String fieldName, Object value) { + try { + var field = findField(target.getClass(), fieldName); + field.setAccessible(true); + field.set(target, value); + } catch (NoSuchFieldException | IllegalAccessException e) { + throw new IllegalStateException("Unable to update OpenTelemetry SDK field: " + fieldName, e); + } + } + private static Field findField(Class type, String fieldName) throws NoSuchFieldException { var current = type; while (current != null) { @@ -648,14 +683,22 @@ private static Field findField(Class type, String fieldName) throws NoSuchFie throw new NoSuchFieldException(fieldName); } - private record DefaultTracerProvider(SdkTracerProviderBuilder sdkBuilder, TracerProvider tracerProvider) { + private record DefaultTracerProvider( + SdkTracerProviderBuilder sdkBuilder, + TracerProvider tracerProvider, + SdkTracerProvider sdkTracerProvider, + DeterministicIdGenerator idGenerator) { - private static DefaultTracerProvider fromSdkBuilder(SdkTracerProviderBuilder sdkBuilder) { - return new DefaultTracerProvider(sdkBuilder, null); + private static DefaultTracerProvider fromSdkBuilder( + SdkTracerProviderBuilder sdkBuilder, DeterministicIdGenerator idGenerator) { + return new DefaultTracerProvider(sdkBuilder, null, null, idGenerator); } - private static DefaultTracerProvider fromTracerProvider(TracerProvider tracerProvider) { - return new DefaultTracerProvider(null, tracerProvider); + private static DefaultTracerProvider fromTracerProvider( + TracerProvider tracerProvider, + SdkTracerProvider sdkTracerProvider, + DeterministicIdGenerator idGenerator) { + return new DefaultTracerProvider(null, tracerProvider, sdkTracerProvider, idGenerator); } } } diff --git a/otel-plugin/src/test/java/io/opentelemetry/javaagent/testing/FakeJavaAgentTracerProvider.java b/otel-plugin/src/test/java/io/opentelemetry/javaagent/testing/FakeJavaAgentTracerProvider.java index f6ca91d22..7d83f2152 100644 --- a/otel-plugin/src/test/java/io/opentelemetry/javaagent/testing/FakeJavaAgentTracerProvider.java +++ b/otel-plugin/src/test/java/io/opentelemetry/javaagent/testing/FakeJavaAgentTracerProvider.java @@ -8,24 +8,24 @@ public final class FakeJavaAgentTracerProvider implements TracerProvider { - private final TracerProvider delegate; + private final TracerProvider agentTracerProvider; - public FakeJavaAgentTracerProvider(TracerProvider delegate) { - this.delegate = delegate; + public FakeJavaAgentTracerProvider(TracerProvider agentTracerProvider) { + this.agentTracerProvider = agentTracerProvider; } @Override public Tracer get(String instrumentationName) { - return delegate.get(instrumentationName); + return agentTracerProvider.get(instrumentationName); } @Override public Tracer get(String instrumentationName, String instrumentationVersion) { - return delegate.get(instrumentationName, instrumentationVersion); + return agentTracerProvider.get(instrumentationName, instrumentationVersion); } @Override public TracerBuilder tracerBuilder(String instrumentationScopeName) { - return delegate.tracerBuilder(instrumentationScopeName); + return agentTracerProvider.tracerBuilder(instrumentationScopeName); } } diff --git a/otel-plugin/src/test/java/software/amazon/lambda/durable/otel/OtelPluginIntegrationTest.java b/otel-plugin/src/test/java/software/amazon/lambda/durable/otel/OtelPluginIntegrationTest.java index 6c76cf278..87a1e1cbf 100644 --- a/otel-plugin/src/test/java/software/amazon/lambda/durable/otel/OtelPluginIntegrationTest.java +++ b/otel-plugin/src/test/java/software/amazon/lambda/durable/otel/OtelPluginIntegrationTest.java @@ -132,7 +132,7 @@ void defaultConstructor_usesDefaultOtlpExporter_whenGlobalProviderIsNotSdk() { } @Test - void defaultConstructor_usesJavaAgentGlobalTracerProviderDirectly_whenSdkCannotBeCopied() { + void defaultConstructor_usesJavaAgentGlobalTracerProviderDirectly_andOverridesIdGenerator() { GlobalOpenTelemetry.resetForTest(); OtlpGrpcSpanExporter.reset(); var globalExporter = InMemorySpanExporter.create(); diff --git a/otel-plugin/src/test/java/software/amazon/lambda/durable/otel/OtelPluginTest.java b/otel-plugin/src/test/java/software/amazon/lambda/durable/otel/OtelPluginTest.java index 97a038dba..964691d62 100644 --- a/otel-plugin/src/test/java/software/amazon/lambda/durable/otel/OtelPluginTest.java +++ b/otel-plugin/src/test/java/software/amazon/lambda/durable/otel/OtelPluginTest.java @@ -94,7 +94,7 @@ void defaultConstructor_usesDefaultOtlpExporter_whenGlobalProviderIsNotSdk() { } @Test - void defaultConstructor_usesJavaAgentGlobalTracerProviderDirectly_whenSdkCannotBeCopied() { + void defaultConstructor_usesJavaAgentGlobalTracerProviderDirectly_andOverridesIdGenerator() { GlobalOpenTelemetry.resetForTest(); OtlpGrpcSpanExporter.reset(); var globalExporter = InMemorySpanExporter.create(); @@ -127,6 +127,13 @@ public ContextPropagators getPropagators() { assertEquals(2, spans.size()); assertTrue(spans.stream().anyMatch(span -> span.getName().equals("invocation"))); assertTrue(spans.stream().anyMatch(span -> span.getName().equals("step"))); + var expectedIds = new DeterministicIdGenerator(); + expectedIds.setDurableExecutionArn("arn:exec1"); + var stepSpan = spans.stream() + .filter(span -> span.getName().equals("step")) + .findFirst() + .orElseThrow(); + assertEquals(expectedIds.generateSpanIdForOperation("op-1"), stepSpan.getSpanId()); assertTrue(OtlpGrpcSpanExporter.getFinishedSpanItems().isEmpty()); } From 5085a3b786923b61cbce9d1b51d1c948ab0b3b2d Mon Sep 17 00:00:00 2001 From: Frank Chen Date: Mon, 13 Jul 2026 22:56:55 -0700 Subject: [PATCH 17/36] Install Java agent ID generator in provider branch --- .../software/amazon/lambda/durable/otel/OtelPlugin.java | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/otel-plugin/src/main/java/software/amazon/lambda/durable/otel/OtelPlugin.java b/otel-plugin/src/main/java/software/amazon/lambda/durable/otel/OtelPlugin.java index a96794ecb..cd62e04e6 100644 --- a/otel-plugin/src/main/java/software/amazon/lambda/durable/otel/OtelPlugin.java +++ b/otel-plugin/src/main/java/software/amazon/lambda/durable/otel/OtelPlugin.java @@ -186,6 +186,9 @@ private OtelPlugin( this.tracer = sdkTracerProvider.get(INSTRUMENTATION_NAME); } else { this.sdkTracerProvider = defaultTracerProvider.sdkTracerProvider(); + if (sdkTracerProvider != null) { + overrideIdGenerator(sdkTracerProvider, idGenerator); + } this.tracer = defaultTracerProvider.tracerProvider().get(INSTRUMENTATION_NAME); } @@ -533,10 +536,9 @@ private static DefaultTracerProvider getDefaultTracerProvider() { var globalTracerProvider = GlobalOpenTelemetry.getTracerProvider(); if (isJavaAgentTracerProvider(globalTracerProvider)) { var sdkTracerProvider = getJavaAgentSdkTracerProvider(globalTracerProvider); - overrideIdGenerator(sdkTracerProvider, idGenerator); logger.info( "OtelPlugin initialized from existing GlobalOpenTelemetry Java agent tracer provider {}; " - + "deterministic span IDs installed through reflection. Cause: {}", + + "deterministic span IDs will be installed through reflection. Cause: {}", globalTracerProvider.getClass().getName(), e.getMessage()); return DefaultTracerProvider.fromTracerProvider(globalTracerProvider, sdkTracerProvider, idGenerator); From cd264467cebf607c8844467f23bd68157155da80 Mon Sep 17 00:00:00 2001 From: Frank Chen Date: Mon, 13 Jul 2026 23:09:26 -0700 Subject: [PATCH 18/36] Use OTel autoconfigure SPI for agent ID generation --- .../OtelXRayDefaultConstructorExample.java | 2 +- otel-plugin/README.md | 11 ++-- otel-plugin/pom.xml | 6 ++ .../lambda/durable/otel/OtelPlugin.java | 60 ++++++++++--------- ...inAutoConfigurationCustomizerProvider.java | 26 ++++++++ ...re.spi.AutoConfigurationCustomizerProvider | 1 + .../otel/OtelPluginIntegrationTest.java | 3 +- .../lambda/durable/otel/OtelPluginTest.java | 55 ++++++++++++++++- 8 files changed, 130 insertions(+), 34 deletions(-) create mode 100644 otel-plugin/src/main/java/software/amazon/lambda/durable/otel/OtelPluginAutoConfigurationCustomizerProvider.java create mode 100644 otel-plugin/src/main/resources/META-INF/services/io.opentelemetry.sdk.autoconfigure.spi.AutoConfigurationCustomizerProvider diff --git a/examples/src/main/java/software/amazon/lambda/durable/examples/otel/OtelXRayDefaultConstructorExample.java b/examples/src/main/java/software/amazon/lambda/durable/examples/otel/OtelXRayDefaultConstructorExample.java index fed15671a..5f5ee86f9 100644 --- a/examples/src/main/java/software/amazon/lambda/durable/examples/otel/OtelXRayDefaultConstructorExample.java +++ b/examples/src/main/java/software/amazon/lambda/durable/examples/otel/OtelXRayDefaultConstructorExample.java @@ -13,7 +13,7 @@ * OTel + X-Ray example that uses the no-arg plugin constructor. * *

    {@link OtelPlugin#OtelPlugin()} copies a global SDK provider when possible, or uses the ADOT Java agent provider - * directly after installing deterministic span ID generation. + * directly with deterministic span ID generation installed by the plugin's OpenTelemetry autoconfigure SPI. */ @ExampleTemplate(tracing = true) public class OtelXRayDefaultConstructorExample extends DurableHandler { diff --git a/otel-plugin/README.md b/otel-plugin/README.md index f3990575b..fee12fa75 100644 --- a/otel-plugin/README.md +++ b/otel-plugin/README.md @@ -46,7 +46,7 @@ You also need the OpenTelemetry SDK and an exporter: ### 1. ADOT Lambda Layer -This plugin uses the [AWS Distro for OpenTelemetry (ADOT) Lambda layer](https://aws-otel.github.io/docs/getting-started/lambda) for trace export. `OtelPlugin()` copies a global SDK provider when possible, or uses the ADOT Java agent provider directly after installing deterministic span ID generation. If no usable global provider exists, it creates the default OTLP gRPC exporter and sends spans to a reachable OTLP collector. +This plugin uses the [AWS Distro for OpenTelemetry (ADOT) Lambda layer](https://aws-otel.github.io/docs/getting-started/lambda) for trace export. `OtelPlugin()` copies a global SDK provider when possible, or uses the ADOT Java agent provider directly with deterministic span ID generation installed through the OpenTelemetry `AutoConfigurationCustomizerProvider` SPI. If no usable global provider exists, it creates the default OTLP gRPC exporter and sends spans to a reachable OTLP collector. The layer ARN follows the format: @@ -66,6 +66,7 @@ MyFunction: Environment: Variables: AWS_LAMBDA_EXEC_WRAPPER: /opt/otel-instrument + OTEL_JAVAAGENT_EXTENSIONS: /var/task/lib/aws-durable-execution-sdk-java-plugin-otel-.jar ``` **AWS CLI:** @@ -74,9 +75,11 @@ MyFunction: aws lambda update-function-configuration \ --function-name your-function-name \ --layers "arn:aws:lambda::615299751070:layer:AWSOpenTelemetryDistroJava:15" \ - --environment "Variables={AWS_LAMBDA_EXEC_WRAPPER=/opt/otel-instrument}" + --environment "Variables={AWS_LAMBDA_EXEC_WRAPPER=/opt/otel-instrument,OTEL_JAVAAGENT_EXTENSIONS=/var/task/lib/aws-durable-execution-sdk-java-plugin-otel-.jar}" ``` +Set `OTEL_JAVAAGENT_EXTENSIONS` to the deployed jar that contains this plugin's `META-INF/services/io.opentelemetry.sdk.autoconfigure.spi.AutoConfigurationCustomizerProvider` entry. If you shade the plugin into your application jar, point the variable to that shaded jar instead. + ### 2. AWS X-Ray Active Tracing Enable active tracing on your Lambda function so the `_X_AMZN_TRACE_ID` environment variable is populated at invocation time. The plugin uses this header to derive deterministic trace IDs that remain consistent across all invocations of the same durable execution. @@ -220,7 +223,7 @@ new OtelPlugin(tracerProviderBuilder, contextExtractor, enableMdc); | Parameter | Description | Default | |-----------|-------------|---------| -| `tracerProviderBuilder` | `SdkTracerProviderBuilder` with your exporter/processor configured | Copies global SDK provider when possible, or uses the ADOT Java agent provider directly | +| `tracerProviderBuilder` | `SdkTracerProviderBuilder` with your exporter/processor configured | Copies global SDK provider when possible, or uses the ADOT Java agent provider configured by the plugin SPI | | `contextExtractor` | Extracts parent trace context from the Lambda environment | `XRayContextExtractor` | | `enableMdc` | If true, injects `traceId`/`spanId`/`traceSampled` into SLF4J MDC | `true` | @@ -245,7 +248,7 @@ After deploying your function with the plugin configured: | Traces appear but are fragmented | X-Ray active tracing not enabled on the Lambda function | | Missing spans for some operations | Sampling is configured below 1.0 | | `_X_AMZN_TRACE_ID` not populated | X-Ray active tracing not enabled | -| Plugin spans missing but Lambda/runtime spans appear | The ADOT wrapper did not initialize `GlobalOpenTelemetry`, or the fallback OTLP exporter cannot reach a collector | +| Plugin spans missing but Lambda/runtime spans appear | The ADOT wrapper did not initialize `GlobalOpenTelemetry`, the plugin jar was not configured in `OTEL_JAVAAGENT_EXTENSIONS`, or the fallback OTLP exporter cannot reach a collector | > **Note on ADOT wrapper:** Use `AWS_LAMBDA_EXEC_WRAPPER=/opt/otel-instrument` with the `AWSOpenTelemetryDistroJava` layer. The older `/opt/otel-handler` path is not valid for this layer and can fail before the Java handler starts. diff --git a/otel-plugin/pom.xml b/otel-plugin/pom.xml index 949122af3..fbdd6126a 100644 --- a/otel-plugin/pom.xml +++ b/otel-plugin/pom.xml @@ -40,6 +40,12 @@ ${opentelemetry.version} provided + + io.opentelemetry + opentelemetry-sdk-extension-autoconfigure-spi + ${opentelemetry.version} + provided + diff --git a/otel-plugin/src/main/java/software/amazon/lambda/durable/otel/OtelPlugin.java b/otel-plugin/src/main/java/software/amazon/lambda/durable/otel/OtelPlugin.java index cd62e04e6..91d6c5519 100644 --- a/otel-plugin/src/main/java/software/amazon/lambda/durable/otel/OtelPlugin.java +++ b/otel-plugin/src/main/java/software/amazon/lambda/durable/otel/OtelPlugin.java @@ -19,7 +19,6 @@ import io.opentelemetry.context.Scope; import io.opentelemetry.sdk.common.Clock; import io.opentelemetry.sdk.resources.Resource; -import io.opentelemetry.sdk.trace.IdGenerator; import io.opentelemetry.sdk.trace.SdkTracerProvider; import io.opentelemetry.sdk.trace.SdkTracerProviderBuilder; import io.opentelemetry.sdk.trace.SpanLimits; @@ -74,8 +73,9 @@ * * *

    When using {@link #OtelPlugin()}, the plugin copies a global SDK provider when one exists. If ADOT exposes a Java - * agent bridge provider, the plugin uses that provider directly after installing its deterministic ID generator through - * reflection. Otherwise it creates a default OTLP exporter from the application classpath. + * agent bridge provider, the plugin uses that provider directly with the deterministic ID generator installed by + * {@link OtelPluginAutoConfigurationCustomizerProvider}. Otherwise it creates a default OTLP exporter from the + * application classpath. * *

    Thread-safe: uses {@link ConcurrentHashMap} for span/scope storage since the SDK runs user code on multiple * threads. @@ -132,8 +132,8 @@ public OtelPlugin(SdkTracerProviderBuilder tracerProviderBuilder) { * Creates an OTel plugin with default settings: X-Ray context extraction, MDC enabled, and default OTLP export. * *

    If {@code GlobalOpenTelemetry} is backed by the SDK, the plugin copies that provider's export pipeline. If - * ADOT exposes a Java agent bridge provider, the plugin uses that provider directly after installing deterministic - * ID generation into the underlying SDK provider. Otherwise, the plugin creates + * ADOT exposes a Java agent bridge provider, the plugin uses that provider directly with deterministic ID + * generation installed by {@link OtelPluginAutoConfigurationCustomizerProvider}. Otherwise, the plugin creates * {@code OtlpGrpcSpanExporter.getDefault()} from the application classpath. */ public OtelPlugin() { @@ -186,9 +186,6 @@ private OtelPlugin( this.tracer = sdkTracerProvider.get(INSTRUMENTATION_NAME); } else { this.sdkTracerProvider = defaultTracerProvider.sdkTracerProvider(); - if (sdkTracerProvider != null) { - overrideIdGenerator(sdkTracerProvider, idGenerator); - } this.tracer = defaultTracerProvider.tracerProvider().get(INSTRUMENTATION_NAME); } @@ -536,12 +533,10 @@ private static DefaultTracerProvider getDefaultTracerProvider() { var globalTracerProvider = GlobalOpenTelemetry.getTracerProvider(); if (isJavaAgentTracerProvider(globalTracerProvider)) { var sdkTracerProvider = getJavaAgentSdkTracerProvider(globalTracerProvider); - logger.info( - "OtelPlugin initialized from existing GlobalOpenTelemetry Java agent tracer provider {}; " - + "deterministic span IDs will be installed through reflection. Cause: {}", - globalTracerProvider.getClass().getName(), - e.getMessage()); - return DefaultTracerProvider.fromTracerProvider(globalTracerProvider, sdkTracerProvider, idGenerator); + var javaAgentIdGenerator = OtelPluginAutoConfigurationCustomizerProvider.idGenerator(); + logJavaAgentInitialization(globalTracerProvider, sdkTracerProvider, javaAgentIdGenerator, e); + return DefaultTracerProvider.fromTracerProvider( + globalTracerProvider, sdkTracerProvider, javaAgentIdGenerator); } else { logger.warn( "OtelPlugin could not initialize from GlobalOpenTelemetry; using default OTLP exporter. " @@ -620,10 +615,31 @@ private static SdkTracerProvider getJavaAgentSdkTracerProvider(TracerProvider tr throw new IllegalStateException("Java agent tracer provider field agentTracerProvider is not a TracerProvider"); } - private static void overrideIdGenerator(SdkTracerProvider sdkTracerProvider, IdGenerator idGenerator) { + private static void logJavaAgentInitialization( + TracerProvider globalTracerProvider, + SdkTracerProvider sdkTracerProvider, + DeterministicIdGenerator idGenerator, + IllegalStateException cause) { + if (hasIdGenerator(sdkTracerProvider, idGenerator)) { + logger.info( + "OtelPlugin initialized from existing GlobalOpenTelemetry Java agent tracer provider {}; " + + "deterministic span IDs installed through AutoConfigurationCustomizerProvider. Cause: {}", + globalTracerProvider.getClass().getName(), + cause.getMessage()); + return; + } + logger.warn( + "OtelPlugin initialized from existing GlobalOpenTelemetry Java agent tracer provider {}, but the " + + "agent SDK provider is not using the durable execution ID generator. Configure this plugin " + + "jar with OTEL_JAVAAGENT_EXTENSIONS so AutoConfigurationCustomizerProvider can install " + + "deterministic span IDs. Cause: {}", + globalTracerProvider.getClass().getName(), + cause.getMessage()); + } + + private static boolean hasIdGenerator(SdkTracerProvider sdkTracerProvider, DeterministicIdGenerator idGenerator) { var sharedState = getField(sdkTracerProvider, "sharedState", Object.class); - setField(sharedState, "idGenerator", idGenerator); - setField(sharedState, "idGeneratorSafeToSkipIdValidation", false); + return getField(sharedState, "idGenerator", Object.class) == idGenerator; } private static SdkTracerProvider unobfuscateSdkTracerProvider(TracerProvider tracerProvider) { @@ -663,16 +679,6 @@ private static T getField(Object target, String fieldName, Class fieldTyp } } - private static void setField(Object target, String fieldName, Object value) { - try { - var field = findField(target.getClass(), fieldName); - field.setAccessible(true); - field.set(target, value); - } catch (NoSuchFieldException | IllegalAccessException e) { - throw new IllegalStateException("Unable to update OpenTelemetry SDK field: " + fieldName, e); - } - } - private static Field findField(Class type, String fieldName) throws NoSuchFieldException { var current = type; while (current != null) { diff --git a/otel-plugin/src/main/java/software/amazon/lambda/durable/otel/OtelPluginAutoConfigurationCustomizerProvider.java b/otel-plugin/src/main/java/software/amazon/lambda/durable/otel/OtelPluginAutoConfigurationCustomizerProvider.java new file mode 100644 index 000000000..7973f5a8f --- /dev/null +++ b/otel-plugin/src/main/java/software/amazon/lambda/durable/otel/OtelPluginAutoConfigurationCustomizerProvider.java @@ -0,0 +1,26 @@ +// Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +// SPDX-License-Identifier: Apache-2.0 +package software.amazon.lambda.durable.otel; + +import io.opentelemetry.sdk.autoconfigure.spi.AutoConfigurationCustomizer; +import io.opentelemetry.sdk.autoconfigure.spi.AutoConfigurationCustomizerProvider; + +/** + * Installs the durable-execution ID generator when the OpenTelemetry Java agent auto-configures the SDK. + * + * @deprecated This is a preview API that is experimental and may be changed or removed in future releases. + */ +@Deprecated +public final class OtelPluginAutoConfigurationCustomizerProvider implements AutoConfigurationCustomizerProvider { + + private static final DeterministicIdGenerator ID_GENERATOR = new DeterministicIdGenerator(); + + @Override + public void customize(AutoConfigurationCustomizer autoConfiguration) { + autoConfiguration.addTracerProviderCustomizer((builder, config) -> builder.setIdGenerator(ID_GENERATOR)); + } + + static DeterministicIdGenerator idGenerator() { + return ID_GENERATOR; + } +} diff --git a/otel-plugin/src/main/resources/META-INF/services/io.opentelemetry.sdk.autoconfigure.spi.AutoConfigurationCustomizerProvider b/otel-plugin/src/main/resources/META-INF/services/io.opentelemetry.sdk.autoconfigure.spi.AutoConfigurationCustomizerProvider new file mode 100644 index 000000000..3aa2fb591 --- /dev/null +++ b/otel-plugin/src/main/resources/META-INF/services/io.opentelemetry.sdk.autoconfigure.spi.AutoConfigurationCustomizerProvider @@ -0,0 +1 @@ +software.amazon.lambda.durable.otel.OtelPluginAutoConfigurationCustomizerProvider diff --git a/otel-plugin/src/test/java/software/amazon/lambda/durable/otel/OtelPluginIntegrationTest.java b/otel-plugin/src/test/java/software/amazon/lambda/durable/otel/OtelPluginIntegrationTest.java index 87a1e1cbf..2e55974a0 100644 --- a/otel-plugin/src/test/java/software/amazon/lambda/durable/otel/OtelPluginIntegrationTest.java +++ b/otel-plugin/src/test/java/software/amazon/lambda/durable/otel/OtelPluginIntegrationTest.java @@ -132,11 +132,12 @@ void defaultConstructor_usesDefaultOtlpExporter_whenGlobalProviderIsNotSdk() { } @Test - void defaultConstructor_usesJavaAgentGlobalTracerProviderDirectly_andOverridesIdGenerator() { + void defaultConstructor_usesJavaAgentGlobalTracerProviderDirectly_withAutoConfiguredIdGenerator() { GlobalOpenTelemetry.resetForTest(); OtlpGrpcSpanExporter.reset(); var globalExporter = InMemorySpanExporter.create(); var sdkTracerProvider = SdkTracerProvider.builder() + .setIdGenerator(OtelPluginAutoConfigurationCustomizerProvider.idGenerator()) .addSpanProcessor(SimpleSpanProcessor.create(globalExporter)) .build(); var javaAgentTracerProvider = new FakeJavaAgentTracerProvider(sdkTracerProvider); diff --git a/otel-plugin/src/test/java/software/amazon/lambda/durable/otel/OtelPluginTest.java b/otel-plugin/src/test/java/software/amazon/lambda/durable/otel/OtelPluginTest.java index 964691d62..1a486c33e 100644 --- a/otel-plugin/src/test/java/software/amazon/lambda/durable/otel/OtelPluginTest.java +++ b/otel-plugin/src/test/java/software/amazon/lambda/durable/otel/OtelPluginTest.java @@ -3,6 +3,10 @@ package software.amazon.lambda.durable.otel; import static org.junit.jupiter.api.Assertions.*; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.when; import io.opentelemetry.api.GlobalOpenTelemetry; import io.opentelemetry.api.OpenTelemetry; @@ -16,13 +20,21 @@ import io.opentelemetry.exporter.otlp.trace.OtlpGrpcSpanExporter; import io.opentelemetry.javaagent.testing.FakeJavaAgentTracerProvider; import io.opentelemetry.sdk.OpenTelemetrySdk; +import io.opentelemetry.sdk.autoconfigure.spi.AutoConfigurationCustomizer; +import io.opentelemetry.sdk.autoconfigure.spi.AutoConfigurationCustomizerProvider; +import io.opentelemetry.sdk.autoconfigure.spi.ConfigProperties; import io.opentelemetry.sdk.testing.exporter.InMemorySpanExporter; import io.opentelemetry.sdk.trace.SdkTracerProvider; +import io.opentelemetry.sdk.trace.SdkTracerProviderBuilder; import io.opentelemetry.sdk.trace.export.SimpleSpanProcessor; import java.time.Instant; +import java.util.ServiceLoader; +import java.util.concurrent.TimeUnit; +import java.util.function.BiFunction; import org.junit.jupiter.api.AfterEach; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; +import org.mockito.ArgumentCaptor; import software.amazon.lambda.durable.execution.SuspendExecutionException; import software.amazon.lambda.durable.plugin.*; @@ -94,11 +106,12 @@ void defaultConstructor_usesDefaultOtlpExporter_whenGlobalProviderIsNotSdk() { } @Test - void defaultConstructor_usesJavaAgentGlobalTracerProviderDirectly_andOverridesIdGenerator() { + void defaultConstructor_usesJavaAgentGlobalTracerProviderDirectly_withAutoConfiguredIdGenerator() { GlobalOpenTelemetry.resetForTest(); OtlpGrpcSpanExporter.reset(); var globalExporter = InMemorySpanExporter.create(); var sdkTracerProvider = SdkTracerProvider.builder() + .setIdGenerator(OtelPluginAutoConfigurationCustomizerProvider.idGenerator()) .addSpanProcessor(SimpleSpanProcessor.create(globalExporter)) .build(); var javaAgentTracerProvider = new FakeJavaAgentTracerProvider(sdkTracerProvider); @@ -137,6 +150,46 @@ public ContextPropagators getPropagators() { assertTrue(OtlpGrpcSpanExporter.getFinishedSpanItems().isEmpty()); } + @Test + void autoConfigurationCustomizerProvider_installsSharedDeterministicIdGenerator() { + var exporter = InMemorySpanExporter.create(); + var autoConfiguration = mock(AutoConfigurationCustomizer.class); + when(autoConfiguration.addTracerProviderCustomizer(any())).thenReturn(autoConfiguration); + + new OtelPluginAutoConfigurationCustomizerProvider().customize(autoConfiguration); + + @SuppressWarnings("unchecked") + var customizer = ArgumentCaptor.forClass(BiFunction.class); + verify(autoConfiguration).addTracerProviderCustomizer(customizer.capture()); + + var idGenerator = OtelPluginAutoConfigurationCustomizerProvider.idGenerator(); + idGenerator.setDurableExecutionArn("arn:spi"); + idGenerator.setNextSpanOperationId("op-spi"); + + @SuppressWarnings("unchecked") + var tracerProviderCustomizer = + (BiFunction) + customizer.getValue(); + var tracerProvider = tracerProviderCustomizer + .apply(SdkTracerProvider.builder().addSpanProcessor(SimpleSpanProcessor.create(exporter)), null) + .build(); + + var span = tracerProvider.get("test").spanBuilder("step").startSpan(); + span.end(); + tracerProvider.forceFlush().join(5, TimeUnit.SECONDS); + + var spans = exporter.getFinishedSpanItems(); + assertEquals(1, spans.size()); + assertEquals( + idGenerator.generateSpanIdForOperation("op-spi"), spans.get(0).getSpanId()); + } + + @Test + void autoConfigurationCustomizerProvider_isRegisteredAsServiceProvider() { + assertTrue(ServiceLoader.load(AutoConfigurationCustomizerProvider.class).stream() + .anyMatch(provider -> provider.type().equals(OtelPluginAutoConfigurationCustomizerProvider.class))); + } + @Test void invocationStart_usesCurrentSpanContext_whenExtractorReturnsNull() { var traceId = "5759e988bd862e3fe1be46a994272793"; From d149fe6ed79acbe045945f8cd3f1631ea5ab35fc Mon Sep 17 00:00:00 2001 From: Frank Chen Date: Mon, 13 Jul 2026 23:21:53 -0700 Subject: [PATCH 19/36] Configure OTel agent extension in examples template --- examples/README.md | 1 + examples/generate-template.py | 19 +++++++++++++++++-- 2 files changed, 18 insertions(+), 2 deletions(-) diff --git a/examples/README.md b/examples/README.md index 600484a67..9940c17e5 100644 --- a/examples/README.md +++ b/examples/README.md @@ -48,6 +48,7 @@ The SAM template configures: - `DurableConfig` with `ExecutionTimeout` and `RetentionPeriodInDays` - CloudWatch log groups for Lambda functions with 7 days of retention - IAM permissions for `lambda:CheckpointDurableExecutions` and `lambda:GetDurableExecutionState` +- ADOT tracing examples with `AWS_LAMBDA_EXEC_WRAPPER` and `OTEL_JAVAAGENT_EXTENSIONS` so the OTel plugin SPI is loaded by the Java agent `template.yaml` is generated from the Java example handlers and is intentionally not checked in. Re-run `python3 generate-template.py` after adding or removing a deployable example handler. diff --git a/examples/generate-template.py b/examples/generate-template.py index 087c90013..bb1a3b8d2 100755 --- a/examples/generate-template.py +++ b/examples/generate-template.py @@ -4,6 +4,7 @@ import argparse import re +import xml.etree.ElementTree as ET from dataclasses import dataclass from pathlib import Path @@ -13,6 +14,18 @@ EXAMPLE_PACKAGE_ROOT = SOURCE_ROOT / "software/amazon/lambda/durable/examples" DEFAULT_OUTPUT = EXAMPLES_DIR / "template.yaml" TEMPLATE_ANNOTATION = "ExampleTemplate" +POM_NAMESPACE = {"m": "http://maven.apache.org/POM/4.0.0"} + + +def read_examples_jar_path() -> str: + root = ET.parse(EXAMPLES_DIR / "pom.xml").getroot() + artifact_id = root.findtext("m:artifactId", namespaces=POM_NAMESPACE) + version = root.findtext("m:version", namespaces=POM_NAMESPACE) + if version is None: + version = root.findtext("m:parent/m:version", namespaces=POM_NAMESPACE) + if artifact_id is None or version is None: + raise ValueError("Unable to read examples artifactId/version from pom.xml") + return f"/var/task/lib/{artifact_id}-{version}.jar" @dataclass(frozen=True) @@ -100,7 +113,7 @@ def discover_examples() -> list[ExampleFunction]: return examples -def emit_function(lines: list[str], example: ExampleFunction) -> None: +def emit_function(lines: list[str], example: ExampleFunction, java_agent_extension_path: str) -> None: lines.extend( [ f" {example.logical_id}:", @@ -127,6 +140,7 @@ def emit_function(lines: list[str], example: ExampleFunction) -> None: " Environment:", " Variables:", " AWS_LAMBDA_EXEC_WRAPPER: /opt/otel-instrument", + f" OTEL_JAVAAGENT_EXTENSIONS: {java_agent_extension_path}", ] ) lines.append("") @@ -152,6 +166,7 @@ def emit_log_group(lines: list[str], example: ExampleFunction) -> None: def render_template(examples: list[ExampleFunction]) -> str: + java_agent_extension_path = read_examples_jar_path() lines = [ "# This file is generated by examples/generate-template.py. Do not edit it by hand.", 'AWSTemplateFormatVersion: "2010-09-09"', @@ -203,7 +218,7 @@ def render_template(examples: list[ExampleFunction]) -> str: for example in examples: emit_log_group(lines, example) - emit_function(lines, example) + emit_function(lines, example, java_agent_extension_path) lines.append("Outputs:") for index, example in enumerate(examples): From 58270e5c55c1c7cb55665a0c39f2a87b12d357b7 Mon Sep 17 00:00:00 2001 From: Frank Chen Date: Mon, 13 Jul 2026 23:33:36 -0700 Subject: [PATCH 20/36] Handle ADOT agent provider classloader split --- .../otel/DeterministicIdGenerator.java | 45 ++++++++++++++++- .../lambda/durable/otel/OtelPlugin.java | 50 +++++++++++++++++-- .../testing/FakeJavaAgentTracerProvider.java | 14 ++++-- .../otel/DeterministicIdGeneratorTest.java | 19 +++++++ .../otel/OtelPluginIntegrationTest.java | 7 ++- .../lambda/durable/otel/OtelPluginTest.java | 7 ++- 6 files changed, 128 insertions(+), 14 deletions(-) diff --git a/otel-plugin/src/main/java/software/amazon/lambda/durable/otel/DeterministicIdGenerator.java b/otel-plugin/src/main/java/software/amazon/lambda/durable/otel/DeterministicIdGenerator.java index 255af2d5e..19cb4d6e0 100644 --- a/otel-plugin/src/main/java/software/amazon/lambda/durable/otel/DeterministicIdGenerator.java +++ b/otel-plugin/src/main/java/software/amazon/lambda/durable/otel/DeterministicIdGenerator.java @@ -30,6 +30,10 @@ public class DeterministicIdGenerator implements IdGenerator { private static final IdGenerator RANDOM = IdGenerator.random(); + private static final String PROPERTY_PREFIX = "software.amazon.lambda.durable.otel."; + private static final String EXTRACTED_TRACE_ID_PROPERTY = PROPERTY_PREFIX + "extractedTraceId"; + private static final String DURABLE_EXECUTION_ARN_PROPERTY = PROPERTY_PREFIX + "durableExecutionArn"; + private static final String PENDING_SPAN_OPERATION_ID_PROPERTY_PREFIX = PROPERTY_PREFIX + "pendingSpanOperationId."; private final AtomicReference extractedTraceId = new AtomicReference<>(null); private final AtomicReference arnDerivedTraceId = new AtomicReference<>(null); @@ -44,6 +48,7 @@ public class DeterministicIdGenerator implements IdGenerator { */ public void setExtractedTraceId(String traceId) { this.extractedTraceId.set(traceId); + setOrClearProperty(EXTRACTED_TRACE_ID_PROPERTY, traceId); } /** @@ -54,7 +59,8 @@ public void setExtractedTraceId(String traceId) { */ public void setDurableExecutionArn(String arn) { this.durableExecutionArn.set(arn); - this.arnDerivedTraceId.set(generateTraceIdFromArn(arn)); + this.arnDerivedTraceId.set(arn != null ? generateTraceIdFromArn(arn) : null); + setOrClearProperty(DURABLE_EXECUTION_ARN_PROPERTY, arn); } /** @@ -64,6 +70,7 @@ public void setDurableExecutionArn(String arn) { */ public void setNextSpanOperationId(String operationId) { this.pendingSpanOperationId.set(operationId); + setOrClearProperty(pendingSpanOperationIdProperty(), operationId); } /** @@ -80,11 +87,18 @@ public String generateSpanIdForOperation(String operationId) { public String generateTraceId() { // Priority 1: extracted from X-Ray header (backend propagates same Root across invocations) var extracted = extractedTraceId.get(); + if (extracted == null) { + extracted = System.getProperty(EXTRACTED_TRACE_ID_PROPERTY); + } if (extracted != null) { return extracted; } // Priority 2: deterministic from execution ARN (local tests, non-Lambda) var arnDerived = arnDerivedTraceId.get(); + if (arnDerived == null) { + var arn = System.getProperty(DURABLE_EXECUTION_ARN_PROPERTY); + arnDerived = arn != null ? generateTraceIdFromArn(arn) : null; + } if (arnDerived != null) { return arnDerived; } @@ -95,8 +109,12 @@ public String generateTraceId() { @Override public String generateSpanId() { var operationId = pendingSpanOperationId.get(); + if (operationId == null) { + operationId = System.getProperty(pendingSpanOperationIdProperty()); + } if (operationId != null) { pendingSpanOperationId.remove(); + System.clearProperty(pendingSpanOperationIdProperty()); return generateSpanIdFromOperation(operationId); } return RANDOM.generateSpanId(); @@ -113,11 +131,36 @@ private String generateTraceIdFromArn(String arn) { */ private String generateSpanIdFromOperation(String operationId) { var arn = durableExecutionArn.get(); + if (arn == null) { + arn = System.getProperty(DURABLE_EXECUTION_ARN_PROPERTY); + } var input = arn != null ? arn + ":" + operationId : operationId; var hash = sha256(input); return hash.substring(0, 16); } + static void clearSharedStateForTest() { + System.clearProperty(EXTRACTED_TRACE_ID_PROPERTY); + System.clearProperty(DURABLE_EXECUTION_ARN_PROPERTY); + System.getProperties().stringPropertyNames().stream() + .filter(name -> name.startsWith(PENDING_SPAN_OPERATION_ID_PROPERTY_PREFIX)) + .toList() + .forEach(System::clearProperty); + } + + private static String pendingSpanOperationIdProperty() { + return PENDING_SPAN_OPERATION_ID_PROPERTY_PREFIX + + Thread.currentThread().getId(); + } + + private static void setOrClearProperty(String key, String value) { + if (value == null) { + System.clearProperty(key); + } else { + System.setProperty(key, value); + } + } + private static String sha256(String input) { try { var digest = MessageDigest.getInstance("SHA-256"); diff --git a/otel-plugin/src/main/java/software/amazon/lambda/durable/otel/OtelPlugin.java b/otel-plugin/src/main/java/software/amazon/lambda/durable/otel/OtelPlugin.java index 91d6c5519..b22f64ae8 100644 --- a/otel-plugin/src/main/java/software/amazon/lambda/durable/otel/OtelPlugin.java +++ b/otel-plugin/src/main/java/software/amazon/lambda/durable/otel/OtelPlugin.java @@ -211,6 +211,8 @@ public void onInvocationStart(InvocationInfo info) { if (extractedContext != null) { // Use the X-Ray trace ID — backend propagates same Root across all invocations idGenerator.setExtractedTraceId(extractedContext.traceId()); + } else { + idGenerator.setExtractedTraceId(null); } // If no extracted context, idGenerator falls back to ARN-derived trace ID @@ -605,14 +607,37 @@ private static boolean isJavaAgentTracerProvider(TracerProvider tracerProvider) } private static SdkTracerProvider getJavaAgentSdkTracerProvider(TracerProvider tracerProvider) { - var agentTracerProvider = getField(tracerProvider, "agentTracerProvider", Object.class); + Object agentTracerProvider; + try { + agentTracerProvider = getField(tracerProvider, "agentTracerProvider", Object.class); + } catch (IllegalStateException e) { + logger.warn( + "OtelPlugin could not inspect Java agent tracer provider {}; using the global provider directly.", + tracerProvider.getClass().getName(), + e); + return null; + } + if (agentTracerProvider instanceof SdkTracerProvider sdkTracerProvider) { return sdkTracerProvider; } if (agentTracerProvider instanceof TracerProvider nestedTracerProvider) { - return unobfuscateSdkTracerProvider(nestedTracerProvider); + try { + return unobfuscateSdkTracerProvider(nestedTracerProvider); + } catch (IllegalStateException e) { + logger.warn( + "OtelPlugin could not inspect nested Java agent SDK tracer provider {}; using the global " + + "provider directly.", + nestedTracerProvider.getClass().getName(), + e); + return null; + } } - throw new IllegalStateException("Java agent tracer provider field agentTracerProvider is not a TracerProvider"); + logger.info( + "OtelPlugin could not inspect Java agent SDK tracer provider because field agentTracerProvider has " + + "type {}; using the global provider directly.", + agentTracerProvider != null ? agentTracerProvider.getClass().getName() : "null"); + return null; } private static void logJavaAgentInitialization( @@ -620,6 +645,16 @@ private static void logJavaAgentInitialization( SdkTracerProvider sdkTracerProvider, DeterministicIdGenerator idGenerator, IllegalStateException cause) { + if (sdkTracerProvider == null) { + logger.info( + "OtelPlugin initialized from existing GlobalOpenTelemetry Java agent tracer provider {}, but the " + + "agent SDK provider is not visible to the application class loader, so deterministic " + + "ID generator installation cannot be verified. If durable operation span IDs are not " + + "deterministic, configure this plugin jar with OTEL_JAVAAGENT_EXTENSIONS. Cause: {}", + globalTracerProvider.getClass().getName(), + cause.getMessage()); + return; + } if (hasIdGenerator(sdkTracerProvider, idGenerator)) { logger.info( "OtelPlugin initialized from existing GlobalOpenTelemetry Java agent tracer provider {}; " @@ -638,8 +673,13 @@ private static void logJavaAgentInitialization( } private static boolean hasIdGenerator(SdkTracerProvider sdkTracerProvider, DeterministicIdGenerator idGenerator) { - var sharedState = getField(sdkTracerProvider, "sharedState", Object.class); - return getField(sharedState, "idGenerator", Object.class) == idGenerator; + try { + var sharedState = getField(sdkTracerProvider, "sharedState", Object.class); + return getField(sharedState, "idGenerator", Object.class) == idGenerator; + } catch (IllegalStateException e) { + logger.warn("OtelPlugin could not verify the Java agent SDK ID generator", e); + return false; + } } private static SdkTracerProvider unobfuscateSdkTracerProvider(TracerProvider tracerProvider) { diff --git a/otel-plugin/src/test/java/io/opentelemetry/javaagent/testing/FakeJavaAgentTracerProvider.java b/otel-plugin/src/test/java/io/opentelemetry/javaagent/testing/FakeJavaAgentTracerProvider.java index 7d83f2152..ed88eeafc 100644 --- a/otel-plugin/src/test/java/io/opentelemetry/javaagent/testing/FakeJavaAgentTracerProvider.java +++ b/otel-plugin/src/test/java/io/opentelemetry/javaagent/testing/FakeJavaAgentTracerProvider.java @@ -8,24 +8,30 @@ public final class FakeJavaAgentTracerProvider implements TracerProvider { - private final TracerProvider agentTracerProvider; + private final Object agentTracerProvider; + private final TracerProvider delegate; public FakeJavaAgentTracerProvider(TracerProvider agentTracerProvider) { + this(agentTracerProvider, agentTracerProvider); + } + + public FakeJavaAgentTracerProvider(Object agentTracerProvider, TracerProvider delegate) { this.agentTracerProvider = agentTracerProvider; + this.delegate = delegate; } @Override public Tracer get(String instrumentationName) { - return agentTracerProvider.get(instrumentationName); + return delegate.get(instrumentationName); } @Override public Tracer get(String instrumentationName, String instrumentationVersion) { - return agentTracerProvider.get(instrumentationName, instrumentationVersion); + return delegate.get(instrumentationName, instrumentationVersion); } @Override public TracerBuilder tracerBuilder(String instrumentationScopeName) { - return agentTracerProvider.tracerBuilder(instrumentationScopeName); + return delegate.tracerBuilder(instrumentationScopeName); } } diff --git a/otel-plugin/src/test/java/software/amazon/lambda/durable/otel/DeterministicIdGeneratorTest.java b/otel-plugin/src/test/java/software/amazon/lambda/durable/otel/DeterministicIdGeneratorTest.java index bc1a5a90d..6133a3ddb 100644 --- a/otel-plugin/src/test/java/software/amazon/lambda/durable/otel/DeterministicIdGeneratorTest.java +++ b/otel-plugin/src/test/java/software/amazon/lambda/durable/otel/DeterministicIdGeneratorTest.java @@ -4,6 +4,7 @@ import static org.junit.jupiter.api.Assertions.*; +import org.junit.jupiter.api.AfterEach; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; @@ -13,9 +14,15 @@ class DeterministicIdGeneratorTest { @BeforeEach void setUp() { + DeterministicIdGenerator.clearSharedStateForTest(); generator = new DeterministicIdGenerator(); } + @AfterEach + void tearDown() { + DeterministicIdGenerator.clearSharedStateForTest(); + } + @Test void generateTraceId_withoutArn_returnsRandom() { var id1 = generator.generateTraceId(); @@ -120,6 +127,18 @@ void generateSpanIdForOperation_isDeterministic() { assertEquals(id1, id2); } + @Test + void generatedIds_areSharedAcrossGeneratorInstances() { + var pluginGenerator = new DeterministicIdGenerator(); + var agentGenerator = new DeterministicIdGenerator(); + + pluginGenerator.setDurableExecutionArn("arn:exec1"); + pluginGenerator.setNextSpanOperationId("op-1"); + + assertEquals(pluginGenerator.generateTraceId(), agentGenerator.generateTraceId()); + assertEquals(pluginGenerator.generateSpanIdForOperation("op-1"), agentGenerator.generateSpanId()); + } + @Test void traceId_isValidHex() { generator.setDurableExecutionArn("arn:exec1"); diff --git a/otel-plugin/src/test/java/software/amazon/lambda/durable/otel/OtelPluginIntegrationTest.java b/otel-plugin/src/test/java/software/amazon/lambda/durable/otel/OtelPluginIntegrationTest.java index 2e55974a0..7ef8399d8 100644 --- a/otel-plugin/src/test/java/software/amazon/lambda/durable/otel/OtelPluginIntegrationTest.java +++ b/otel-plugin/src/test/java/software/amazon/lambda/durable/otel/OtelPluginIntegrationTest.java @@ -41,6 +41,7 @@ class OtelPluginIntegrationTest { @BeforeEach void setUp() { + DeterministicIdGenerator.clearSharedStateForTest(); spanExporter = InMemorySpanExporter.create(); var plugin = new OtelPlugin( @@ -55,6 +56,7 @@ void setUp() { void tearDown() { GlobalOpenTelemetry.resetForTest(); OtlpGrpcSpanExporter.reset(); + DeterministicIdGenerator.clearSharedStateForTest(); } @Test @@ -136,11 +138,12 @@ void defaultConstructor_usesJavaAgentGlobalTracerProviderDirectly_withAutoConfig GlobalOpenTelemetry.resetForTest(); OtlpGrpcSpanExporter.reset(); var globalExporter = InMemorySpanExporter.create(); + var javaAgentIdGenerator = new DeterministicIdGenerator(); var sdkTracerProvider = SdkTracerProvider.builder() - .setIdGenerator(OtelPluginAutoConfigurationCustomizerProvider.idGenerator()) + .setIdGenerator(javaAgentIdGenerator) .addSpanProcessor(SimpleSpanProcessor.create(globalExporter)) .build(); - var javaAgentTracerProvider = new FakeJavaAgentTracerProvider(sdkTracerProvider); + var javaAgentTracerProvider = new FakeJavaAgentTracerProvider(new Object(), sdkTracerProvider); GlobalOpenTelemetry.set(new OpenTelemetry() { @Override public io.opentelemetry.api.trace.TracerProvider getTracerProvider() { diff --git a/otel-plugin/src/test/java/software/amazon/lambda/durable/otel/OtelPluginTest.java b/otel-plugin/src/test/java/software/amazon/lambda/durable/otel/OtelPluginTest.java index 1a486c33e..715e7d4cb 100644 --- a/otel-plugin/src/test/java/software/amazon/lambda/durable/otel/OtelPluginTest.java +++ b/otel-plugin/src/test/java/software/amazon/lambda/durable/otel/OtelPluginTest.java @@ -45,6 +45,7 @@ class OtelPluginTest { @BeforeEach void setUp() { + DeterministicIdGenerator.clearSharedStateForTest(); spanExporter = InMemorySpanExporter.create(); plugin = new OtelPlugin( @@ -57,6 +58,7 @@ void setUp() { void tearDown() { GlobalOpenTelemetry.resetForTest(); OtlpGrpcSpanExporter.reset(); + DeterministicIdGenerator.clearSharedStateForTest(); } @Test @@ -110,11 +112,12 @@ void defaultConstructor_usesJavaAgentGlobalTracerProviderDirectly_withAutoConfig GlobalOpenTelemetry.resetForTest(); OtlpGrpcSpanExporter.reset(); var globalExporter = InMemorySpanExporter.create(); + var javaAgentIdGenerator = new DeterministicIdGenerator(); var sdkTracerProvider = SdkTracerProvider.builder() - .setIdGenerator(OtelPluginAutoConfigurationCustomizerProvider.idGenerator()) + .setIdGenerator(javaAgentIdGenerator) .addSpanProcessor(SimpleSpanProcessor.create(globalExporter)) .build(); - var javaAgentTracerProvider = new FakeJavaAgentTracerProvider(sdkTracerProvider); + var javaAgentTracerProvider = new FakeJavaAgentTracerProvider(new Object(), sdkTracerProvider); GlobalOpenTelemetry.set(new OpenTelemetry() { @Override public io.opentelemetry.api.trace.TracerProvider getTracerProvider() { From 34429f0b0150513fed22bb6851046361ddfb0dc9 Mon Sep 17 00:00:00 2001 From: Frank Chen Date: Mon, 13 Jul 2026 23:38:52 -0700 Subject: [PATCH 21/36] Require OTel SPI for default plugin constructor --- .../lambda/durable/otel/OtelPlugin.java | 87 +++++++++---------- ...inAutoConfigurationCustomizerProvider.java | 15 ++++ .../otel/OtelPluginIntegrationTest.java | 7 +- .../lambda/durable/otel/OtelPluginTest.java | 65 +++++++++++++- 4 files changed, 126 insertions(+), 48 deletions(-) diff --git a/otel-plugin/src/main/java/software/amazon/lambda/durable/otel/OtelPlugin.java b/otel-plugin/src/main/java/software/amazon/lambda/durable/otel/OtelPlugin.java index b22f64ae8..07c104dd0 100644 --- a/otel-plugin/src/main/java/software/amazon/lambda/durable/otel/OtelPlugin.java +++ b/otel-plugin/src/main/java/software/amazon/lambda/durable/otel/OtelPlugin.java @@ -17,15 +17,11 @@ import io.opentelemetry.api.trace.TracerProvider; import io.opentelemetry.context.Context; import io.opentelemetry.context.Scope; -import io.opentelemetry.sdk.common.Clock; import io.opentelemetry.sdk.resources.Resource; import io.opentelemetry.sdk.trace.SdkTracerProvider; import io.opentelemetry.sdk.trace.SdkTracerProviderBuilder; -import io.opentelemetry.sdk.trace.SpanLimits; -import io.opentelemetry.sdk.trace.SpanProcessor; import io.opentelemetry.sdk.trace.export.SimpleSpanProcessor; import io.opentelemetry.sdk.trace.export.SpanExporter; -import io.opentelemetry.sdk.trace.samplers.Sampler; import io.opentelemetry.semconv.ServiceAttributes; import java.lang.reflect.Field; import java.lang.reflect.InvocationTargetException; @@ -33,7 +29,6 @@ import java.time.Instant; import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.TimeUnit; -import java.util.function.Supplier; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import software.amazon.lambda.durable.plugin.DurableExecutionPlugin; @@ -72,10 +67,9 @@ *

  • Tracing: Active (to populate {@code _X_AMZN_TRACE_ID}) * * - *

    When using {@link #OtelPlugin()}, the plugin copies a global SDK provider when one exists. If ADOT exposes a Java - * agent bridge provider, the plugin uses that provider directly with the deterministic ID generator installed by - * {@link OtelPluginAutoConfigurationCustomizerProvider}. Otherwise it creates a default OTLP exporter from the - * application classpath. + *

    When using {@link #OtelPlugin()}, the plugin uses the global provider directly when one exists and assumes the + * deterministic ID generator was installed by {@link OtelPluginAutoConfigurationCustomizerProvider}. Otherwise it + * creates a default OTLP exporter from the application classpath. * *

    Thread-safe: uses {@link ConcurrentHashMap} for span/scope storage since the SDK runs user code on multiple * threads. @@ -131,10 +125,9 @@ public OtelPlugin(SdkTracerProviderBuilder tracerProviderBuilder) { /** * Creates an OTel plugin with default settings: X-Ray context extraction, MDC enabled, and default OTLP export. * - *

    If {@code GlobalOpenTelemetry} is backed by the SDK, the plugin copies that provider's export pipeline. If - * ADOT exposes a Java agent bridge provider, the plugin uses that provider directly with deterministic ID - * generation installed by {@link OtelPluginAutoConfigurationCustomizerProvider}. Otherwise, the plugin creates - * {@code OtlpGrpcSpanExporter.getDefault()} from the application classpath. + *

    If {@code GlobalOpenTelemetry} is configured, the plugin uses that provider directly and assumes deterministic + * ID generation was installed by {@link OtelPluginAutoConfigurationCustomizerProvider}. Otherwise, the plugin + * creates {@code OtlpGrpcSpanExporter.getDefault()} from the application classpath. */ public OtelPlugin() { this(getDefaultTracerProvider()); @@ -179,10 +172,12 @@ private OtelPlugin( var resource = Resource.create(Attributes.of(ServiceAttributes.SERVICE_NAME, "invocation")); defaultTracerProvider.sdkBuilder().addResource(resource); - this.sdkTracerProvider = defaultTracerProvider - .sdkBuilder() - .setIdGenerator(idGenerator) - .build(); + var sdkBuilder = defaultTracerProvider.sdkBuilder(); + if (defaultTracerProvider.installIdGenerator()) { + sdkBuilder.setIdGenerator(idGenerator); + } + + this.sdkTracerProvider = sdkBuilder.build(); this.tracer = sdkTracerProvider.get(INSTRUMENTATION_NAME); } else { this.sdkTracerProvider = defaultTracerProvider.sdkTracerProvider(); @@ -526,19 +521,21 @@ private static ExtractedContext extractCurrentSpanContext() { } private static DefaultTracerProvider getDefaultTracerProvider() { + validateAutoConfigurationCustomizerProviderInstalled(); + var idGenerator = new DeterministicIdGenerator(); + var globalTracerProvider = GlobalOpenTelemetry.getTracerProvider(); try { - var globalBuilder = getGlobalTracerProviderBuilder(); - logger.info("OtelPlugin initialized from existing GlobalOpenTelemetry SDK tracer provider"); - return DefaultTracerProvider.fromSdkBuilder(globalBuilder, idGenerator); + var sdkTracerProvider = getGlobalSdkTracerProvider(globalTracerProvider); + logger.info("OtelPlugin initialized from existing GlobalOpenTelemetry SDK tracer provider; assuming " + + "deterministic span IDs were installed through AutoConfigurationCustomizerProvider"); + return DefaultTracerProvider.fromTracerProvider(globalTracerProvider, sdkTracerProvider, idGenerator); } catch (IllegalStateException e) { - var globalTracerProvider = GlobalOpenTelemetry.getTracerProvider(); if (isJavaAgentTracerProvider(globalTracerProvider)) { var sdkTracerProvider = getJavaAgentSdkTracerProvider(globalTracerProvider); var javaAgentIdGenerator = OtelPluginAutoConfigurationCustomizerProvider.idGenerator(); logJavaAgentInitialization(globalTracerProvider, sdkTracerProvider, javaAgentIdGenerator, e); - return DefaultTracerProvider.fromTracerProvider( - globalTracerProvider, sdkTracerProvider, javaAgentIdGenerator); + return DefaultTracerProvider.fromTracerProvider(globalTracerProvider, sdkTracerProvider, idGenerator); } else { logger.warn( "OtelPlugin could not initialize from GlobalOpenTelemetry; using default OTLP exporter. " @@ -549,22 +546,20 @@ private static DefaultTracerProvider getDefaultTracerProvider() { var fallbackBuilder = SdkTracerProvider.builder() .addSpanProcessor(SimpleSpanProcessor.create(createDefaultOtlpExporter())); - logger.info( - "OtelPlugin initialized with a new default OTLP tracer provider; ensure an OTLP collector is reachable"); - return DefaultTracerProvider.fromSdkBuilder(fallbackBuilder, idGenerator); + logger.info("OtelPlugin initialized with a new default OTLP tracer provider without overriding its ID " + + "generator; ensure an OTLP collector is reachable"); + return DefaultTracerProvider.fromSdkBuilderWithoutIdGenerator(fallbackBuilder, idGenerator); } } - private static SdkTracerProviderBuilder getGlobalTracerProviderBuilder() { - var sdkTracerProvider = getGlobalSdkTracerProvider(); - var sharedState = getField(sdkTracerProvider, "sharedState", Object.class); - - return SdkTracerProvider.builder() - .setClock(getField(sharedState, "clock", Clock.class)) - .setResource(getField(sharedState, "resource", Resource.class)) - .setSpanLimits(getSpanLimitsSupplier(sharedState)) - .setSampler(getField(sharedState, "sampler", Sampler.class)) - .addSpanProcessor(getField(sharedState, "activeSpanProcessor", SpanProcessor.class)); + private static void validateAutoConfigurationCustomizerProviderInstalled() { + if (OtelPluginAutoConfigurationCustomizerProvider.isInstalled()) { + return; + } + throw new IllegalStateException( + "OtelPlugin() requires OtelPluginAutoConfigurationCustomizerProvider to be installed by the " + + "OpenTelemetry Java agent. Package this plugin jar as an agent extension and set " + + "OTEL_JAVAAGENT_EXTENSIONS to that jar before constructing OtelPlugin()."); } private static SpanExporter createDefaultOtlpExporter() { @@ -594,8 +589,7 @@ private static SpanExporter asSpanExporter(Method getDefault) { } } - private static SdkTracerProvider getGlobalSdkTracerProvider() { - var tracerProvider = GlobalOpenTelemetry.getTracerProvider(); + private static SdkTracerProvider getGlobalSdkTracerProvider(TracerProvider tracerProvider) { if (tracerProvider instanceof SdkTracerProvider sdkTracerProvider) { return sdkTracerProvider; } @@ -704,11 +698,6 @@ private static SdkTracerProvider unobfuscateSdkTracerProvider(TracerProvider tra } } - @SuppressWarnings("unchecked") - private static Supplier getSpanLimitsSupplier(Object sharedState) { - return (Supplier) getField(sharedState, "spanLimitsSupplier", Supplier.class); - } - private static T getField(Object target, String fieldName, Class fieldType) { try { var field = findField(target.getClass(), fieldName); @@ -735,18 +724,24 @@ private record DefaultTracerProvider( SdkTracerProviderBuilder sdkBuilder, TracerProvider tracerProvider, SdkTracerProvider sdkTracerProvider, - DeterministicIdGenerator idGenerator) { + DeterministicIdGenerator idGenerator, + boolean installIdGenerator) { private static DefaultTracerProvider fromSdkBuilder( SdkTracerProviderBuilder sdkBuilder, DeterministicIdGenerator idGenerator) { - return new DefaultTracerProvider(sdkBuilder, null, null, idGenerator); + return new DefaultTracerProvider(sdkBuilder, null, null, idGenerator, true); + } + + private static DefaultTracerProvider fromSdkBuilderWithoutIdGenerator( + SdkTracerProviderBuilder sdkBuilder, DeterministicIdGenerator idGenerator) { + return new DefaultTracerProvider(sdkBuilder, null, null, idGenerator, false); } private static DefaultTracerProvider fromTracerProvider( TracerProvider tracerProvider, SdkTracerProvider sdkTracerProvider, DeterministicIdGenerator idGenerator) { - return new DefaultTracerProvider(null, tracerProvider, sdkTracerProvider, idGenerator); + return new DefaultTracerProvider(null, tracerProvider, sdkTracerProvider, idGenerator, false); } } } diff --git a/otel-plugin/src/main/java/software/amazon/lambda/durable/otel/OtelPluginAutoConfigurationCustomizerProvider.java b/otel-plugin/src/main/java/software/amazon/lambda/durable/otel/OtelPluginAutoConfigurationCustomizerProvider.java index 7973f5a8f..08b205d7b 100644 --- a/otel-plugin/src/main/java/software/amazon/lambda/durable/otel/OtelPluginAutoConfigurationCustomizerProvider.java +++ b/otel-plugin/src/main/java/software/amazon/lambda/durable/otel/OtelPluginAutoConfigurationCustomizerProvider.java @@ -13,14 +13,29 @@ @Deprecated public final class OtelPluginAutoConfigurationCustomizerProvider implements AutoConfigurationCustomizerProvider { + private static final String INSTALLED_PROPERTY = + "software.amazon.lambda.durable.otel.autoConfigurationCustomizerProviderInstalled"; private static final DeterministicIdGenerator ID_GENERATOR = new DeterministicIdGenerator(); @Override public void customize(AutoConfigurationCustomizer autoConfiguration) { + markInstalled(); autoConfiguration.addTracerProviderCustomizer((builder, config) -> builder.setIdGenerator(ID_GENERATOR)); } static DeterministicIdGenerator idGenerator() { return ID_GENERATOR; } + + static boolean isInstalled() { + return Boolean.getBoolean(INSTALLED_PROPERTY); + } + + static void markInstalled() { + System.setProperty(INSTALLED_PROPERTY, Boolean.TRUE.toString()); + } + + static void resetInstalledForTest() { + System.clearProperty(INSTALLED_PROPERTY); + } } diff --git a/otel-plugin/src/test/java/software/amazon/lambda/durable/otel/OtelPluginIntegrationTest.java b/otel-plugin/src/test/java/software/amazon/lambda/durable/otel/OtelPluginIntegrationTest.java index 7ef8399d8..f5aeeef2c 100644 --- a/otel-plugin/src/test/java/software/amazon/lambda/durable/otel/OtelPluginIntegrationTest.java +++ b/otel-plugin/src/test/java/software/amazon/lambda/durable/otel/OtelPluginIntegrationTest.java @@ -42,6 +42,7 @@ class OtelPluginIntegrationTest { @BeforeEach void setUp() { DeterministicIdGenerator.clearSharedStateForTest(); + OtelPluginAutoConfigurationCustomizerProvider.resetInstalledForTest(); spanExporter = InMemorySpanExporter.create(); var plugin = new OtelPlugin( @@ -57,6 +58,7 @@ void tearDown() { GlobalOpenTelemetry.resetForTest(); OtlpGrpcSpanExporter.reset(); DeterministicIdGenerator.clearSharedStateForTest(); + OtelPluginAutoConfigurationCustomizerProvider.resetInstalledForTest(); } @Test @@ -83,7 +85,8 @@ void simpleStep_producesInvocationAndOperationAndAttemptSpans() { } @Test - void defaultConstructor_copiesGlobalSdkTracerProviderPipeline() { + void defaultConstructor_usesGlobalSdkTracerProviderDirectly() { + OtelPluginAutoConfigurationCustomizerProvider.markInstalled(); GlobalOpenTelemetry.resetForTest(); var globalExporter = InMemorySpanExporter.create(); var globalTracerProvider = SdkTracerProvider.builder() @@ -113,6 +116,7 @@ void defaultConstructor_copiesGlobalSdkTracerProviderPipeline() { @Test void defaultConstructor_usesDefaultOtlpExporter_whenGlobalProviderIsNotSdk() { + OtelPluginAutoConfigurationCustomizerProvider.markInstalled(); GlobalOpenTelemetry.resetForTest(); OtlpGrpcSpanExporter.reset(); @@ -135,6 +139,7 @@ void defaultConstructor_usesDefaultOtlpExporter_whenGlobalProviderIsNotSdk() { @Test void defaultConstructor_usesJavaAgentGlobalTracerProviderDirectly_withAutoConfiguredIdGenerator() { + OtelPluginAutoConfigurationCustomizerProvider.markInstalled(); GlobalOpenTelemetry.resetForTest(); OtlpGrpcSpanExporter.reset(); var globalExporter = InMemorySpanExporter.create(); diff --git a/otel-plugin/src/test/java/software/amazon/lambda/durable/otel/OtelPluginTest.java b/otel-plugin/src/test/java/software/amazon/lambda/durable/otel/OtelPluginTest.java index 715e7d4cb..e6b5fcd57 100644 --- a/otel-plugin/src/test/java/software/amazon/lambda/durable/otel/OtelPluginTest.java +++ b/otel-plugin/src/test/java/software/amazon/lambda/durable/otel/OtelPluginTest.java @@ -24,12 +24,14 @@ import io.opentelemetry.sdk.autoconfigure.spi.AutoConfigurationCustomizerProvider; import io.opentelemetry.sdk.autoconfigure.spi.ConfigProperties; import io.opentelemetry.sdk.testing.exporter.InMemorySpanExporter; +import io.opentelemetry.sdk.trace.IdGenerator; import io.opentelemetry.sdk.trace.SdkTracerProvider; import io.opentelemetry.sdk.trace.SdkTracerProviderBuilder; import io.opentelemetry.sdk.trace.export.SimpleSpanProcessor; import java.time.Instant; import java.util.ServiceLoader; import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicInteger; import java.util.function.BiFunction; import org.junit.jupiter.api.AfterEach; import org.junit.jupiter.api.BeforeEach; @@ -46,6 +48,7 @@ class OtelPluginTest { @BeforeEach void setUp() { DeterministicIdGenerator.clearSharedStateForTest(); + OtelPluginAutoConfigurationCustomizerProvider.resetInstalledForTest(); spanExporter = InMemorySpanExporter.create(); plugin = new OtelPlugin( @@ -59,10 +62,22 @@ void tearDown() { GlobalOpenTelemetry.resetForTest(); OtlpGrpcSpanExporter.reset(); DeterministicIdGenerator.clearSharedStateForTest(); + OtelPluginAutoConfigurationCustomizerProvider.resetInstalledForTest(); } @Test - void defaultConstructor_copiesGlobalSdkTracerProviderPipeline() { + void defaultConstructor_throwsWhenAutoConfigurationCustomizerProviderIsNotInstalled() { + GlobalOpenTelemetry.resetForTest(); + + var error = assertThrows(IllegalStateException.class, OtelPlugin::new); + + assertTrue(error.getMessage().contains("OtelPluginAutoConfigurationCustomizerProvider")); + assertTrue(error.getMessage().contains("OTEL_JAVAAGENT_EXTENSIONS")); + } + + @Test + void defaultConstructor_usesGlobalSdkTracerProviderDirectly() { + OtelPluginAutoConfigurationCustomizerProvider.markInstalled(); GlobalOpenTelemetry.resetForTest(); var globalExporter = InMemorySpanExporter.create(); var globalTracerProvider = SdkTracerProvider.builder() @@ -87,8 +102,37 @@ void defaultConstructor_copiesGlobalSdkTracerProviderPipeline() { assertTrue(spans.stream().allMatch(span -> span.getTraceId().equals(traceId))); } + @Test + void defaultConstructor_doesNotReplaceGlobalSdkTracerProviderIdGenerator() { + OtelPluginAutoConfigurationCustomizerProvider.markInstalled(); + GlobalOpenTelemetry.resetForTest(); + var globalExporter = InMemorySpanExporter.create(); + var idGenerator = new CountingIdGenerator(); + var globalTracerProvider = SdkTracerProvider.builder() + .setIdGenerator(idGenerator) + .addSpanProcessor(SimpleSpanProcessor.create(globalExporter)) + .build(); + OpenTelemetrySdk.builder().setTracerProvider(globalTracerProvider).buildAndRegisterGlobal(); + + var defaultPlugin = new OtelPlugin(); + defaultPlugin.onInvocationStart(new InvocationInfo("req-1", "arn:exec1", true)); + defaultPlugin.onOperationStart( + new OperationInfo("op-1", "step", "STEP", "Step", null, Instant.now(), null, false)); + defaultPlugin.onOperationEnd(new OperationEndInfo( + "op-1", "step", "STEP", "Step", null, Instant.now(), Instant.now(), "SUCCEEDED", false, null)); + defaultPlugin.onInvocationEnd( + new InvocationEndInfo("req-1", "arn:exec1", true, InvocationStatus.SUCCEEDED, null)); + + var stepSpan = globalExporter.getFinishedSpanItems().stream() + .filter(span -> span.getName().equals("step")) + .findFirst() + .orElseThrow(); + assertEquals("0000000000000002", stepSpan.getSpanId()); + } + @Test void defaultConstructor_usesDefaultOtlpExporter_whenGlobalProviderIsNotSdk() { + OtelPluginAutoConfigurationCustomizerProvider.markInstalled(); GlobalOpenTelemetry.resetForTest(); OtlpGrpcSpanExporter.reset(); @@ -109,6 +153,7 @@ void defaultConstructor_usesDefaultOtlpExporter_whenGlobalProviderIsNotSdk() { @Test void defaultConstructor_usesJavaAgentGlobalTracerProviderDirectly_withAutoConfiguredIdGenerator() { + OtelPluginAutoConfigurationCustomizerProvider.markInstalled(); GlobalOpenTelemetry.resetForTest(); OtlpGrpcSpanExporter.reset(); var globalExporter = InMemorySpanExporter.create(); @@ -155,11 +200,13 @@ public ContextPropagators getPropagators() { @Test void autoConfigurationCustomizerProvider_installsSharedDeterministicIdGenerator() { + OtelPluginAutoConfigurationCustomizerProvider.resetInstalledForTest(); var exporter = InMemorySpanExporter.create(); var autoConfiguration = mock(AutoConfigurationCustomizer.class); when(autoConfiguration.addTracerProviderCustomizer(any())).thenReturn(autoConfiguration); new OtelPluginAutoConfigurationCustomizerProvider().customize(autoConfiguration); + assertTrue(OtelPluginAutoConfigurationCustomizerProvider.isInstalled()); @SuppressWarnings("unchecked") var customizer = ArgumentCaptor.forClass(BiFunction.class); @@ -187,6 +234,22 @@ void autoConfigurationCustomizerProvider_installsSharedDeterministicIdGenerator( idGenerator.generateSpanIdForOperation("op-spi"), spans.get(0).getSpanId()); } + private static final class CountingIdGenerator implements IdGenerator { + + private final AtomicInteger traceIds = new AtomicInteger(); + private final AtomicInteger spanIds = new AtomicInteger(); + + @Override + public String generateTraceId() { + return String.format("%032x", traceIds.incrementAndGet()); + } + + @Override + public String generateSpanId() { + return String.format("%016x", spanIds.incrementAndGet()); + } + } + @Test void autoConfigurationCustomizerProvider_isRegisteredAsServiceProvider() { assertTrue(ServiceLoader.load(AutoConfigurationCustomizerProvider.class).stream() From ffb8c74b35ca92704b097cea9db8d46483016f62 Mon Sep 17 00:00:00 2001 From: Frank Chen Date: Mon, 13 Jul 2026 23:44:28 -0700 Subject: [PATCH 22/36] Rely on OTel SPI for default plugin provider --- .../lambda/durable/otel/OtelPlugin.java | 161 ++++++------------ .../otel/OtelPluginIntegrationTest.java | 23 --- .../lambda/durable/otel/OtelPluginTest.java | 32 ++-- 3 files changed, 63 insertions(+), 153 deletions(-) diff --git a/otel-plugin/src/main/java/software/amazon/lambda/durable/otel/OtelPlugin.java b/otel-plugin/src/main/java/software/amazon/lambda/durable/otel/OtelPlugin.java index 07c104dd0..a0c3c0451 100644 --- a/otel-plugin/src/main/java/software/amazon/lambda/durable/otel/OtelPlugin.java +++ b/otel-plugin/src/main/java/software/amazon/lambda/durable/otel/OtelPlugin.java @@ -20,12 +20,9 @@ import io.opentelemetry.sdk.resources.Resource; import io.opentelemetry.sdk.trace.SdkTracerProvider; import io.opentelemetry.sdk.trace.SdkTracerProviderBuilder; -import io.opentelemetry.sdk.trace.export.SimpleSpanProcessor; -import io.opentelemetry.sdk.trace.export.SpanExporter; import io.opentelemetry.semconv.ServiceAttributes; import java.lang.reflect.Field; import java.lang.reflect.InvocationTargetException; -import java.lang.reflect.Method; import java.time.Instant; import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.TimeUnit; @@ -67,9 +64,8 @@ *

  • Tracing: Active (to populate {@code _X_AMZN_TRACE_ID}) * * - *

    When using {@link #OtelPlugin()}, the plugin uses the global provider directly when one exists and assumes the - * deterministic ID generator was installed by {@link OtelPluginAutoConfigurationCustomizerProvider}. Otherwise it - * creates a default OTLP exporter from the application classpath. + *

    When using {@link #OtelPlugin()}, the plugin requires {@link OtelPluginAutoConfigurationCustomizerProvider} to + * have been installed by the OpenTelemetry Java agent and uses the global provider directly. * *

    Thread-safe: uses {@link ConcurrentHashMap} for span/scope storage since the SDK runs user code on multiple * threads. @@ -123,14 +119,13 @@ public OtelPlugin(SdkTracerProviderBuilder tracerProviderBuilder) { } /** - * Creates an OTel plugin with default settings: X-Ray context extraction, MDC enabled, and default OTLP export. + * Creates an OTel plugin with default settings: X-Ray context extraction and MDC enabled. * - *

    If {@code GlobalOpenTelemetry} is configured, the plugin uses that provider directly and assumes deterministic - * ID generation was installed by {@link OtelPluginAutoConfigurationCustomizerProvider}. Otherwise, the plugin - * creates {@code OtlpGrpcSpanExporter.getDefault()} from the application classpath. + *

    Uses {@code GlobalOpenTelemetry} directly and assumes deterministic ID generation was installed by + * {@link OtelPluginAutoConfigurationCustomizerProvider}. */ public OtelPlugin() { - this(getDefaultTracerProvider()); + this(getDefaultTracerProvider(), OtelPluginAutoConfigurationCustomizerProvider.idGenerator()); } /** @@ -152,42 +147,29 @@ public OtelPlugin(SdkTracerProviderBuilder tracerProviderBuilder, ContextExtract */ public OtelPlugin( SdkTracerProviderBuilder tracerProviderBuilder, ContextExtractor contextExtractor, boolean enableMdc) { - this( - DefaultTracerProvider.fromSdkBuilder(tracerProviderBuilder, new DeterministicIdGenerator()), - contextExtractor, - enableMdc); - } - - private OtelPlugin(DefaultTracerProvider defaultTracerProvider) { - this(defaultTracerProvider, new XRayContextExtractor(), true); - } - - private OtelPlugin( - DefaultTracerProvider defaultTracerProvider, ContextExtractor contextExtractor, boolean enableMdc) { - this.idGenerator = defaultTracerProvider.idGenerator(); + this.idGenerator = new DeterministicIdGenerator(); - if (defaultTracerProvider.sdkBuilder() != null) { - // Set service.name to "invocation" — X-Ray uses this as the display name for SERVER spans, - // creating a separate service node in the trace map labeled "invocation". - var resource = Resource.create(Attributes.of(ServiceAttributes.SERVICE_NAME, "invocation")); - defaultTracerProvider.sdkBuilder().addResource(resource); - - var sdkBuilder = defaultTracerProvider.sdkBuilder(); - if (defaultTracerProvider.installIdGenerator()) { - sdkBuilder.setIdGenerator(idGenerator); - } - - this.sdkTracerProvider = sdkBuilder.build(); - this.tracer = sdkTracerProvider.get(INSTRUMENTATION_NAME); - } else { - this.sdkTracerProvider = defaultTracerProvider.sdkTracerProvider(); - this.tracer = defaultTracerProvider.tracerProvider().get(INSTRUMENTATION_NAME); - } + // Set service.name to "invocation" — X-Ray uses this as the display name for SERVER spans, + // creating a separate service node in the trace map labeled "invocation". + var resource = Resource.create(Attributes.of(ServiceAttributes.SERVICE_NAME, "invocation")); + tracerProviderBuilder.addResource(resource); + this.sdkTracerProvider = + tracerProviderBuilder.setIdGenerator(idGenerator).build(); + this.tracer = sdkTracerProvider.get(INSTRUMENTATION_NAME); this.contextExtractor = contextExtractor; this.enableMdc = enableMdc; } + private OtelPlugin(TracerProvider tracerProvider, DeterministicIdGenerator idGenerator) { + this.idGenerator = idGenerator; + this.sdkTracerProvider = getSdkTracerProviderForFlush(tracerProvider); + this.tracer = tracerProvider.get(INSTRUMENTATION_NAME); + + this.contextExtractor = new XRayContextExtractor(); + this.enableMdc = true; + } + // ─── Invocation hooks ──────────────────────────────────────────────── @Override @@ -520,35 +502,33 @@ private static ExtractedContext extractCurrentSpanContext() { return new ExtractedContext(spanContext.getTraceId(), spanContext.getSpanId()); } - private static DefaultTracerProvider getDefaultTracerProvider() { + private static TracerProvider getDefaultTracerProvider() { validateAutoConfigurationCustomizerProviderInstalled(); - var idGenerator = new DeterministicIdGenerator(); + var idGenerator = OtelPluginAutoConfigurationCustomizerProvider.idGenerator(); var globalTracerProvider = GlobalOpenTelemetry.getTracerProvider(); try { var sdkTracerProvider = getGlobalSdkTracerProvider(globalTracerProvider); logger.info("OtelPlugin initialized from existing GlobalOpenTelemetry SDK tracer provider; assuming " + "deterministic span IDs were installed through AutoConfigurationCustomizerProvider"); - return DefaultTracerProvider.fromTracerProvider(globalTracerProvider, sdkTracerProvider, idGenerator); + if (!hasIdGenerator(sdkTracerProvider, idGenerator)) { + logger.warn("OtelPlugin initialized from existing GlobalOpenTelemetry SDK tracer provider, but the " + + "provider is not using the durable execution ID generator. Configure this plugin " + + "jar with OTEL_JAVAAGENT_EXTENSIONS so AutoConfigurationCustomizerProvider can " + + "install deterministic span IDs."); + } + return globalTracerProvider; } catch (IllegalStateException e) { if (isJavaAgentTracerProvider(globalTracerProvider)) { var sdkTracerProvider = getJavaAgentSdkTracerProvider(globalTracerProvider); var javaAgentIdGenerator = OtelPluginAutoConfigurationCustomizerProvider.idGenerator(); logJavaAgentInitialization(globalTracerProvider, sdkTracerProvider, javaAgentIdGenerator, e); - return DefaultTracerProvider.fromTracerProvider(globalTracerProvider, sdkTracerProvider, idGenerator); - } else { - logger.warn( - "OtelPlugin could not initialize from GlobalOpenTelemetry; using default OTLP exporter. " - + "Cause: {}", - e.getMessage(), - e); + return globalTracerProvider; } - - var fallbackBuilder = SdkTracerProvider.builder() - .addSpanProcessor(SimpleSpanProcessor.create(createDefaultOtlpExporter())); - logger.info("OtelPlugin initialized with a new default OTLP tracer provider without overriding its ID " - + "generator; ensure an OTLP collector is reachable"); - return DefaultTracerProvider.fromSdkBuilderWithoutIdGenerator(fallbackBuilder, idGenerator); + throw new IllegalStateException( + "OtelPlugin() requires GlobalOpenTelemetry to be initialized by " + + "OtelPluginAutoConfigurationCustomizerProvider through the OpenTelemetry Java agent.", + e); } } @@ -562,33 +542,6 @@ private static void validateAutoConfigurationCustomizerProviderInstalled() { + "OTEL_JAVAAGENT_EXTENSIONS to that jar before constructing OtelPlugin()."); } - private static SpanExporter createDefaultOtlpExporter() { - try { - var exporterClass = Class.forName("io.opentelemetry.exporter.otlp.trace.OtlpGrpcSpanExporter"); - var getDefault = exporterClass.getMethod("getDefault"); - return asSpanExporter(getDefault); - } catch (ClassNotFoundException e) { - throw new IllegalStateException( - "OtelPlugin() requires either GlobalOpenTelemetry backed by the OpenTelemetry SDK or " - + "opentelemetry-exporter-otlp on the application classpath.", - e); - } catch (NoSuchMethodException e) { - throw new IllegalStateException("Unable to create default OTLP exporter", e); - } - } - - private static SpanExporter asSpanExporter(Method getDefault) { - try { - var exporter = getDefault.invoke(null); - if (exporter instanceof SpanExporter spanExporter) { - return spanExporter; - } - throw new IllegalStateException("Default OTLP exporter is not an OpenTelemetry SpanExporter"); - } catch (IllegalAccessException | InvocationTargetException e) { - throw new IllegalStateException("Unable to create default OTLP exporter", e); - } - } - private static SdkTracerProvider getGlobalSdkTracerProvider(TracerProvider tracerProvider) { if (tracerProvider instanceof SdkTracerProvider sdkTracerProvider) { return sdkTracerProvider; @@ -600,6 +553,21 @@ private static boolean isJavaAgentTracerProvider(TracerProvider tracerProvider) return tracerProvider.getClass().getName().startsWith("io.opentelemetry.javaagent."); } + private static SdkTracerProvider getSdkTracerProviderForFlush(TracerProvider tracerProvider) { + try { + if (isJavaAgentTracerProvider(tracerProvider)) { + return getJavaAgentSdkTracerProvider(tracerProvider); + } + return getGlobalSdkTracerProvider(tracerProvider); + } catch (IllegalStateException e) { + logger.warn( + "OtelPlugin could not inspect SDK tracer provider for forceFlush; spans will rely on the global " + + "provider's own flushing.", + e); + return null; + } + } + private static SdkTracerProvider getJavaAgentSdkTracerProvider(TracerProvider tracerProvider) { Object agentTracerProvider; try { @@ -719,29 +687,4 @@ private static Field findField(Class type, String fieldName) throws NoSuchFie } throw new NoSuchFieldException(fieldName); } - - private record DefaultTracerProvider( - SdkTracerProviderBuilder sdkBuilder, - TracerProvider tracerProvider, - SdkTracerProvider sdkTracerProvider, - DeterministicIdGenerator idGenerator, - boolean installIdGenerator) { - - private static DefaultTracerProvider fromSdkBuilder( - SdkTracerProviderBuilder sdkBuilder, DeterministicIdGenerator idGenerator) { - return new DefaultTracerProvider(sdkBuilder, null, null, idGenerator, true); - } - - private static DefaultTracerProvider fromSdkBuilderWithoutIdGenerator( - SdkTracerProviderBuilder sdkBuilder, DeterministicIdGenerator idGenerator) { - return new DefaultTracerProvider(sdkBuilder, null, null, idGenerator, false); - } - - private static DefaultTracerProvider fromTracerProvider( - TracerProvider tracerProvider, - SdkTracerProvider sdkTracerProvider, - DeterministicIdGenerator idGenerator) { - return new DefaultTracerProvider(null, tracerProvider, sdkTracerProvider, idGenerator, false); - } - } } diff --git a/otel-plugin/src/test/java/software/amazon/lambda/durable/otel/OtelPluginIntegrationTest.java b/otel-plugin/src/test/java/software/amazon/lambda/durable/otel/OtelPluginIntegrationTest.java index f5aeeef2c..fce4a02c6 100644 --- a/otel-plugin/src/test/java/software/amazon/lambda/durable/otel/OtelPluginIntegrationTest.java +++ b/otel-plugin/src/test/java/software/amazon/lambda/durable/otel/OtelPluginIntegrationTest.java @@ -114,29 +114,6 @@ void defaultConstructor_usesGlobalSdkTracerProviderDirectly() { assertTrue(spans.stream().allMatch(span -> span.getTraceId().equals(traceId))); } - @Test - void defaultConstructor_usesDefaultOtlpExporter_whenGlobalProviderIsNotSdk() { - OtelPluginAutoConfigurationCustomizerProvider.markInstalled(); - GlobalOpenTelemetry.resetForTest(); - OtlpGrpcSpanExporter.reset(); - - var defaultConfig = - DurableConfig.builder().withPlugins(new OtelPlugin()).build(); - var runner = LocalDurableTestRunner.create( - String.class, - (input, ctx) -> ctx.step("default-otlp-step", String.class, stepCtx -> "Hello " + input), - defaultConfig); - - var result = runner.runUntilComplete("World"); - assertEquals(ExecutionStatus.SUCCEEDED, result.getStatus()); - - var spans = OtlpGrpcSpanExporter.getFinishedSpanItems(); - assertTrue(spans.size() >= 3, "Expected at least 3 spans, got " + spans.size()); - assertSpanExists(spans, "invocation"); - assertSpanExists(spans, "default-otlp-step"); - assertSpanExists(spans, "default-otlp-step attempt 1"); - } - @Test void defaultConstructor_usesJavaAgentGlobalTracerProviderDirectly_withAutoConfiguredIdGenerator() { OtelPluginAutoConfigurationCustomizerProvider.markInstalled(); diff --git a/otel-plugin/src/test/java/software/amazon/lambda/durable/otel/OtelPluginTest.java b/otel-plugin/src/test/java/software/amazon/lambda/durable/otel/OtelPluginTest.java index e6b5fcd57..cf4c3a94f 100644 --- a/otel-plugin/src/test/java/software/amazon/lambda/durable/otel/OtelPluginTest.java +++ b/otel-plugin/src/test/java/software/amazon/lambda/durable/otel/OtelPluginTest.java @@ -75,6 +75,17 @@ void defaultConstructor_throwsWhenAutoConfigurationCustomizerProviderIsNotInstal assertTrue(error.getMessage().contains("OTEL_JAVAAGENT_EXTENSIONS")); } + @Test + void defaultConstructor_throwsWhenGlobalOpenTelemetryIsNotInitializedBySpi() { + OtelPluginAutoConfigurationCustomizerProvider.markInstalled(); + GlobalOpenTelemetry.resetForTest(); + + var error = assertThrows(IllegalStateException.class, OtelPlugin::new); + + assertTrue(error.getMessage().contains("GlobalOpenTelemetry")); + assertTrue(error.getMessage().contains("OtelPluginAutoConfigurationCustomizerProvider")); + } + @Test void defaultConstructor_usesGlobalSdkTracerProviderDirectly() { OtelPluginAutoConfigurationCustomizerProvider.markInstalled(); @@ -130,27 +141,6 @@ void defaultConstructor_doesNotReplaceGlobalSdkTracerProviderIdGenerator() { assertEquals("0000000000000002", stepSpan.getSpanId()); } - @Test - void defaultConstructor_usesDefaultOtlpExporter_whenGlobalProviderIsNotSdk() { - OtelPluginAutoConfigurationCustomizerProvider.markInstalled(); - GlobalOpenTelemetry.resetForTest(); - OtlpGrpcSpanExporter.reset(); - - var defaultPlugin = new OtelPlugin(); - defaultPlugin.onInvocationStart(new InvocationInfo("req-1", "arn:exec1", true)); - defaultPlugin.onOperationStart( - new OperationInfo("op-1", "step", "STEP", "Step", null, Instant.now(), null, false)); - defaultPlugin.onOperationEnd(new OperationEndInfo( - "op-1", "step", "STEP", "Step", null, Instant.now(), Instant.now(), "SUCCEEDED", false, null)); - defaultPlugin.onInvocationEnd( - new InvocationEndInfo("req-1", "arn:exec1", true, InvocationStatus.SUCCEEDED, null)); - - var spans = OtlpGrpcSpanExporter.getFinishedSpanItems(); - assertEquals(2, spans.size()); - assertTrue(spans.stream().anyMatch(span -> span.getName().equals("invocation"))); - assertTrue(spans.stream().anyMatch(span -> span.getName().equals("step"))); - } - @Test void defaultConstructor_usesJavaAgentGlobalTracerProviderDirectly_withAutoConfiguredIdGenerator() { OtelPluginAutoConfigurationCustomizerProvider.markInstalled(); From 1a7ba9dbc4fe35daffaa81a1bf5e4af08e70f91a Mon Sep 17 00:00:00 2001 From: Frank Chen Date: Mon, 13 Jul 2026 23:52:29 -0700 Subject: [PATCH 23/36] Avoid loading OTel SPI provider from plugin constructor --- .../OtelXRayDefaultConstructorExample.java | 4 +- otel-plugin/README.md | 4 +- .../lambda/durable/otel/OtelPlugin.java | 64 ++++++------------- ...inAutoConfigurationCustomizerProvider.java | 16 +---- .../OtelPluginAutoConfigurationState.java | 23 +++++++ .../otel/OtelPluginIntegrationTest.java | 10 +-- .../lambda/durable/otel/OtelPluginTest.java | 18 +++--- 7 files changed, 62 insertions(+), 77 deletions(-) create mode 100644 otel-plugin/src/main/java/software/amazon/lambda/durable/otel/OtelPluginAutoConfigurationState.java diff --git a/examples/src/main/java/software/amazon/lambda/durable/examples/otel/OtelXRayDefaultConstructorExample.java b/examples/src/main/java/software/amazon/lambda/durable/examples/otel/OtelXRayDefaultConstructorExample.java index 5f5ee86f9..9ccc70673 100644 --- a/examples/src/main/java/software/amazon/lambda/durable/examples/otel/OtelXRayDefaultConstructorExample.java +++ b/examples/src/main/java/software/amazon/lambda/durable/examples/otel/OtelXRayDefaultConstructorExample.java @@ -12,8 +12,8 @@ /** * OTel + X-Ray example that uses the no-arg plugin constructor. * - *

    {@link OtelPlugin#OtelPlugin()} copies a global SDK provider when possible, or uses the ADOT Java agent provider - * directly with deterministic span ID generation installed by the plugin's OpenTelemetry autoconfigure SPI. + *

    {@link OtelPlugin#OtelPlugin()} uses the global provider initialized by the ADOT Java agent with deterministic + * span ID generation installed by the plugin's OpenTelemetry autoconfigure SPI. */ @ExampleTemplate(tracing = true) public class OtelXRayDefaultConstructorExample extends DurableHandler { diff --git a/otel-plugin/README.md b/otel-plugin/README.md index fee12fa75..bd2f44106 100644 --- a/otel-plugin/README.md +++ b/otel-plugin/README.md @@ -46,7 +46,7 @@ You also need the OpenTelemetry SDK and an exporter: ### 1. ADOT Lambda Layer -This plugin uses the [AWS Distro for OpenTelemetry (ADOT) Lambda layer](https://aws-otel.github.io/docs/getting-started/lambda) for trace export. `OtelPlugin()` copies a global SDK provider when possible, or uses the ADOT Java agent provider directly with deterministic span ID generation installed through the OpenTelemetry `AutoConfigurationCustomizerProvider` SPI. If no usable global provider exists, it creates the default OTLP gRPC exporter and sends spans to a reachable OTLP collector. +This plugin uses the [AWS Distro for OpenTelemetry (ADOT) Lambda layer](https://aws-otel.github.io/docs/getting-started/lambda) for trace export. `OtelPlugin()` uses the global provider initialized by the ADOT Java agent and requires deterministic span ID generation to be installed through the OpenTelemetry `AutoConfigurationCustomizerProvider` SPI. The layer ARN follows the format: @@ -248,7 +248,7 @@ After deploying your function with the plugin configured: | Traces appear but are fragmented | X-Ray active tracing not enabled on the Lambda function | | Missing spans for some operations | Sampling is configured below 1.0 | | `_X_AMZN_TRACE_ID` not populated | X-Ray active tracing not enabled | -| Plugin spans missing but Lambda/runtime spans appear | The ADOT wrapper did not initialize `GlobalOpenTelemetry`, the plugin jar was not configured in `OTEL_JAVAAGENT_EXTENSIONS`, or the fallback OTLP exporter cannot reach a collector | +| Plugin spans missing but Lambda/runtime spans appear | The ADOT wrapper did not initialize `GlobalOpenTelemetry`, or the plugin jar was not configured in `OTEL_JAVAAGENT_EXTENSIONS` | > **Note on ADOT wrapper:** Use `AWS_LAMBDA_EXEC_WRAPPER=/opt/otel-instrument` with the `AWSOpenTelemetryDistroJava` layer. The older `/opt/otel-handler` path is not valid for this layer and can fail before the Java handler starts. diff --git a/otel-plugin/src/main/java/software/amazon/lambda/durable/otel/OtelPlugin.java b/otel-plugin/src/main/java/software/amazon/lambda/durable/otel/OtelPlugin.java index a0c3c0451..8c54202c4 100644 --- a/otel-plugin/src/main/java/software/amazon/lambda/durable/otel/OtelPlugin.java +++ b/otel-plugin/src/main/java/software/amazon/lambda/durable/otel/OtelPlugin.java @@ -64,7 +64,7 @@ *

  • Tracing: Active (to populate {@code _X_AMZN_TRACE_ID}) * * - *

    When using {@link #OtelPlugin()}, the plugin requires {@link OtelPluginAutoConfigurationCustomizerProvider} to + *

    When using {@link #OtelPlugin()}, the plugin requires {@code OtelPluginAutoConfigurationCustomizerProvider} to * have been installed by the OpenTelemetry Java agent and uses the global provider directly. * *

    Thread-safe: uses {@link ConcurrentHashMap} for span/scope storage since the SDK runs user code on multiple @@ -122,10 +122,10 @@ public OtelPlugin(SdkTracerProviderBuilder tracerProviderBuilder) { * Creates an OTel plugin with default settings: X-Ray context extraction and MDC enabled. * *

    Uses {@code GlobalOpenTelemetry} directly and assumes deterministic ID generation was installed by - * {@link OtelPluginAutoConfigurationCustomizerProvider}. + * {@code OtelPluginAutoConfigurationCustomizerProvider}. */ public OtelPlugin() { - this(getDefaultTracerProvider(), OtelPluginAutoConfigurationCustomizerProvider.idGenerator()); + this(getDefaultTracerProvider(), createDefaultIdGenerator()); } /** @@ -505,24 +505,16 @@ private static ExtractedContext extractCurrentSpanContext() { private static TracerProvider getDefaultTracerProvider() { validateAutoConfigurationCustomizerProviderInstalled(); - var idGenerator = OtelPluginAutoConfigurationCustomizerProvider.idGenerator(); var globalTracerProvider = GlobalOpenTelemetry.getTracerProvider(); try { - var sdkTracerProvider = getGlobalSdkTracerProvider(globalTracerProvider); + getGlobalSdkTracerProvider(globalTracerProvider); logger.info("OtelPlugin initialized from existing GlobalOpenTelemetry SDK tracer provider; assuming " + "deterministic span IDs were installed through AutoConfigurationCustomizerProvider"); - if (!hasIdGenerator(sdkTracerProvider, idGenerator)) { - logger.warn("OtelPlugin initialized from existing GlobalOpenTelemetry SDK tracer provider, but the " - + "provider is not using the durable execution ID generator. Configure this plugin " - + "jar with OTEL_JAVAAGENT_EXTENSIONS so AutoConfigurationCustomizerProvider can " - + "install deterministic span IDs."); - } return globalTracerProvider; } catch (IllegalStateException e) { if (isJavaAgentTracerProvider(globalTracerProvider)) { var sdkTracerProvider = getJavaAgentSdkTracerProvider(globalTracerProvider); - var javaAgentIdGenerator = OtelPluginAutoConfigurationCustomizerProvider.idGenerator(); - logJavaAgentInitialization(globalTracerProvider, sdkTracerProvider, javaAgentIdGenerator, e); + logJavaAgentInitialization(globalTracerProvider, sdkTracerProvider, e); return globalTracerProvider; } throw new IllegalStateException( @@ -532,8 +524,15 @@ private static TracerProvider getDefaultTracerProvider() { } } + private static DeterministicIdGenerator createDefaultIdGenerator() { + // This is intentionally a separate instance from the SPI provider's generator. The Java agent extension and + // application may load this plugin in different class loaders, so DeterministicIdGenerator bridges invocation + // state through system properties that the SPI-installed generator can read when spans are started. + return new DeterministicIdGenerator(); + } + private static void validateAutoConfigurationCustomizerProviderInstalled() { - if (OtelPluginAutoConfigurationCustomizerProvider.isInstalled()) { + if (OtelPluginAutoConfigurationState.isInstalled()) { return; } throw new IllegalStateException( @@ -603,47 +602,24 @@ private static SdkTracerProvider getJavaAgentSdkTracerProvider(TracerProvider tr } private static void logJavaAgentInitialization( - TracerProvider globalTracerProvider, - SdkTracerProvider sdkTracerProvider, - DeterministicIdGenerator idGenerator, - IllegalStateException cause) { + TracerProvider globalTracerProvider, SdkTracerProvider sdkTracerProvider, IllegalStateException cause) { if (sdkTracerProvider == null) { logger.info( "OtelPlugin initialized from existing GlobalOpenTelemetry Java agent tracer provider {}, but the " - + "agent SDK provider is not visible to the application class loader, so deterministic " - + "ID generator installation cannot be verified. If durable operation span IDs are not " - + "deterministic, configure this plugin jar with OTEL_JAVAAGENT_EXTENSIONS. Cause: {}", - globalTracerProvider.getClass().getName(), - cause.getMessage()); - return; - } - if (hasIdGenerator(sdkTracerProvider, idGenerator)) { - logger.info( - "OtelPlugin initialized from existing GlobalOpenTelemetry Java agent tracer provider {}; " - + "deterministic span IDs installed through AutoConfigurationCustomizerProvider. Cause: {}", + + "agent SDK provider is not visible to the application class loader. Assuming " + + "deterministic span IDs were installed through AutoConfigurationCustomizerProvider. " + + "Cause: {}", globalTracerProvider.getClass().getName(), cause.getMessage()); return; } - logger.warn( - "OtelPlugin initialized from existing GlobalOpenTelemetry Java agent tracer provider {}, but the " - + "agent SDK provider is not using the durable execution ID generator. Configure this plugin " - + "jar with OTEL_JAVAAGENT_EXTENSIONS so AutoConfigurationCustomizerProvider can install " - + "deterministic span IDs. Cause: {}", + logger.info( + "OtelPlugin initialized from existing GlobalOpenTelemetry Java agent tracer provider {}; assuming " + + "deterministic span IDs were installed through AutoConfigurationCustomizerProvider. Cause: {}", globalTracerProvider.getClass().getName(), cause.getMessage()); } - private static boolean hasIdGenerator(SdkTracerProvider sdkTracerProvider, DeterministicIdGenerator idGenerator) { - try { - var sharedState = getField(sdkTracerProvider, "sharedState", Object.class); - return getField(sharedState, "idGenerator", Object.class) == idGenerator; - } catch (IllegalStateException e) { - logger.warn("OtelPlugin could not verify the Java agent SDK ID generator", e); - return false; - } - } - private static SdkTracerProvider unobfuscateSdkTracerProvider(TracerProvider tracerProvider) { var providerClassName = tracerProvider.getClass().getName(); try { diff --git a/otel-plugin/src/main/java/software/amazon/lambda/durable/otel/OtelPluginAutoConfigurationCustomizerProvider.java b/otel-plugin/src/main/java/software/amazon/lambda/durable/otel/OtelPluginAutoConfigurationCustomizerProvider.java index 08b205d7b..63ebf8db1 100644 --- a/otel-plugin/src/main/java/software/amazon/lambda/durable/otel/OtelPluginAutoConfigurationCustomizerProvider.java +++ b/otel-plugin/src/main/java/software/amazon/lambda/durable/otel/OtelPluginAutoConfigurationCustomizerProvider.java @@ -13,29 +13,15 @@ @Deprecated public final class OtelPluginAutoConfigurationCustomizerProvider implements AutoConfigurationCustomizerProvider { - private static final String INSTALLED_PROPERTY = - "software.amazon.lambda.durable.otel.autoConfigurationCustomizerProviderInstalled"; private static final DeterministicIdGenerator ID_GENERATOR = new DeterministicIdGenerator(); @Override public void customize(AutoConfigurationCustomizer autoConfiguration) { - markInstalled(); + OtelPluginAutoConfigurationState.markInstalled(); autoConfiguration.addTracerProviderCustomizer((builder, config) -> builder.setIdGenerator(ID_GENERATOR)); } static DeterministicIdGenerator idGenerator() { return ID_GENERATOR; } - - static boolean isInstalled() { - return Boolean.getBoolean(INSTALLED_PROPERTY); - } - - static void markInstalled() { - System.setProperty(INSTALLED_PROPERTY, Boolean.TRUE.toString()); - } - - static void resetInstalledForTest() { - System.clearProperty(INSTALLED_PROPERTY); - } } diff --git a/otel-plugin/src/main/java/software/amazon/lambda/durable/otel/OtelPluginAutoConfigurationState.java b/otel-plugin/src/main/java/software/amazon/lambda/durable/otel/OtelPluginAutoConfigurationState.java new file mode 100644 index 000000000..765489b69 --- /dev/null +++ b/otel-plugin/src/main/java/software/amazon/lambda/durable/otel/OtelPluginAutoConfigurationState.java @@ -0,0 +1,23 @@ +// Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +// SPDX-License-Identifier: Apache-2.0 +package software.amazon.lambda.durable.otel; + +final class OtelPluginAutoConfigurationState { + + private static final String INSTALLED_PROPERTY = + "software.amazon.lambda.durable.otel.autoConfigurationCustomizerProviderInstalled"; + + private OtelPluginAutoConfigurationState() {} + + static boolean isInstalled() { + return Boolean.getBoolean(INSTALLED_PROPERTY); + } + + static void markInstalled() { + System.setProperty(INSTALLED_PROPERTY, Boolean.TRUE.toString()); + } + + static void resetInstalledForTest() { + System.clearProperty(INSTALLED_PROPERTY); + } +} diff --git a/otel-plugin/src/test/java/software/amazon/lambda/durable/otel/OtelPluginIntegrationTest.java b/otel-plugin/src/test/java/software/amazon/lambda/durable/otel/OtelPluginIntegrationTest.java index fce4a02c6..90d2cb5fb 100644 --- a/otel-plugin/src/test/java/software/amazon/lambda/durable/otel/OtelPluginIntegrationTest.java +++ b/otel-plugin/src/test/java/software/amazon/lambda/durable/otel/OtelPluginIntegrationTest.java @@ -42,7 +42,7 @@ class OtelPluginIntegrationTest { @BeforeEach void setUp() { DeterministicIdGenerator.clearSharedStateForTest(); - OtelPluginAutoConfigurationCustomizerProvider.resetInstalledForTest(); + OtelPluginAutoConfigurationState.resetInstalledForTest(); spanExporter = InMemorySpanExporter.create(); var plugin = new OtelPlugin( @@ -58,7 +58,7 @@ void tearDown() { GlobalOpenTelemetry.resetForTest(); OtlpGrpcSpanExporter.reset(); DeterministicIdGenerator.clearSharedStateForTest(); - OtelPluginAutoConfigurationCustomizerProvider.resetInstalledForTest(); + OtelPluginAutoConfigurationState.resetInstalledForTest(); } @Test @@ -86,7 +86,7 @@ void simpleStep_producesInvocationAndOperationAndAttemptSpans() { @Test void defaultConstructor_usesGlobalSdkTracerProviderDirectly() { - OtelPluginAutoConfigurationCustomizerProvider.markInstalled(); + OtelPluginAutoConfigurationState.markInstalled(); GlobalOpenTelemetry.resetForTest(); var globalExporter = InMemorySpanExporter.create(); var globalTracerProvider = SdkTracerProvider.builder() @@ -115,8 +115,8 @@ void defaultConstructor_usesGlobalSdkTracerProviderDirectly() { } @Test - void defaultConstructor_usesJavaAgentGlobalTracerProviderDirectly_withAutoConfiguredIdGenerator() { - OtelPluginAutoConfigurationCustomizerProvider.markInstalled(); + void defaultConstructor_usesJavaAgentGlobalTracerProviderDirectly_withSeparateAutoConfiguredIdGenerator() { + OtelPluginAutoConfigurationState.markInstalled(); GlobalOpenTelemetry.resetForTest(); OtlpGrpcSpanExporter.reset(); var globalExporter = InMemorySpanExporter.create(); diff --git a/otel-plugin/src/test/java/software/amazon/lambda/durable/otel/OtelPluginTest.java b/otel-plugin/src/test/java/software/amazon/lambda/durable/otel/OtelPluginTest.java index cf4c3a94f..1838596c3 100644 --- a/otel-plugin/src/test/java/software/amazon/lambda/durable/otel/OtelPluginTest.java +++ b/otel-plugin/src/test/java/software/amazon/lambda/durable/otel/OtelPluginTest.java @@ -48,7 +48,7 @@ class OtelPluginTest { @BeforeEach void setUp() { DeterministicIdGenerator.clearSharedStateForTest(); - OtelPluginAutoConfigurationCustomizerProvider.resetInstalledForTest(); + OtelPluginAutoConfigurationState.resetInstalledForTest(); spanExporter = InMemorySpanExporter.create(); plugin = new OtelPlugin( @@ -62,7 +62,7 @@ void tearDown() { GlobalOpenTelemetry.resetForTest(); OtlpGrpcSpanExporter.reset(); DeterministicIdGenerator.clearSharedStateForTest(); - OtelPluginAutoConfigurationCustomizerProvider.resetInstalledForTest(); + OtelPluginAutoConfigurationState.resetInstalledForTest(); } @Test @@ -77,7 +77,7 @@ void defaultConstructor_throwsWhenAutoConfigurationCustomizerProviderIsNotInstal @Test void defaultConstructor_throwsWhenGlobalOpenTelemetryIsNotInitializedBySpi() { - OtelPluginAutoConfigurationCustomizerProvider.markInstalled(); + OtelPluginAutoConfigurationState.markInstalled(); GlobalOpenTelemetry.resetForTest(); var error = assertThrows(IllegalStateException.class, OtelPlugin::new); @@ -88,7 +88,7 @@ void defaultConstructor_throwsWhenGlobalOpenTelemetryIsNotInitializedBySpi() { @Test void defaultConstructor_usesGlobalSdkTracerProviderDirectly() { - OtelPluginAutoConfigurationCustomizerProvider.markInstalled(); + OtelPluginAutoConfigurationState.markInstalled(); GlobalOpenTelemetry.resetForTest(); var globalExporter = InMemorySpanExporter.create(); var globalTracerProvider = SdkTracerProvider.builder() @@ -115,7 +115,7 @@ void defaultConstructor_usesGlobalSdkTracerProviderDirectly() { @Test void defaultConstructor_doesNotReplaceGlobalSdkTracerProviderIdGenerator() { - OtelPluginAutoConfigurationCustomizerProvider.markInstalled(); + OtelPluginAutoConfigurationState.markInstalled(); GlobalOpenTelemetry.resetForTest(); var globalExporter = InMemorySpanExporter.create(); var idGenerator = new CountingIdGenerator(); @@ -142,8 +142,8 @@ void defaultConstructor_doesNotReplaceGlobalSdkTracerProviderIdGenerator() { } @Test - void defaultConstructor_usesJavaAgentGlobalTracerProviderDirectly_withAutoConfiguredIdGenerator() { - OtelPluginAutoConfigurationCustomizerProvider.markInstalled(); + void defaultConstructor_usesJavaAgentGlobalTracerProviderDirectly_withSeparateAutoConfiguredIdGenerator() { + OtelPluginAutoConfigurationState.markInstalled(); GlobalOpenTelemetry.resetForTest(); OtlpGrpcSpanExporter.reset(); var globalExporter = InMemorySpanExporter.create(); @@ -190,13 +190,13 @@ public ContextPropagators getPropagators() { @Test void autoConfigurationCustomizerProvider_installsSharedDeterministicIdGenerator() { - OtelPluginAutoConfigurationCustomizerProvider.resetInstalledForTest(); + OtelPluginAutoConfigurationState.resetInstalledForTest(); var exporter = InMemorySpanExporter.create(); var autoConfiguration = mock(AutoConfigurationCustomizer.class); when(autoConfiguration.addTracerProviderCustomizer(any())).thenReturn(autoConfiguration); new OtelPluginAutoConfigurationCustomizerProvider().customize(autoConfiguration); - assertTrue(OtelPluginAutoConfigurationCustomizerProvider.isInstalled()); + assertTrue(OtelPluginAutoConfigurationState.isInstalled()); @SuppressWarnings("unchecked") var customizer = ArgumentCaptor.forClass(BiFunction.class); From 187a46302f374ca019d6b71a009bd4e779380e26 Mon Sep 17 00:00:00 2001 From: Frank Chen Date: Mon, 13 Jul 2026 23:55:40 -0700 Subject: [PATCH 24/36] Package OTel autoconfigure SPI at runtime --- otel-plugin/pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/otel-plugin/pom.xml b/otel-plugin/pom.xml index fbdd6126a..c4df0b013 100644 --- a/otel-plugin/pom.xml +++ b/otel-plugin/pom.xml @@ -40,11 +40,11 @@ ${opentelemetry.version} provided + io.opentelemetry opentelemetry-sdk-extension-autoconfigure-spi ${opentelemetry.version} - provided From 3229eba12c5dba6021c79a59ac520f1f97455d01 Mon Sep 17 00:00:00 2001 From: Frank Chen Date: Tue, 14 Jul 2026 00:04:26 -0700 Subject: [PATCH 25/36] Point ADOT extension config to plugin jar --- examples/generate-template.py | 12 ++++---- otel-plugin/README.md | 2 +- otel-plugin/pom.xml | 3 +- .../lambda/durable/otel/OtelPlugin.java | 30 ++++++++++++++++++- ...inAutoConfigurationCustomizerProvider.java | 11 +++++++ 5 files changed, 49 insertions(+), 9 deletions(-) diff --git a/examples/generate-template.py b/examples/generate-template.py index bb1a3b8d2..01738dffa 100755 --- a/examples/generate-template.py +++ b/examples/generate-template.py @@ -17,15 +17,14 @@ POM_NAMESPACE = {"m": "http://maven.apache.org/POM/4.0.0"} -def read_examples_jar_path() -> str: +def read_otel_plugin_jar_path() -> str: root = ET.parse(EXAMPLES_DIR / "pom.xml").getroot() - artifact_id = root.findtext("m:artifactId", namespaces=POM_NAMESPACE) version = root.findtext("m:version", namespaces=POM_NAMESPACE) if version is None: version = root.findtext("m:parent/m:version", namespaces=POM_NAMESPACE) - if artifact_id is None or version is None: - raise ValueError("Unable to read examples artifactId/version from pom.xml") - return f"/var/task/lib/{artifact_id}-{version}.jar" + if version is None: + raise ValueError("Unable to read examples version from pom.xml") + return f"/var/task/lib/aws-durable-execution-sdk-java-plugin-otel-{version}.jar" @dataclass(frozen=True) @@ -140,6 +139,7 @@ def emit_function(lines: list[str], example: ExampleFunction, java_agent_extensi " Environment:", " Variables:", " AWS_LAMBDA_EXEC_WRAPPER: /opt/otel-instrument", + f' JAVA_TOOL_OPTIONS: "-Dotel.javaagent.extensions={java_agent_extension_path}"', f" OTEL_JAVAAGENT_EXTENSIONS: {java_agent_extension_path}", ] ) @@ -166,7 +166,7 @@ def emit_log_group(lines: list[str], example: ExampleFunction) -> None: def render_template(examples: list[ExampleFunction]) -> str: - java_agent_extension_path = read_examples_jar_path() + java_agent_extension_path = read_otel_plugin_jar_path() lines = [ "# This file is generated by examples/generate-template.py. Do not edit it by hand.", 'AWSTemplateFormatVersion: "2010-09-09"', diff --git a/otel-plugin/README.md b/otel-plugin/README.md index bd2f44106..df69e5ee9 100644 --- a/otel-plugin/README.md +++ b/otel-plugin/README.md @@ -78,7 +78,7 @@ aws lambda update-function-configuration \ --environment "Variables={AWS_LAMBDA_EXEC_WRAPPER=/opt/otel-instrument,OTEL_JAVAAGENT_EXTENSIONS=/var/task/lib/aws-durable-execution-sdk-java-plugin-otel-.jar}" ``` -Set `OTEL_JAVAAGENT_EXTENSIONS` to the deployed jar that contains this plugin's `META-INF/services/io.opentelemetry.sdk.autoconfigure.spi.AutoConfigurationCustomizerProvider` entry. If you shade the plugin into your application jar, point the variable to that shaded jar instead. +Set `OTEL_JAVAAGENT_EXTENSIONS` to the deployed OTel plugin jar that contains this plugin's `META-INF/services/io.opentelemetry.sdk.autoconfigure.spi.AutoConfigurationCustomizerProvider` entry. Prefer the standalone plugin dependency jar under `/var/task/lib` so the Java agent loads the extension against its own OpenTelemetry SPI classes. ### 2. AWS X-Ray Active Tracing diff --git a/otel-plugin/pom.xml b/otel-plugin/pom.xml index c4df0b013..bf79192ab 100644 --- a/otel-plugin/pom.xml +++ b/otel-plugin/pom.xml @@ -40,11 +40,12 @@ ${opentelemetry.version} provided - + io.opentelemetry opentelemetry-sdk-extension-autoconfigure-spi ${opentelemetry.version} + provided diff --git a/otel-plugin/src/main/java/software/amazon/lambda/durable/otel/OtelPlugin.java b/otel-plugin/src/main/java/software/amazon/lambda/durable/otel/OtelPlugin.java index 8c54202c4..823e9926e 100644 --- a/otel-plugin/src/main/java/software/amazon/lambda/durable/otel/OtelPlugin.java +++ b/otel-plugin/src/main/java/software/amazon/lambda/durable/otel/OtelPlugin.java @@ -23,6 +23,8 @@ import io.opentelemetry.semconv.ServiceAttributes; import java.lang.reflect.Field; import java.lang.reflect.InvocationTargetException; +import java.nio.file.Files; +import java.nio.file.Path; import java.time.Instant; import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.TimeUnit; @@ -538,7 +540,33 @@ private static void validateAutoConfigurationCustomizerProviderInstalled() { throw new IllegalStateException( "OtelPlugin() requires OtelPluginAutoConfigurationCustomizerProvider to be installed by the " + "OpenTelemetry Java agent. Package this plugin jar as an agent extension and set " - + "OTEL_JAVAAGENT_EXTENSIONS to that jar before constructing OtelPlugin()."); + + "OTEL_JAVAAGENT_EXTENSIONS or -Dotel.javaagent.extensions to that jar before constructing " + + "OtelPlugin(). " + + javaAgentExtensionsDiagnostic()); + } + + private static String javaAgentExtensionsDiagnostic() { + var propertyValue = System.getProperty("otel.javaagent.extensions"); + var environmentValue = System.getenv("OTEL_JAVAAGENT_EXTENSIONS"); + var configuredPath = propertyValue != null ? propertyValue : environmentValue; + return "otel.javaagent.extensions=" + + valueOrUnset(propertyValue) + + ", OTEL_JAVAAGENT_EXTENSIONS=" + + valueOrUnset(environmentValue) + + ", configured extension path exists=" + + extensionPathExists(configuredPath); + } + + private static String valueOrUnset(String value) { + return value != null ? value : ""; + } + + private static boolean extensionPathExists(String configuredPath) { + if (configuredPath == null || configuredPath.isBlank()) { + return false; + } + var firstPath = configuredPath.split(",", 2)[0]; + return Files.exists(Path.of(firstPath)); } private static SdkTracerProvider getGlobalSdkTracerProvider(TracerProvider tracerProvider) { diff --git a/otel-plugin/src/main/java/software/amazon/lambda/durable/otel/OtelPluginAutoConfigurationCustomizerProvider.java b/otel-plugin/src/main/java/software/amazon/lambda/durable/otel/OtelPluginAutoConfigurationCustomizerProvider.java index 63ebf8db1..93cc32203 100644 --- a/otel-plugin/src/main/java/software/amazon/lambda/durable/otel/OtelPluginAutoConfigurationCustomizerProvider.java +++ b/otel-plugin/src/main/java/software/amazon/lambda/durable/otel/OtelPluginAutoConfigurationCustomizerProvider.java @@ -18,10 +18,21 @@ public final class OtelPluginAutoConfigurationCustomizerProvider implements Auto @Override public void customize(AutoConfigurationCustomizer autoConfiguration) { OtelPluginAutoConfigurationState.markInstalled(); + System.err.println("OtelPluginAutoConfigurationCustomizerProvider installed from " + codeSourceLocation()); autoConfiguration.addTracerProviderCustomizer((builder, config) -> builder.setIdGenerator(ID_GENERATOR)); } static DeterministicIdGenerator idGenerator() { return ID_GENERATOR; } + + private static String codeSourceLocation() { + var codeSource = OtelPluginAutoConfigurationCustomizerProvider.class + .getProtectionDomain() + .getCodeSource(); + if (codeSource == null || codeSource.getLocation() == null) { + return "unknown location"; + } + return codeSource.getLocation().toString(); + } } From 4adeaf0607c9d0631a86c39dc60a785fc7c326c5 Mon Sep 17 00:00:00 2001 From: Frank Chen Date: Tue, 14 Jul 2026 00:14:50 -0700 Subject: [PATCH 26/36] Package OTel plugin jar for ADOT extension --- examples/README.md | 2 ++ examples/pom.xml | 24 ++++++++++++++++++++++++ 2 files changed, 26 insertions(+) diff --git a/examples/README.md b/examples/README.md index 9940c17e5..e9933b092 100644 --- a/examples/README.md +++ b/examples/README.md @@ -52,6 +52,8 @@ The SAM template configures: `template.yaml` is generated from the Java example handlers and is intentionally not checked in. Re-run `python3 generate-template.py` after adding or removing a deployable example handler. +The examples package copies the OTel plugin jar into `lib/` so the ADOT Java agent can load it as an extension from the generated `OTEL_JAVAAGENT_EXTENSIONS` path. + ## Invoke Deployed Functions ```bash diff --git a/examples/pom.xml b/examples/pom.xml index e84d61003..298e02b3c 100644 --- a/examples/pom.xml +++ b/examples/pom.xml @@ -154,6 +154,30 @@ + + org.apache.maven.plugins + maven-dependency-plugin + 3.9.0 + + + copy-otel-javaagent-extension + process-resources + + copy + + + + + ${project.groupId} + aws-durable-execution-sdk-java-plugin-otel + ${project.version} + ${project.build.outputDirectory}/lib + + + + + + org.apache.maven.plugins maven-source-plugin From 8549deec99a7aeb5efef0a6b4b8a293d4c9affc8 Mon Sep 17 00:00:00 2001 From: Frank Chen Date: Tue, 14 Jul 2026 00:21:38 -0700 Subject: [PATCH 27/36] Remove OTel SPI debug helper --- ...PluginAutoConfigurationCustomizerProvider.java | 15 --------------- .../lambda/durable/otel/OtelPluginTest.java | 9 +++++---- 2 files changed, 5 insertions(+), 19 deletions(-) diff --git a/otel-plugin/src/main/java/software/amazon/lambda/durable/otel/OtelPluginAutoConfigurationCustomizerProvider.java b/otel-plugin/src/main/java/software/amazon/lambda/durable/otel/OtelPluginAutoConfigurationCustomizerProvider.java index 93cc32203..23ac19ecd 100644 --- a/otel-plugin/src/main/java/software/amazon/lambda/durable/otel/OtelPluginAutoConfigurationCustomizerProvider.java +++ b/otel-plugin/src/main/java/software/amazon/lambda/durable/otel/OtelPluginAutoConfigurationCustomizerProvider.java @@ -18,21 +18,6 @@ public final class OtelPluginAutoConfigurationCustomizerProvider implements Auto @Override public void customize(AutoConfigurationCustomizer autoConfiguration) { OtelPluginAutoConfigurationState.markInstalled(); - System.err.println("OtelPluginAutoConfigurationCustomizerProvider installed from " + codeSourceLocation()); autoConfiguration.addTracerProviderCustomizer((builder, config) -> builder.setIdGenerator(ID_GENERATOR)); } - - static DeterministicIdGenerator idGenerator() { - return ID_GENERATOR; - } - - private static String codeSourceLocation() { - var codeSource = OtelPluginAutoConfigurationCustomizerProvider.class - .getProtectionDomain() - .getCodeSource(); - if (codeSource == null || codeSource.getLocation() == null) { - return "unknown location"; - } - return codeSource.getLocation().toString(); - } } diff --git a/otel-plugin/src/test/java/software/amazon/lambda/durable/otel/OtelPluginTest.java b/otel-plugin/src/test/java/software/amazon/lambda/durable/otel/OtelPluginTest.java index 1838596c3..dada15e9c 100644 --- a/otel-plugin/src/test/java/software/amazon/lambda/durable/otel/OtelPluginTest.java +++ b/otel-plugin/src/test/java/software/amazon/lambda/durable/otel/OtelPluginTest.java @@ -202,9 +202,9 @@ void autoConfigurationCustomizerProvider_installsSharedDeterministicIdGenerator( var customizer = ArgumentCaptor.forClass(BiFunction.class); verify(autoConfiguration).addTracerProviderCustomizer(customizer.capture()); - var idGenerator = OtelPluginAutoConfigurationCustomizerProvider.idGenerator(); - idGenerator.setDurableExecutionArn("arn:spi"); - idGenerator.setNextSpanOperationId("op-spi"); + var pluginGenerator = new DeterministicIdGenerator(); + pluginGenerator.setDurableExecutionArn("arn:spi"); + pluginGenerator.setNextSpanOperationId("op-spi"); @SuppressWarnings("unchecked") var tracerProviderCustomizer = @@ -221,7 +221,8 @@ void autoConfigurationCustomizerProvider_installsSharedDeterministicIdGenerator( var spans = exporter.getFinishedSpanItems(); assertEquals(1, spans.size()); assertEquals( - idGenerator.generateSpanIdForOperation("op-spi"), spans.get(0).getSpanId()); + pluginGenerator.generateSpanIdForOperation("op-spi"), + spans.get(0).getSpanId()); } private static final class CountingIdGenerator implements IdGenerator { From 01317a7078a00fa4ebb18a426e131ef37cd4f503 Mon Sep 17 00:00:00 2001 From: Zhongke Chen Date: Tue, 14 Jul 2026 07:45:37 +0000 Subject: [PATCH 28/36] Limit ADOT Java agent to default constructor example --- examples/README.md | 5 +++-- examples/generate-template.py | 19 ++++++++++++++----- .../durable/examples/ExampleTemplate.java | 2 ++ .../OtelXRayDefaultConstructorExample.java | 2 +- 4 files changed, 20 insertions(+), 8 deletions(-) diff --git a/examples/README.md b/examples/README.md index e9933b092..47d0f82dc 100644 --- a/examples/README.md +++ b/examples/README.md @@ -48,11 +48,12 @@ The SAM template configures: - `DurableConfig` with `ExecutionTimeout` and `RetentionPeriodInDays` - CloudWatch log groups for Lambda functions with 7 days of retention - IAM permissions for `lambda:CheckpointDurableExecutions` and `lambda:GetDurableExecutionState` -- ADOT tracing examples with `AWS_LAMBDA_EXEC_WRAPPER` and `OTEL_JAVAAGENT_EXTENSIONS` so the OTel plugin SPI is loaded by the Java agent +- ADOT tracing examples with active X-Ray tracing and the ADOT Lambda layer +- The default-constructor OTel example with `AWS_LAMBDA_EXEC_WRAPPER` and `OTEL_JAVAAGENT_EXTENSIONS` so the OTel plugin SPI is loaded by the Java agent `template.yaml` is generated from the Java example handlers and is intentionally not checked in. Re-run `python3 generate-template.py` after adding or removing a deployable example handler. -The examples package copies the OTel plugin jar into `lib/` so the ADOT Java agent can load it as an extension from the generated `OTEL_JAVAAGENT_EXTENSIONS` path. +The examples package copies the OTel plugin jar into `lib/` so the ADOT Java agent can load it as an extension for examples that enable `OTEL_JAVAAGENT_EXTENSIONS`. ## Invoke Deployed Functions diff --git a/examples/generate-template.py b/examples/generate-template.py index 01738dffa..ac1d5fdb5 100755 --- a/examples/generate-template.py +++ b/examples/generate-template.py @@ -34,6 +34,7 @@ class ExampleFunction: suffix: str condition: str | None tracing: bool + java_agent: bool @property def logical_id(self) -> str: @@ -70,24 +71,26 @@ def is_top_level_durable_handler(source: str, class_name: str) -> bool: return bool(match and "extends DurableHandler" in match.group("header")) -def read_template_annotation(source: str, class_name: str) -> tuple[str | None, bool]: +def read_template_annotation(source: str, class_name: str) -> tuple[str | None, bool, bool]: class_match = re.search(rf"public\s+(?:final\s+)?class\s+{class_name}\b", source) if not class_match: - return None, False + return None, False, False prefix = source[: class_match.start()] matches = list( re.finditer(rf"@(?:[A-Za-z_][\w.]*\.)?{TEMPLATE_ANNOTATION}\s*(?:\((?P.*?)\))?", prefix, re.DOTALL) ) if not matches: - return None, False + return None, False, False body = matches[-1].group("body") or "" condition_match = re.search(r'condition\s*=\s*"([^"]+)"', body) tracing_match = re.search(r"tracing\s*=\s*(true|false)", body) + java_agent_match = re.search(r"javaAgent\s*=\s*(true|false)", body) condition = condition_match.group(1) if condition_match else None tracing = tracing_match.group(1) == "true" if tracing_match else False - return condition, tracing + java_agent = java_agent_match.group(1) == "true" if java_agent_match else False + return condition, tracing, java_agent def discover_examples() -> list[ExampleFunction]: @@ -98,7 +101,7 @@ def discover_examples() -> list[ExampleFunction]: if not is_top_level_durable_handler(source, class_name): continue - condition, tracing = read_template_annotation(source, class_name) + condition, tracing, java_agent = read_template_annotation(source, class_name) package_name = read_package(source, path) examples.append( ExampleFunction( @@ -107,6 +110,7 @@ def discover_examples() -> list[ExampleFunction]: suffix=kebab_case(class_name), condition=condition, tracing=tracing, + java_agent=java_agent, ) ) return examples @@ -136,6 +140,11 @@ def emit_function(lines: list[str], example: ExampleFunction, java_agent_extensi " Tracing: Active", " Layers:", " - !Sub arn:aws:lambda:${AWS::Region}:615299751070:layer:AWSOpenTelemetryDistroJava:15", + ] + ) + if example.java_agent: + lines.extend( + [ " Environment:", " Variables:", " AWS_LAMBDA_EXEC_WRAPPER: /opt/otel-instrument", diff --git a/examples/src/main/java/software/amazon/lambda/durable/examples/ExampleTemplate.java b/examples/src/main/java/software/amazon/lambda/durable/examples/ExampleTemplate.java index 898634884..bd7d92bc0 100644 --- a/examples/src/main/java/software/amazon/lambda/durable/examples/ExampleTemplate.java +++ b/examples/src/main/java/software/amazon/lambda/durable/examples/ExampleTemplate.java @@ -14,4 +14,6 @@ String condition() default ""; boolean tracing() default false; + + boolean javaAgent() default false; } diff --git a/examples/src/main/java/software/amazon/lambda/durable/examples/otel/OtelXRayDefaultConstructorExample.java b/examples/src/main/java/software/amazon/lambda/durable/examples/otel/OtelXRayDefaultConstructorExample.java index 9ccc70673..f6e4efec1 100644 --- a/examples/src/main/java/software/amazon/lambda/durable/examples/otel/OtelXRayDefaultConstructorExample.java +++ b/examples/src/main/java/software/amazon/lambda/durable/examples/otel/OtelXRayDefaultConstructorExample.java @@ -15,7 +15,7 @@ *

    {@link OtelPlugin#OtelPlugin()} uses the global provider initialized by the ADOT Java agent with deterministic * span ID generation installed by the plugin's OpenTelemetry autoconfigure SPI. */ -@ExampleTemplate(tracing = true) +@ExampleTemplate(tracing = true, javaAgent = true) public class OtelXRayDefaultConstructorExample extends DurableHandler { @Override From f40792af1418ed5aa8e806f8ec16c53394c0c976 Mon Sep 17 00:00:00 2001 From: Frank Chen Date: Tue, 14 Jul 2026 10:18:21 -0700 Subject: [PATCH 29/36] Update OTel examples for ADOT Java agent --- examples/README.md | 2 +- examples/pom.xml | 12 -- .../durable/examples/general/OtelExample.java | 3 +- .../examples/otel/OtelXRayExamples.java | 15 +- .../examples/otel/OtelXRayStepExample.java | 18 +- .../examples/otel/OtelXRayWaitExample.java | 18 +- .../CloudBasedOtelIntegrationTest.java | 6 +- .../otel/OtelXRayExampleTestSupport.java | 30 ++++ .../otel/OtelXRayStepExampleTest.java | 12 ++ .../otel/OtelXRayWaitExampleTest.java | 12 ++ otel-plugin/README.md | 13 +- .../lambda/durable/otel/OtelPlugin.java | 157 +++--------------- .../otlp/trace/OtlpGrpcSpanExporter.java | 44 ----- .../testing/FakeJavaAgentTracerProvider.java | 8 +- .../otel/OtelPluginIntegrationTest.java | 6 +- .../lambda/durable/otel/OtelPluginTest.java | 6 +- 16 files changed, 106 insertions(+), 256 deletions(-) create mode 100644 examples/src/test/java/software/amazon/lambda/durable/examples/otel/OtelXRayExampleTestSupport.java delete mode 100644 otel-plugin/src/test/java/io/opentelemetry/exporter/otlp/trace/OtlpGrpcSpanExporter.java diff --git a/examples/README.md b/examples/README.md index 47d0f82dc..399e91c9f 100644 --- a/examples/README.md +++ b/examples/README.md @@ -49,7 +49,7 @@ The SAM template configures: - CloudWatch log groups for Lambda functions with 7 days of retention - IAM permissions for `lambda:CheckpointDurableExecutions` and `lambda:GetDurableExecutionState` - ADOT tracing examples with active X-Ray tracing and the ADOT Lambda layer -- The default-constructor OTel example with `AWS_LAMBDA_EXEC_WRAPPER` and `OTEL_JAVAAGENT_EXTENSIONS` so the OTel plugin SPI is loaded by the Java agent +- OTel examples that use `new OtelPlugin()` with `AWS_LAMBDA_EXEC_WRAPPER` and `OTEL_JAVAAGENT_EXTENSIONS` so the OTel plugin SPI is loaded by the Java agent `template.yaml` is generated from the Java example handlers and is intentionally not checked in. Re-run `python3 generate-template.py` after adding or removing a deployable example handler. diff --git a/examples/pom.xml b/examples/pom.xml index 298e02b3c..27771e5cf 100644 --- a/examples/pom.xml +++ b/examples/pom.xml @@ -50,18 +50,6 @@ 1.63.0 - - - io.opentelemetry - opentelemetry-exporter-otlp - 1.64.0 - - - io.grpc - grpc-netty-shaded - 1.82.1 - - com.amazonaws diff --git a/examples/src/main/java/software/amazon/lambda/durable/examples/general/OtelExample.java b/examples/src/main/java/software/amazon/lambda/durable/examples/general/OtelExample.java index 94b2be751..69629aeff 100644 --- a/examples/src/main/java/software/amazon/lambda/durable/examples/general/OtelExample.java +++ b/examples/src/main/java/software/amazon/lambda/durable/examples/general/OtelExample.java @@ -22,8 +22,7 @@ *

  • Logging exporter (spans printed to stdout → CloudWatch Logs) * * - *

    In production, replace {@code LoggingSpanExporter} with {@code OtlpGrpcSpanExporter} to send spans to an OTLP - * collector (X-Ray, Datadog, etc.). + *

    In production, replace {@code LoggingSpanExporter} with the exporter for your observability backend. * *

    Expected trace structure: * diff --git a/examples/src/main/java/software/amazon/lambda/durable/examples/otel/OtelXRayExamples.java b/examples/src/main/java/software/amazon/lambda/durable/examples/otel/OtelXRayExamples.java index e1e58b431..55a9188d8 100644 --- a/examples/src/main/java/software/amazon/lambda/durable/examples/otel/OtelXRayExamples.java +++ b/examples/src/main/java/software/amazon/lambda/durable/examples/otel/OtelXRayExamples.java @@ -2,7 +2,7 @@ // SPDX-License-Identifier: Apache-2.0 package software.amazon.lambda.durable.examples.otel; -import io.opentelemetry.exporter.otlp.trace.OtlpGrpcSpanExporter; +import io.opentelemetry.exporter.logging.LoggingSpanExporter; import io.opentelemetry.sdk.trace.SdkTracerProvider; import io.opentelemetry.sdk.trace.export.SimpleSpanProcessor; import java.util.List; @@ -20,10 +20,9 @@ public final class OtelXRayExamples { private OtelXRayExamples() {} - private static DurableConfig otelConfig() { - var otlpExporter = OtlpGrpcSpanExporter.getDefault(); - var otelPlugin = - new OtelPlugin(SdkTracerProvider.builder().addSpanProcessor(SimpleSpanProcessor.create(otlpExporter))); + private static DurableConfig localOtelConfig() { + var otelPlugin = new OtelPlugin( + SdkTracerProvider.builder().addSpanProcessor(SimpleSpanProcessor.create(LoggingSpanExporter.create()))); return DurableConfig.builder().withPlugins(otelPlugin).build(); } @@ -32,7 +31,7 @@ public static class MapExample extends DurableHandler { @Override protected DurableConfig createConfiguration() { - return otelConfig(); + return localOtelConfig(); } @Override @@ -56,7 +55,7 @@ public static class ParallelExample extends DurableHandlerExports spans via OTLP to the ADOT Lambda Layer collector, which forwards them to X-Ray. Requires: + *

    Exports spans through the ADOT Java agent global OpenTelemetry provider. Requires: * *

      *
    • {@code Tracing: Active} on the Lambda function - *
    • ADOT Lambda Layer added to the function (for the OTLP collector) + *
    • ADOT Lambda Layer added to the function + *
    • {@code OTEL_JAVAAGENT_EXTENSIONS} pointing at the OTel plugin jar *
    * *

    Expected trace structure in X-Ray: @@ -32,18 +30,12 @@ * └── transform attempt 1 * */ -@ExampleTemplate(tracing = true) +@ExampleTemplate(tracing = true, javaAgent = true) public class OtelXRayStepExample extends DurableHandler { @Override protected DurableConfig createConfiguration() { - // OTLP exporter sends spans to the ADOT collector (localhost:4317 by default) - var otlpExporter = OtlpGrpcSpanExporter.getDefault(); - - var otelPlugin = - new OtelPlugin(SdkTracerProvider.builder().addSpanProcessor(SimpleSpanProcessor.create(otlpExporter))); - - return DurableConfig.builder().withPlugins(otelPlugin).build(); + return DurableConfig.builder().withPlugins(new OtelPlugin()).build(); } @Override diff --git a/examples/src/main/java/software/amazon/lambda/durable/examples/otel/OtelXRayWaitExample.java b/examples/src/main/java/software/amazon/lambda/durable/examples/otel/OtelXRayWaitExample.java index 4ee8e6551..a614e5c6a 100644 --- a/examples/src/main/java/software/amazon/lambda/durable/examples/otel/OtelXRayWaitExample.java +++ b/examples/src/main/java/software/amazon/lambda/durable/examples/otel/OtelXRayWaitExample.java @@ -2,9 +2,6 @@ // SPDX-License-Identifier: Apache-2.0 package software.amazon.lambda.durable.examples.otel; -import io.opentelemetry.exporter.otlp.trace.OtlpGrpcSpanExporter; -import io.opentelemetry.sdk.trace.SdkTracerProvider; -import io.opentelemetry.sdk.trace.export.SimpleSpanProcessor; import java.time.Duration; import software.amazon.lambda.durable.DurableConfig; import software.amazon.lambda.durable.DurableContext; @@ -23,11 +20,12 @@ *

  • Invocation 2: replays "before-wait" (no-op) → wait completes → "after-wait" step runs * * - *

    Exports spans via OTLP to the ADOT Lambda Layer collector. Requires: + *

    Exports spans through the ADOT Java agent global OpenTelemetry provider. Requires: * *

      *
    • {@code Tracing: Active} on the Lambda function - *
    • ADOT Lambda Layer added to the function (for the OTLP collector) + *
    • ADOT Lambda Layer added to the function + *
    • {@code OTEL_JAVAAGENT_EXTENSIONS} pointing at the OTel plugin jar *
    * *

    Expected trace structure in X-Ray (all under one trace ID — backend propagates same Root): @@ -44,18 +42,12 @@ * └── durable.step:after-wait [attempt 1] * */ -@ExampleTemplate(tracing = true) +@ExampleTemplate(tracing = true, javaAgent = true) public class OtelXRayWaitExample extends DurableHandler { @Override protected DurableConfig createConfiguration() { - // OTLP exporter sends spans to the ADOT collector (localhost:4317 by default) - var otlpExporter = OtlpGrpcSpanExporter.getDefault(); - - var otelPlugin = - new OtelPlugin(SdkTracerProvider.builder().addSpanProcessor(SimpleSpanProcessor.create(otlpExporter))); - - return DurableConfig.builder().withPlugins(otelPlugin).build(); + return DurableConfig.builder().withPlugins(new OtelPlugin()).build(); } @Override diff --git a/examples/src/test/java/software/amazon/lambda/durable/examples/CloudBasedOtelIntegrationTest.java b/examples/src/test/java/software/amazon/lambda/durable/examples/CloudBasedOtelIntegrationTest.java index eb8a706c5..8455b43b0 100644 --- a/examples/src/test/java/software/amazon/lambda/durable/examples/CloudBasedOtelIntegrationTest.java +++ b/examples/src/test/java/software/amazon/lambda/durable/examples/CloudBasedOtelIntegrationTest.java @@ -38,8 +38,8 @@ *

    These tests deploy Lambda functions configured with: * *

      - *
    • OpenTelemetry Durable Plugin with OTLP gRPC exporter - *
    • ADOT collector layer (OTLP receiver → X-Ray exporter) + *
    • OpenTelemetry Durable Plugin using the ADOT Java agent global provider + *
    • ADOT Java agent layer with the plugin jar loaded through {@code OTEL_JAVAAGENT_EXTENSIONS} *
    • Active X-Ray tracing *
    * @@ -257,7 +257,7 @@ void waitAndResume_producesUnifiedTraceAcrossInvocations() throws Exception { } @Test - void defaultConstructorWithGlobalProvider_producesDurableSpansInXRay() throws Exception { + void defaultConstructorWithAdotJavaAgent_producesDurableSpansInXRay() throws Exception { var startTime = Instant.now(); var runner = CloudDurableTestRunner.create( diff --git a/examples/src/test/java/software/amazon/lambda/durable/examples/otel/OtelXRayExampleTestSupport.java b/examples/src/test/java/software/amazon/lambda/durable/examples/otel/OtelXRayExampleTestSupport.java new file mode 100644 index 000000000..b1f1a5744 --- /dev/null +++ b/examples/src/test/java/software/amazon/lambda/durable/examples/otel/OtelXRayExampleTestSupport.java @@ -0,0 +1,30 @@ +// Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +// SPDX-License-Identifier: Apache-2.0 +package software.amazon.lambda.durable.examples.otel; + +import io.opentelemetry.api.GlobalOpenTelemetry; +import io.opentelemetry.exporter.logging.LoggingSpanExporter; +import io.opentelemetry.sdk.OpenTelemetrySdk; +import io.opentelemetry.sdk.trace.SdkTracerProvider; +import io.opentelemetry.sdk.trace.export.SimpleSpanProcessor; + +final class OtelXRayExampleTestSupport { + + private static final String SPI_INSTALLED_PROPERTY = + "software.amazon.lambda.durable.otel.autoConfigurationCustomizerProviderInstalled"; + + private OtelXRayExampleTestSupport() {} + + static void installGlobalOpenTelemetry() { + System.setProperty(SPI_INSTALLED_PROPERTY, Boolean.TRUE.toString()); + var tracerProvider = SdkTracerProvider.builder() + .addSpanProcessor(SimpleSpanProcessor.create(LoggingSpanExporter.create())) + .build(); + OpenTelemetrySdk.builder().setTracerProvider(tracerProvider).buildAndRegisterGlobal(); + } + + static void resetGlobalOpenTelemetry() { + GlobalOpenTelemetry.resetForTest(); + System.clearProperty(SPI_INSTALLED_PROPERTY); + } +} diff --git a/examples/src/test/java/software/amazon/lambda/durable/examples/otel/OtelXRayStepExampleTest.java b/examples/src/test/java/software/amazon/lambda/durable/examples/otel/OtelXRayStepExampleTest.java index 1bae61686..dffe99080 100644 --- a/examples/src/test/java/software/amazon/lambda/durable/examples/otel/OtelXRayStepExampleTest.java +++ b/examples/src/test/java/software/amazon/lambda/durable/examples/otel/OtelXRayStepExampleTest.java @@ -4,6 +4,8 @@ import static org.junit.jupiter.api.Assertions.*; +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; import software.amazon.lambda.durable.examples.types.GreetingRequest; import software.amazon.lambda.durable.model.ExecutionStatus; @@ -11,6 +13,16 @@ class OtelXRayStepExampleTest { + @BeforeEach + void setUp() { + OtelXRayExampleTestSupport.installGlobalOpenTelemetry(); + } + + @AfterEach + void tearDown() { + OtelXRayExampleTestSupport.resetGlobalOpenTelemetry(); + } + @Test void testSimpleSteps_succeeds() { var handler = new OtelXRayStepExample(); diff --git a/examples/src/test/java/software/amazon/lambda/durable/examples/otel/OtelXRayWaitExampleTest.java b/examples/src/test/java/software/amazon/lambda/durable/examples/otel/OtelXRayWaitExampleTest.java index 4d46dc2bb..ac42d31d9 100644 --- a/examples/src/test/java/software/amazon/lambda/durable/examples/otel/OtelXRayWaitExampleTest.java +++ b/examples/src/test/java/software/amazon/lambda/durable/examples/otel/OtelXRayWaitExampleTest.java @@ -4,6 +4,8 @@ import static org.junit.jupiter.api.Assertions.*; +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; import software.amazon.lambda.durable.examples.types.GreetingRequest; import software.amazon.lambda.durable.model.ExecutionStatus; @@ -11,6 +13,16 @@ class OtelXRayWaitExampleTest { + @BeforeEach + void setUp() { + OtelXRayExampleTestSupport.installGlobalOpenTelemetry(); + } + + @AfterEach + void tearDown() { + OtelXRayExampleTestSupport.resetGlobalOpenTelemetry(); + } + @Test void testFirstInvocation_suspendsOnWait() { var handler = new OtelXRayWaitExample(); diff --git a/otel-plugin/README.md b/otel-plugin/README.md index df69e5ee9..c818395e5 100644 --- a/otel-plugin/README.md +++ b/otel-plugin/README.md @@ -10,7 +10,7 @@ OpenTelemetry instrumentation plugin for the AWS Lambda Durable Execution SDK fo - **Span-per-Operation**: Each durable operation (step, wait, map, etc.) gets its own span with accurate timing - **Attempt Spans**: Each user function execution (step attempt, child context run) gets a span, including retries - **Log Correlation**: Injects `traceId`, `spanId`, and `traceSampled` into SLF4J MDC for end-to-end observability -- **Self-Contained Setup**: `new OtelPlugin()` can export to the ADOT collector without handler-side OpenTelemetry initialization +- **ADOT Java Agent Setup**: `new OtelPlugin()` uses the ADOT Java agent global OpenTelemetry provider without handler-side initialization ## Installation @@ -22,7 +22,7 @@ OpenTelemetry instrumentation plugin for the AWS Lambda Durable Execution SDK fo ``` -You also need the OpenTelemetry SDK and an exporter: +If you configure your own `SdkTracerProviderBuilder`, add the OpenTelemetry SDK and an exporter: ```xml @@ -32,11 +32,14 @@ You also need the OpenTelemetry SDK and an exporter: io.opentelemetry - opentelemetry-exporter-otlp + opentelemetry-exporter-logging 1.63.0 ``` +For `new OtelPlugin()` on Lambda, use the ADOT Java agent setup below instead of initializing an exporter in your +handler. + ## Quick Start using X-Ray/CloudWatch Tracing 1. Add the ADOT Lambda Layer to your function @@ -208,7 +211,7 @@ These appear automatically in structured log output (Log4j2 JSON, Logback JSON) ### Constructor Options ```java -// Default: X-Ray context extraction, MDC enabled, default OTLP export +// Default: X-Ray context extraction, MDC enabled, ADOT Java agent global provider new OtelPlugin(); // Custom tracer provider pipeline @@ -223,7 +226,7 @@ new OtelPlugin(tracerProviderBuilder, contextExtractor, enableMdc); | Parameter | Description | Default | |-----------|-------------|---------| -| `tracerProviderBuilder` | `SdkTracerProviderBuilder` with your exporter/processor configured | Copies global SDK provider when possible, or uses the ADOT Java agent provider configured by the plugin SPI | +| `tracerProviderBuilder` | `SdkTracerProviderBuilder` with your exporter/processor configured | Not used by `new OtelPlugin()`; the default constructor uses the ADOT Java agent provider configured by the plugin SPI | | `contextExtractor` | Extracts parent trace context from the Lambda environment | `XRayContextExtractor` | | `enableMdc` | If true, injects `traceId`/`spanId`/`traceSampled` into SLF4J MDC | `true` | diff --git a/otel-plugin/src/main/java/software/amazon/lambda/durable/otel/OtelPlugin.java b/otel-plugin/src/main/java/software/amazon/lambda/durable/otel/OtelPlugin.java index 823e9926e..306c613ce 100644 --- a/otel-plugin/src/main/java/software/amazon/lambda/durable/otel/OtelPlugin.java +++ b/otel-plugin/src/main/java/software/amazon/lambda/durable/otel/OtelPlugin.java @@ -21,8 +21,6 @@ import io.opentelemetry.sdk.trace.SdkTracerProvider; import io.opentelemetry.sdk.trace.SdkTracerProviderBuilder; import io.opentelemetry.semconv.ServiceAttributes; -import java.lang.reflect.Field; -import java.lang.reflect.InvocationTargetException; import java.nio.file.Files; import java.nio.file.Path; import java.time.Instant; @@ -62,7 +60,7 @@ *

    Requires the ADOT Lambda Layer for trace export. Configure with: * *

      - *
    • Lambda Layer: {@code AWSOpenTelemetryDistroJava} (provides the OTLP collector extension) + *
    • Lambda Layer: {@code AWSOpenTelemetryDistroJava} (provides the ADOT Java agent and export pipeline) *
    • Tracing: Active (to populate {@code _X_AMZN_TRACE_ID}) *
    * @@ -106,12 +104,13 @@ public class OtelPlugin implements DurableExecutionPlugin { *

    Uses the provided tracer provider builder. Customers configure exporters and span processors on the builder — * the plugin handles ID generation. * - *

    For ADOT layer usage, configure with an OTLP exporter: + *

    For ADOT Java agent usage, prefer {@link #OtelPlugin()} with the plugin jar configured through + * {@code OTEL_JAVAAGENT_EXTENSIONS}. Use this builder constructor when you want to own the exporter pipeline: * *

    {@code
    -     * var otlpExporter = OtlpGrpcSpanExporter.getDefault(); // sends to localhost:4317
    +     * var exporter = LoggingSpanExporter.create();
          * var plugin = new OtelPlugin(
    -     *     SdkTracerProvider.builder().addSpanProcessor(SimpleSpanProcessor.create(otlpExporter)));
    +     *     SdkTracerProvider.builder().addSpanProcessor(SimpleSpanProcessor.create(exporter)));
          * }
    * * @param tracerProviderBuilder the tracer provider builder (ID generator will be overridden) @@ -199,7 +198,7 @@ public void onInvocationStart(InvocationInfo info) { Context parentContext; if (extractedContext != null && extractedContext.parentSpanId() != null) { // X-Ray header has parent — create the invocation span as a child of that segment. - // This connects our OTLP-exported spans to the Lambda service's X-Ray segments. + // This connects plugin spans to the Lambda service's X-Ray segments. var parentSpanContext = SpanContext.createFromRemoteParent( extractedContext.traceId(), extractedContext.parentSpanId(), @@ -508,22 +507,15 @@ private static TracerProvider getDefaultTracerProvider() { validateAutoConfigurationCustomizerProviderInstalled(); var globalTracerProvider = GlobalOpenTelemetry.getTracerProvider(); - try { - getGlobalSdkTracerProvider(globalTracerProvider); - logger.info("OtelPlugin initialized from existing GlobalOpenTelemetry SDK tracer provider; assuming " - + "deterministic span IDs were installed through AutoConfigurationCustomizerProvider"); - return globalTracerProvider; - } catch (IllegalStateException e) { - if (isJavaAgentTracerProvider(globalTracerProvider)) { - var sdkTracerProvider = getJavaAgentSdkTracerProvider(globalTracerProvider); - logJavaAgentInitialization(globalTracerProvider, sdkTracerProvider, e); - return globalTracerProvider; - } - throw new IllegalStateException( - "OtelPlugin() requires GlobalOpenTelemetry to be initialized by " - + "OtelPluginAutoConfigurationCustomizerProvider through the OpenTelemetry Java agent.", - e); + if (globalTracerProvider == TracerProvider.noop()) { + throw new IllegalStateException("OtelPlugin() requires GlobalOpenTelemetry to be initialized by " + + "OtelPluginAutoConfigurationCustomizerProvider through the OpenTelemetry Java agent."); } + logger.info( + "OtelPlugin initialized from existing GlobalOpenTelemetry tracer provider {}; assuming deterministic " + + "span IDs were installed through AutoConfigurationCustomizerProvider", + globalTracerProvider.getClass().getName()); + return globalTracerProvider; } private static DeterministicIdGenerator createDefaultIdGenerator() { @@ -569,126 +561,15 @@ private static boolean extensionPathExists(String configuredPath) { return Files.exists(Path.of(firstPath)); } - private static SdkTracerProvider getGlobalSdkTracerProvider(TracerProvider tracerProvider) { - if (tracerProvider instanceof SdkTracerProvider sdkTracerProvider) { - return sdkTracerProvider; - } - return unobfuscateSdkTracerProvider(tracerProvider); - } - - private static boolean isJavaAgentTracerProvider(TracerProvider tracerProvider) { - return tracerProvider.getClass().getName().startsWith("io.opentelemetry.javaagent."); - } - private static SdkTracerProvider getSdkTracerProviderForFlush(TracerProvider tracerProvider) { - try { - if (isJavaAgentTracerProvider(tracerProvider)) { - return getJavaAgentSdkTracerProvider(tracerProvider); - } - return getGlobalSdkTracerProvider(tracerProvider); - } catch (IllegalStateException e) { - logger.warn( - "OtelPlugin could not inspect SDK tracer provider for forceFlush; spans will rely on the global " - + "provider's own flushing.", - e); - return null; - } - } - - private static SdkTracerProvider getJavaAgentSdkTracerProvider(TracerProvider tracerProvider) { - Object agentTracerProvider; - try { - agentTracerProvider = getField(tracerProvider, "agentTracerProvider", Object.class); - } catch (IllegalStateException e) { - logger.warn( - "OtelPlugin could not inspect Java agent tracer provider {}; using the global provider directly.", - tracerProvider.getClass().getName(), - e); - return null; - } - - if (agentTracerProvider instanceof SdkTracerProvider sdkTracerProvider) { + if (tracerProvider instanceof SdkTracerProvider sdkTracerProvider) { return sdkTracerProvider; } - if (agentTracerProvider instanceof TracerProvider nestedTracerProvider) { - try { - return unobfuscateSdkTracerProvider(nestedTracerProvider); - } catch (IllegalStateException e) { - logger.warn( - "OtelPlugin could not inspect nested Java agent SDK tracer provider {}; using the global " - + "provider directly.", - nestedTracerProvider.getClass().getName(), - e); - return null; - } - } logger.info( - "OtelPlugin could not inspect Java agent SDK tracer provider because field agentTracerProvider has " - + "type {}; using the global provider directly.", - agentTracerProvider != null ? agentTracerProvider.getClass().getName() : "null"); + "OtelPlugin forceFlush is not available because GlobalOpenTelemetry provider {} is not an " + + "SdkTracerProvider visible to the application class loader; spans will rely on the " + + "provider's own flushing.", + tracerProvider.getClass().getName()); return null; } - - private static void logJavaAgentInitialization( - TracerProvider globalTracerProvider, SdkTracerProvider sdkTracerProvider, IllegalStateException cause) { - if (sdkTracerProvider == null) { - logger.info( - "OtelPlugin initialized from existing GlobalOpenTelemetry Java agent tracer provider {}, but the " - + "agent SDK provider is not visible to the application class loader. Assuming " - + "deterministic span IDs were installed through AutoConfigurationCustomizerProvider. " - + "Cause: {}", - globalTracerProvider.getClass().getName(), - cause.getMessage()); - return; - } - logger.info( - "OtelPlugin initialized from existing GlobalOpenTelemetry Java agent tracer provider {}; assuming " - + "deterministic span IDs were installed through AutoConfigurationCustomizerProvider. Cause: {}", - globalTracerProvider.getClass().getName(), - cause.getMessage()); - } - - private static SdkTracerProvider unobfuscateSdkTracerProvider(TracerProvider tracerProvider) { - var providerClassName = tracerProvider.getClass().getName(); - try { - var unobfuscate = tracerProvider.getClass().getDeclaredMethod("unobfuscate"); - unobfuscate.setAccessible(true); - var sdkTracerProvider = unobfuscate.invoke(tracerProvider); - if (sdkTracerProvider instanceof SdkTracerProvider result) { - return result; - } - throw new IllegalStateException( - "GlobalOpenTelemetry provider " + providerClassName + " is not backed by an SdkTracerProvider"); - } catch (NoSuchMethodException e) { - throw new IllegalStateException( - "GlobalOpenTelemetry provider " + providerClassName + " does not expose unobfuscate(); " - + "ADOT Java instrumentation may not have initialized an SDK provider.", - e); - } catch (IllegalAccessException | InvocationTargetException e) { - throw new IllegalStateException( - "Unable to invoke unobfuscate() on GlobalOpenTelemetry provider " + providerClassName, e); - } - } - - private static T getField(Object target, String fieldName, Class fieldType) { - try { - var field = findField(target.getClass(), fieldName); - field.setAccessible(true); - return fieldType.cast(field.get(target)); - } catch (NoSuchFieldException | IllegalAccessException e) { - throw new IllegalStateException("Unable to copy OpenTelemetry SDK field: " + fieldName, e); - } - } - - private static Field findField(Class type, String fieldName) throws NoSuchFieldException { - var current = type; - while (current != null) { - try { - return current.getDeclaredField(fieldName); - } catch (NoSuchFieldException e) { - current = current.getSuperclass(); - } - } - throw new NoSuchFieldException(fieldName); - } } diff --git a/otel-plugin/src/test/java/io/opentelemetry/exporter/otlp/trace/OtlpGrpcSpanExporter.java b/otel-plugin/src/test/java/io/opentelemetry/exporter/otlp/trace/OtlpGrpcSpanExporter.java deleted file mode 100644 index c0c3cdb10..000000000 --- a/otel-plugin/src/test/java/io/opentelemetry/exporter/otlp/trace/OtlpGrpcSpanExporter.java +++ /dev/null @@ -1,44 +0,0 @@ -// Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. -// SPDX-License-Identifier: Apache-2.0 -package io.opentelemetry.exporter.otlp.trace; - -import io.opentelemetry.sdk.common.CompletableResultCode; -import io.opentelemetry.sdk.trace.data.SpanData; -import io.opentelemetry.sdk.trace.export.SpanExporter; -import java.util.ArrayList; -import java.util.Collection; -import java.util.List; -import java.util.concurrent.CopyOnWriteArrayList; - -public final class OtlpGrpcSpanExporter implements SpanExporter { - - private static final List FINISHED_SPANS = new CopyOnWriteArrayList<>(); - - public static SpanExporter getDefault() { - return new OtlpGrpcSpanExporter(); - } - - public static List getFinishedSpanItems() { - return new ArrayList<>(FINISHED_SPANS); - } - - public static void reset() { - FINISHED_SPANS.clear(); - } - - @Override - public CompletableResultCode export(Collection spans) { - FINISHED_SPANS.addAll(spans); - return CompletableResultCode.ofSuccess(); - } - - @Override - public CompletableResultCode flush() { - return CompletableResultCode.ofSuccess(); - } - - @Override - public CompletableResultCode shutdown() { - return CompletableResultCode.ofSuccess(); - } -} diff --git a/otel-plugin/src/test/java/io/opentelemetry/javaagent/testing/FakeJavaAgentTracerProvider.java b/otel-plugin/src/test/java/io/opentelemetry/javaagent/testing/FakeJavaAgentTracerProvider.java index ed88eeafc..db99b5bdb 100644 --- a/otel-plugin/src/test/java/io/opentelemetry/javaagent/testing/FakeJavaAgentTracerProvider.java +++ b/otel-plugin/src/test/java/io/opentelemetry/javaagent/testing/FakeJavaAgentTracerProvider.java @@ -8,16 +8,10 @@ public final class FakeJavaAgentTracerProvider implements TracerProvider { - private final Object agentTracerProvider; private final TracerProvider delegate; public FakeJavaAgentTracerProvider(TracerProvider agentTracerProvider) { - this(agentTracerProvider, agentTracerProvider); - } - - public FakeJavaAgentTracerProvider(Object agentTracerProvider, TracerProvider delegate) { - this.agentTracerProvider = agentTracerProvider; - this.delegate = delegate; + this.delegate = agentTracerProvider; } @Override diff --git a/otel-plugin/src/test/java/software/amazon/lambda/durable/otel/OtelPluginIntegrationTest.java b/otel-plugin/src/test/java/software/amazon/lambda/durable/otel/OtelPluginIntegrationTest.java index 90d2cb5fb..200660024 100644 --- a/otel-plugin/src/test/java/software/amazon/lambda/durable/otel/OtelPluginIntegrationTest.java +++ b/otel-plugin/src/test/java/software/amazon/lambda/durable/otel/OtelPluginIntegrationTest.java @@ -8,7 +8,6 @@ import io.opentelemetry.api.OpenTelemetry; import io.opentelemetry.api.trace.StatusCode; import io.opentelemetry.context.propagation.ContextPropagators; -import io.opentelemetry.exporter.otlp.trace.OtlpGrpcSpanExporter; import io.opentelemetry.javaagent.testing.FakeJavaAgentTracerProvider; import io.opentelemetry.sdk.OpenTelemetrySdk; import io.opentelemetry.sdk.testing.exporter.InMemorySpanExporter; @@ -56,7 +55,6 @@ void setUp() { @AfterEach void tearDown() { GlobalOpenTelemetry.resetForTest(); - OtlpGrpcSpanExporter.reset(); DeterministicIdGenerator.clearSharedStateForTest(); OtelPluginAutoConfigurationState.resetInstalledForTest(); } @@ -118,14 +116,13 @@ void defaultConstructor_usesGlobalSdkTracerProviderDirectly() { void defaultConstructor_usesJavaAgentGlobalTracerProviderDirectly_withSeparateAutoConfiguredIdGenerator() { OtelPluginAutoConfigurationState.markInstalled(); GlobalOpenTelemetry.resetForTest(); - OtlpGrpcSpanExporter.reset(); var globalExporter = InMemorySpanExporter.create(); var javaAgentIdGenerator = new DeterministicIdGenerator(); var sdkTracerProvider = SdkTracerProvider.builder() .setIdGenerator(javaAgentIdGenerator) .addSpanProcessor(SimpleSpanProcessor.create(globalExporter)) .build(); - var javaAgentTracerProvider = new FakeJavaAgentTracerProvider(new Object(), sdkTracerProvider); + var javaAgentTracerProvider = new FakeJavaAgentTracerProvider(sdkTracerProvider); GlobalOpenTelemetry.set(new OpenTelemetry() { @Override public io.opentelemetry.api.trace.TracerProvider getTracerProvider() { @@ -153,7 +150,6 @@ public ContextPropagators getPropagators() { assertSpanExists(spans, "invocation"); assertSpanExists(spans, "javaagent-step"); assertSpanExists(spans, "javaagent-step attempt 1"); - assertTrue(OtlpGrpcSpanExporter.getFinishedSpanItems().isEmpty()); } @Test diff --git a/otel-plugin/src/test/java/software/amazon/lambda/durable/otel/OtelPluginTest.java b/otel-plugin/src/test/java/software/amazon/lambda/durable/otel/OtelPluginTest.java index dada15e9c..1646ab4ef 100644 --- a/otel-plugin/src/test/java/software/amazon/lambda/durable/otel/OtelPluginTest.java +++ b/otel-plugin/src/test/java/software/amazon/lambda/durable/otel/OtelPluginTest.java @@ -17,7 +17,6 @@ import io.opentelemetry.api.trace.TraceFlags; import io.opentelemetry.api.trace.TraceState; import io.opentelemetry.context.propagation.ContextPropagators; -import io.opentelemetry.exporter.otlp.trace.OtlpGrpcSpanExporter; import io.opentelemetry.javaagent.testing.FakeJavaAgentTracerProvider; import io.opentelemetry.sdk.OpenTelemetrySdk; import io.opentelemetry.sdk.autoconfigure.spi.AutoConfigurationCustomizer; @@ -60,7 +59,6 @@ void setUp() { @AfterEach void tearDown() { GlobalOpenTelemetry.resetForTest(); - OtlpGrpcSpanExporter.reset(); DeterministicIdGenerator.clearSharedStateForTest(); OtelPluginAutoConfigurationState.resetInstalledForTest(); } @@ -145,14 +143,13 @@ void defaultConstructor_doesNotReplaceGlobalSdkTracerProviderIdGenerator() { void defaultConstructor_usesJavaAgentGlobalTracerProviderDirectly_withSeparateAutoConfiguredIdGenerator() { OtelPluginAutoConfigurationState.markInstalled(); GlobalOpenTelemetry.resetForTest(); - OtlpGrpcSpanExporter.reset(); var globalExporter = InMemorySpanExporter.create(); var javaAgentIdGenerator = new DeterministicIdGenerator(); var sdkTracerProvider = SdkTracerProvider.builder() .setIdGenerator(javaAgentIdGenerator) .addSpanProcessor(SimpleSpanProcessor.create(globalExporter)) .build(); - var javaAgentTracerProvider = new FakeJavaAgentTracerProvider(new Object(), sdkTracerProvider); + var javaAgentTracerProvider = new FakeJavaAgentTracerProvider(sdkTracerProvider); GlobalOpenTelemetry.set(new OpenTelemetry() { @Override public io.opentelemetry.api.trace.TracerProvider getTracerProvider() { @@ -185,7 +182,6 @@ public ContextPropagators getPropagators() { .findFirst() .orElseThrow(); assertEquals(expectedIds.generateSpanIdForOperation("op-1"), stepSpan.getSpanId()); - assertTrue(OtlpGrpcSpanExporter.getFinishedSpanItems().isEmpty()); } @Test From 1cad798479a2db3388b178b354586be8fe63f19f Mon Sep 17 00:00:00 2001 From: Silan He <16982279+SilanHe@users.noreply.github.com> Date: Thu, 16 Jul 2026 14:43:38 -0700 Subject: [PATCH 30/36] fix(otel): Use operation startTimestamp for continuation spans (#542) Set the span start time to the operation's original startTimestamp when creating continuation spans in onOperationEnd. Previously the span defaulted to the current wall-clock time, which misrepresented the actual duration of cross-invocation operations. Add a test to verify the continuation span uses the operation's startTimestamp. Co-authored-by: hsilan --- .../lambda/durable/otel/OtelPlugin.java | 4 ++++ .../lambda/durable/otel/OtelPluginTest.java | 24 +++++++++++++++++++ 2 files changed, 28 insertions(+) diff --git a/otel-plugin/src/main/java/software/amazon/lambda/durable/otel/OtelPlugin.java b/otel-plugin/src/main/java/software/amazon/lambda/durable/otel/OtelPlugin.java index 306c613ce..01d05410d 100644 --- a/otel-plugin/src/main/java/software/amazon/lambda/durable/otel/OtelPlugin.java +++ b/otel-plugin/src/main/java/software/amazon/lambda/durable/otel/OtelPlugin.java @@ -342,6 +342,10 @@ public void onOperationEnd(OperationEndInfo info) { .setAttribute(DURABLE_OPERATION_ID, info.id()) .setAttribute(DURABLE_OPERATION_TYPE, info.type()); + if (info.startTimestamp() != null) { + spanBuilder.setStartTimestamp(info.startTimestamp()); + } + if (info.name() != null) { spanBuilder.setAttribute(DURABLE_OPERATION_NAME, info.name()); } diff --git a/otel-plugin/src/test/java/software/amazon/lambda/durable/otel/OtelPluginTest.java b/otel-plugin/src/test/java/software/amazon/lambda/durable/otel/OtelPluginTest.java index 1646ab4ef..3c4e2c9e9 100644 --- a/otel-plugin/src/test/java/software/amazon/lambda/durable/otel/OtelPluginTest.java +++ b/otel-plugin/src/test/java/software/amazon/lambda/durable/otel/OtelPluginTest.java @@ -739,6 +739,30 @@ void operationEnd_withoutMatchingStart_createsContinuationSpanWithLink() { assertFalse(continuationSpan.getLinks().isEmpty(), "Continuation span should have a Link"); } + @Test + void operationEnd_withoutMatchingStart_usesOperationStartTimestamp() { + plugin.onInvocationStart(new InvocationInfo("req-1", "arn:exec1", true)); + + var operationStart = Instant.parse("2026-01-15T08:30:00Z"); + var operationEnd = Instant.parse("2026-01-15T09:00:00Z"); + + // onOperationEnd without a prior onOperationStart — continuation span + plugin.onOperationEnd(new OperationEndInfo( + "op-wait-1", "my-wait", "WAIT", "Wait", null, operationStart, operationEnd, "SUCCEEDED", false, null)); + + plugin.onInvocationEnd(new InvocationEndInfo("req-1", "arn:exec1", true, InvocationStatus.SUCCEEDED, null)); + + var continuationSpan = spanExporter.getFinishedSpanItems().stream() + .filter(s -> s.getName().contains("wait")) + .findFirst() + .orElseThrow(); + + assertEquals( + operationStart.toEpochMilli(), + continuationSpan.getStartEpochNanos() / 1_000_000, + "Continuation span should use the operation's startTimestamp"); + } + @Test void operationEnd_withoutMatchingStart_withError_setsErrorStatus() { plugin.onInvocationStart(new InvocationInfo("req-1", "arn:exec1", true)); From 75a0d61149104fdd5b01d6590c8f1c522f0f8b21 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 20 Jul 2026 14:13:30 -0700 Subject: [PATCH 31/36] [maven]: Bump com.fasterxml.jackson:jackson-bom from 2.21.4 to 2.22.1 (#551) Bumps [com.fasterxml.jackson:jackson-bom](https://github.com/FasterXML/jackson-bom) from 2.21.4 to 2.22.1. - [Commits](https://github.com/FasterXML/jackson-bom/compare/jackson-bom-2.21.4...jackson-bom-2.22.1) --- updated-dependencies: - dependency-name: com.fasterxml.jackson:jackson-bom dependency-version: 2.22.1 dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index 28e7273b7..9e9759bcd 100644 --- a/pom.xml +++ b/pom.xml @@ -52,7 +52,7 @@ 17 UTF-8 2.46.21 - 2.21.4 + 2.22.1 6.1.2 5.23.0 2.0.18 From b82abe92beea42d0d6670cc4355f0384bd69cf80 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 20 Jul 2026 21:15:18 +0000 Subject: [PATCH 32/36] [maven]: Bump software.amazon.awssdk:bom from 2.46.21 to 2.48.3 (#549) Bumps software.amazon.awssdk:bom from 2.46.21 to 2.48.3. --- updated-dependencies: - dependency-name: software.amazon.awssdk:bom dependency-version: 2.48.3 dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index 9e9759bcd..207042a7d 100644 --- a/pom.xml +++ b/pom.xml @@ -51,7 +51,7 @@ 17 17 UTF-8 - 2.46.21 + 2.48.3 2.22.1 6.1.2 5.23.0 From 45f004b5b0bdfdd2be9d77a30beccf3b73af3f6b Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 20 Jul 2026 14:15:56 -0700 Subject: [PATCH 33/36] [github-actions]: Bump slackapi/slack-github-action (#548) Bumps [slackapi/slack-github-action](https://github.com/slackapi/slack-github-action) from 3.0.5 to 4.0.0. - [Release notes](https://github.com/slackapi/slack-github-action/releases) - [Changelog](https://github.com/slackapi/slack-github-action/blob/main/CHANGELOG.md) - [Commits](https://github.com/slackapi/slack-github-action/compare/0d95c9a7becc1e6e297d76df9bc735c44f4cbcbc...dcb1066f776dd043e64d0e8ba94ca15cc7e1875d) --- updated-dependencies: - dependency-name: slackapi/slack-github-action dependency-version: 4.0.0 dependency-type: direct:production update-type: version-update:semver-major ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- .github/workflows/notify-discussions.yml | 2 +- .github/workflows/notify-issues.yml | 2 +- .github/workflows/notify-pr.yml | 2 +- .github/workflows/notify-release.yml | 2 +- 4 files changed, 4 insertions(+), 4 deletions(-) diff --git a/.github/workflows/notify-discussions.yml b/.github/workflows/notify-discussions.yml index 9d94d04a5..ef9fc58c8 100644 --- a/.github/workflows/notify-discussions.yml +++ b/.github/workflows/notify-discussions.yml @@ -11,7 +11,7 @@ jobs: runs-on: ubuntu-latest steps: - name: Send discussion notification to Slack - uses: slackapi/slack-github-action@0d95c9a7becc1e6e297d76df9bc735c44f4cbcbc # v3.0.5 + uses: slackapi/slack-github-action@dcb1066f776dd043e64d0e8ba94ca15cc7e1875d # v4.0.0 with: webhook: ${{ secrets.SLACK_WEBHOOK_URL_DISCUSSION }} webhook-type: incoming-webhook diff --git a/.github/workflows/notify-issues.yml b/.github/workflows/notify-issues.yml index 1c6ace8d9..8e8c9f40f 100644 --- a/.github/workflows/notify-issues.yml +++ b/.github/workflows/notify-issues.yml @@ -11,7 +11,7 @@ jobs: runs-on: ubuntu-latest steps: - name: Send issue notification to Slack - uses: slackapi/slack-github-action@0d95c9a7becc1e6e297d76df9bc735c44f4cbcbc # v3.0.5 + uses: slackapi/slack-github-action@dcb1066f776dd043e64d0e8ba94ca15cc7e1875d # v4.0.0 with: webhook: ${{ secrets.SLACK_WEBHOOK_URL_ISSUE }} webhook-type: incoming-webhook diff --git a/.github/workflows/notify-pr.yml b/.github/workflows/notify-pr.yml index e82b93b44..31fcfadf6 100644 --- a/.github/workflows/notify-pr.yml +++ b/.github/workflows/notify-pr.yml @@ -12,7 +12,7 @@ jobs: steps: - name: Send pull request notification to Slack if: github.event.pull_request.draft == false - uses: slackapi/slack-github-action@0d95c9a7becc1e6e297d76df9bc735c44f4cbcbc # v3.0.5 + uses: slackapi/slack-github-action@dcb1066f776dd043e64d0e8ba94ca15cc7e1875d # v4.0.0 with: webhook: ${{ secrets.SLACK_WEBHOOK_URL_PR }} webhook-type: incoming-webhook diff --git a/.github/workflows/notify-release.yml b/.github/workflows/notify-release.yml index 83b190e41..bbee7fd0a 100644 --- a/.github/workflows/notify-release.yml +++ b/.github/workflows/notify-release.yml @@ -17,7 +17,7 @@ jobs: runs-on: ubuntu-latest steps: - name: Send release notification to Slack - uses: slackapi/slack-github-action@0d95c9a7becc1e6e297d76df9bc735c44f4cbcbc # v3.0.5 + uses: slackapi/slack-github-action@dcb1066f776dd043e64d0e8ba94ca15cc7e1875d # v4.0.0 with: webhook: ${{ secrets.SLACK_WEBHOOK_URL_RELEASE }} webhook-type: incoming-webhook From 448512013db48269f33122610250b65d950ae641 Mon Sep 17 00:00:00 2001 From: ALEX WANG <47266819+wangyb-A@users.noreply.github.com> Date: Tue, 21 Jul 2026 10:45:24 -0700 Subject: [PATCH 34/36] feat(conformance): add Java conformance test package (step, wait suites) (#552) * feat(conformance): add Java conformance test package (step, wait suites) + CI Adds the aws-durable-execution-sdk-java-conformance-tests module: Java handlers + SAM templates for the step and wait conformance suites, exercised by the language-agnostic runner (aws-durable-execution-conformance-tests). - 20 step handlers + 5 wait handlers using the SDK's real API (built-in StepContext.getAttempt() for retry counting -- no DynamoDB side channel) - template_step.yaml / template_wait.yaml map handlers to requirement IDs via TestingMetadata.TestDescription - scripts/inject_execution_role.py (ported from the JS conformance package) lets CI reuse TEST_LAMBDA_EXECUTION_ROLE_ARN instead of creating IAM roles - .github/workflows/conformance-tests.yml runs one parallel matrix job per suite on PRs touching the package; add future suites via matrix.suite * ci: trigger conformance tests on sdk/** and root pom changes --------- Co-authored-by: Alex Wang --- .github/workflows/conformance-tests.yml | 113 ++++++ conformance-tests/.gitignore | 5 + conformance-tests/README.md | 112 ++++++ conformance-tests/pom.xml | 96 +++++ .../scripts/inject_execution_role.py | 71 ++++ .../src/main/java/step/StepAndWaitReplay.java | 18 + .../main/java/step/StepAtMostOnceNoRetry.java | 34 ++ .../java/step/StepAtMostOnceWithRetry.java | 46 +++ .../src/main/java/step/StepBasic.java | 15 + .../src/main/java/step/StepComplexObject.java | 24 ++ .../src/main/java/step/StepCustomSerdes.java | 35 ++ .../src/main/java/step/StepDefaultRetry.java | 22 ++ .../src/main/java/step/StepErrorCaught.java | 31 ++ .../src/main/java/step/StepLogging.java | 20 + .../src/main/java/step/StepNested.java | 17 + .../src/main/java/step/StepNullResult.java | 15 + .../java/step/StepReplayRethrowsFailed.java | 36 ++ .../java/step/StepReplaySkipsSucceeded.java | 21 + .../main/java/step/StepRetryCustomConfig.java | 33 ++ .../main/java/step/StepRetryExhaustion.java | 28 ++ .../main/java/step/StepRetryNonRetryable.java | 38 ++ .../java/step/StepRetrySpecificException.java | 42 ++ .../src/main/java/step/StepWithError.java | 25 ++ .../src/main/java/step/StepWithName.java | 15 + .../src/main/java/step/StepWithRetry.java | 34 ++ .../src/main/java/wait/WaitBasic.java | 17 + .../src/main/java/wait/WaitDurationUnits.java | 17 + .../src/main/java/wait/WaitLongDuration.java | 17 + .../java/wait/WaitMultipleSequential.java | 19 + .../src/main/java/wait/WaitWithName.java | 17 + conformance-tests/template_step.yaml | 374 ++++++++++++++++++ conformance-tests/template_wait.yaml | 129 ++++++ pom.xml | 1 + 33 files changed, 1537 insertions(+) create mode 100644 .github/workflows/conformance-tests.yml create mode 100644 conformance-tests/.gitignore create mode 100644 conformance-tests/README.md create mode 100644 conformance-tests/pom.xml create mode 100644 conformance-tests/scripts/inject_execution_role.py create mode 100644 conformance-tests/src/main/java/step/StepAndWaitReplay.java create mode 100644 conformance-tests/src/main/java/step/StepAtMostOnceNoRetry.java create mode 100644 conformance-tests/src/main/java/step/StepAtMostOnceWithRetry.java create mode 100644 conformance-tests/src/main/java/step/StepBasic.java create mode 100644 conformance-tests/src/main/java/step/StepComplexObject.java create mode 100644 conformance-tests/src/main/java/step/StepCustomSerdes.java create mode 100644 conformance-tests/src/main/java/step/StepDefaultRetry.java create mode 100644 conformance-tests/src/main/java/step/StepErrorCaught.java create mode 100644 conformance-tests/src/main/java/step/StepLogging.java create mode 100644 conformance-tests/src/main/java/step/StepNested.java create mode 100644 conformance-tests/src/main/java/step/StepNullResult.java create mode 100644 conformance-tests/src/main/java/step/StepReplayRethrowsFailed.java create mode 100644 conformance-tests/src/main/java/step/StepReplaySkipsSucceeded.java create mode 100644 conformance-tests/src/main/java/step/StepRetryCustomConfig.java create mode 100644 conformance-tests/src/main/java/step/StepRetryExhaustion.java create mode 100644 conformance-tests/src/main/java/step/StepRetryNonRetryable.java create mode 100644 conformance-tests/src/main/java/step/StepRetrySpecificException.java create mode 100644 conformance-tests/src/main/java/step/StepWithError.java create mode 100644 conformance-tests/src/main/java/step/StepWithName.java create mode 100644 conformance-tests/src/main/java/step/StepWithRetry.java create mode 100644 conformance-tests/src/main/java/wait/WaitBasic.java create mode 100644 conformance-tests/src/main/java/wait/WaitDurationUnits.java create mode 100644 conformance-tests/src/main/java/wait/WaitLongDuration.java create mode 100644 conformance-tests/src/main/java/wait/WaitMultipleSequential.java create mode 100644 conformance-tests/src/main/java/wait/WaitWithName.java create mode 100644 conformance-tests/template_step.yaml create mode 100644 conformance-tests/template_wait.yaml diff --git a/.github/workflows/conformance-tests.yml b/.github/workflows/conformance-tests.yml new file mode 100644 index 000000000..878afd930 --- /dev/null +++ b/.github/workflows/conformance-tests.yml @@ -0,0 +1,113 @@ +name: Conformance Tests + +# Full-integration conformance run: builds the SDK + Java handlers, installs the +# pinned language-agnostic runner from PyPI, then deploys + invokes + validates +# one SAM stack per suite. Each suite runs as its own parallel matrix job. To add +# a suite later, add its name to `strategy.matrix.suite` (and ship its +# template_.yaml + handlers under conformance-tests/). + +on: + pull_request: + branches: [main] + paths: + - "sdk/**" + - "pom.xml" + - "conformance-tests/**" + - ".github/workflows/conformance-tests.yml" + workflow_dispatch: + +concurrency: + group: ${{ github.head_ref || github.ref_name || github.run_id }}-conformance + cancel-in-progress: true + +permissions: + contents: read + id-token: write # Required for AWS OIDC credentials + +jobs: + conformance: + name: conformance (${{ matrix.suite }}) + runs-on: ubuntu-latest + env: + AWS_REGION: us-west-2 + strategy: + fail-fast: false + matrix: + suite: [step, wait] + defaults: + run: + working-directory: conformance-tests + steps: + - name: Checkout repository + uses: actions/checkout@v7 + + - name: Setup Java 21 + uses: actions/setup-java@v5 + with: + distribution: corretto + java-version: 21 + cache: maven + + - name: Setup AWS SAM CLI + uses: aws-actions/setup-sam@v3 + with: + use-installer: true + + - name: Build SDK and conformance handlers + working-directory: . + run: mvn -B -q -Dmaven.test.skip=true install --file pom.xml + + - name: Setup Python + uses: actions/setup-python@v5 + with: + python-version: "3.14" + + - name: Install conformance runner + run: pip install aws-durable-execution-conformance-tests==0.1.0 + + - name: Configure AWS credentials + uses: aws-actions/configure-aws-credentials@517a711dbcd0e402f90c77e7e2f81e849156e31d # v6.2.2 + with: + # SAM-capable deploy role (has CloudFormation/S3/IAM), same role the + # SAM-based e2e tests assume. + role-to-assume: "${{ secrets.TEST_ROLE_ARN }}" + role-session-name: java-sdk-conformance-test + aws-region: ${{ env.AWS_REGION }} + allowed-account-ids: ${{ secrets.TEST_ACCOUNT_ID }} + + - name: Inject Lambda execution role into template + env: + ROLE_ARN: ${{ secrets.TEST_LAMBDA_EXECUTION_ROLE_ARN }} + run: | + if [ -z "$ROLE_ARN" ]; then + echo "TEST_LAMBDA_EXECUTION_ROLE_ARN not set; template will create its own role." + exit 0 + fi + # Point every function at the pre-existing execution role and drop the + # self-created DurableFunctionRole. Mutates only the CI checkout; the + # checked-in template stays self-contained for local runs. + python3 scripts/inject_execution_role.py \ + --template template_${{ matrix.suite }}.yaml \ + --role-arn "$ROLE_ARN" + + - name: Run conformance suite + run: | + python -m aws_durable_execution_conformance_tests.app \ + --template template_${{ matrix.suite }}.yaml \ + --language java \ + --suite ${{ matrix.suite }} \ + --name conformance-java-${{ matrix.suite }}-${{ github.run_id }} \ + --region ${{ env.AWS_REGION }} \ + --history-dir history-${{ matrix.suite }} \ + --report console junit \ + --report-file report-${{ matrix.suite }} + + - name: Upload conformance report + if: always() + uses: actions/upload-artifact@v4 + with: + name: conformance-report-${{ matrix.suite }} + path: | + conformance-tests/report-${{ matrix.suite }}.xml + conformance-tests/history-${{ matrix.suite }}/ + if-no-files-found: warn diff --git a/conformance-tests/.gitignore b/conformance-tests/.gitignore new file mode 100644 index 000000000..e5d1d8205 --- /dev/null +++ b/conformance-tests/.gitignore @@ -0,0 +1,5 @@ +target/ +.aws-sam/ +history-*/ +report-*.xml +report-*.json diff --git a/conformance-tests/README.md b/conformance-tests/README.md new file mode 100644 index 000000000..0cddc5a89 --- /dev/null +++ b/conformance-tests/README.md @@ -0,0 +1,112 @@ +# Durable Execution Java SDK — Conformance Tests + +Cross-SDK **conformance test handlers** for the Durable Execution Java SDK. +These handlers are deployed as AWS Lambda functions and exercised by the +language-agnostic conformance runner +[`aws-durable-execution-conformance-tests`](https://pypi.org/project/aws-durable-execution-conformance-tests/), +which invokes each function, pulls its execution history, and asserts it matches +the shared requirement specification. + +The runner and the requirement specifications (the `test-requirements/` YAML files) +are **not** in this repo — they live in +[`aws/aws-durable-execution-conformance-tests`](https://github.com/aws/aws-durable-execution-conformance-tests). +This package owns only the **Java handlers** and the **SAM templates** that wire +them to requirement IDs. + +## Layout + +``` +src/main/java/ + step/ # one handler class per scenario (context.step) + StepBasic.java + StepWithRetry.java + ... + wait/ # wait / timer scenarios +template_step.yaml # SAM template: maps each handler -> requirement ID(s) +template_wait.yaml +scripts/ + inject_execution_role.py # CI helper: use a pre-existing execution role +``` + +Each SAM template is a self-contained deployment for one **suite** (operation +category). Suites are added incrementally; today this package ships `step` and +`wait`. + +## How a handler maps to a requirement + +The link is the `TestingMetadata.TestDescription` field on each function in the +SAM template — a list of requirement IDs the handler satisfies: + +```yaml +StepBasic: + Type: AWS::Serverless::Function + TestingMetadata: + TestDescription: ["1-1"] # <- requirement ID(s) in the conformance repo + Properties: + CodeUri: . + Handler: step.StepBasic # <- package.ClassName extending DurableHandler + Role: + Fn::GetAtt: [DurableFunctionRole, Arn] + DurableConfig: + RetentionPeriodInDays: 7 + ExecutionTimeout: 300 +``` + +The runner invokes the function once per requirement ID using that requirement's +`Input`, then diffs the resulting execution history against the requirement's +`ExpectedExecutionHistory`. + +## Authoring a new test case + +1. **Find (or add) the requirement.** Requirement IDs and their expected history + live in the conformance repo under `test-requirements//.yaml`. If + the behavior isn't specified yet, propose it there first. +2. **Write the handler.** Add `src/main/java//.java` + extending `DurableHandler`. Use the SDK's **real API** — never + hand-roll logic to force the expected result. A handler that fails because + the SDK is non-compliant is a valid, valuable outcome; that failure is the + signal. +3. **Register it in the template.** Add an `AWS::Serverless::Function` entry to + `template_.yaml` with `Handler: .` and the + matching `TestDescription: [""]`. +4. **Rebuild and run** (below). + +## Running locally + +Prerequisites: Java 21, Maven, Python ≥ 3.14, the AWS SAM CLI, and AWS +credentials for an account where the Durable Execution service is available. + +```bash +# 1. Build the SDK + these handlers (from the repo root) +mvn -B -Dmaven.test.skip=true install + +# 2. Install the conformance runner (pinned) +pip install aws-durable-execution-conformance-tests==0.1.0 + +# 3. Deploy + invoke + validate one suite +cd conformance-tests +python -m aws_durable_execution_conformance_tests.app \ + --template template_step.yaml \ + --language java \ + --suite step \ + --name conformance-java-step-local \ + --region us-west-2 \ + --history-dir history-step \ + --report junit --report-file report-step +``` + +The runner deploys the template via SAM, invokes each function once per +requirement ID, and reports `PASSED` / `FAILED` / `UNCOVERED` per requirement. A +non-zero exit means at least one requirement failed. + +## CI + +`.github/workflows/conformance-tests.yml` runs the same flow on pull requests +and on manual dispatch, one parallel job per suite (a build matrix). It assumes +AWS credentials via OIDC using the repository's existing integration secrets. + +Before deploying, CI runs `scripts/inject_execution_role.py` to point every +function at the pre-existing execution role (`TEST_LAMBDA_EXECUTION_ROLE_ARN`) +and drop the template's self-created `DurableFunctionRole` — so CI deploys +don't create IAM roles. This rewrites only CI's checkout; the checked-in +template stays self-contained for local runs. diff --git a/conformance-tests/pom.xml b/conformance-tests/pom.xml new file mode 100644 index 000000000..c13e4eeb4 --- /dev/null +++ b/conformance-tests/pom.xml @@ -0,0 +1,96 @@ + + + 4.0.0 + + + software.amazon.lambda.durable + aws-durable-execution-sdk-java-parent + 2.1.1-SNAPSHOT + + + aws-durable-execution-sdk-java-conformance-tests + jar + + AWS Lambda Durable Execution SDK Conformance Tests + Cross-SDK conformance test handlers for the AWS Lambda Durable Execution SDK + https://github.com/aws/aws-durable-execution-sdk-java + + + scm:git:https://github.com/aws/aws-durable-execution-sdk-java.git + scm:git:https://github.com/aws/aws-durable-execution-sdk-java.git + https://github.com/aws/aws-durable-execution-sdk-java + + + + + + software.amazon.lambda.durable + aws-durable-execution-sdk-java + ${project.version} + + + + + com.amazonaws + aws-lambda-java-core + + + + + org.apache.logging.log4j + log4j-slf4j2-impl + 2.26.1 + + + org.apache.logging.log4j + log4j-core + 2.26.1 + + + org.apache.logging.log4j + log4j-layout-template-json + 2.26.1 + + + + + + + + org.apache.maven.plugins + maven-shade-plugin + 3.6.2 + + false + + + + org.apache.logging.log4j + log4j-transform-maven-shade-plugin-extensions + 0.2.0 + + + + + package + + shade + + + + + + + + + + + + + + diff --git a/conformance-tests/scripts/inject_execution_role.py b/conformance-tests/scripts/inject_execution_role.py new file mode 100644 index 000000000..d4a54d692 --- /dev/null +++ b/conformance-tests/scripts/inject_execution_role.py @@ -0,0 +1,71 @@ +#!/usr/bin/env python3 +"""Inject a pre-existing Lambda execution role into a conformance SAM template. + +Rewrites the given template in place so that every AWS::Serverless::Function +uses the provided execution role ARN, and removes the self-created +DurableFunctionRole resource. Used by CI to avoid creating an IAM role per +deploy; the checked-in template stays self-contained for local runs. + +Usage: + python3 scripts/inject_execution_role.py --template template_step.yaml \ + --role-arn arn:aws:iam::123456789012:role/my-execution-role + +Requires PyYAML (already a dependency of the conformance runner). +""" + +from __future__ import annotations + +import argparse +import sys + +import yaml + +SELF_CREATED_ROLE = "DurableFunctionRole" +FUNCTION_TYPE = "AWS::Serverless::Function" + + +def inject(template_path: str, role_arn: str) -> int: + """Rewrite template_path in place; return the number of functions updated.""" + with open(template_path, encoding="utf-8") as f: + doc = yaml.safe_load(f) + + resources = doc.get("Resources") + if not isinstance(resources, dict): + raise SystemExit(f"{template_path}: no Resources section found") + + resources.pop(SELF_CREATED_ROLE, None) + + updated = 0 + for resource in resources.values(): + if resource.get("Type") == FUNCTION_TYPE: + resource.setdefault("Properties", {})["Role"] = role_arn + updated += 1 + + if updated == 0: + raise SystemExit(f"{template_path}: no {FUNCTION_TYPE} resources found") + + with open(template_path, "w", encoding="utf-8") as f: + yaml.safe_dump(doc, f, sort_keys=False) + + return updated + + +def main() -> None: + parser = argparse.ArgumentParser(description=__doc__.splitlines()[0]) + parser.add_argument( + "--template", + required=True, + help="Path to the SAM template to rewrite in place.", + ) + parser.add_argument( + "--role-arn", + required=True, + help="Execution role ARN to set on every serverless function.", + ) + args = parser.parse_args() + updated = inject(args.template, args.role_arn) + print(f"Injected execution role into {updated} functions in {args.template}") + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/conformance-tests/src/main/java/step/StepAndWaitReplay.java b/conformance-tests/src/main/java/step/StepAndWaitReplay.java new file mode 100644 index 000000000..f8970c39b --- /dev/null +++ b/conformance-tests/src/main/java/step/StepAndWaitReplay.java @@ -0,0 +1,18 @@ +// Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +// SPDX-License-Identifier: Apache-2.0 +package step; + +import java.time.Duration; +import software.amazon.lambda.durable.DurableContext; +import software.amazon.lambda.durable.DurableHandler; + +/** 1-8: Step and wait with replay */ +public class StepAndWaitReplay extends DurableHandler { + + @Override + public String handleRequest(Object input, DurableContext context) { + String result = context.step("compute", String.class, stepCtx -> "computed"); + context.wait(null, Duration.ofSeconds(2)); + return result; + } +} diff --git a/conformance-tests/src/main/java/step/StepAtMostOnceNoRetry.java b/conformance-tests/src/main/java/step/StepAtMostOnceNoRetry.java new file mode 100644 index 000000000..f20437ad8 --- /dev/null +++ b/conformance-tests/src/main/java/step/StepAtMostOnceNoRetry.java @@ -0,0 +1,34 @@ +// Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +// SPDX-License-Identifier: Apache-2.0 +package step; + +import software.amazon.lambda.durable.DurableContext; +import software.amazon.lambda.durable.DurableHandler; +import software.amazon.lambda.durable.config.StepConfig; +import software.amazon.lambda.durable.config.StepSemantics; +import software.amazon.lambda.durable.retry.RetryStrategies; + +/** 1-17: AtMostOnce interrupted (no retry) */ +public class StepAtMostOnceNoRetry extends DurableHandler { + + @Override + public String handleRequest(Object input, DurableContext context) { + return context.step( + "at_most_once_flaky_step", + String.class, + stepCtx -> { + System.out.println(input); + System.out.flush(); + try { + Thread.sleep(1000); + } catch (InterruptedException ignored) { + } + System.exit(1); + return "unreachable"; + }, + StepConfig.builder() + .semanticsPerRetry(StepSemantics.AT_MOST_ONCE_PER_RETRY) + .retryStrategy(RetryStrategies.Presets.NO_RETRY) + .build()); + } +} diff --git a/conformance-tests/src/main/java/step/StepAtMostOnceWithRetry.java b/conformance-tests/src/main/java/step/StepAtMostOnceWithRetry.java new file mode 100644 index 000000000..f507b2840 --- /dev/null +++ b/conformance-tests/src/main/java/step/StepAtMostOnceWithRetry.java @@ -0,0 +1,46 @@ +// Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +// SPDX-License-Identifier: Apache-2.0 +package step; + +import java.time.Duration; +import software.amazon.lambda.durable.DurableContext; +import software.amazon.lambda.durable.DurableHandler; +import software.amazon.lambda.durable.config.StepConfig; +import software.amazon.lambda.durable.config.StepSemantics; +import software.amazon.lambda.durable.retry.RetryDecision; + +/** 1-18: AtMostOnce interrupted (with retry, succeeds on second attempt) */ +public class StepAtMostOnceWithRetry extends DurableHandler { + + @Override + public String handleRequest(Object input, DurableContext context) { + return context.step( + "at-most-once-retry", + String.class, + stepCtx -> { + // Print input to stdout each time step executes + System.out.println(input); + System.out.flush(); + + // Use the SDK's built-in attempt number (1-based at runtime). + // With AT_MOST_ONCE_PER_RETRY the interrupted first attempt is + // checkpointed as a retry, so the re-executed step sees a higher + // attempt number and succeeds. + if (stepCtx.getAttempt() < 2) { + try { + Thread.sleep(1000); + } catch (InterruptedException ignored) { + } + System.exit(1); + } + return "succeeded on second attempt"; + }, + StepConfig.builder() + .semanticsPerRetry(StepSemantics.AT_MOST_ONCE_PER_RETRY) + .retryStrategy((error, attempt) -> { + if (attempt >= 3) return RetryDecision.fail(); + return RetryDecision.retry(Duration.ofSeconds(1)); + }) + .build()); + } +} diff --git a/conformance-tests/src/main/java/step/StepBasic.java b/conformance-tests/src/main/java/step/StepBasic.java new file mode 100644 index 000000000..abbdb782c --- /dev/null +++ b/conformance-tests/src/main/java/step/StepBasic.java @@ -0,0 +1,15 @@ +// Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +// SPDX-License-Identifier: Apache-2.0 +package step; + +import software.amazon.lambda.durable.DurableContext; +import software.amazon.lambda.durable.DurableHandler; + +/** 1-1: Step basic (succeeds on first attempt) */ +public class StepBasic extends DurableHandler { + + @Override + public String handleRequest(String input, DurableContext context) { + return context.step("greet", String.class, stepCtx -> "Hello, " + input + "!"); + } +} diff --git a/conformance-tests/src/main/java/step/StepComplexObject.java b/conformance-tests/src/main/java/step/StepComplexObject.java new file mode 100644 index 000000000..7547289ef --- /dev/null +++ b/conformance-tests/src/main/java/step/StepComplexObject.java @@ -0,0 +1,24 @@ +// Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +// SPDX-License-Identifier: Apache-2.0 +package step; + +import java.util.List; +import java.util.Map; +import software.amazon.lambda.durable.DurableContext; +import software.amazon.lambda.durable.DurableHandler; + +/** 1-4: Returning complex object */ +@SuppressWarnings("unchecked") +public class StepComplexObject extends DurableHandler, Map> { + + @Override + public Map handleRequest(Map input, DurableContext context) { + return context.step("build-response", Map.class, stepCtx -> { + String name = (String) input.get("name"); + List tags = (List) input.get("tags"); + return Map.of( + "user", Map.of("name", name, "tags", tags), + "count", tags.size()); + }); + } +} diff --git a/conformance-tests/src/main/java/step/StepCustomSerdes.java b/conformance-tests/src/main/java/step/StepCustomSerdes.java new file mode 100644 index 000000000..dcd6d7c2f --- /dev/null +++ b/conformance-tests/src/main/java/step/StepCustomSerdes.java @@ -0,0 +1,35 @@ +// Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +// SPDX-License-Identifier: Apache-2.0 +package step; + +import software.amazon.lambda.durable.DurableContext; +import software.amazon.lambda.durable.DurableHandler; +import software.amazon.lambda.durable.TypeToken; +import software.amazon.lambda.durable.config.StepConfig; +import software.amazon.lambda.durable.serde.SerDes; + +/** 1-6: Custom serdes (per-step) - uppercases result on serialization */ +public class StepCustomSerdes extends DurableHandler { + + private static final SerDes UPPERCASE_SERDES = new SerDes() { + @Override + public String serialize(Object value) { + return value != null ? ((String) value).toUpperCase() : null; + } + + @Override + @SuppressWarnings("unchecked") + public T deserialize(String data, TypeToken typeToken) { + return (T) data; + } + }; + + @Override + public String handleRequest(String input, DurableContext context) { + return context.step( + "uppercase", + String.class, + stepCtx -> input, + StepConfig.builder().serDes(UPPERCASE_SERDES).build()); + } +} diff --git a/conformance-tests/src/main/java/step/StepDefaultRetry.java b/conformance-tests/src/main/java/step/StepDefaultRetry.java new file mode 100644 index 000000000..09cf6382d --- /dev/null +++ b/conformance-tests/src/main/java/step/StepDefaultRetry.java @@ -0,0 +1,22 @@ +// Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +// SPDX-License-Identifier: Apache-2.0 +package step; + +import software.amazon.lambda.durable.DurableContext; +import software.amazon.lambda.durable.DurableHandler; + +/** 1-13: Default retry strategy (no explicit config) */ +public class StepDefaultRetry extends DurableHandler { + + @Override + public String handleRequest(Object input, DurableContext context) { + return context.step("default-retry", String.class, stepCtx -> { + // Use the SDK's built-in attempt number (1-based at runtime): + // fail on the first two attempts, succeed on the third. + if (stepCtx.getAttempt() < 3) { + throw new RuntimeException("Attempt " + stepCtx.getAttempt() + " failed"); + } + return "recovered"; + }); + } +} diff --git a/conformance-tests/src/main/java/step/StepErrorCaught.java b/conformance-tests/src/main/java/step/StepErrorCaught.java new file mode 100644 index 000000000..c63686d7f --- /dev/null +++ b/conformance-tests/src/main/java/step/StepErrorCaught.java @@ -0,0 +1,31 @@ +// Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +// SPDX-License-Identifier: Apache-2.0 +package step; + +import software.amazon.lambda.durable.DurableContext; +import software.amazon.lambda.durable.DurableHandler; +import software.amazon.lambda.durable.config.StepConfig; +import software.amazon.lambda.durable.retry.RetryStrategies; + +/** 1-20: Error caught and handled (try/catch) */ +public class StepErrorCaught extends DurableHandler { + + @Override + public String handleRequest(Object input, DurableContext context) { + try { + context.step( + "failing-step", + String.class, + stepCtx -> { + throw new RuntimeException("Something went wrong"); + }, + StepConfig.builder() + .retryStrategy(RetryStrategies.Presets.NO_RETRY) + .build()); + } catch (RuntimeException e) { + // Error caught, continue with fallback + } + + return context.step("fallback-step", String.class, stepCtx -> "fallback_result"); + } +} diff --git a/conformance-tests/src/main/java/step/StepLogging.java b/conformance-tests/src/main/java/step/StepLogging.java new file mode 100644 index 000000000..4fef7c946 --- /dev/null +++ b/conformance-tests/src/main/java/step/StepLogging.java @@ -0,0 +1,20 @@ +// Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +// SPDX-License-Identifier: Apache-2.0 +package step; + +import software.amazon.lambda.durable.DurableContext; +import software.amazon.lambda.durable.DurableHandler; + +/** 1-7: Step with context logger */ +public class StepLogging extends DurableHandler { + + @Override + public String handleRequest(String input, DurableContext context) { + return context.step("greet", String.class, stepCtx -> { + stepCtx.getLogger().info("Greeting step started for: {}", input); + String greeting = "Hello, " + input + "!"; + stepCtx.getLogger().info("Greeting step completed with: {}", greeting); + return greeting; + }); + } +} diff --git a/conformance-tests/src/main/java/step/StepNested.java b/conformance-tests/src/main/java/step/StepNested.java new file mode 100644 index 000000000..122974354 --- /dev/null +++ b/conformance-tests/src/main/java/step/StepNested.java @@ -0,0 +1,17 @@ +// Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +// SPDX-License-Identifier: Apache-2.0 +package step; + +import software.amazon.lambda.durable.DurableContext; +import software.amazon.lambda.durable.DurableHandler; + +/** 1-3: Sequential steps where second depends on first */ +public class StepNested extends DurableHandler { + + @Override + public String handleRequest(Object input, DurableContext context) { + String result1 = context.step("step-one", String.class, stepCtx -> "first"); + String result2 = context.step("step-two", String.class, stepCtx -> result1 + "_second"); + return result2; + } +} diff --git a/conformance-tests/src/main/java/step/StepNullResult.java b/conformance-tests/src/main/java/step/StepNullResult.java new file mode 100644 index 000000000..126bfc309 --- /dev/null +++ b/conformance-tests/src/main/java/step/StepNullResult.java @@ -0,0 +1,15 @@ +// Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +// SPDX-License-Identifier: Apache-2.0 +package step; + +import software.amazon.lambda.durable.DurableContext; +import software.amazon.lambda.durable.DurableHandler; + +/** 1-5: Undefined/null result */ +public class StepNullResult extends DurableHandler { + + @Override + public Object handleRequest(Object input, DurableContext context) { + return context.step("do-nothing", Object.class, stepCtx -> null); + } +} diff --git a/conformance-tests/src/main/java/step/StepReplayRethrowsFailed.java b/conformance-tests/src/main/java/step/StepReplayRethrowsFailed.java new file mode 100644 index 000000000..08b76a5b3 --- /dev/null +++ b/conformance-tests/src/main/java/step/StepReplayRethrowsFailed.java @@ -0,0 +1,36 @@ +// Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +// SPDX-License-Identifier: Apache-2.0 +package step; + +import java.time.Duration; +import software.amazon.lambda.durable.DurableContext; +import software.amazon.lambda.durable.DurableHandler; +import software.amazon.lambda.durable.config.StepConfig; +import software.amazon.lambda.durable.retry.RetryStrategies; + +/** 1-10: Replay re-throws failed step */ +public class StepReplayRethrowsFailed extends DurableHandler { + + @Override + public String handleRequest(Object input, DurableContext context) { + String errorMessage = ""; + + try { + context.step( + "failing-step", + String.class, + stepCtx -> { + stepCtx.getLogger().info("step executed"); + throw new RuntimeException("Something went wrong"); + }, + StepConfig.builder() + .retryStrategy(RetryStrategies.Presets.NO_RETRY) + .build()); + } catch (RuntimeException e) { + errorMessage = e.getCause() != null ? e.getCause().getMessage() : e.getMessage(); + } + + context.wait(null, Duration.ofSeconds(1)); + return "caught: " + errorMessage; + } +} diff --git a/conformance-tests/src/main/java/step/StepReplaySkipsSucceeded.java b/conformance-tests/src/main/java/step/StepReplaySkipsSucceeded.java new file mode 100644 index 000000000..a590a6083 --- /dev/null +++ b/conformance-tests/src/main/java/step/StepReplaySkipsSucceeded.java @@ -0,0 +1,21 @@ +// Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +// SPDX-License-Identifier: Apache-2.0 +package step; + +import java.time.Duration; +import software.amazon.lambda.durable.DurableContext; +import software.amazon.lambda.durable.DurableHandler; + +/** 1-9: Replay skips succeeded step */ +public class StepReplaySkipsSucceeded extends DurableHandler { + + @Override + public String handleRequest(Object input, DurableContext context) { + String result = context.step("cached-step", String.class, stepCtx -> { + stepCtx.getLogger().info("step executed"); + return "cached_value"; + }); + context.wait(null, Duration.ofSeconds(1)); + return result; + } +} diff --git a/conformance-tests/src/main/java/step/StepRetryCustomConfig.java b/conformance-tests/src/main/java/step/StepRetryCustomConfig.java new file mode 100644 index 000000000..66195b72f --- /dev/null +++ b/conformance-tests/src/main/java/step/StepRetryCustomConfig.java @@ -0,0 +1,33 @@ +// Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +// SPDX-License-Identifier: Apache-2.0 +package step; + +import java.time.Duration; +import software.amazon.lambda.durable.DurableContext; +import software.amazon.lambda.durable.DurableHandler; +import software.amazon.lambda.durable.config.StepConfig; +import software.amazon.lambda.durable.retry.JitterStrategy; +import software.amazon.lambda.durable.retry.RetryStrategies; + +/** 1-14: Retry with custom config (2s initial, 3x backoff, no jitter) */ +public class StepRetryCustomConfig extends DurableHandler { + + @Override + public String handleRequest(Object input, DurableContext context) { + return context.step( + "custom-retry", + String.class, + stepCtx -> { + // Use the SDK's built-in attempt number (1-based at runtime): + // fail on the first two attempts, succeed on the third. + if (stepCtx.getAttempt() < 3) { + throw new RuntimeException("Attempt " + stepCtx.getAttempt() + " failed"); + } + return "finally succeeded"; + }, + StepConfig.builder() + .retryStrategy(RetryStrategies.exponentialBackoff( + 5, Duration.ofSeconds(2), Duration.ofSeconds(60), 3.0, JitterStrategy.NONE)) + .build()); + } +} diff --git a/conformance-tests/src/main/java/step/StepRetryExhaustion.java b/conformance-tests/src/main/java/step/StepRetryExhaustion.java new file mode 100644 index 000000000..919ad4d80 --- /dev/null +++ b/conformance-tests/src/main/java/step/StepRetryExhaustion.java @@ -0,0 +1,28 @@ +// Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +// SPDX-License-Identifier: Apache-2.0 +package step; + +import java.time.Duration; +import software.amazon.lambda.durable.DurableContext; +import software.amazon.lambda.durable.DurableHandler; +import software.amazon.lambda.durable.config.StepConfig; +import software.amazon.lambda.durable.retry.JitterStrategy; +import software.amazon.lambda.durable.retry.RetryStrategies; + +/** 1-12: Retry exhaustion (max attempts) */ +public class StepRetryExhaustion extends DurableHandler { + + @Override + public String handleRequest(Object input, DurableContext context) { + return context.step( + "always-fails", + String.class, + stepCtx -> { + throw new RuntimeException("Always fails"); + }, + StepConfig.builder() + .retryStrategy(RetryStrategies.exponentialBackoff( + 4, Duration.ofSeconds(1), Duration.ofSeconds(10), 1.0, JitterStrategy.NONE)) + .build()); + } +} diff --git a/conformance-tests/src/main/java/step/StepRetryNonRetryable.java b/conformance-tests/src/main/java/step/StepRetryNonRetryable.java new file mode 100644 index 000000000..9546519c9 --- /dev/null +++ b/conformance-tests/src/main/java/step/StepRetryNonRetryable.java @@ -0,0 +1,38 @@ +// Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +// SPDX-License-Identifier: Apache-2.0 +package step; + +import java.time.Duration; +import software.amazon.lambda.durable.DurableContext; +import software.amazon.lambda.durable.DurableHandler; +import software.amazon.lambda.durable.config.StepConfig; +import software.amazon.lambda.durable.retry.RetryDecision; + +/** 1-16: Retry specific exception (non-retryable fails) */ +public class StepRetryNonRetryable extends DurableHandler { + + public static class TransientError extends RuntimeException { + public TransientError(String message) { + super(message); + } + } + + @Override + public String handleRequest(Object input, DurableContext context) { + return context.step( + "non-retryable", + String.class, + stepCtx -> { + throw new TransientError("Temporary failure"); + }, + StepConfig.builder() + .retryStrategy((error, attempt) -> { + // Only retry ValidationError, not TransientError + if (error.getClass().getSimpleName().equals("ValidationError") && attempt < 3) { + return RetryDecision.retry(Duration.ofSeconds(1)); + } + return RetryDecision.fail(); + }) + .build()); + } +} diff --git a/conformance-tests/src/main/java/step/StepRetrySpecificException.java b/conformance-tests/src/main/java/step/StepRetrySpecificException.java new file mode 100644 index 000000000..b5420b35b --- /dev/null +++ b/conformance-tests/src/main/java/step/StepRetrySpecificException.java @@ -0,0 +1,42 @@ +// Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +// SPDX-License-Identifier: Apache-2.0 +package step; + +import java.time.Duration; +import software.amazon.lambda.durable.DurableContext; +import software.amazon.lambda.durable.DurableHandler; +import software.amazon.lambda.durable.config.StepConfig; +import software.amazon.lambda.durable.retry.RetryDecision; + +/** 1-15: Retry specific exception (retries only TransientError) */ +public class StepRetrySpecificException extends DurableHandler { + + public static class TransientError extends RuntimeException { + public TransientError(String message) { + super(message); + } + } + + @Override + public String handleRequest(Object input, DurableContext context) { + return context.step( + "specific-retry", + String.class, + stepCtx -> { + // Use the SDK's built-in attempt number (1-based at runtime): + // fail on the first attempt, succeed on the second. + if (stepCtx.getAttempt() < 2) { + throw new TransientError("Temporary failure"); + } + return "recovered from transient"; + }, + StepConfig.builder() + .retryStrategy((error, attempt) -> { + if (error instanceof TransientError && attempt < 3) { + return RetryDecision.retry(Duration.ofSeconds(1)); + } + return RetryDecision.fail(); + }) + .build()); + } +} diff --git a/conformance-tests/src/main/java/step/StepWithError.java b/conformance-tests/src/main/java/step/StepWithError.java new file mode 100644 index 000000000..3c209dc1b --- /dev/null +++ b/conformance-tests/src/main/java/step/StepWithError.java @@ -0,0 +1,25 @@ +// Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +// SPDX-License-Identifier: Apache-2.0 +package step; + +import software.amazon.lambda.durable.DurableContext; +import software.amazon.lambda.durable.DurableHandler; +import software.amazon.lambda.durable.config.StepConfig; +import software.amazon.lambda.durable.retry.RetryStrategies; + +/** 1-19: Step with error (fails permanently) */ +public class StepWithError extends DurableHandler { + + @Override + public String handleRequest(Object input, DurableContext context) { + return context.step( + "failing-step", + String.class, + stepCtx -> { + throw new RuntimeException("Something went wrong"); + }, + StepConfig.builder() + .retryStrategy(RetryStrategies.Presets.NO_RETRY) + .build()); + } +} diff --git a/conformance-tests/src/main/java/step/StepWithName.java b/conformance-tests/src/main/java/step/StepWithName.java new file mode 100644 index 000000000..a540d5ffe --- /dev/null +++ b/conformance-tests/src/main/java/step/StepWithName.java @@ -0,0 +1,15 @@ +// Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +// SPDX-License-Identifier: Apache-2.0 +package step; + +import software.amazon.lambda.durable.DurableContext; +import software.amazon.lambda.durable.DurableHandler; + +/** 1-2: Step with name */ +public class StepWithName extends DurableHandler { + + @Override + public String handleRequest(String input, DurableContext context) { + return context.step("custom_step_name", String.class, stepCtx -> "Hello, " + input + "!"); + } +} diff --git a/conformance-tests/src/main/java/step/StepWithRetry.java b/conformance-tests/src/main/java/step/StepWithRetry.java new file mode 100644 index 000000000..0a5f35dc9 --- /dev/null +++ b/conformance-tests/src/main/java/step/StepWithRetry.java @@ -0,0 +1,34 @@ +// Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +// SPDX-License-Identifier: Apache-2.0 +package step; + +import java.time.Duration; +import software.amazon.lambda.durable.DurableContext; +import software.amazon.lambda.durable.DurableHandler; +import software.amazon.lambda.durable.config.StepConfig; +import software.amazon.lambda.durable.retry.RetryDecision; + +/** 1-11: Step with retry (fails then succeeds) */ +public class StepWithRetry extends DurableHandler { + + @Override + public String handleRequest(Object input, DurableContext context) { + return context.step( + "retry-step", + String.class, + stepCtx -> { + // Use the SDK's built-in attempt number (1-based at runtime): + // fail on the first attempt, succeed on the second. + if (stepCtx.getAttempt() < 2) { + throw new RuntimeException("Attempt " + stepCtx.getAttempt() + " failed"); + } + return "Operation succeeded"; + }, + StepConfig.builder() + .retryStrategy((error, attempt) -> { + if (attempt >= 3) return RetryDecision.fail(); + return RetryDecision.retry(Duration.ofSeconds(1)); + }) + .build()); + } +} diff --git a/conformance-tests/src/main/java/wait/WaitBasic.java b/conformance-tests/src/main/java/wait/WaitBasic.java new file mode 100644 index 000000000..bf9b3414c --- /dev/null +++ b/conformance-tests/src/main/java/wait/WaitBasic.java @@ -0,0 +1,17 @@ +// Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +// SPDX-License-Identifier: Apache-2.0 +package wait; + +import java.time.Duration; +import software.amazon.lambda.durable.DurableContext; +import software.amazon.lambda.durable.DurableHandler; + +/** 2-1: Wait basic */ +public class WaitBasic extends DurableHandler { + + @Override + public String handleRequest(Object input, DurableContext context) { + context.wait(null, Duration.ofSeconds(2)); + return "Wait completed"; + } +} diff --git a/conformance-tests/src/main/java/wait/WaitDurationUnits.java b/conformance-tests/src/main/java/wait/WaitDurationUnits.java new file mode 100644 index 000000000..b0086349e --- /dev/null +++ b/conformance-tests/src/main/java/wait/WaitDurationUnits.java @@ -0,0 +1,17 @@ +// Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +// SPDX-License-Identifier: Apache-2.0 +package wait; + +import java.time.Duration; +import software.amazon.lambda.durable.DurableContext; +import software.amazon.lambda.durable.DurableHandler; + +/** 2-4: Wait with different duration units (minutes) */ +public class WaitDurationUnits extends DurableHandler { + + @Override + public String handleRequest(Object input, DurableContext context) { + context.wait(null, Duration.ofMinutes(1)); + return "Wait with minutes completed"; + } +} diff --git a/conformance-tests/src/main/java/wait/WaitLongDuration.java b/conformance-tests/src/main/java/wait/WaitLongDuration.java new file mode 100644 index 000000000..01fc1d6f6 --- /dev/null +++ b/conformance-tests/src/main/java/wait/WaitLongDuration.java @@ -0,0 +1,17 @@ +// Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +// SPDX-License-Identifier: Apache-2.0 +package wait; + +import java.time.Duration; +import software.amazon.lambda.durable.DurableContext; +import software.amazon.lambda.durable.DurableHandler; + +/** 2-5: Wait with long duration (1 hour) */ +public class WaitLongDuration extends DurableHandler { + + @Override + public String handleRequest(Object input, DurableContext context) { + context.wait(null, Duration.ofHours(1)); + return "Wait with hours completed"; + } +} diff --git a/conformance-tests/src/main/java/wait/WaitMultipleSequential.java b/conformance-tests/src/main/java/wait/WaitMultipleSequential.java new file mode 100644 index 000000000..ba461a8af --- /dev/null +++ b/conformance-tests/src/main/java/wait/WaitMultipleSequential.java @@ -0,0 +1,19 @@ +// Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +// SPDX-License-Identifier: Apache-2.0 +package wait; + +import java.time.Duration; +import java.util.Map; +import software.amazon.lambda.durable.DurableContext; +import software.amazon.lambda.durable.DurableHandler; + +/** 2-3: Multiple sequential waits */ +public class WaitMultipleSequential extends DurableHandler> { + + @Override + public Map handleRequest(Object input, DurableContext context) { + context.wait("wait-1", Duration.ofSeconds(2)); + context.wait("wait-2", Duration.ofSeconds(2)); + return Map.of("completedWaits", 2); + } +} diff --git a/conformance-tests/src/main/java/wait/WaitWithName.java b/conformance-tests/src/main/java/wait/WaitWithName.java new file mode 100644 index 000000000..462b1c33b --- /dev/null +++ b/conformance-tests/src/main/java/wait/WaitWithName.java @@ -0,0 +1,17 @@ +// Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +// SPDX-License-Identifier: Apache-2.0 +package wait; + +import java.time.Duration; +import software.amazon.lambda.durable.DurableContext; +import software.amazon.lambda.durable.DurableHandler; + +/** 2-2: Wait with name */ +public class WaitWithName extends DurableHandler { + + @Override + public String handleRequest(Object input, DurableContext context) { + context.wait("custom_wait_name", Duration.ofSeconds(2)); + return "Wait with name completed"; + } +} diff --git a/conformance-tests/template_step.yaml b/conformance-tests/template_step.yaml new file mode 100644 index 000000000..c3870ce0a --- /dev/null +++ b/conformance-tests/template_step.yaml @@ -0,0 +1,374 @@ +AWSTemplateFormatVersion: '2010-09-09' +Transform: AWS::Serverless-2016-10-31 +Description: Durable Execution Conformance Test Examples - Java (Step) + +Parameters: + Architecture: + Type: String + Default: arm64 + Description: Lambda Function Architecture + AllowedValues: + - x86_64 + - arm64 + JavaVersion: + Type: String + Default: 'java21' + Description: Java runtime version + +Globals: + Function: + Timeout: 60 + MemorySize: 512 + Runtime: + Ref: JavaVersion + Architectures: + - Ref: Architecture + +Resources: + DurableFunctionRole: + Type: AWS::IAM::Role + Properties: + AssumeRolePolicyDocument: + Version: '2012-10-17' + Statement: + - Effect: Allow + Principal: + Service: lambda.amazonaws.com + Action: sts:AssumeRole + ManagedPolicyArns: + - arn:aws:iam::aws:policy/service-role/AWSLambdaBasicExecutionRole + Policies: + - PolicyName: DurableExecutionPolicy + PolicyDocument: + Version: '2012-10-17' + Statement: + - Effect: Allow + Action: + - lambda:CheckpointDurableExecution + - lambda:GetDurableExecutionState + Resource: '*' + + StepBasic: + Type: AWS::Serverless::Function + TestingMetadata: + TestDescription: ["1-1"] + Properties: + CodeUri: . + Handler: step.StepBasic + Description: Step basic (succeeds on first attempt) + Role: + Fn::GetAtt: + - DurableFunctionRole + - Arn + DurableConfig: + RetentionPeriodInDays: 7 + ExecutionTimeout: 300 + + StepWithName: + Type: AWS::Serverless::Function + TestingMetadata: + TestDescription: ["1-2"] + Properties: + CodeUri: . + Handler: step.StepWithName + Description: Step with explicit name parameter + Role: + Fn::GetAtt: + - DurableFunctionRole + - Arn + DurableConfig: + RetentionPeriodInDays: 7 + ExecutionTimeout: 300 + + StepNested: + Type: AWS::Serverless::Function + TestingMetadata: + TestDescription: ["1-3"] + Properties: + CodeUri: . + Handler: step.StepNested + Description: Sequential steps where second depends on first + Role: + Fn::GetAtt: + - DurableFunctionRole + - Arn + DurableConfig: + RetentionPeriodInDays: 7 + ExecutionTimeout: 300 + + StepComplexObject: + Type: AWS::Serverless::Function + TestingMetadata: + TestDescription: ["1-4"] + Properties: + CodeUri: . + Handler: step.StepComplexObject + Description: Returning complex object with nested structure + Role: + Fn::GetAtt: + - DurableFunctionRole + - Arn + DurableConfig: + RetentionPeriodInDays: 7 + ExecutionTimeout: 300 + + StepNullResult: + Type: AWS::Serverless::Function + TestingMetadata: + TestDescription: ["1-5"] + Properties: + CodeUri: . + Handler: step.StepNullResult + Description: Undefined/null result + Role: + Fn::GetAtt: + - DurableFunctionRole + - Arn + DurableConfig: + RetentionPeriodInDays: 7 + ExecutionTimeout: 300 + + StepCustomSerdes: + Type: AWS::Serverless::Function + TestingMetadata: + TestDescription: ["1-6"] + Properties: + CodeUri: . + Handler: step.StepCustomSerdes + Description: Custom serdes (per-step) transforms to uppercase + Role: + Fn::GetAtt: + - DurableFunctionRole + - Arn + DurableConfig: + RetentionPeriodInDays: 7 + ExecutionTimeout: 300 + + StepLogging: + Type: AWS::Serverless::Function + TestingMetadata: + TestDescription: ["1-7"] + Properties: + CodeUri: . + Handler: step.StepLogging + Description: Step with context logger + Role: + Fn::GetAtt: + - DurableFunctionRole + - Arn + Environment: + Variables: + # Log4j2's default configuration logs at ERROR, which suppresses the + # logger.info calls that requirement 1-7 asserts on. Raise it to INFO. + JAVA_TOOL_OPTIONS: -Dorg.apache.logging.log4j.level=INFO + DurableConfig: + RetentionPeriodInDays: 7 + ExecutionTimeout: 300 + + StepAndWaitReplay: + Type: AWS::Serverless::Function + TestingMetadata: + TestDescription: ["1-8"] + Properties: + CodeUri: . + Handler: step.StepAndWaitReplay + Description: Step and wait with replay + Role: + Fn::GetAtt: + - DurableFunctionRole + - Arn + DurableConfig: + RetentionPeriodInDays: 7 + ExecutionTimeout: 300 + + StepReplaySkipsSucceeded: + Type: AWS::Serverless::Function + TestingMetadata: + TestDescription: ["1-9"] + Properties: + CodeUri: . + Handler: step.StepReplaySkipsSucceeded + Description: Replay skips succeeded step + Role: + Fn::GetAtt: + - DurableFunctionRole + - Arn + DurableConfig: + RetentionPeriodInDays: 7 + ExecutionTimeout: 300 + + StepReplayRethrowsFailed: + Type: AWS::Serverless::Function + TestingMetadata: + TestDescription: ["1-10"] + Properties: + CodeUri: . + Handler: step.StepReplayRethrowsFailed + Description: Replay re-throws failed step + Role: + Fn::GetAtt: + - DurableFunctionRole + - Arn + DurableConfig: + RetentionPeriodInDays: 7 + ExecutionTimeout: 300 + + StepWithRetry: + Type: AWS::Serverless::Function + TestingMetadata: + TestDescription: ["1-11"] + Properties: + CodeUri: . + Handler: step.StepWithRetry + Description: Step with retry (fails then succeeds) + Role: + Fn::GetAtt: + - DurableFunctionRole + - Arn + DurableConfig: + RetentionPeriodInDays: 7 + ExecutionTimeout: 300 + + StepRetryExhaustion: + Type: AWS::Serverless::Function + TestingMetadata: + TestDescription: ["1-12"] + Properties: + CodeUri: . + Handler: step.StepRetryExhaustion + Description: Retry exhaustion (max attempts) + Role: + Fn::GetAtt: + - DurableFunctionRole + - Arn + DurableConfig: + RetentionPeriodInDays: 7 + ExecutionTimeout: 300 + + StepDefaultRetry: + Type: AWS::Serverless::Function + TestingMetadata: + TestDescription: ["1-13"] + Properties: + CodeUri: . + Handler: step.StepDefaultRetry + Description: Default retry strategy + Role: + Fn::GetAtt: + - DurableFunctionRole + - Arn + DurableConfig: + RetentionPeriodInDays: 7 + ExecutionTimeout: 300 + + StepRetryCustomConfig: + Type: AWS::Serverless::Function + TestingMetadata: + TestDescription: ["1-14"] + Properties: + CodeUri: . + Handler: step.StepRetryCustomConfig + Description: Retry with custom config (fixed interval and backoff) + Role: + Fn::GetAtt: + - DurableFunctionRole + - Arn + DurableConfig: + RetentionPeriodInDays: 7 + ExecutionTimeout: 300 + + StepRetrySpecificException: + Type: AWS::Serverless::Function + TestingMetadata: + TestDescription: ["1-15"] + Properties: + CodeUri: . + Handler: step.StepRetrySpecificException + Description: Retry specific exception + Role: + Fn::GetAtt: + - DurableFunctionRole + - Arn + DurableConfig: + RetentionPeriodInDays: 7 + ExecutionTimeout: 300 + + StepRetryNonRetryable: + Type: AWS::Serverless::Function + TestingMetadata: + TestDescription: ["1-16"] + Properties: + CodeUri: . + Handler: step.StepRetryNonRetryable + Description: Retry specific exception (non-retryable fails) + Role: + Fn::GetAtt: + - DurableFunctionRole + - Arn + DurableConfig: + RetentionPeriodInDays: 7 + ExecutionTimeout: 300 + + StepAtMostOnceNoRetry: + Type: AWS::Serverless::Function + TestingMetadata: + TestDescription: ["1-17"] + Properties: + CodeUri: . + Handler: step.StepAtMostOnceNoRetry + Description: Step with AtMostOncePerRetry semantics (interrupted, no retry) + Role: + Fn::GetAtt: + - DurableFunctionRole + - Arn + DurableConfig: + RetentionPeriodInDays: 7 + ExecutionTimeout: 300 + + StepAtMostOnceWithRetry: + Type: AWS::Serverless::Function + TestingMetadata: + TestDescription: ["1-18"] + Properties: + CodeUri: . + Handler: step.StepAtMostOnceWithRetry + Description: AtMostOnce interrupted (with retry, succeeds on second attempt) + Role: + Fn::GetAtt: + - DurableFunctionRole + - Arn + DurableConfig: + RetentionPeriodInDays: 7 + ExecutionTimeout: 300 + + StepWithError: + Type: AWS::Serverless::Function + TestingMetadata: + TestDescription: ["1-19"] + Properties: + CodeUri: . + Handler: step.StepWithError + Description: Step with error (fails permanently) + Role: + Fn::GetAtt: + - DurableFunctionRole + - Arn + DurableConfig: + RetentionPeriodInDays: 7 + ExecutionTimeout: 300 + + StepErrorCaught: + Type: AWS::Serverless::Function + TestingMetadata: + TestDescription: ["1-20"] + Properties: + CodeUri: . + Handler: step.StepErrorCaught + Description: Error caught and handled (try/catch) + Role: + Fn::GetAtt: + - DurableFunctionRole + - Arn + DurableConfig: + RetentionPeriodInDays: 7 + ExecutionTimeout: 300 diff --git a/conformance-tests/template_wait.yaml b/conformance-tests/template_wait.yaml new file mode 100644 index 000000000..a775d4bc8 --- /dev/null +++ b/conformance-tests/template_wait.yaml @@ -0,0 +1,129 @@ +AWSTemplateFormatVersion: '2010-09-09' +Transform: AWS::Serverless-2016-10-31 +Description: Durable Execution Conformance Test Examples - Java (Wait) + +Parameters: + Architecture: + Type: String + Default: arm64 + Description: Lambda Function Architecture + AllowedValues: + - x86_64 + - arm64 + JavaVersion: + Type: String + Default: 'java21' + Description: Java runtime version + +Globals: + Function: + Timeout: 60 + MemorySize: 512 + Runtime: + Ref: JavaVersion + Architectures: + - Ref: Architecture + +Resources: + DurableFunctionRole: + Type: AWS::IAM::Role + Properties: + AssumeRolePolicyDocument: + Version: '2012-10-17' + Statement: + - Effect: Allow + Principal: + Service: lambda.amazonaws.com + Action: sts:AssumeRole + ManagedPolicyArns: + - arn:aws:iam::aws:policy/service-role/AWSLambdaBasicExecutionRole + Policies: + - PolicyName: DurableExecutionPolicy + PolicyDocument: + Version: '2012-10-17' + Statement: + - Effect: Allow + Action: + - lambda:CheckpointDurableExecution + - lambda:GetDurableExecutionState + Resource: '*' + + WaitBasic: + Type: AWS::Serverless::Function + TestingMetadata: + TestDescription: ["2-1"] + Properties: + CodeUri: . + Handler: wait.WaitBasic + Description: Wait basic + Role: + Fn::GetAtt: + - DurableFunctionRole + - Arn + DurableConfig: + RetentionPeriodInDays: 7 + ExecutionTimeout: 300 + + WaitWithName: + Type: AWS::Serverless::Function + TestingMetadata: + TestDescription: ["2-2"] + Properties: + CodeUri: . + Handler: wait.WaitWithName + Description: Wait with explicit name parameter + Role: + Fn::GetAtt: + - DurableFunctionRole + - Arn + DurableConfig: + RetentionPeriodInDays: 7 + ExecutionTimeout: 300 + + WaitMultipleSequential: + Type: AWS::Serverless::Function + TestingMetadata: + TestDescription: ["2-3"] + Properties: + CodeUri: . + Handler: wait.WaitMultipleSequential + Description: Multiple sequential waits + Role: + Fn::GetAtt: + - DurableFunctionRole + - Arn + DurableConfig: + RetentionPeriodInDays: 7 + ExecutionTimeout: 300 + + WaitDurationUnits: + Type: AWS::Serverless::Function + TestingMetadata: + TestDescription: ["2-4"] + Properties: + CodeUri: . + Handler: wait.WaitDurationUnits + Description: Wait with different duration units (minutes) + Role: + Fn::GetAtt: + - DurableFunctionRole + - Arn + DurableConfig: + RetentionPeriodInDays: 7 + ExecutionTimeout: 3700 + + WaitLongDuration: + Type: AWS::Serverless::Function + TestingMetadata: + TestDescription: ["2-5"] + Properties: + CodeUri: . + Handler: wait.WaitLongDuration + Description: Wait with long duration (1 hour) + Role: + Fn::GetAtt: + - DurableFunctionRole + - Arn + DurableConfig: + RetentionPeriodInDays: 7 + ExecutionTimeout: 7200 diff --git a/pom.xml b/pom.xml index 207042a7d..648053c05 100644 --- a/pom.xml +++ b/pom.xml @@ -44,6 +44,7 @@ sdk-integration-tests otel-plugin examples + conformance-tests coverage-report From 9f5e58ae4f49274c0b2f4a6598418a4596b61fc7 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Tue, 21 Jul 2026 13:17:41 -0700 Subject: [PATCH 35/36] [maven]: Bump io.opentelemetry:opentelemetry-exporter-logging (#550) --- examples/pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/examples/pom.xml b/examples/pom.xml index 27771e5cf..da063dbd1 100644 --- a/examples/pom.xml +++ b/examples/pom.xml @@ -47,7 +47,7 @@ io.opentelemetry opentelemetry-exporter-logging - 1.63.0 + 1.64.0 From 903a436f16d2ab938b6a736590a216190018cc86 Mon Sep 17 00:00:00 2001 From: Ayushi Ahjolia Date: Tue, 21 Jul 2026 13:25:12 -0700 Subject: [PATCH 36/36] feat(examples): Parameterize ADOT layer ARN by region --- .github/workflows/e2e-tests.yml | 30 ++++++++++++++++++- examples/README.md | 13 ++++++++ examples/generate-template.py | 10 ++++++- .../durable/examples/general/OtelExample.java | 10 +++---- .../examples/otel/OtelXRayWaitExample.java | 16 +++++----- otel-plugin/README.md | 8 +++-- 6 files changed, 69 insertions(+), 18 deletions(-) diff --git a/.github/workflows/e2e-tests.yml b/.github/workflows/e2e-tests.yml index 4a4bd63fb..dcc1eae27 100644 --- a/.github/workflows/e2e-tests.yml +++ b/.github/workflows/e2e-tests.yml @@ -71,6 +71,34 @@ jobs: - name: Generate SAM template run: python3 generate-template.py working-directory: ./examples + - name: Resolve latest ADOT Java layer ARN + env: + GH_TOKEN: ${{ github.token }} + run: | + # The ADOT Java layer is regional: account ID and version vary per region. + # Resolve the current ARN for AWS_REGION from the release notes table (region | ARN). + ADOT_LAYER_ARN=$( + gh api repos/aws-observability/aws-otel-java-instrumentation/releases/latest \ + --jq .body | + awk -F '|' -v region="$AWS_REGION" ' + $2 ~ "^[[:space:]]*" region "[[:space:]]*$" { + gsub(/^[[:space:]]+|[[:space:]]+$/, "", $3) + print $3 + exit + } + ' + ) + if [ -z "$ADOT_LAYER_ARN" ]; then + echo "Could not resolve the latest ADOT Java layer for $AWS_REGION" + exit 1 + fi + aws lambda get-layer-version-by-arn \ + --arn "$ADOT_LAYER_ARN" \ + --region "$AWS_REGION" \ + --query LayerVersionArn \ + --output text > /dev/null + echo "Using ADOT Java layer $ADOT_LAYER_ARN" + echo "ADOT_LAYER_ARN=$ADOT_LAYER_ARN" >> "$GITHUB_ENV" - name: sam build env: MAVEN_OPTS: -DskipTests=true -Dmaven.test.skip=true @@ -87,7 +115,7 @@ jobs: run: | sam deploy --stack-name Java${{ matrix.java }}-JavaSDKCloudBasedIntegrationTestStack \ --resolve-image-repos --resolve-s3 --parameter-overrides \ - 'ParameterKey=Architecture,ParameterValue=x86_64 ParameterKey=JavaVersion,ParameterValue=java${{ matrix.java }} ParameterKey=FunctionNamePrefix,ParameterValue=Java${{ matrix.java }}- ParameterKey=RoleArn,ParameterValue=${{ secrets.TEST_LAMBDA_EXECUTION_ROLE_ARN }}' + 'ParameterKey=Architecture,ParameterValue=x86_64 ParameterKey=JavaVersion,ParameterValue=java${{ matrix.java }} ParameterKey=FunctionNamePrefix,ParameterValue=Java${{ matrix.java }}- ParameterKey=RoleArn,ParameterValue=${{ secrets.TEST_LAMBDA_EXECUTION_ROLE_ARN }} ParameterKey=AdotLayerArn,ParameterValue='"$ADOT_LAYER_ARN" working-directory: ./examples - name: Cloud Based Integration Tests run: | diff --git a/examples/README.md b/examples/README.md index 399e91c9f..fa24c9cc3 100644 --- a/examples/README.md +++ b/examples/README.md @@ -55,6 +55,19 @@ The SAM template configures: The examples package copies the OTel plugin jar into `lib/` so the ADOT Java agent can load it as an extension for examples that enable `OTEL_JAVAAGENT_EXTENSIONS`. +### ADOT layer region + +The ADOT layer is regional — its account ID and version vary by region — so the template exposes it as the `AdotLayerArn` parameter. The default targets `us-west-2` (the region used by the e2e tests), and the e2e workflow resolves the current ARN for its region automatically. + +If you deploy the tracing examples to any other region, override the parameter with that region's ARN, otherwise the deploy fails CloudFormation's `ResourceExistenceCheck` on a cross-region layer: + +```bash +sam deploy --parameter-overrides \ + AdotLayerArn=arn:aws:lambda:us-east-1:615299751070:layer:AWSOpenTelemetryDistroJava:16 +``` + +Find the current ARN for your region in the [ADOT Java instrumentation releases](https://github.com/aws-observability/aws-otel-java-instrumentation/releases/latest). You can also persist the override in your (git-ignored) `samconfig.toml` under `parameter_overrides`. + ## Invoke Deployed Functions ```bash diff --git a/examples/generate-template.py b/examples/generate-template.py index ac1d5fdb5..6b1607fe6 100755 --- a/examples/generate-template.py +++ b/examples/generate-template.py @@ -139,7 +139,7 @@ def emit_function(lines: list[str], example: ExampleFunction, java_agent_extensi [ " Tracing: Active", " Layers:", - " - !Sub arn:aws:lambda:${AWS::Region}:615299751070:layer:AWSOpenTelemetryDistroJava:15", + " - !Ref AdotLayerArn", ] ) if example.java_agent: @@ -201,6 +201,14 @@ def render_template(examples: list[ExampleFunction]) -> str: " RoleArn:", " Type: String", " Description: IAM Role ARN for Lambda function execution", + " AdotLayerArn:", + " Type: String", + " Default: arn:aws:lambda:us-west-2:615299751070:layer:AWSOpenTelemetryDistroJava:16", + " Description: >-", + " ARN of the ADOT (AWS Distro for OpenTelemetry) Lambda layer used by the tracing examples.", + " The layer is regional: its account ID and version vary by region, so override this per", + " deployment region. The default targets us-west-2 (the region used by e2e tests); CI", + " resolves the latest ARN for its region.", "", "Conditions:", " IsJava21OrLater:", diff --git a/examples/src/main/java/software/amazon/lambda/durable/examples/general/OtelExample.java b/examples/src/main/java/software/amazon/lambda/durable/examples/general/OtelExample.java index 69629aeff..a50931e8e 100644 --- a/examples/src/main/java/software/amazon/lambda/durable/examples/general/OtelExample.java +++ b/examples/src/main/java/software/amazon/lambda/durable/examples/general/OtelExample.java @@ -27,11 +27,11 @@ *

    Expected trace structure: * *

    - * durable.invocation
    - * ├── durable.step:create-greeting [attempt 1]
    - * ├── durable.step:create-greeting (operation, backfilled)
    - * ├── durable.step:transform [attempt 1]
    - * └── durable.step:transform (operation, backfilled)
    + * invocation
    + * ├── create-greeting
    + * │   └── create-greeting attempt 1
    + * └── transform
    + *     └── transform attempt 1
      * 
    */ public class OtelExample extends DurableHandler { diff --git a/examples/src/main/java/software/amazon/lambda/durable/examples/otel/OtelXRayWaitExample.java b/examples/src/main/java/software/amazon/lambda/durable/examples/otel/OtelXRayWaitExample.java index a614e5c6a..55fa5b659 100644 --- a/examples/src/main/java/software/amazon/lambda/durable/examples/otel/OtelXRayWaitExample.java +++ b/examples/src/main/java/software/amazon/lambda/durable/examples/otel/OtelXRayWaitExample.java @@ -32,14 +32,14 @@ * *
      * Trace (single trace ID across both invocations)
    - * ├── durable.invocation (invocation 1)
    - * │   ├── durable.step:before-wait
    - * │   │   └── durable.step:before-wait [attempt 1]
    - * │   └── durable.wait:pause (ended as PENDING)
    - * └── durable.invocation (invocation 2)
    - *     ├── durable.wait:pause (completed)
    - *     └── durable.step:after-wait
    - *         └── durable.step:after-wait [attempt 1]
    + * ├── invocation (invocation 1)
    + * │   ├── before-wait
    + * │   │   └── before-wait attempt 1
    + * │   └── pause (ended as PENDING)
    + * └── invocation (invocation 2)
    + *     ├── pause (completed)
    + *     └── after-wait
    + *         └── after-wait attempt 1
      * 
    */ @ExampleTemplate(tracing = true, javaAgent = true) diff --git a/otel-plugin/README.md b/otel-plugin/README.md index c818395e5..141218916 100644 --- a/otel-plugin/README.md +++ b/otel-plugin/README.md @@ -54,9 +54,11 @@ This plugin uses the [AWS Distro for OpenTelemetry (ADOT) Lambda layer](https:// The layer ARN follows the format: ``` -arn:aws:lambda::615299751070:layer:AWSOpenTelemetryDistroJava:15 +arn:aws:lambda::615299751070:layer:AWSOpenTelemetryDistroJava:16 ``` +> **Note:** This layer is regional — the account ID and version vary by region. Use the ARN for your deployment region; find the current per-region ARN in the [ADOT Java instrumentation releases](https://github.com/aws-observability/aws-otel-java-instrumentation/releases/latest). + **CloudFormation / SAM:** ```yaml @@ -65,7 +67,7 @@ MyFunction: Properties: Tracing: Active Layers: - - !Sub arn:aws:lambda:${AWS::Region}:615299751070:layer:AWSOpenTelemetryDistroJava:15 + - !Sub arn:aws:lambda:${AWS::Region}:615299751070:layer:AWSOpenTelemetryDistroJava:16 Environment: Variables: AWS_LAMBDA_EXEC_WRAPPER: /opt/otel-instrument @@ -77,7 +79,7 @@ MyFunction: ```bash aws lambda update-function-configuration \ --function-name your-function-name \ - --layers "arn:aws:lambda::615299751070:layer:AWSOpenTelemetryDistroJava:15" \ + --layers "arn:aws:lambda::615299751070:layer:AWSOpenTelemetryDistroJava:16" \ --environment "Variables={AWS_LAMBDA_EXEC_WRAPPER=/opt/otel-instrument,OTEL_JAVAAGENT_EXTENSIONS=/var/task/lib/aws-durable-execution-sdk-java-plugin-otel-.jar}" ```