From 6bcf170b03aeaa19a4d813f0c5980e5801ceca58 Mon Sep 17 00:00:00 2001 From: John Viegas Date: Wed, 29 Jul 2026 12:50:52 -0700 Subject: [PATCH 1/2] test(sdk-benchmarks): Add cold-start benchmarks for SdkWarmUp priming --- .../awssdk/benchmark/BenchmarkRunner.java | 8 +- .../coldstart/ColdStartMockServer.java | 101 ++++++++++++++ .../coldstart/ColdStartRequests.java | 70 ++++++++++ .../SdkClientColdStartBenchmark.java | 30 ++++ .../V2ColdStartAfterWarmUpBenchmark.java | 128 ++++++++++++++++++ .../V2ColdStartNoWarmUpBenchmark.java | 124 +++++++++++++++++ .../V2SdkWarmUpExecutionTimeBenchmark.java | 73 ++++++++++ 7 files changed, 533 insertions(+), 1 deletion(-) create mode 100644 test/sdk-benchmarks/src/main/java/software/amazon/awssdk/benchmark/coldstart/ColdStartMockServer.java create mode 100644 test/sdk-benchmarks/src/main/java/software/amazon/awssdk/benchmark/coldstart/ColdStartRequests.java create mode 100644 test/sdk-benchmarks/src/main/java/software/amazon/awssdk/benchmark/coldstart/SdkClientColdStartBenchmark.java create mode 100644 test/sdk-benchmarks/src/main/java/software/amazon/awssdk/benchmark/coldstart/V2ColdStartAfterWarmUpBenchmark.java create mode 100644 test/sdk-benchmarks/src/main/java/software/amazon/awssdk/benchmark/coldstart/V2ColdStartNoWarmUpBenchmark.java create mode 100644 test/sdk-benchmarks/src/main/java/software/amazon/awssdk/benchmark/coldstart/V2SdkWarmUpExecutionTimeBenchmark.java diff --git a/test/sdk-benchmarks/src/main/java/software/amazon/awssdk/benchmark/BenchmarkRunner.java b/test/sdk-benchmarks/src/main/java/software/amazon/awssdk/benchmark/BenchmarkRunner.java index 0b28b61a0e4e..82e17212d0de 100644 --- a/test/sdk-benchmarks/src/main/java/software/amazon/awssdk/benchmark/BenchmarkRunner.java +++ b/test/sdk-benchmarks/src/main/java/software/amazon/awssdk/benchmark/BenchmarkRunner.java @@ -51,8 +51,11 @@ import software.amazon.awssdk.benchmark.apicall.protocol.QueryProtocolBenchmark; import software.amazon.awssdk.benchmark.apicall.protocol.SmithyRpcV2ProtocolBenchmark; import software.amazon.awssdk.benchmark.apicall.protocol.XmlProtocolBenchmark; +import software.amazon.awssdk.benchmark.coldstart.V2ColdStartAfterWarmUpBenchmark; +import software.amazon.awssdk.benchmark.coldstart.V2ColdStartNoWarmUpBenchmark; import software.amazon.awssdk.benchmark.coldstart.V2DefaultClientCreationBenchmark; import software.amazon.awssdk.benchmark.coldstart.V2OptimizedClientCreationBenchmark; +import software.amazon.awssdk.benchmark.coldstart.V2SdkWarmUpExecutionTimeBenchmark; import software.amazon.awssdk.benchmark.enhanced.dynamodb.EnhancedClientDeleteV1MapperComparisonBenchmark; import software.amazon.awssdk.benchmark.enhanced.dynamodb.EnhancedClientGetOverheadBenchmark; import software.amazon.awssdk.benchmark.enhanced.dynamodb.EnhancedClientGetV1MapperComparisonBenchmark; @@ -87,7 +90,10 @@ public class BenchmarkRunner { private static final List COLD_START_BENCHMARKS = Arrays.asList( V2OptimizedClientCreationBenchmark.class.getSimpleName(), - V2DefaultClientCreationBenchmark.class.getSimpleName()); + V2DefaultClientCreationBenchmark.class.getSimpleName(), + V2ColdStartNoWarmUpBenchmark.class.getSimpleName(), + V2ColdStartAfterWarmUpBenchmark.class.getSimpleName(), + V2SdkWarmUpExecutionTimeBenchmark.class.getSimpleName()); private static final List MAPPER_BENCHMARKS = Arrays.asList( EnhancedClientGetOverheadBenchmark.class.getSimpleName(), diff --git a/test/sdk-benchmarks/src/main/java/software/amazon/awssdk/benchmark/coldstart/ColdStartMockServer.java b/test/sdk-benchmarks/src/main/java/software/amazon/awssdk/benchmark/coldstart/ColdStartMockServer.java new file mode 100644 index 000000000000..818956ff1a36 --- /dev/null +++ b/test/sdk-benchmarks/src/main/java/software/amazon/awssdk/benchmark/coldstart/ColdStartMockServer.java @@ -0,0 +1,101 @@ +/* + * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"). + * You may not use this file except in compliance with the License. + * A copy of the License is located at + * + * http://aws.amazon.com/apache2.0 + * + * or in the "license" file accompanying this file. This file is distributed + * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing + * permissions and limitations under the License. + */ + +package software.amazon.awssdk.benchmark.coldstart; + +import java.io.IOException; +import java.io.InputStream; +import java.net.URI; +import javax.servlet.http.HttpServlet; +import javax.servlet.http.HttpServletRequest; +import javax.servlet.http.HttpServletResponse; +import org.eclipse.jetty.server.Connector; +import org.eclipse.jetty.server.Server; +import org.eclipse.jetty.server.ServerConnector; +import org.eclipse.jetty.servlet.ServletContextHandler; +import org.eclipse.jetty.servlet.ServletHolder; +import software.amazon.awssdk.benchmark.utils.BenchmarkUtils; +import software.amazon.awssdk.utils.IoUtils; + +/** + * Lightweight plain-HTTP Jetty server that answers every request with a fixed status, body and content type, so a real + * service client can complete an operation without leaving the machine. + * + *

This mirrors {@code apicall.protocol.ProtocolRoundtripServer}, which is package-private to its own package. It is + * duplicated here rather than shared so the cold-start benchmarks do not force a visibility change on the throughput + * benchmarks that use the original. + * + *

Only the plain HTTP connector is exposed. That is deliberate: the cold-start benchmarks take a single measurement per + * JVM, and TLS handshake plus trust-all keystore setup would add variance to the one sample that matters. + */ +final class ColdStartMockServer { + + private final Server server; + private final int port; + + ColdStartMockServer(byte[] responseBody, String contentType) throws IOException { + port = BenchmarkUtils.getUnusedPort(); + server = new Server(); + ServerConnector connector = new ServerConnector(server); + connector.setPort(port); + server.setConnectors(new Connector[] {connector}); + + ServletContextHandler context = new ServletContextHandler(server, "/", ServletContextHandler.SESSIONS); + context.addServlet(new ServletHolder(new FixedResponseServlet(responseBody, contentType)), "/*"); + server.setHandler(context); + } + + void start() throws Exception { + server.start(); + } + + void stop() throws Exception { + server.stop(); + } + + URI getHttpUri() { + return URI.create("http://localhost:" + port); + } + + static byte[] loadFixture(String path) throws IOException { + try (InputStream is = ColdStartMockServer.class.getClassLoader().getResourceAsStream("fixtures/" + path)) { + if (is == null) { + throw new IOException("Fixture not found: fixtures/" + path); + } + return IoUtils.toByteArray(is); + } + } + + /** + * Stateless: returns the same response for every method, path and body, and does not validate the request. + */ + private static final class FixedResponseServlet extends HttpServlet { + private final byte[] body; + private final String contentType; + + private FixedResponseServlet(byte[] body, String contentType) { + this.body = body; + this.contentType = contentType; + } + + @Override + protected void service(HttpServletRequest req, HttpServletResponse resp) throws IOException { + resp.setStatus(200); + resp.setContentLength(body.length); + resp.setContentType(contentType); + resp.getOutputStream().write(body); + } + } +} diff --git a/test/sdk-benchmarks/src/main/java/software/amazon/awssdk/benchmark/coldstart/ColdStartRequests.java b/test/sdk-benchmarks/src/main/java/software/amazon/awssdk/benchmark/coldstart/ColdStartRequests.java new file mode 100644 index 000000000000..4d2f2d7408bd --- /dev/null +++ b/test/sdk-benchmarks/src/main/java/software/amazon/awssdk/benchmark/coldstart/ColdStartRequests.java @@ -0,0 +1,70 @@ +/* + * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"). + * You may not use this file except in compliance with the License. + * A copy of the License is located at + * + * http://aws.amazon.com/apache2.0 + * + * or in the "license" file accompanying this file. This file is distributed + * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing + * permissions and limitations under the License. + */ + +package software.amazon.awssdk.benchmark.coldstart; + +import java.util.HashMap; +import java.util.Map; +import software.amazon.awssdk.core.SdkBytes; +import software.amazon.awssdk.services.dynamodb.model.AttributeValue; +import software.amazon.awssdk.services.dynamodb.model.PutItemRequest; + +/** + * The request used by every cold-start arm, so the arms differ only in whether {@code SdkWarmUp.prime} ran. + * + *

The item shape matches {@code fixtures/json-protocol/putitem-response.json} and the item built by + * {@code apicall.protocol.V2JsonRoundtripBenchmark}, covering the scalar, set, map and list attribute types so the + * marshaller and unmarshaller do a representative amount of first-call work. + */ +final class ColdStartRequests { + + static final String FIXTURE = "json-protocol/putitem-response.json"; + static final String CONTENT_TYPE = "application/x-amz-json-1.0"; + + private ColdStartRequests() { + } + + static PutItemRequest putItemRequest() { + return PutItemRequest.builder() + .tableName("benchmark-table") + .item(itemMap()) + .build(); + } + + private static Map itemMap() { + Map item = new HashMap<>(); + item.put("pk", AttributeValue.fromS("benchmark-key")); + item.put("sk", AttributeValue.fromN("100")); + item.put("stringField", AttributeValue.fromS("test-value")); + item.put("numberField", AttributeValue.fromN("123.456")); + item.put("binaryField", AttributeValue.fromB(SdkBytes.fromUtf8String("hello world"))); + item.put("stringSetField", AttributeValue.builder().ss("value1", "value2", "value3").build()); + item.put("numberSetField", AttributeValue.builder().ns("1.1", "2.2", "3.3").build()); + item.put("boolField", AttributeValue.fromBool(false)); + item.put("nullField", AttributeValue.builder().nul(true).build()); + Map deep = new HashMap<>(); + deep.put("level2", AttributeValue.fromN("999")); + Map nested = new HashMap<>(); + nested.put("nested", AttributeValue.fromS("nested-value")); + nested.put("deepNested", AttributeValue.fromM(deep)); + item.put("mapField", AttributeValue.fromM(nested)); + item.put("listField", AttributeValue.builder().l( + AttributeValue.fromS("item1"), + AttributeValue.fromN("42"), + AttributeValue.fromBool(true), + AttributeValue.builder().nul(true).build()).build()); + return item; + } +} diff --git a/test/sdk-benchmarks/src/main/java/software/amazon/awssdk/benchmark/coldstart/SdkClientColdStartBenchmark.java b/test/sdk-benchmarks/src/main/java/software/amazon/awssdk/benchmark/coldstart/SdkClientColdStartBenchmark.java new file mode 100644 index 000000000000..4b198cfa807e --- /dev/null +++ b/test/sdk-benchmarks/src/main/java/software/amazon/awssdk/benchmark/coldstart/SdkClientColdStartBenchmark.java @@ -0,0 +1,30 @@ +/* + * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"). + * You may not use this file except in compliance with the License. + * A copy of the License is located at + * + * http://aws.amazon.com/apache2.0 + * + * or in the "license" file accompanying this file. This file is distributed + * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing + * permissions and limitations under the License. + */ + +package software.amazon.awssdk.benchmark.coldstart; + +import org.openjdk.jmh.infra.Blackhole; + +/** + * Measured unit for the cold-start benchmarks: build a service client and complete its first operation. + * + *

Separate from {@link SdkClientCreationBenchmark}, whose unit is construction only. The first operation is where the + * marshaller, unmarshaller, signer, endpoint rules and interceptor chain are first exercised, so construction alone would + * not represent what an application pays on its first request. + */ +public interface SdkClientColdStartBenchmark { + + void coldFirstCall(Blackhole blackhole) throws Exception; +} diff --git a/test/sdk-benchmarks/src/main/java/software/amazon/awssdk/benchmark/coldstart/V2ColdStartAfterWarmUpBenchmark.java b/test/sdk-benchmarks/src/main/java/software/amazon/awssdk/benchmark/coldstart/V2ColdStartAfterWarmUpBenchmark.java new file mode 100644 index 000000000000..5b820cee78f1 --- /dev/null +++ b/test/sdk-benchmarks/src/main/java/software/amazon/awssdk/benchmark/coldstart/V2ColdStartAfterWarmUpBenchmark.java @@ -0,0 +1,128 @@ +/* + * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"). + * You may not use this file except in compliance with the License. + * A copy of the License is located at + * + * http://aws.amazon.com/apache2.0 + * + * or in the "license" file accompanying this file. This file is distributed + * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing + * permissions and limitations under the License. + */ + +package software.amazon.awssdk.benchmark.coldstart; + +import static software.amazon.awssdk.benchmark.coldstart.ColdStartRequests.CONTENT_TYPE; +import static software.amazon.awssdk.benchmark.coldstart.ColdStartRequests.FIXTURE; +import static software.amazon.awssdk.benchmark.coldstart.ColdStartRequests.putItemRequest; + +import java.util.Collection; +import java.util.concurrent.TimeUnit; +import org.openjdk.jmh.annotations.Benchmark; +import org.openjdk.jmh.annotations.BenchmarkMode; +import org.openjdk.jmh.annotations.Fork; +import org.openjdk.jmh.annotations.Level; +import org.openjdk.jmh.annotations.Measurement; +import org.openjdk.jmh.annotations.Mode; +import org.openjdk.jmh.annotations.OutputTimeUnit; +import org.openjdk.jmh.annotations.Scope; +import org.openjdk.jmh.annotations.Setup; +import org.openjdk.jmh.annotations.State; +import org.openjdk.jmh.annotations.TearDown; +import org.openjdk.jmh.annotations.Warmup; +import org.openjdk.jmh.infra.Blackhole; +import org.openjdk.jmh.profile.StackProfiler; +import org.openjdk.jmh.results.RunResult; +import org.openjdk.jmh.runner.Runner; +import org.openjdk.jmh.runner.RunnerException; +import org.openjdk.jmh.runner.options.CommandLineOptionException; +import org.openjdk.jmh.runner.options.CommandLineOptions; +import org.openjdk.jmh.runner.options.Options; +import org.openjdk.jmh.runner.options.OptionsBuilder; +import software.amazon.awssdk.auth.credentials.AwsBasicCredentials; +import software.amazon.awssdk.auth.credentials.StaticCredentialsProvider; +import software.amazon.awssdk.core.crac.SdkWarmUp; +import software.amazon.awssdk.http.apache5.Apache5HttpClient; +import software.amazon.awssdk.regions.Region; +import software.amazon.awssdk.services.dynamodb.DynamoDbClient; + +/** + * Primed arm: identical to {@link V2ColdStartNoWarmUpBenchmark} except that {@link SdkWarmUp#prime(Class[])} runs in + * {@code @Setup}, outside the timing window. The measured number is therefore post-prime construction plus post-prime + * first call, and the difference from the baseline arm is the first-call work that priming front-loads. + * + *

Uses the targeted overload rather than {@code SdkWarmUp.prime()}. The no-arg form warms every + * {@code SdkWarmUpProvider} on the classpath, which on this module's classpath includes services unrelated to this + * measurement. + * + *

Priming also warms each sync {@code SdkHttpService} on the classpath by sending a {@code GET} to the regional STS + * endpoint, which is intended: exercising a real request is how the SPI lookup, TLS stack and connection machinery get + * initialized ahead of time. Those calls happen in {@code @Setup}, outside the measured window, and are best effort: + * a failure never fails the benchmark, but it does affect the score. On a host without network access the HTTP client + * stack is only partially warmed, the measured first call is slower, and the primed-vs-no-prime delta silently shrinks. + * Only compare the two arms when both ran in the same network environment. The cost of priming itself is measured + * separately by {@link V2SdkWarmUpExecutionTimeBenchmark}. + * + *

The provider primes DynamoDB with {@code ListBackups} while the measured call is {@code PutItem}. That is + * deliberate: the measured delta is the cross-operation transfer benefit, the realistic case, since shared work + * (signing, HTTP, base marshalling) dominates over operation-specific marshallers. + * + *

See {@link V2ColdStartNoWarmUpBenchmark} for why the JMH parameters below differ from the rest of this module and why + * they must not be overridden from the command line. + */ +@State(Scope.Benchmark) +@BenchmarkMode(Mode.SingleShotTime) +@OutputTimeUnit(TimeUnit.MILLISECONDS) +@Warmup(iterations = 0) +@Measurement(iterations = 1) +@Fork(20) +public class V2ColdStartAfterWarmUpBenchmark implements SdkClientColdStartBenchmark { + + private ColdStartMockServer server; + private DynamoDbClient client; + + @Setup(Level.Trial) + public void setup() throws Exception { + server = new ColdStartMockServer(ColdStartMockServer.loadFixture(FIXTURE), CONTENT_TYPE); + server.start(); + + SdkWarmUp.prime(DynamoDbClient.class); + } + + @Override + @Benchmark + public void coldFirstCall(Blackhole blackhole) throws Exception { + client = DynamoDbClient.builder() + .endpointOverride(server.getHttpUri()) + .region(Region.US_EAST_1) + .credentialsProvider(StaticCredentialsProvider.create( + AwsBasicCredentials.create("test", "test"))) + .httpClient(Apache5HttpClient.create()) + .endpointDiscoveryEnabled(false) + .build(); + + blackhole.consume(client.putItem(putItemRequest())); + } + + @TearDown(Level.Trial) + public void tearDown() throws Exception { + if (client != null) { + client.close(); + } + if (server != null) { + server.stop(); + } + } + + public static void main(String... args) throws RunnerException, CommandLineOptionException { + Options opt = new OptionsBuilder() + .parent(new CommandLineOptions()) + .include(V2ColdStartAfterWarmUpBenchmark.class.getSimpleName()) + .addProfiler(StackProfiler.class) + .build(); + Collection run = new Runner(opt).run(); + } +} diff --git a/test/sdk-benchmarks/src/main/java/software/amazon/awssdk/benchmark/coldstart/V2ColdStartNoWarmUpBenchmark.java b/test/sdk-benchmarks/src/main/java/software/amazon/awssdk/benchmark/coldstart/V2ColdStartNoWarmUpBenchmark.java new file mode 100644 index 000000000000..1a66d25dc124 --- /dev/null +++ b/test/sdk-benchmarks/src/main/java/software/amazon/awssdk/benchmark/coldstart/V2ColdStartNoWarmUpBenchmark.java @@ -0,0 +1,124 @@ +/* + * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"). + * You may not use this file except in compliance with the License. + * A copy of the License is located at + * + * http://aws.amazon.com/apache2.0 + * + * or in the "license" file accompanying this file. This file is distributed + * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing + * permissions and limitations under the License. + */ + +package software.amazon.awssdk.benchmark.coldstart; + +import static software.amazon.awssdk.benchmark.coldstart.ColdStartRequests.CONTENT_TYPE; +import static software.amazon.awssdk.benchmark.coldstart.ColdStartRequests.FIXTURE; +import static software.amazon.awssdk.benchmark.coldstart.ColdStartRequests.putItemRequest; + +import java.util.Collection; +import java.util.concurrent.TimeUnit; +import org.openjdk.jmh.annotations.Benchmark; +import org.openjdk.jmh.annotations.BenchmarkMode; +import org.openjdk.jmh.annotations.Fork; +import org.openjdk.jmh.annotations.Level; +import org.openjdk.jmh.annotations.Measurement; +import org.openjdk.jmh.annotations.Mode; +import org.openjdk.jmh.annotations.OutputTimeUnit; +import org.openjdk.jmh.annotations.Scope; +import org.openjdk.jmh.annotations.Setup; +import org.openjdk.jmh.annotations.State; +import org.openjdk.jmh.annotations.TearDown; +import org.openjdk.jmh.annotations.Warmup; +import org.openjdk.jmh.infra.Blackhole; +import org.openjdk.jmh.profile.StackProfiler; +import org.openjdk.jmh.results.RunResult; +import org.openjdk.jmh.runner.Runner; +import org.openjdk.jmh.runner.RunnerException; +import org.openjdk.jmh.runner.options.CommandLineOptionException; +import org.openjdk.jmh.runner.options.CommandLineOptions; +import org.openjdk.jmh.runner.options.Options; +import org.openjdk.jmh.runner.options.OptionsBuilder; +import software.amazon.awssdk.auth.credentials.AwsBasicCredentials; +import software.amazon.awssdk.auth.credentials.StaticCredentialsProvider; +import software.amazon.awssdk.http.apache5.Apache5HttpClient; +import software.amazon.awssdk.regions.Region; +import software.amazon.awssdk.services.dynamodb.DynamoDbClient; + +/** + * Baseline arm: build a {@link DynamoDbClient} and complete its first operation in a JVM where + * {@code SdkWarmUp.prime} never ran. Compare against {@link V2ColdStartAfterWarmUpBenchmark}, whose only difference is that + * priming happened in {@code @Setup}, outside the timing window. + * + *

The operation goes to an in-process Jetty server on localhost over plain HTTP with an explicit endpoint override and + * static credentials, so the measurement excludes DNS, TLS to AWS and service latency, and isolates SDK-side + * initialization: class loading, builder and configuration resolution, interceptor chain assembly, endpoint rule + * evaluation, auth scheme and signer setup, marshaller and unmarshaller construction, and HTTP client construction. + * + *

The JMH parameters below deliberately differ from every other benchmark in this module. + * {@code SdkWarmUp} keeps its primed state in statics with no reset hook, so priming does real work at most once per JVM + * and only the first invocation after it is genuinely cold. {@link Mode#SingleShotTime} with zero warmup iterations and a + * single measurement iteration yields exactly one measured invocation per fork; the high fork count supplies the samples. + * For {@code SingleShotTime} a JMH warmup iteration is itself a full invocation, so zero warmup is required, not cosmetic. + * + *

Do not override these from the command line. JMH CLI flags such as {@code -wi 3 -i 3 -f 1} take precedence + * over these annotations and would silently make the measured invocations warm, which makes both arms converge. + * + *

This benchmark cannot speak to Lambda SnapStart economics. Under SnapStart the priming cost is paid once at + * checkpoint and amortized across every restore; JMH has no checkpoint or restore, so here it is paid in the same JVM + * that is measured. + */ +@State(Scope.Benchmark) +@BenchmarkMode(Mode.SingleShotTime) +@OutputTimeUnit(TimeUnit.MILLISECONDS) +@Warmup(iterations = 0) +@Measurement(iterations = 1) +@Fork(20) +public class V2ColdStartNoWarmUpBenchmark implements SdkClientColdStartBenchmark { + + private ColdStartMockServer server; + private DynamoDbClient client; + + @Setup(Level.Trial) + public void setup() throws Exception { + server = new ColdStartMockServer(ColdStartMockServer.loadFixture(FIXTURE), CONTENT_TYPE); + server.start(); + } + + @Override + @Benchmark + public void coldFirstCall(Blackhole blackhole) throws Exception { + client = DynamoDbClient.builder() + .endpointOverride(server.getHttpUri()) + .region(Region.US_EAST_1) + .credentialsProvider(StaticCredentialsProvider.create( + AwsBasicCredentials.create("test", "test"))) + .httpClient(Apache5HttpClient.create()) + .endpointDiscoveryEnabled(false) + .build(); + + blackhole.consume(client.putItem(putItemRequest())); + } + + @TearDown(Level.Trial) + public void tearDown() throws Exception { + if (client != null) { + client.close(); + } + if (server != null) { + server.stop(); + } + } + + public static void main(String... args) throws RunnerException, CommandLineOptionException { + Options opt = new OptionsBuilder() + .parent(new CommandLineOptions()) + .include(V2ColdStartNoWarmUpBenchmark.class.getSimpleName()) + .addProfiler(StackProfiler.class) + .build(); + Collection run = new Runner(opt).run(); + } +} diff --git a/test/sdk-benchmarks/src/main/java/software/amazon/awssdk/benchmark/coldstart/V2SdkWarmUpExecutionTimeBenchmark.java b/test/sdk-benchmarks/src/main/java/software/amazon/awssdk/benchmark/coldstart/V2SdkWarmUpExecutionTimeBenchmark.java new file mode 100644 index 000000000000..eff3acb651b8 --- /dev/null +++ b/test/sdk-benchmarks/src/main/java/software/amazon/awssdk/benchmark/coldstart/V2SdkWarmUpExecutionTimeBenchmark.java @@ -0,0 +1,73 @@ +/* + * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"). + * You may not use this file except in compliance with the License. + * A copy of the License is located at + * + * http://aws.amazon.com/apache2.0 + * + * or in the "license" file accompanying this file. This file is distributed + * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing + * permissions and limitations under the License. + */ + +package software.amazon.awssdk.benchmark.coldstart; + +import java.util.Collection; +import java.util.concurrent.TimeUnit; +import org.openjdk.jmh.annotations.Benchmark; +import org.openjdk.jmh.annotations.BenchmarkMode; +import org.openjdk.jmh.annotations.Fork; +import org.openjdk.jmh.annotations.Measurement; +import org.openjdk.jmh.annotations.Mode; +import org.openjdk.jmh.annotations.OutputTimeUnit; +import org.openjdk.jmh.annotations.Scope; +import org.openjdk.jmh.annotations.State; +import org.openjdk.jmh.annotations.Warmup; +import org.openjdk.jmh.profile.StackProfiler; +import org.openjdk.jmh.results.RunResult; +import org.openjdk.jmh.runner.Runner; +import org.openjdk.jmh.runner.RunnerException; +import org.openjdk.jmh.runner.options.CommandLineOptionException; +import org.openjdk.jmh.runner.options.CommandLineOptions; +import org.openjdk.jmh.runner.options.Options; +import org.openjdk.jmh.runner.options.OptionsBuilder; +import software.amazon.awssdk.core.crac.SdkWarmUp; +import software.amazon.awssdk.services.dynamodb.DynamoDbClient; + +/** + * Measures how long {@link SdkWarmUp#prime(Class[])} takes to run: the one-time upfront work an application pays + * during initialization to get the first-call saving that {@link V2ColdStartAfterWarmUpBenchmark} shows against + * {@link V2ColdStartNoWarmUpBenchmark}. + * + *

Priming makes a real STS {@code GET} per sync HTTP client on the classpath, so the score includes network time, + * varies by host, and is an upper bound (this module has several HTTP clients; typical apps have one). Do not add it + * to {@code baseline.json}. + * + *

See {@link V2ColdStartNoWarmUpBenchmark} for the single-shot JMH parameters. Priming is once-per-JVM, so only the + * first invocation in a fork is a real measurement. + */ +@State(Scope.Benchmark) +@BenchmarkMode(Mode.SingleShotTime) +@OutputTimeUnit(TimeUnit.MILLISECONDS) +@Warmup(iterations = 0) +@Measurement(iterations = 1) +@Fork(20) +public class V2SdkWarmUpExecutionTimeBenchmark { + + @Benchmark + public void prime() throws Exception { + SdkWarmUp.prime(DynamoDbClient.class); + } + + public static void main(String... args) throws RunnerException, CommandLineOptionException { + Options opt = new OptionsBuilder() + .parent(new CommandLineOptions()) + .include(V2SdkWarmUpExecutionTimeBenchmark.class.getSimpleName()) + .addProfiler(StackProfiler.class) + .build(); + Collection run = new Runner(opt).run(); + } +} From 9058f3673cea9a6f155148f0c21b4e1dd9a8dcfc Mon Sep 17 00:00:00 2001 From: John Viegas Date: Wed, 29 Jul 2026 13:11:23 -0700 Subject: [PATCH 2/2] refactor(sdk-benchmarks): Unify duplicate Jetty mock servers into MockHttpServer --- .../protocol/ProtocolRoundtripServer.java | 70 ----------------- .../protocol/ProtocolRoundtripServlet.java | 39 ---------- .../protocol/V1CborRoundtripBenchmark.java | 7 +- .../protocol/V1Ec2RoundtripBenchmark.java | 9 +-- .../protocol/V1JsonRoundtripBenchmark.java | 9 +-- .../protocol/V1QueryRoundtripBenchmark.java | 9 +-- .../V1RestJsonRoundtripBenchmark.java | 9 +-- .../protocol/V1RestXmlRoundtripBenchmark.java | 9 +-- .../protocol/V2CborRoundtripBenchmark.java | 7 +- .../protocol/V2Ec2RoundtripBenchmark.java | 9 +-- .../protocol/V2JsonRoundtripBenchmark.java | 9 +-- .../protocol/V2QueryRoundtripBenchmark.java | 9 +-- .../V2RestJsonRoundtripBenchmark.java | 9 +-- .../protocol/V2RestXmlRoundtripBenchmark.java | 9 +-- .../coldstart/ColdStartRequests.java | 70 ----------------- .../SdkClientColdStartBenchmark.java | 30 ------- .../V2ColdStartAfterWarmUpBenchmark.java | 78 +++++++++++-------- .../V2ColdStartNoWarmUpBenchmark.java | 76 +++++++++++------- .../V2SdkWarmUpExecutionTimeBenchmark.java | 13 +--- .../MockHttpServer.java} | 29 +++---- 20 files changed, 154 insertions(+), 355 deletions(-) delete mode 100644 test/sdk-benchmarks/src/main/java/software/amazon/awssdk/benchmark/apicall/protocol/ProtocolRoundtripServer.java delete mode 100644 test/sdk-benchmarks/src/main/java/software/amazon/awssdk/benchmark/apicall/protocol/ProtocolRoundtripServlet.java delete mode 100644 test/sdk-benchmarks/src/main/java/software/amazon/awssdk/benchmark/coldstart/ColdStartRequests.java delete mode 100644 test/sdk-benchmarks/src/main/java/software/amazon/awssdk/benchmark/coldstart/SdkClientColdStartBenchmark.java rename test/sdk-benchmarks/src/main/java/software/amazon/awssdk/benchmark/{coldstart/ColdStartMockServer.java => utils/MockHttpServer.java} (70%) diff --git a/test/sdk-benchmarks/src/main/java/software/amazon/awssdk/benchmark/apicall/protocol/ProtocolRoundtripServer.java b/test/sdk-benchmarks/src/main/java/software/amazon/awssdk/benchmark/apicall/protocol/ProtocolRoundtripServer.java deleted file mode 100644 index 96911c4e9442..000000000000 --- a/test/sdk-benchmarks/src/main/java/software/amazon/awssdk/benchmark/apicall/protocol/ProtocolRoundtripServer.java +++ /dev/null @@ -1,70 +0,0 @@ -/* - * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. - * - * Licensed under the Apache License, Version 2.0 (the "License"). - * You may not use this file except in compliance with the License. - * A copy of the License is located at - * - * http://aws.amazon.com/apache2.0 - * - * or in the "license" file accompanying this file. This file is distributed - * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either - * express or implied. See the License for the specific language governing - * permissions and limitations under the License. - */ - -package software.amazon.awssdk.benchmark.apicall.protocol; - -import java.io.IOException; -import java.io.InputStream; -import java.net.URI; -import org.eclipse.jetty.server.Connector; -import org.eclipse.jetty.server.Server; -import org.eclipse.jetty.server.ServerConnector; -import org.eclipse.jetty.servlet.ServletContextHandler; -import org.eclipse.jetty.servlet.ServletHolder; -import software.amazon.awssdk.benchmark.utils.BenchmarkUtils; -import software.amazon.awssdk.utils.IoUtils; - -/** - * Lightweight Jetty server for protocol roundtrip benchmarks. - */ -class ProtocolRoundtripServer { - - private final Server server; - private final int port; - - ProtocolRoundtripServer(ProtocolRoundtripServlet servlet) throws IOException { - port = BenchmarkUtils.getUnusedPort(); - server = new Server(); - ServerConnector connector = new ServerConnector(server); - connector.setPort(port); - server.setConnectors(new Connector[] {connector}); - - ServletContextHandler context = new ServletContextHandler(server, "/", ServletContextHandler.SESSIONS); - context.addServlet(new ServletHolder(servlet), "/*"); - server.setHandler(context); - } - - void start() throws Exception { - server.start(); - } - - void stop() throws Exception { - server.stop(); - } - - URI getHttpUri() { - return URI.create("http://localhost:" + port); - } - - static byte[] loadFixture(String path) throws IOException { - try (InputStream is = ProtocolRoundtripServer.class.getClassLoader() - .getResourceAsStream("fixtures/" + path)) { - if (is == null) { - throw new IOException("Fixture not found: fixtures/" + path); - } - return IoUtils.toByteArray(is); - } - } -} diff --git a/test/sdk-benchmarks/src/main/java/software/amazon/awssdk/benchmark/apicall/protocol/ProtocolRoundtripServlet.java b/test/sdk-benchmarks/src/main/java/software/amazon/awssdk/benchmark/apicall/protocol/ProtocolRoundtripServlet.java deleted file mode 100644 index abcce14534ad..000000000000 --- a/test/sdk-benchmarks/src/main/java/software/amazon/awssdk/benchmark/apicall/protocol/ProtocolRoundtripServlet.java +++ /dev/null @@ -1,39 +0,0 @@ -/* - * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. - * - * Licensed under the Apache License, Version 2.0 (the "License"). - * You may not use this file except in compliance with the License. - * A copy of the License is located at - * - * http://aws.amazon.com/apache2.0 - * - * or in the "license" file accompanying this file. This file is distributed - * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either - * express or implied. See the License for the specific language governing - * permissions and limitations under the License. - */ - -package software.amazon.awssdk.benchmark.apicall.protocol; - -import java.io.IOException; -import javax.servlet.http.HttpServlet; -import javax.servlet.http.HttpServletRequest; -import javax.servlet.http.HttpServletResponse; - -class ProtocolRoundtripServlet extends HttpServlet { - private final byte[] body; - private final String contentType; - - ProtocolRoundtripServlet(byte[] body, String contentType) { - this.body = body; - this.contentType = contentType; - } - - @Override - protected void service(HttpServletRequest req, HttpServletResponse resp) throws IOException { - resp.setStatus(200); - resp.setContentLength(body.length); - resp.setContentType(contentType); - resp.getOutputStream().write(body); - } -} diff --git a/test/sdk-benchmarks/src/main/java/software/amazon/awssdk/benchmark/apicall/protocol/V1CborRoundtripBenchmark.java b/test/sdk-benchmarks/src/main/java/software/amazon/awssdk/benchmark/apicall/protocol/V1CborRoundtripBenchmark.java index eb7dfccee20c..e54d004cbb59 100644 --- a/test/sdk-benchmarks/src/main/java/software/amazon/awssdk/benchmark/apicall/protocol/V1CborRoundtripBenchmark.java +++ b/test/sdk-benchmarks/src/main/java/software/amazon/awssdk/benchmark/apicall/protocol/V1CborRoundtripBenchmark.java @@ -41,6 +41,7 @@ import org.openjdk.jmh.annotations.TearDown; import org.openjdk.jmh.annotations.Warmup; import org.openjdk.jmh.infra.Blackhole; +import software.amazon.awssdk.benchmark.utils.MockHttpServer; /** * V1 roundtrip benchmark for SmithyRpcV2 CBOR protocol using CloudWatch GetMetricData via HTTP servlet. @@ -53,16 +54,14 @@ @OutputTimeUnit(TimeUnit.SECONDS) public class V1CborRoundtripBenchmark { - private ProtocolRoundtripServer server; + private MockHttpServer server; private AmazonCloudWatch client; @Setup(Level.Trial) public void setup() throws Exception { byte[] response = createCborResponseFixture(); - ProtocolRoundtripServlet servlet = new ProtocolRoundtripServlet(response, "application/cbor"); - - server = new ProtocolRoundtripServer(servlet); + server = new MockHttpServer(response, "application/cbor"); server.start(); client = AmazonCloudWatchClientBuilder.standard() diff --git a/test/sdk-benchmarks/src/main/java/software/amazon/awssdk/benchmark/apicall/protocol/V1Ec2RoundtripBenchmark.java b/test/sdk-benchmarks/src/main/java/software/amazon/awssdk/benchmark/apicall/protocol/V1Ec2RoundtripBenchmark.java index 6ae6df995a99..e3b0b06b5fb1 100644 --- a/test/sdk-benchmarks/src/main/java/software/amazon/awssdk/benchmark/apicall/protocol/V1Ec2RoundtripBenchmark.java +++ b/test/sdk-benchmarks/src/main/java/software/amazon/awssdk/benchmark/apicall/protocol/V1Ec2RoundtripBenchmark.java @@ -36,6 +36,7 @@ import org.openjdk.jmh.annotations.TearDown; import org.openjdk.jmh.annotations.Warmup; import org.openjdk.jmh.infra.Blackhole; +import software.amazon.awssdk.benchmark.utils.MockHttpServer; /** * V1 roundtrip benchmark for EC2 protocol using EC2 DescribeInstances via HTTP servlet. @@ -48,16 +49,14 @@ @OutputTimeUnit(TimeUnit.SECONDS) public class V1Ec2RoundtripBenchmark { - private ProtocolRoundtripServer server; + private MockHttpServer server; private AmazonEC2 client; @Setup(Level.Trial) public void setup() throws Exception { - byte[] response = ProtocolRoundtripServer.loadFixture("ec2-protocol/describe-instances-response.xml"); + byte[] response = MockHttpServer.loadFixture("ec2-protocol/describe-instances-response.xml"); - ProtocolRoundtripServlet servlet = new ProtocolRoundtripServlet(response, "text/xml"); - - server = new ProtocolRoundtripServer(servlet); + server = new MockHttpServer(response, "text/xml"); server.start(); client = AmazonEC2ClientBuilder.standard() diff --git a/test/sdk-benchmarks/src/main/java/software/amazon/awssdk/benchmark/apicall/protocol/V1JsonRoundtripBenchmark.java b/test/sdk-benchmarks/src/main/java/software/amazon/awssdk/benchmark/apicall/protocol/V1JsonRoundtripBenchmark.java index 58146050360c..f0b1bde7450c 100644 --- a/test/sdk-benchmarks/src/main/java/software/amazon/awssdk/benchmark/apicall/protocol/V1JsonRoundtripBenchmark.java +++ b/test/sdk-benchmarks/src/main/java/software/amazon/awssdk/benchmark/apicall/protocol/V1JsonRoundtripBenchmark.java @@ -39,6 +39,7 @@ import org.openjdk.jmh.annotations.TearDown; import org.openjdk.jmh.annotations.Warmup; import org.openjdk.jmh.infra.Blackhole; +import software.amazon.awssdk.benchmark.utils.MockHttpServer; /** * V1 roundtrip benchmark for JSON protocol (aws-json) using DynamoDB PutItem via HTTP servlet. @@ -51,16 +52,14 @@ @OutputTimeUnit(TimeUnit.SECONDS) public class V1JsonRoundtripBenchmark { - private ProtocolRoundtripServer server; + private MockHttpServer server; private AmazonDynamoDB client; @Setup(Level.Trial) public void setup() throws Exception { - byte[] response = ProtocolRoundtripServer.loadFixture("json-protocol/putitem-response.json"); + byte[] response = MockHttpServer.loadFixture("json-protocol/putitem-response.json"); - ProtocolRoundtripServlet servlet = new ProtocolRoundtripServlet(response, "application/x-amz-json-1.0"); - - server = new ProtocolRoundtripServer(servlet); + server = new MockHttpServer(response, "application/x-amz-json-1.0"); server.start(); client = AmazonDynamoDBClientBuilder.standard() diff --git a/test/sdk-benchmarks/src/main/java/software/amazon/awssdk/benchmark/apicall/protocol/V1QueryRoundtripBenchmark.java b/test/sdk-benchmarks/src/main/java/software/amazon/awssdk/benchmark/apicall/protocol/V1QueryRoundtripBenchmark.java index 3da508d4ff5c..318a9269c91e 100644 --- a/test/sdk-benchmarks/src/main/java/software/amazon/awssdk/benchmark/apicall/protocol/V1QueryRoundtripBenchmark.java +++ b/test/sdk-benchmarks/src/main/java/software/amazon/awssdk/benchmark/apicall/protocol/V1QueryRoundtripBenchmark.java @@ -35,6 +35,7 @@ import org.openjdk.jmh.annotations.TearDown; import org.openjdk.jmh.annotations.Warmup; import org.openjdk.jmh.infra.Blackhole; +import software.amazon.awssdk.benchmark.utils.MockHttpServer; /** * V1 roundtrip benchmark for Query protocol using STS AssumeRole via HTTP servlet. @@ -47,16 +48,14 @@ @OutputTimeUnit(TimeUnit.SECONDS) public class V1QueryRoundtripBenchmark { - private ProtocolRoundtripServer server; + private MockHttpServer server; private AWSSecurityTokenService client; @Setup(Level.Trial) public void setup() throws Exception { - byte[] response = ProtocolRoundtripServer.loadFixture("query-protocol/assumerole-response.xml"); + byte[] response = MockHttpServer.loadFixture("query-protocol/assumerole-response.xml"); - ProtocolRoundtripServlet servlet = new ProtocolRoundtripServlet(response, "text/xml"); - - server = new ProtocolRoundtripServer(servlet); + server = new MockHttpServer(response, "text/xml"); server.start(); client = AWSSecurityTokenServiceClientBuilder.standard() diff --git a/test/sdk-benchmarks/src/main/java/software/amazon/awssdk/benchmark/apicall/protocol/V1RestJsonRoundtripBenchmark.java b/test/sdk-benchmarks/src/main/java/software/amazon/awssdk/benchmark/apicall/protocol/V1RestJsonRoundtripBenchmark.java index 5bc6d9ca080f..f0f532b430ea 100644 --- a/test/sdk-benchmarks/src/main/java/software/amazon/awssdk/benchmark/apicall/protocol/V1RestJsonRoundtripBenchmark.java +++ b/test/sdk-benchmarks/src/main/java/software/amazon/awssdk/benchmark/apicall/protocol/V1RestJsonRoundtripBenchmark.java @@ -42,6 +42,7 @@ import org.openjdk.jmh.annotations.TearDown; import org.openjdk.jmh.annotations.Warmup; import org.openjdk.jmh.infra.Blackhole; +import software.amazon.awssdk.benchmark.utils.MockHttpServer; /** * V1 roundtrip benchmark for REST-JSON protocol using Lambda CreateFunction via HTTP servlet. @@ -54,16 +55,14 @@ @OutputTimeUnit(TimeUnit.SECONDS) public class V1RestJsonRoundtripBenchmark { - private ProtocolRoundtripServer server; + private MockHttpServer server; private AWSLambda client; @Setup(Level.Trial) public void setup() throws Exception { - byte[] response = ProtocolRoundtripServer.loadFixture("rest-json-protocol/createfunction-response.json"); + byte[] response = MockHttpServer.loadFixture("rest-json-protocol/createfunction-response.json"); - ProtocolRoundtripServlet servlet = new ProtocolRoundtripServlet(response, "application/json"); - - server = new ProtocolRoundtripServer(servlet); + server = new MockHttpServer(response, "application/json"); server.start(); client = AWSLambdaClientBuilder.standard() diff --git a/test/sdk-benchmarks/src/main/java/software/amazon/awssdk/benchmark/apicall/protocol/V1RestXmlRoundtripBenchmark.java b/test/sdk-benchmarks/src/main/java/software/amazon/awssdk/benchmark/apicall/protocol/V1RestXmlRoundtripBenchmark.java index ff42e19ada68..91d4376c26ae 100644 --- a/test/sdk-benchmarks/src/main/java/software/amazon/awssdk/benchmark/apicall/protocol/V1RestXmlRoundtripBenchmark.java +++ b/test/sdk-benchmarks/src/main/java/software/amazon/awssdk/benchmark/apicall/protocol/V1RestXmlRoundtripBenchmark.java @@ -45,6 +45,7 @@ import org.openjdk.jmh.annotations.TearDown; import org.openjdk.jmh.annotations.Warmup; import org.openjdk.jmh.infra.Blackhole; +import software.amazon.awssdk.benchmark.utils.MockHttpServer; /** * V1 roundtrip benchmark for REST-XML protocol using CloudFront CreateDistribution via HTTP servlet. @@ -57,16 +58,14 @@ @OutputTimeUnit(TimeUnit.SECONDS) public class V1RestXmlRoundtripBenchmark { - private ProtocolRoundtripServer server; + private MockHttpServer server; private AmazonCloudFront client; @Setup(Level.Trial) public void setup() throws Exception { - byte[] response = ProtocolRoundtripServer.loadFixture("rest-xml-protocol/create-distribution-response.xml"); + byte[] response = MockHttpServer.loadFixture("rest-xml-protocol/create-distribution-response.xml"); - ProtocolRoundtripServlet servlet = new ProtocolRoundtripServlet(response, "text/xml"); - - server = new ProtocolRoundtripServer(servlet); + server = new MockHttpServer(response, "text/xml"); server.start(); client = AmazonCloudFrontClientBuilder.standard() diff --git a/test/sdk-benchmarks/src/main/java/software/amazon/awssdk/benchmark/apicall/protocol/V2CborRoundtripBenchmark.java b/test/sdk-benchmarks/src/main/java/software/amazon/awssdk/benchmark/apicall/protocol/V2CborRoundtripBenchmark.java index f2f3e60a740b..a7b0cf78b3a0 100644 --- a/test/sdk-benchmarks/src/main/java/software/amazon/awssdk/benchmark/apicall/protocol/V2CborRoundtripBenchmark.java +++ b/test/sdk-benchmarks/src/main/java/software/amazon/awssdk/benchmark/apicall/protocol/V2CborRoundtripBenchmark.java @@ -30,6 +30,7 @@ import org.openjdk.jmh.annotations.TearDown; import org.openjdk.jmh.annotations.Warmup; import org.openjdk.jmh.infra.Blackhole; +import software.amazon.awssdk.benchmark.utils.MockHttpServer; import software.amazon.awssdk.auth.credentials.AwsBasicCredentials; import software.amazon.awssdk.auth.credentials.StaticCredentialsProvider; import software.amazon.awssdk.http.apache5.Apache5HttpClient; @@ -53,16 +54,14 @@ @OutputTimeUnit(TimeUnit.SECONDS) public class V2CborRoundtripBenchmark { - private ProtocolRoundtripServer server; + private MockHttpServer server; private CloudWatchClient client; @Setup(Level.Trial) public void setup() throws Exception { byte[] response = createCborResponseFixture(); - ProtocolRoundtripServlet servlet = new ProtocolRoundtripServlet(response, "application/cbor"); - - server = new ProtocolRoundtripServer(servlet); + server = new MockHttpServer(response, "application/cbor"); server.start(); client = CloudWatchClient.builder() diff --git a/test/sdk-benchmarks/src/main/java/software/amazon/awssdk/benchmark/apicall/protocol/V2Ec2RoundtripBenchmark.java b/test/sdk-benchmarks/src/main/java/software/amazon/awssdk/benchmark/apicall/protocol/V2Ec2RoundtripBenchmark.java index 3dfdd9cd74d2..9b23335c9818 100644 --- a/test/sdk-benchmarks/src/main/java/software/amazon/awssdk/benchmark/apicall/protocol/V2Ec2RoundtripBenchmark.java +++ b/test/sdk-benchmarks/src/main/java/software/amazon/awssdk/benchmark/apicall/protocol/V2Ec2RoundtripBenchmark.java @@ -29,6 +29,7 @@ import org.openjdk.jmh.annotations.TearDown; import org.openjdk.jmh.annotations.Warmup; import org.openjdk.jmh.infra.Blackhole; +import software.amazon.awssdk.benchmark.utils.MockHttpServer; import software.amazon.awssdk.auth.credentials.AwsBasicCredentials; import software.amazon.awssdk.auth.credentials.StaticCredentialsProvider; import software.amazon.awssdk.http.apache5.Apache5HttpClient; @@ -48,16 +49,14 @@ @OutputTimeUnit(TimeUnit.SECONDS) public class V2Ec2RoundtripBenchmark { - private ProtocolRoundtripServer server; + private MockHttpServer server; private Ec2Client client; @Setup(Level.Trial) public void setup() throws Exception { - byte[] response = ProtocolRoundtripServer.loadFixture("ec2-protocol/describe-instances-response.xml"); + byte[] response = MockHttpServer.loadFixture("ec2-protocol/describe-instances-response.xml"); - ProtocolRoundtripServlet servlet = new ProtocolRoundtripServlet(response, "text/xml"); - - server = new ProtocolRoundtripServer(servlet); + server = new MockHttpServer(response, "text/xml"); server.start(); client = Ec2Client.builder() diff --git a/test/sdk-benchmarks/src/main/java/software/amazon/awssdk/benchmark/apicall/protocol/V2JsonRoundtripBenchmark.java b/test/sdk-benchmarks/src/main/java/software/amazon/awssdk/benchmark/apicall/protocol/V2JsonRoundtripBenchmark.java index 6d23fc581f6e..f96c8a44a42f 100644 --- a/test/sdk-benchmarks/src/main/java/software/amazon/awssdk/benchmark/apicall/protocol/V2JsonRoundtripBenchmark.java +++ b/test/sdk-benchmarks/src/main/java/software/amazon/awssdk/benchmark/apicall/protocol/V2JsonRoundtripBenchmark.java @@ -31,6 +31,7 @@ import org.openjdk.jmh.annotations.TearDown; import org.openjdk.jmh.annotations.Warmup; import org.openjdk.jmh.infra.Blackhole; +import software.amazon.awssdk.benchmark.utils.MockHttpServer; import software.amazon.awssdk.auth.credentials.AwsBasicCredentials; import software.amazon.awssdk.auth.credentials.StaticCredentialsProvider; import software.amazon.awssdk.core.SdkBytes; @@ -51,16 +52,14 @@ @OutputTimeUnit(TimeUnit.SECONDS) public class V2JsonRoundtripBenchmark { - private ProtocolRoundtripServer server; + private MockHttpServer server; private DynamoDbClient client; @Setup(Level.Trial) public void setup() throws Exception { - byte[] response = ProtocolRoundtripServer.loadFixture("json-protocol/putitem-response.json"); + byte[] response = MockHttpServer.loadFixture("json-protocol/putitem-response.json"); - ProtocolRoundtripServlet servlet = new ProtocolRoundtripServlet(response, "application/x-amz-json-1.0"); - - server = new ProtocolRoundtripServer(servlet); + server = new MockHttpServer(response, "application/x-amz-json-1.0"); server.start(); client = DynamoDbClient.builder() diff --git a/test/sdk-benchmarks/src/main/java/software/amazon/awssdk/benchmark/apicall/protocol/V2QueryRoundtripBenchmark.java b/test/sdk-benchmarks/src/main/java/software/amazon/awssdk/benchmark/apicall/protocol/V2QueryRoundtripBenchmark.java index aff4d9c23cd2..a03e94f2808b 100644 --- a/test/sdk-benchmarks/src/main/java/software/amazon/awssdk/benchmark/apicall/protocol/V2QueryRoundtripBenchmark.java +++ b/test/sdk-benchmarks/src/main/java/software/amazon/awssdk/benchmark/apicall/protocol/V2QueryRoundtripBenchmark.java @@ -29,6 +29,7 @@ import org.openjdk.jmh.annotations.TearDown; import org.openjdk.jmh.annotations.Warmup; import org.openjdk.jmh.infra.Blackhole; +import software.amazon.awssdk.benchmark.utils.MockHttpServer; import software.amazon.awssdk.auth.credentials.AwsBasicCredentials; import software.amazon.awssdk.auth.credentials.StaticCredentialsProvider; import software.amazon.awssdk.http.apache5.Apache5HttpClient; @@ -47,16 +48,14 @@ @OutputTimeUnit(TimeUnit.SECONDS) public class V2QueryRoundtripBenchmark { - private ProtocolRoundtripServer server; + private MockHttpServer server; private StsClient client; @Setup(Level.Trial) public void setup() throws Exception { - byte[] response = ProtocolRoundtripServer.loadFixture("query-protocol/assumerole-response.xml"); + byte[] response = MockHttpServer.loadFixture("query-protocol/assumerole-response.xml"); - ProtocolRoundtripServlet servlet = new ProtocolRoundtripServlet(response, "text/xml"); - - server = new ProtocolRoundtripServer(servlet); + server = new MockHttpServer(response, "text/xml"); server.start(); client = StsClient.builder() diff --git a/test/sdk-benchmarks/src/main/java/software/amazon/awssdk/benchmark/apicall/protocol/V2RestJsonRoundtripBenchmark.java b/test/sdk-benchmarks/src/main/java/software/amazon/awssdk/benchmark/apicall/protocol/V2RestJsonRoundtripBenchmark.java index 563ad7fd1dd3..c2d48a398ae1 100644 --- a/test/sdk-benchmarks/src/main/java/software/amazon/awssdk/benchmark/apicall/protocol/V2RestJsonRoundtripBenchmark.java +++ b/test/sdk-benchmarks/src/main/java/software/amazon/awssdk/benchmark/apicall/protocol/V2RestJsonRoundtripBenchmark.java @@ -31,6 +31,7 @@ import org.openjdk.jmh.annotations.TearDown; import org.openjdk.jmh.annotations.Warmup; import org.openjdk.jmh.infra.Blackhole; +import software.amazon.awssdk.benchmark.utils.MockHttpServer; import software.amazon.awssdk.auth.credentials.AwsBasicCredentials; import software.amazon.awssdk.auth.credentials.StaticCredentialsProvider; import software.amazon.awssdk.http.apache5.Apache5HttpClient; @@ -54,16 +55,14 @@ @OutputTimeUnit(TimeUnit.SECONDS) public class V2RestJsonRoundtripBenchmark { - private ProtocolRoundtripServer server; + private MockHttpServer server; private LambdaClient client; @Setup(Level.Trial) public void setup() throws Exception { - byte[] response = ProtocolRoundtripServer.loadFixture("rest-json-protocol/createfunction-response.json"); + byte[] response = MockHttpServer.loadFixture("rest-json-protocol/createfunction-response.json"); - ProtocolRoundtripServlet servlet = new ProtocolRoundtripServlet(response, "application/json"); - - server = new ProtocolRoundtripServer(servlet); + server = new MockHttpServer(response, "application/json"); server.start(); client = LambdaClient.builder() diff --git a/test/sdk-benchmarks/src/main/java/software/amazon/awssdk/benchmark/apicall/protocol/V2RestXmlRoundtripBenchmark.java b/test/sdk-benchmarks/src/main/java/software/amazon/awssdk/benchmark/apicall/protocol/V2RestXmlRoundtripBenchmark.java index 04cdea1b56ef..61af837142b4 100644 --- a/test/sdk-benchmarks/src/main/java/software/amazon/awssdk/benchmark/apicall/protocol/V2RestXmlRoundtripBenchmark.java +++ b/test/sdk-benchmarks/src/main/java/software/amazon/awssdk/benchmark/apicall/protocol/V2RestXmlRoundtripBenchmark.java @@ -29,6 +29,7 @@ import org.openjdk.jmh.annotations.TearDown; import org.openjdk.jmh.annotations.Warmup; import org.openjdk.jmh.infra.Blackhole; +import software.amazon.awssdk.benchmark.utils.MockHttpServer; import software.amazon.awssdk.auth.credentials.AwsBasicCredentials; import software.amazon.awssdk.auth.credentials.StaticCredentialsProvider; import software.amazon.awssdk.http.apache5.Apache5HttpClient; @@ -57,16 +58,14 @@ @OutputTimeUnit(TimeUnit.SECONDS) public class V2RestXmlRoundtripBenchmark { - private ProtocolRoundtripServer server; + private MockHttpServer server; private CloudFrontClient client; @Setup(Level.Trial) public void setup() throws Exception { - byte[] response = ProtocolRoundtripServer.loadFixture("rest-xml-protocol/create-distribution-response.xml"); + byte[] response = MockHttpServer.loadFixture("rest-xml-protocol/create-distribution-response.xml"); - ProtocolRoundtripServlet servlet = new ProtocolRoundtripServlet(response, "text/xml"); - - server = new ProtocolRoundtripServer(servlet); + server = new MockHttpServer(response, "text/xml"); server.start(); client = CloudFrontClient.builder() diff --git a/test/sdk-benchmarks/src/main/java/software/amazon/awssdk/benchmark/coldstart/ColdStartRequests.java b/test/sdk-benchmarks/src/main/java/software/amazon/awssdk/benchmark/coldstart/ColdStartRequests.java deleted file mode 100644 index 4d2f2d7408bd..000000000000 --- a/test/sdk-benchmarks/src/main/java/software/amazon/awssdk/benchmark/coldstart/ColdStartRequests.java +++ /dev/null @@ -1,70 +0,0 @@ -/* - * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. - * - * Licensed under the Apache License, Version 2.0 (the "License"). - * You may not use this file except in compliance with the License. - * A copy of the License is located at - * - * http://aws.amazon.com/apache2.0 - * - * or in the "license" file accompanying this file. This file is distributed - * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either - * express or implied. See the License for the specific language governing - * permissions and limitations under the License. - */ - -package software.amazon.awssdk.benchmark.coldstart; - -import java.util.HashMap; -import java.util.Map; -import software.amazon.awssdk.core.SdkBytes; -import software.amazon.awssdk.services.dynamodb.model.AttributeValue; -import software.amazon.awssdk.services.dynamodb.model.PutItemRequest; - -/** - * The request used by every cold-start arm, so the arms differ only in whether {@code SdkWarmUp.prime} ran. - * - *

The item shape matches {@code fixtures/json-protocol/putitem-response.json} and the item built by - * {@code apicall.protocol.V2JsonRoundtripBenchmark}, covering the scalar, set, map and list attribute types so the - * marshaller and unmarshaller do a representative amount of first-call work. - */ -final class ColdStartRequests { - - static final String FIXTURE = "json-protocol/putitem-response.json"; - static final String CONTENT_TYPE = "application/x-amz-json-1.0"; - - private ColdStartRequests() { - } - - static PutItemRequest putItemRequest() { - return PutItemRequest.builder() - .tableName("benchmark-table") - .item(itemMap()) - .build(); - } - - private static Map itemMap() { - Map item = new HashMap<>(); - item.put("pk", AttributeValue.fromS("benchmark-key")); - item.put("sk", AttributeValue.fromN("100")); - item.put("stringField", AttributeValue.fromS("test-value")); - item.put("numberField", AttributeValue.fromN("123.456")); - item.put("binaryField", AttributeValue.fromB(SdkBytes.fromUtf8String("hello world"))); - item.put("stringSetField", AttributeValue.builder().ss("value1", "value2", "value3").build()); - item.put("numberSetField", AttributeValue.builder().ns("1.1", "2.2", "3.3").build()); - item.put("boolField", AttributeValue.fromBool(false)); - item.put("nullField", AttributeValue.builder().nul(true).build()); - Map deep = new HashMap<>(); - deep.put("level2", AttributeValue.fromN("999")); - Map nested = new HashMap<>(); - nested.put("nested", AttributeValue.fromS("nested-value")); - nested.put("deepNested", AttributeValue.fromM(deep)); - item.put("mapField", AttributeValue.fromM(nested)); - item.put("listField", AttributeValue.builder().l( - AttributeValue.fromS("item1"), - AttributeValue.fromN("42"), - AttributeValue.fromBool(true), - AttributeValue.builder().nul(true).build()).build()); - return item; - } -} diff --git a/test/sdk-benchmarks/src/main/java/software/amazon/awssdk/benchmark/coldstart/SdkClientColdStartBenchmark.java b/test/sdk-benchmarks/src/main/java/software/amazon/awssdk/benchmark/coldstart/SdkClientColdStartBenchmark.java deleted file mode 100644 index 4b198cfa807e..000000000000 --- a/test/sdk-benchmarks/src/main/java/software/amazon/awssdk/benchmark/coldstart/SdkClientColdStartBenchmark.java +++ /dev/null @@ -1,30 +0,0 @@ -/* - * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. - * - * Licensed under the Apache License, Version 2.0 (the "License"). - * You may not use this file except in compliance with the License. - * A copy of the License is located at - * - * http://aws.amazon.com/apache2.0 - * - * or in the "license" file accompanying this file. This file is distributed - * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either - * express or implied. See the License for the specific language governing - * permissions and limitations under the License. - */ - -package software.amazon.awssdk.benchmark.coldstart; - -import org.openjdk.jmh.infra.Blackhole; - -/** - * Measured unit for the cold-start benchmarks: build a service client and complete its first operation. - * - *

Separate from {@link SdkClientCreationBenchmark}, whose unit is construction only. The first operation is where the - * marshaller, unmarshaller, signer, endpoint rules and interceptor chain are first exercised, so construction alone would - * not represent what an application pays on its first request. - */ -public interface SdkClientColdStartBenchmark { - - void coldFirstCall(Blackhole blackhole) throws Exception; -} diff --git a/test/sdk-benchmarks/src/main/java/software/amazon/awssdk/benchmark/coldstart/V2ColdStartAfterWarmUpBenchmark.java b/test/sdk-benchmarks/src/main/java/software/amazon/awssdk/benchmark/coldstart/V2ColdStartAfterWarmUpBenchmark.java index 5b820cee78f1..eddfd677c98b 100644 --- a/test/sdk-benchmarks/src/main/java/software/amazon/awssdk/benchmark/coldstart/V2ColdStartAfterWarmUpBenchmark.java +++ b/test/sdk-benchmarks/src/main/java/software/amazon/awssdk/benchmark/coldstart/V2ColdStartAfterWarmUpBenchmark.java @@ -15,11 +15,9 @@ package software.amazon.awssdk.benchmark.coldstart; -import static software.amazon.awssdk.benchmark.coldstart.ColdStartRequests.CONTENT_TYPE; -import static software.amazon.awssdk.benchmark.coldstart.ColdStartRequests.FIXTURE; -import static software.amazon.awssdk.benchmark.coldstart.ColdStartRequests.putItemRequest; - import java.util.Collection; +import java.util.HashMap; +import java.util.Map; import java.util.concurrent.TimeUnit; import org.openjdk.jmh.annotations.Benchmark; import org.openjdk.jmh.annotations.BenchmarkMode; @@ -44,34 +42,19 @@ import org.openjdk.jmh.runner.options.OptionsBuilder; import software.amazon.awssdk.auth.credentials.AwsBasicCredentials; import software.amazon.awssdk.auth.credentials.StaticCredentialsProvider; +import software.amazon.awssdk.benchmark.utils.MockHttpServer; +import software.amazon.awssdk.core.SdkBytes; import software.amazon.awssdk.core.crac.SdkWarmUp; import software.amazon.awssdk.http.apache5.Apache5HttpClient; import software.amazon.awssdk.regions.Region; import software.amazon.awssdk.services.dynamodb.DynamoDbClient; +import software.amazon.awssdk.services.dynamodb.model.AttributeValue; +import software.amazon.awssdk.services.dynamodb.model.PutItemRequest; /** - * Primed arm: identical to {@link V2ColdStartNoWarmUpBenchmark} except that {@link SdkWarmUp#prime(Class[])} runs in - * {@code @Setup}, outside the timing window. The measured number is therefore post-prime construction plus post-prime - * first call, and the difference from the baseline arm is the first-call work that priming front-loads. - * - *

Uses the targeted overload rather than {@code SdkWarmUp.prime()}. The no-arg form warms every - * {@code SdkWarmUpProvider} on the classpath, which on this module's classpath includes services unrelated to this - * measurement. - * - *

Priming also warms each sync {@code SdkHttpService} on the classpath by sending a {@code GET} to the regional STS - * endpoint, which is intended: exercising a real request is how the SPI lookup, TLS stack and connection machinery get - * initialized ahead of time. Those calls happen in {@code @Setup}, outside the measured window, and are best effort: - * a failure never fails the benchmark, but it does affect the score. On a host without network access the HTTP client - * stack is only partially warmed, the measured first call is slower, and the primed-vs-no-prime delta silently shrinks. - * Only compare the two arms when both ran in the same network environment. The cost of priming itself is measured - * separately by {@link V2SdkWarmUpExecutionTimeBenchmark}. - * - *

The provider primes DynamoDB with {@code ListBackups} while the measured call is {@code PutItem}. That is - * deliberate: the measured delta is the cross-operation transfer benefit, the realistic case, since shared work - * (signing, HTTP, base marshalling) dominates over operation-specific marshallers. - * - *

See {@link V2ColdStartNoWarmUpBenchmark} for why the JMH parameters below differ from the rest of this module and why - * they must not be overridden from the command line. + * Same measurement as {@link V2ColdStartNoWarmUpBenchmark}, but {@link SdkWarmUp#prime(Class[])} runs in {@code @Setup} + * (untimed, needs network for the STS warm-up GET). The score difference between the two is the first-call work that + * warm-up front-loads. See {@link V2SdkWarmUpExecutionTimeBenchmark} for how long prime() itself takes. */ @State(Scope.Benchmark) @BenchmarkMode(Mode.SingleShotTime) @@ -79,20 +62,21 @@ @Warmup(iterations = 0) @Measurement(iterations = 1) @Fork(20) -public class V2ColdStartAfterWarmUpBenchmark implements SdkClientColdStartBenchmark { +public class V2ColdStartAfterWarmUpBenchmark { + + private static final String FIXTURE = "json-protocol/putitem-response.json"; + private static final String CONTENT_TYPE = "application/x-amz-json-1.0"; - private ColdStartMockServer server; + private MockHttpServer server; private DynamoDbClient client; @Setup(Level.Trial) public void setup() throws Exception { - server = new ColdStartMockServer(ColdStartMockServer.loadFixture(FIXTURE), CONTENT_TYPE); + server = new MockHttpServer(MockHttpServer.loadFixture(FIXTURE), CONTENT_TYPE); server.start(); SdkWarmUp.prime(DynamoDbClient.class); } - - @Override @Benchmark public void coldFirstCall(Blackhole blackhole) throws Exception { client = DynamoDbClient.builder() @@ -125,4 +109,36 @@ public static void main(String... args) throws RunnerException, CommandLineOptio .build(); Collection run = new Runner(opt).run(); } + + private static PutItemRequest putItemRequest() { + return PutItemRequest.builder() + .tableName("benchmark-table") + .item(itemMap()) + .build(); + } + + private static Map itemMap() { + Map item = new HashMap<>(); + item.put("pk", AttributeValue.fromS("benchmark-key")); + item.put("sk", AttributeValue.fromN("100")); + item.put("stringField", AttributeValue.fromS("test-value")); + item.put("numberField", AttributeValue.fromN("123.456")); + item.put("binaryField", AttributeValue.fromB(SdkBytes.fromUtf8String("hello world"))); + item.put("stringSetField", AttributeValue.builder().ss("value1", "value2", "value3").build()); + item.put("numberSetField", AttributeValue.builder().ns("1.1", "2.2", "3.3").build()); + item.put("boolField", AttributeValue.fromBool(false)); + item.put("nullField", AttributeValue.builder().nul(true).build()); + Map deep = new HashMap<>(); + deep.put("level2", AttributeValue.fromN("999")); + Map nested = new HashMap<>(); + nested.put("nested", AttributeValue.fromS("nested-value")); + nested.put("deepNested", AttributeValue.fromM(deep)); + item.put("mapField", AttributeValue.fromM(nested)); + item.put("listField", AttributeValue.builder().l( + AttributeValue.fromS("item1"), + AttributeValue.fromN("42"), + AttributeValue.fromBool(true), + AttributeValue.builder().nul(true).build()).build()); + return item; + } } diff --git a/test/sdk-benchmarks/src/main/java/software/amazon/awssdk/benchmark/coldstart/V2ColdStartNoWarmUpBenchmark.java b/test/sdk-benchmarks/src/main/java/software/amazon/awssdk/benchmark/coldstart/V2ColdStartNoWarmUpBenchmark.java index 1a66d25dc124..eded601fa2e3 100644 --- a/test/sdk-benchmarks/src/main/java/software/amazon/awssdk/benchmark/coldstart/V2ColdStartNoWarmUpBenchmark.java +++ b/test/sdk-benchmarks/src/main/java/software/amazon/awssdk/benchmark/coldstart/V2ColdStartNoWarmUpBenchmark.java @@ -15,11 +15,9 @@ package software.amazon.awssdk.benchmark.coldstart; -import static software.amazon.awssdk.benchmark.coldstart.ColdStartRequests.CONTENT_TYPE; -import static software.amazon.awssdk.benchmark.coldstart.ColdStartRequests.FIXTURE; -import static software.amazon.awssdk.benchmark.coldstart.ColdStartRequests.putItemRequest; - import java.util.Collection; +import java.util.HashMap; +import java.util.Map; import java.util.concurrent.TimeUnit; import org.openjdk.jmh.annotations.Benchmark; import org.openjdk.jmh.annotations.BenchmarkMode; @@ -44,32 +42,18 @@ import org.openjdk.jmh.runner.options.OptionsBuilder; import software.amazon.awssdk.auth.credentials.AwsBasicCredentials; import software.amazon.awssdk.auth.credentials.StaticCredentialsProvider; +import software.amazon.awssdk.benchmark.utils.MockHttpServer; +import software.amazon.awssdk.core.SdkBytes; import software.amazon.awssdk.http.apache5.Apache5HttpClient; import software.amazon.awssdk.regions.Region; import software.amazon.awssdk.services.dynamodb.DynamoDbClient; +import software.amazon.awssdk.services.dynamodb.model.AttributeValue; +import software.amazon.awssdk.services.dynamodb.model.PutItemRequest; /** - * Baseline arm: build a {@link DynamoDbClient} and complete its first operation in a JVM where - * {@code SdkWarmUp.prime} never ran. Compare against {@link V2ColdStartAfterWarmUpBenchmark}, whose only difference is that - * priming happened in {@code @Setup}, outside the timing window. - * - *

The operation goes to an in-process Jetty server on localhost over plain HTTP with an explicit endpoint override and - * static credentials, so the measurement excludes DNS, TLS to AWS and service latency, and isolates SDK-side - * initialization: class loading, builder and configuration resolution, interceptor chain assembly, endpoint rule - * evaluation, auth scheme and signer setup, marshaller and unmarshaller construction, and HTTP client construction. - * - *

The JMH parameters below deliberately differ from every other benchmark in this module. - * {@code SdkWarmUp} keeps its primed state in statics with no reset hook, so priming does real work at most once per JVM - * and only the first invocation after it is genuinely cold. {@link Mode#SingleShotTime} with zero warmup iterations and a - * single measurement iteration yields exactly one measured invocation per fork; the high fork count supplies the samples. - * For {@code SingleShotTime} a JMH warmup iteration is itself a full invocation, so zero warmup is required, not cosmetic. - * - *

Do not override these from the command line. JMH CLI flags such as {@code -wi 3 -i 3 -f 1} take precedence - * over these annotations and would silently make the measured invocations warm, which makes both arms converge. - * - *

This benchmark cannot speak to Lambda SnapStart economics. Under SnapStart the priming cost is paid once at - * checkpoint and amortized across every restore; JMH has no checkpoint or restore, so here it is paid in the same JVM - * that is measured. + * Baseline: build a {@link DynamoDbClient} and make its first call (to a local mock server) with no warm-up. + * Compare against {@link V2ColdStartAfterWarmUpBenchmark}. Single-shot, zero warmup, high fork count: only the first + * invocation per JVM is cold. Do not override these JMH parameters from the CLI, or both benchmarks converge. */ @State(Scope.Benchmark) @BenchmarkMode(Mode.SingleShotTime) @@ -77,18 +61,20 @@ @Warmup(iterations = 0) @Measurement(iterations = 1) @Fork(20) -public class V2ColdStartNoWarmUpBenchmark implements SdkClientColdStartBenchmark { +public class V2ColdStartNoWarmUpBenchmark { - private ColdStartMockServer server; + private static final String FIXTURE = "json-protocol/putitem-response.json"; + private static final String CONTENT_TYPE = "application/x-amz-json-1.0"; + + private MockHttpServer server; private DynamoDbClient client; @Setup(Level.Trial) public void setup() throws Exception { - server = new ColdStartMockServer(ColdStartMockServer.loadFixture(FIXTURE), CONTENT_TYPE); + server = new MockHttpServer(MockHttpServer.loadFixture(FIXTURE), CONTENT_TYPE); server.start(); } - @Override @Benchmark public void coldFirstCall(Blackhole blackhole) throws Exception { client = DynamoDbClient.builder() @@ -121,4 +107,36 @@ public static void main(String... args) throws RunnerException, CommandLineOptio .build(); Collection run = new Runner(opt).run(); } + + private static PutItemRequest putItemRequest() { + return PutItemRequest.builder() + .tableName("benchmark-table") + .item(itemMap()) + .build(); + } + + private static Map itemMap() { + Map item = new HashMap<>(); + item.put("pk", AttributeValue.fromS("benchmark-key")); + item.put("sk", AttributeValue.fromN("100")); + item.put("stringField", AttributeValue.fromS("test-value")); + item.put("numberField", AttributeValue.fromN("123.456")); + item.put("binaryField", AttributeValue.fromB(SdkBytes.fromUtf8String("hello world"))); + item.put("stringSetField", AttributeValue.builder().ss("value1", "value2", "value3").build()); + item.put("numberSetField", AttributeValue.builder().ns("1.1", "2.2", "3.3").build()); + item.put("boolField", AttributeValue.fromBool(false)); + item.put("nullField", AttributeValue.builder().nul(true).build()); + Map deep = new HashMap<>(); + deep.put("level2", AttributeValue.fromN("999")); + Map nested = new HashMap<>(); + nested.put("nested", AttributeValue.fromS("nested-value")); + nested.put("deepNested", AttributeValue.fromM(deep)); + item.put("mapField", AttributeValue.fromM(nested)); + item.put("listField", AttributeValue.builder().l( + AttributeValue.fromS("item1"), + AttributeValue.fromN("42"), + AttributeValue.fromBool(true), + AttributeValue.builder().nul(true).build()).build()); + return item; + } } diff --git a/test/sdk-benchmarks/src/main/java/software/amazon/awssdk/benchmark/coldstart/V2SdkWarmUpExecutionTimeBenchmark.java b/test/sdk-benchmarks/src/main/java/software/amazon/awssdk/benchmark/coldstart/V2SdkWarmUpExecutionTimeBenchmark.java index eff3acb651b8..7904b539e0d8 100644 --- a/test/sdk-benchmarks/src/main/java/software/amazon/awssdk/benchmark/coldstart/V2SdkWarmUpExecutionTimeBenchmark.java +++ b/test/sdk-benchmarks/src/main/java/software/amazon/awssdk/benchmark/coldstart/V2SdkWarmUpExecutionTimeBenchmark.java @@ -38,16 +38,9 @@ import software.amazon.awssdk.services.dynamodb.DynamoDbClient; /** - * Measures how long {@link SdkWarmUp#prime(Class[])} takes to run: the one-time upfront work an application pays - * during initialization to get the first-call saving that {@link V2ColdStartAfterWarmUpBenchmark} shows against - * {@link V2ColdStartNoWarmUpBenchmark}. - * - *

Priming makes a real STS {@code GET} per sync HTTP client on the classpath, so the score includes network time, - * varies by host, and is an upper bound (this module has several HTTP clients; typical apps have one). Do not add it - * to {@code baseline.json}. - * - *

See {@link V2ColdStartNoWarmUpBenchmark} for the single-shot JMH parameters. Priming is once-per-JVM, so only the - * first invocation in a fork is a real measurement. + * Measures how long {@link SdkWarmUp#prime(Class[])} takes: the upfront price for the first-call saving shown by + * {@link V2ColdStartAfterWarmUpBenchmark}. The score includes real network time (STS GET per HTTP client on the + * classpath), so it varies by host — do not add it to {@code baseline.json}. */ @State(Scope.Benchmark) @BenchmarkMode(Mode.SingleShotTime) diff --git a/test/sdk-benchmarks/src/main/java/software/amazon/awssdk/benchmark/coldstart/ColdStartMockServer.java b/test/sdk-benchmarks/src/main/java/software/amazon/awssdk/benchmark/utils/MockHttpServer.java similarity index 70% rename from test/sdk-benchmarks/src/main/java/software/amazon/awssdk/benchmark/coldstart/ColdStartMockServer.java rename to test/sdk-benchmarks/src/main/java/software/amazon/awssdk/benchmark/utils/MockHttpServer.java index 818956ff1a36..17ae56221a34 100644 --- a/test/sdk-benchmarks/src/main/java/software/amazon/awssdk/benchmark/coldstart/ColdStartMockServer.java +++ b/test/sdk-benchmarks/src/main/java/software/amazon/awssdk/benchmark/utils/MockHttpServer.java @@ -13,7 +13,7 @@ * permissions and limitations under the License. */ -package software.amazon.awssdk.benchmark.coldstart; +package software.amazon.awssdk.benchmark.utils; import java.io.IOException; import java.io.InputStream; @@ -26,26 +26,19 @@ import org.eclipse.jetty.server.ServerConnector; import org.eclipse.jetty.servlet.ServletContextHandler; import org.eclipse.jetty.servlet.ServletHolder; -import software.amazon.awssdk.benchmark.utils.BenchmarkUtils; import software.amazon.awssdk.utils.IoUtils; /** - * Lightweight plain-HTTP Jetty server that answers every request with a fixed status, body and content type, so a real - * service client can complete an operation without leaving the machine. - * - *

This mirrors {@code apicall.protocol.ProtocolRoundtripServer}, which is package-private to its own package. It is - * duplicated here rather than shared so the cold-start benchmarks do not force a visibility change on the throughput - * benchmarks that use the original. - * - *

Only the plain HTTP connector is exposed. That is deliberate: the cold-start benchmarks take a single measurement per - * JVM, and TLS handshake plus trust-all keystore setup would add variance to the one sample that matters. + * Lightweight plain-HTTP Jetty server that answers every request with a fixed 200 status, body and content type, so a + * real service client can complete an operation without leaving the machine. Used by the protocol roundtrip and + * cold-start benchmarks. */ -final class ColdStartMockServer { +public final class MockHttpServer { private final Server server; private final int port; - ColdStartMockServer(byte[] responseBody, String contentType) throws IOException { + public MockHttpServer(byte[] responseBody, String contentType) throws IOException { port = BenchmarkUtils.getUnusedPort(); server = new Server(); ServerConnector connector = new ServerConnector(server); @@ -57,20 +50,20 @@ final class ColdStartMockServer { server.setHandler(context); } - void start() throws Exception { + public void start() throws Exception { server.start(); } - void stop() throws Exception { + public void stop() throws Exception { server.stop(); } - URI getHttpUri() { + public URI getHttpUri() { return URI.create("http://localhost:" + port); } - static byte[] loadFixture(String path) throws IOException { - try (InputStream is = ColdStartMockServer.class.getClassLoader().getResourceAsStream("fixtures/" + path)) { + public static byte[] loadFixture(String path) throws IOException { + try (InputStream is = MockHttpServer.class.getClassLoader().getResourceAsStream("fixtures/" + path)) { if (is == null) { throw new IOException("Fixture not found: fixtures/" + path); }