Skip to content

test(sdk-benchmarks): Add cold-start benchmarks for SdkWarmUp priming #7207

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
joviegas wants to merge 2 commits into
base: feature/master/crac_auto_priming_support
Choose a base branch
from
Open

test(sdk-benchmarks): Add cold-start benchmarks for SdkWarmUp priming #7207

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
6bcf170
test(sdk-benchmarks): Add cold-start benchmarks for SdkWarmUp priming
joviegas Jul 29, 2026
9058f36
refactor(sdk-benchmarks): Unify duplicate Jetty mock servers into Moc…
joviegas Jul 29, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
8 changes: 7 additions & 1 deletion test/sdk-benchmarks/src/main/java/software/amazon/awssdk/benchmark/BenchmarkRunner.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -51,8 +51,11 @@
import software.amazon.awssdk.benchmark.apicall.protocol.QueryProtocolBenchmark;
import software.amazon.awssdk.benchmark.apicall.protocol.SmithyRpcV2ProtocolBenchmark;
import software.amazon.awssdk.benchmark.apicall.protocol.XmlProtocolBenchmark;
import software.amazon.awssdk.benchmark.coldstart.V2ColdStartAfterWarmUpBenchmark;
import software.amazon.awssdk.benchmark.coldstart.V2ColdStartNoWarmUpBenchmark;
import software.amazon.awssdk.benchmark.coldstart.V2DefaultClientCreationBenchmark;
import software.amazon.awssdk.benchmark.coldstart.V2OptimizedClientCreationBenchmark;
import software.amazon.awssdk.benchmark.coldstart.V2SdkWarmUpExecutionTimeBenchmark;
import software.amazon.awssdk.benchmark.enhanced.dynamodb.EnhancedClientDeleteV1MapperComparisonBenchmark;
import software.amazon.awssdk.benchmark.enhanced.dynamodb.EnhancedClientGetOverheadBenchmark;
import software.amazon.awssdk.benchmark.enhanced.dynamodb.EnhancedClientGetV1MapperComparisonBenchmark;
Expand Down Expand Up @@ -87,7 +90,10 @@ public class BenchmarkRunner {

private static final List<String> COLD_START_BENCHMARKS = Arrays.asList(
V2OptimizedClientCreationBenchmark.class.getSimpleName(),
V2DefaultClientCreationBenchmark.class.getSimpleName());
V2DefaultClientCreationBenchmark.class.getSimpleName(),
V2ColdStartNoWarmUpBenchmark.class.getSimpleName(),
V2ColdStartAfterWarmUpBenchmark.class.getSimpleName(),
V2SdkWarmUpExecutionTimeBenchmark.class.getSimpleName());

private static final List<String> MAPPER_BENCHMARKS = Arrays.asList(
EnhancedClientGetOverheadBenchmark.class.getSimpleName(),
Expand Down
70 changes: 0 additions & 70 deletions .../main/java/software/amazon/awssdk/benchmark/apicall/protocol/ProtocolRoundtripServer.java
View file
Open in desktop

This file was deleted.

39 changes: 0 additions & 39 deletions ...main/java/software/amazon/awssdk/benchmark/apicall/protocol/ProtocolRoundtripServlet.java
View file
Open in desktop

This file was deleted.

7 changes: 3 additions & 4 deletions ...main/java/software/amazon/awssdk/benchmark/apicall/protocol/V1CborRoundtripBenchmark.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -41,6 +41,7 @@
import org.openjdk.jmh.annotations.TearDown;
import org.openjdk.jmh.annotations.Warmup;
import org.openjdk.jmh.infra.Blackhole;
import software.amazon.awssdk.benchmark.utils.MockHttpServer;

/**
* V1 roundtrip benchmark for SmithyRpcV2 CBOR protocol using CloudWatch GetMetricData via HTTP servlet.
Expand All @@ -53,16 +54,14 @@
@OutputTimeUnit(TimeUnit.SECONDS)
public class V1CborRoundtripBenchmark {

private ProtocolRoundtripServer server;
private MockHttpServer server;
private AmazonCloudWatch client;

@Setup(Level.Trial)
public void setup() throws Exception {
byte[] response = createCborResponseFixture();

ProtocolRoundtripServlet servlet = new ProtocolRoundtripServlet(response, "application/cbor");

server = new ProtocolRoundtripServer(servlet);
server = new MockHttpServer(response, "application/cbor");
server.start();

client = AmazonCloudWatchClientBuilder.standard()
Expand Down
9 changes: 4 additions & 5 deletions .../main/java/software/amazon/awssdk/benchmark/apicall/protocol/V1Ec2RoundtripBenchmark.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -36,6 +36,7 @@
import org.openjdk.jmh.annotations.TearDown;
import org.openjdk.jmh.annotations.Warmup;
import org.openjdk.jmh.infra.Blackhole;
import software.amazon.awssdk.benchmark.utils.MockHttpServer;

/**
* V1 roundtrip benchmark for EC2 protocol using EC2 DescribeInstances via HTTP servlet.
Expand All @@ -48,16 +49,14 @@
@OutputTimeUnit(TimeUnit.SECONDS)
public class V1Ec2RoundtripBenchmark {

private ProtocolRoundtripServer server;
private MockHttpServer server;
private AmazonEC2 client;

@Setup(Level.Trial)
public void setup() throws Exception {
byte[] response = ProtocolRoundtripServer.loadFixture("ec2-protocol/describe-instances-response.xml");
byte[] response = MockHttpServer.loadFixture("ec2-protocol/describe-instances-response.xml");

ProtocolRoundtripServlet servlet = new ProtocolRoundtripServlet(response, "text/xml");

server = new ProtocolRoundtripServer(servlet);
server = new MockHttpServer(response, "text/xml");
server.start();

client = AmazonEC2ClientBuilder.standard()
Expand Down
9 changes: 4 additions & 5 deletions ...main/java/software/amazon/awssdk/benchmark/apicall/protocol/V1JsonRoundtripBenchmark.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -39,6 +39,7 @@
import org.openjdk.jmh.annotations.TearDown;
import org.openjdk.jmh.annotations.Warmup;
import org.openjdk.jmh.infra.Blackhole;
import software.amazon.awssdk.benchmark.utils.MockHttpServer;

/**
* V1 roundtrip benchmark for JSON protocol (aws-json) using DynamoDB PutItem via HTTP servlet.
Expand All @@ -51,16 +52,14 @@
@OutputTimeUnit(TimeUnit.SECONDS)
public class V1JsonRoundtripBenchmark {

private ProtocolRoundtripServer server;
private MockHttpServer server;
private AmazonDynamoDB client;

@Setup(Level.Trial)
public void setup() throws Exception {
byte[] response = ProtocolRoundtripServer.loadFixture("json-protocol/putitem-response.json");
byte[] response = MockHttpServer.loadFixture("json-protocol/putitem-response.json");

ProtocolRoundtripServlet servlet = new ProtocolRoundtripServlet(response, "application/x-amz-json-1.0");

server = new ProtocolRoundtripServer(servlet);
server = new MockHttpServer(response, "application/x-amz-json-1.0");
server.start();

client = AmazonDynamoDBClientBuilder.standard()
Expand Down
9 changes: 4 additions & 5 deletions ...ain/java/software/amazon/awssdk/benchmark/apicall/protocol/V1QueryRoundtripBenchmark.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -35,6 +35,7 @@
import org.openjdk.jmh.annotations.TearDown;
import org.openjdk.jmh.annotations.Warmup;
import org.openjdk.jmh.infra.Blackhole;
import software.amazon.awssdk.benchmark.utils.MockHttpServer;

/**
* V1 roundtrip benchmark for Query protocol using STS AssumeRole via HTTP servlet.
Expand All @@ -47,16 +48,14 @@
@OutputTimeUnit(TimeUnit.SECONDS)
public class V1QueryRoundtripBenchmark {

private ProtocolRoundtripServer server;
private MockHttpServer server;
private AWSSecurityTokenService client;

@Setup(Level.Trial)
public void setup() throws Exception {
byte[] response = ProtocolRoundtripServer.loadFixture("query-protocol/assumerole-response.xml");
byte[] response = MockHttpServer.loadFixture("query-protocol/assumerole-response.xml");

ProtocolRoundtripServlet servlet = new ProtocolRoundtripServlet(response, "text/xml");

server = new ProtocolRoundtripServer(servlet);
server = new MockHttpServer(response, "text/xml");
server.start();

client = AWSSecurityTokenServiceClientBuilder.standard()
Expand Down
9 changes: 4 additions & 5 deletions .../java/software/amazon/awssdk/benchmark/apicall/protocol/V1RestJsonRoundtripBenchmark.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -42,6 +42,7 @@
import org.openjdk.jmh.annotations.TearDown;
import org.openjdk.jmh.annotations.Warmup;
import org.openjdk.jmh.infra.Blackhole;
import software.amazon.awssdk.benchmark.utils.MockHttpServer;

/**
* V1 roundtrip benchmark for REST-JSON protocol using Lambda CreateFunction via HTTP servlet.
Expand All @@ -54,16 +55,14 @@
@OutputTimeUnit(TimeUnit.SECONDS)
public class V1RestJsonRoundtripBenchmark {

private ProtocolRoundtripServer server;
private MockHttpServer server;
private AWSLambda client;

@Setup(Level.Trial)
public void setup() throws Exception {
byte[] response = ProtocolRoundtripServer.loadFixture("rest-json-protocol/createfunction-response.json");
byte[] response = MockHttpServer.loadFixture("rest-json-protocol/createfunction-response.json");

ProtocolRoundtripServlet servlet = new ProtocolRoundtripServlet(response, "application/json");

server = new ProtocolRoundtripServer(servlet);
server = new MockHttpServer(response, "application/json");
server.start();

client = AWSLambdaClientBuilder.standard()
Expand Down
9 changes: 4 additions & 5 deletions ...n/java/software/amazon/awssdk/benchmark/apicall/protocol/V1RestXmlRoundtripBenchmark.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -45,6 +45,7 @@
import org.openjdk.jmh.annotations.TearDown;
import org.openjdk.jmh.annotations.Warmup;
import org.openjdk.jmh.infra.Blackhole;
import software.amazon.awssdk.benchmark.utils.MockHttpServer;

/**
* V1 roundtrip benchmark for REST-XML protocol using CloudFront CreateDistribution via HTTP servlet.
Expand All @@ -57,16 +58,14 @@
@OutputTimeUnit(TimeUnit.SECONDS)
public class V1RestXmlRoundtripBenchmark {

private ProtocolRoundtripServer server;
private MockHttpServer server;
private AmazonCloudFront client;

@Setup(Level.Trial)
public void setup() throws Exception {
byte[] response = ProtocolRoundtripServer.loadFixture("rest-xml-protocol/create-distribution-response.xml");
byte[] response = MockHttpServer.loadFixture("rest-xml-protocol/create-distribution-response.xml");

ProtocolRoundtripServlet servlet = new ProtocolRoundtripServlet(response, "text/xml");

server = new ProtocolRoundtripServer(servlet);
server = new MockHttpServer(response, "text/xml");
server.start();

client = AmazonCloudFrontClientBuilder.standard()
Expand Down
7 changes: 3 additions & 4 deletions ...main/java/software/amazon/awssdk/benchmark/apicall/protocol/V2CborRoundtripBenchmark.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -30,6 +30,7 @@
import org.openjdk.jmh.annotations.TearDown;
import org.openjdk.jmh.annotations.Warmup;
import org.openjdk.jmh.infra.Blackhole;
import software.amazon.awssdk.benchmark.utils.MockHttpServer;
import software.amazon.awssdk.auth.credentials.AwsBasicCredentials;
import software.amazon.awssdk.auth.credentials.StaticCredentialsProvider;
import software.amazon.awssdk.http.apache5.Apache5HttpClient;
Expand All @@ -53,16 +54,14 @@
@OutputTimeUnit(TimeUnit.SECONDS)
public class V2CborRoundtripBenchmark {

private ProtocolRoundtripServer server;
private MockHttpServer server;
private CloudWatchClient client;

@Setup(Level.Trial)
public void setup() throws Exception {
byte[] response = createCborResponseFixture();

ProtocolRoundtripServlet servlet = new ProtocolRoundtripServlet(response, "application/cbor");

server = new ProtocolRoundtripServer(servlet);
server = new MockHttpServer(response, "application/cbor");
server.start();

client = CloudWatchClient.builder()
Expand Down
9 changes: 4 additions & 5 deletions .../main/java/software/amazon/awssdk/benchmark/apicall/protocol/V2Ec2RoundtripBenchmark.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -29,6 +29,7 @@
import org.openjdk.jmh.annotations.TearDown;
import org.openjdk.jmh.annotations.Warmup;
import org.openjdk.jmh.infra.Blackhole;
import software.amazon.awssdk.benchmark.utils.MockHttpServer;
import software.amazon.awssdk.auth.credentials.AwsBasicCredentials;
import software.amazon.awssdk.auth.credentials.StaticCredentialsProvider;
import software.amazon.awssdk.http.apache5.Apache5HttpClient;
Expand All @@ -48,16 +49,14 @@
@OutputTimeUnit(TimeUnit.SECONDS)
public class V2Ec2RoundtripBenchmark {

private ProtocolRoundtripServer server;
private MockHttpServer server;
private Ec2Client client;

@Setup(Level.Trial)
public void setup() throws Exception {
byte[] response = ProtocolRoundtripServer.loadFixture("ec2-protocol/describe-instances-response.xml");
byte[] response = MockHttpServer.loadFixture("ec2-protocol/describe-instances-response.xml");

ProtocolRoundtripServlet servlet = new ProtocolRoundtripServlet(response, "text/xml");

server = new ProtocolRoundtripServer(servlet);
server = new MockHttpServer(response, "text/xml");
server.start();

client = Ec2Client.builder()
Expand Down
9 changes: 4 additions & 5 deletions ...main/java/software/amazon/awssdk/benchmark/apicall/protocol/V2JsonRoundtripBenchmark.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -31,6 +31,7 @@
import org.openjdk.jmh.annotations.TearDown;
import org.openjdk.jmh.annotations.Warmup;
import org.openjdk.jmh.infra.Blackhole;
import software.amazon.awssdk.benchmark.utils.MockHttpServer;
import software.amazon.awssdk.auth.credentials.AwsBasicCredentials;
import software.amazon.awssdk.auth.credentials.StaticCredentialsProvider;
import software.amazon.awssdk.core.SdkBytes;
Expand All @@ -51,16 +52,14 @@
@OutputTimeUnit(TimeUnit.SECONDS)
public class V2JsonRoundtripBenchmark {

private ProtocolRoundtripServer server;
private MockHttpServer server;
private DynamoDbClient client;

@Setup(Level.Trial)
public void setup() throws Exception {
byte[] response = ProtocolRoundtripServer.loadFixture("json-protocol/putitem-response.json");
byte[] response = MockHttpServer.loadFixture("json-protocol/putitem-response.json");

ProtocolRoundtripServlet servlet = new ProtocolRoundtripServlet(response, "application/x-amz-json-1.0");

server = new ProtocolRoundtripServer(servlet);
server = new MockHttpServer(response, "application/x-amz-json-1.0");
server.start();

client = DynamoDbClient.builder()
Expand Down
Loading
Loading