diff --git a/test/sdk-benchmarks/src/main/java/software/amazon/awssdk/benchmark/BenchmarkRunner.java b/test/sdk-benchmarks/src/main/java/software/amazon/awssdk/benchmark/BenchmarkRunner.java index 0b28b61a0e4e..82e17212d0de 100644 --- a/test/sdk-benchmarks/src/main/java/software/amazon/awssdk/benchmark/BenchmarkRunner.java +++ b/test/sdk-benchmarks/src/main/java/software/amazon/awssdk/benchmark/BenchmarkRunner.java @@ -51,8 +51,11 @@ import software.amazon.awssdk.benchmark.apicall.protocol.QueryProtocolBenchmark; import software.amazon.awssdk.benchmark.apicall.protocol.SmithyRpcV2ProtocolBenchmark; import software.amazon.awssdk.benchmark.apicall.protocol.XmlProtocolBenchmark; +import software.amazon.awssdk.benchmark.coldstart.V2ColdStartAfterWarmUpBenchmark; +import software.amazon.awssdk.benchmark.coldstart.V2ColdStartNoWarmUpBenchmark; import software.amazon.awssdk.benchmark.coldstart.V2DefaultClientCreationBenchmark; import software.amazon.awssdk.benchmark.coldstart.V2OptimizedClientCreationBenchmark; +import software.amazon.awssdk.benchmark.coldstart.V2SdkWarmUpExecutionTimeBenchmark; import software.amazon.awssdk.benchmark.enhanced.dynamodb.EnhancedClientDeleteV1MapperComparisonBenchmark; import software.amazon.awssdk.benchmark.enhanced.dynamodb.EnhancedClientGetOverheadBenchmark; import software.amazon.awssdk.benchmark.enhanced.dynamodb.EnhancedClientGetV1MapperComparisonBenchmark; @@ -87,7 +90,10 @@ public class BenchmarkRunner { private static final List COLD_START_BENCHMARKS = Arrays.asList( V2OptimizedClientCreationBenchmark.class.getSimpleName(), - V2DefaultClientCreationBenchmark.class.getSimpleName()); + V2DefaultClientCreationBenchmark.class.getSimpleName(), + V2ColdStartNoWarmUpBenchmark.class.getSimpleName(), + V2ColdStartAfterWarmUpBenchmark.class.getSimpleName(), + V2SdkWarmUpExecutionTimeBenchmark.class.getSimpleName()); private static final List MAPPER_BENCHMARKS = Arrays.asList( EnhancedClientGetOverheadBenchmark.class.getSimpleName(), diff --git a/test/sdk-benchmarks/src/main/java/software/amazon/awssdk/benchmark/apicall/protocol/ProtocolRoundtripServer.java b/test/sdk-benchmarks/src/main/java/software/amazon/awssdk/benchmark/apicall/protocol/ProtocolRoundtripServer.java deleted file mode 100644 index 96911c4e9442..000000000000 --- a/test/sdk-benchmarks/src/main/java/software/amazon/awssdk/benchmark/apicall/protocol/ProtocolRoundtripServer.java +++ /dev/null @@ -1,70 +0,0 @@ -/* - * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. - * - * Licensed under the Apache License, Version 2.0 (the "License"). - * You may not use this file except in compliance with the License. - * A copy of the License is located at - * - * http://aws.amazon.com/apache2.0 - * - * or in the "license" file accompanying this file. This file is distributed - * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either - * express or implied. See the License for the specific language governing - * permissions and limitations under the License. - */ - -package software.amazon.awssdk.benchmark.apicall.protocol; - -import java.io.IOException; -import java.io.InputStream; -import java.net.URI; -import org.eclipse.jetty.server.Connector; -import org.eclipse.jetty.server.Server; -import org.eclipse.jetty.server.ServerConnector; -import org.eclipse.jetty.servlet.ServletContextHandler; -import org.eclipse.jetty.servlet.ServletHolder; -import software.amazon.awssdk.benchmark.utils.BenchmarkUtils; -import software.amazon.awssdk.utils.IoUtils; - -/** - * Lightweight Jetty server for protocol roundtrip benchmarks. - */ -class ProtocolRoundtripServer { - - private final Server server; - private final int port; - - ProtocolRoundtripServer(ProtocolRoundtripServlet servlet) throws IOException { - port = BenchmarkUtils.getUnusedPort(); - server = new Server(); - ServerConnector connector = new ServerConnector(server); - connector.setPort(port); - server.setConnectors(new Connector[] {connector}); - - ServletContextHandler context = new ServletContextHandler(server, "/", ServletContextHandler.SESSIONS); - context.addServlet(new ServletHolder(servlet), "/*"); - server.setHandler(context); - } - - void start() throws Exception { - server.start(); - } - - void stop() throws Exception { - server.stop(); - } - - URI getHttpUri() { - return URI.create("http://localhost:" + port); - } - - static byte[] loadFixture(String path) throws IOException { - try (InputStream is = ProtocolRoundtripServer.class.getClassLoader() - .getResourceAsStream("fixtures/" + path)) { - if (is == null) { - throw new IOException("Fixture not found: fixtures/" + path); - } - return IoUtils.toByteArray(is); - } - } -} diff --git a/test/sdk-benchmarks/src/main/java/software/amazon/awssdk/benchmark/apicall/protocol/ProtocolRoundtripServlet.java b/test/sdk-benchmarks/src/main/java/software/amazon/awssdk/benchmark/apicall/protocol/ProtocolRoundtripServlet.java deleted file mode 100644 index abcce14534ad..000000000000 --- a/test/sdk-benchmarks/src/main/java/software/amazon/awssdk/benchmark/apicall/protocol/ProtocolRoundtripServlet.java +++ /dev/null @@ -1,39 +0,0 @@ -/* - * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. - * - * Licensed under the Apache License, Version 2.0 (the "License"). - * You may not use this file except in compliance with the License. - * A copy of the License is located at - * - * http://aws.amazon.com/apache2.0 - * - * or in the "license" file accompanying this file. This file is distributed - * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either - * express or implied. See the License for the specific language governing - * permissions and limitations under the License. - */ - -package software.amazon.awssdk.benchmark.apicall.protocol; - -import java.io.IOException; -import javax.servlet.http.HttpServlet; -import javax.servlet.http.HttpServletRequest; -import javax.servlet.http.HttpServletResponse; - -class ProtocolRoundtripServlet extends HttpServlet { - private final byte[] body; - private final String contentType; - - ProtocolRoundtripServlet(byte[] body, String contentType) { - this.body = body; - this.contentType = contentType; - } - - @Override - protected void service(HttpServletRequest req, HttpServletResponse resp) throws IOException { - resp.setStatus(200); - resp.setContentLength(body.length); - resp.setContentType(contentType); - resp.getOutputStream().write(body); - } -} diff --git a/test/sdk-benchmarks/src/main/java/software/amazon/awssdk/benchmark/apicall/protocol/V1CborRoundtripBenchmark.java b/test/sdk-benchmarks/src/main/java/software/amazon/awssdk/benchmark/apicall/protocol/V1CborRoundtripBenchmark.java index eb7dfccee20c..e54d004cbb59 100644 --- a/test/sdk-benchmarks/src/main/java/software/amazon/awssdk/benchmark/apicall/protocol/V1CborRoundtripBenchmark.java +++ b/test/sdk-benchmarks/src/main/java/software/amazon/awssdk/benchmark/apicall/protocol/V1CborRoundtripBenchmark.java @@ -41,6 +41,7 @@ import org.openjdk.jmh.annotations.TearDown; import org.openjdk.jmh.annotations.Warmup; import org.openjdk.jmh.infra.Blackhole; +import software.amazon.awssdk.benchmark.utils.MockHttpServer; /** * V1 roundtrip benchmark for SmithyRpcV2 CBOR protocol using CloudWatch GetMetricData via HTTP servlet. @@ -53,16 +54,14 @@ @OutputTimeUnit(TimeUnit.SECONDS) public class V1CborRoundtripBenchmark { - private ProtocolRoundtripServer server; + private MockHttpServer server; private AmazonCloudWatch client; @Setup(Level.Trial) public void setup() throws Exception { byte[] response = createCborResponseFixture(); - ProtocolRoundtripServlet servlet = new ProtocolRoundtripServlet(response, "application/cbor"); - - server = new ProtocolRoundtripServer(servlet); + server = new MockHttpServer(response, "application/cbor"); server.start(); client = AmazonCloudWatchClientBuilder.standard() diff --git a/test/sdk-benchmarks/src/main/java/software/amazon/awssdk/benchmark/apicall/protocol/V1Ec2RoundtripBenchmark.java b/test/sdk-benchmarks/src/main/java/software/amazon/awssdk/benchmark/apicall/protocol/V1Ec2RoundtripBenchmark.java index 6ae6df995a99..e3b0b06b5fb1 100644 --- a/test/sdk-benchmarks/src/main/java/software/amazon/awssdk/benchmark/apicall/protocol/V1Ec2RoundtripBenchmark.java +++ b/test/sdk-benchmarks/src/main/java/software/amazon/awssdk/benchmark/apicall/protocol/V1Ec2RoundtripBenchmark.java @@ -36,6 +36,7 @@ import org.openjdk.jmh.annotations.TearDown; import org.openjdk.jmh.annotations.Warmup; import org.openjdk.jmh.infra.Blackhole; +import software.amazon.awssdk.benchmark.utils.MockHttpServer; /** * V1 roundtrip benchmark for EC2 protocol using EC2 DescribeInstances via HTTP servlet. @@ -48,16 +49,14 @@ @OutputTimeUnit(TimeUnit.SECONDS) public class V1Ec2RoundtripBenchmark { - private ProtocolRoundtripServer server; + private MockHttpServer server; private AmazonEC2 client; @Setup(Level.Trial) public void setup() throws Exception { - byte[] response = ProtocolRoundtripServer.loadFixture("ec2-protocol/describe-instances-response.xml"); + byte[] response = MockHttpServer.loadFixture("ec2-protocol/describe-instances-response.xml"); - ProtocolRoundtripServlet servlet = new ProtocolRoundtripServlet(response, "text/xml"); - - server = new ProtocolRoundtripServer(servlet); + server = new MockHttpServer(response, "text/xml"); server.start(); client = AmazonEC2ClientBuilder.standard() diff --git a/test/sdk-benchmarks/src/main/java/software/amazon/awssdk/benchmark/apicall/protocol/V1JsonRoundtripBenchmark.java b/test/sdk-benchmarks/src/main/java/software/amazon/awssdk/benchmark/apicall/protocol/V1JsonRoundtripBenchmark.java index 58146050360c..f0b1bde7450c 100644 --- a/test/sdk-benchmarks/src/main/java/software/amazon/awssdk/benchmark/apicall/protocol/V1JsonRoundtripBenchmark.java +++ b/test/sdk-benchmarks/src/main/java/software/amazon/awssdk/benchmark/apicall/protocol/V1JsonRoundtripBenchmark.java @@ -39,6 +39,7 @@ import org.openjdk.jmh.annotations.TearDown; import org.openjdk.jmh.annotations.Warmup; import org.openjdk.jmh.infra.Blackhole; +import software.amazon.awssdk.benchmark.utils.MockHttpServer; /** * V1 roundtrip benchmark for JSON protocol (aws-json) using DynamoDB PutItem via HTTP servlet. @@ -51,16 +52,14 @@ @OutputTimeUnit(TimeUnit.SECONDS) public class V1JsonRoundtripBenchmark { - private ProtocolRoundtripServer server; + private MockHttpServer server; private AmazonDynamoDB client; @Setup(Level.Trial) public void setup() throws Exception { - byte[] response = ProtocolRoundtripServer.loadFixture("json-protocol/putitem-response.json"); + byte[] response = MockHttpServer.loadFixture("json-protocol/putitem-response.json"); - ProtocolRoundtripServlet servlet = new ProtocolRoundtripServlet(response, "application/x-amz-json-1.0"); - - server = new ProtocolRoundtripServer(servlet); + server = new MockHttpServer(response, "application/x-amz-json-1.0"); server.start(); client = AmazonDynamoDBClientBuilder.standard() diff --git a/test/sdk-benchmarks/src/main/java/software/amazon/awssdk/benchmark/apicall/protocol/V1QueryRoundtripBenchmark.java b/test/sdk-benchmarks/src/main/java/software/amazon/awssdk/benchmark/apicall/protocol/V1QueryRoundtripBenchmark.java index 3da508d4ff5c..318a9269c91e 100644 --- a/test/sdk-benchmarks/src/main/java/software/amazon/awssdk/benchmark/apicall/protocol/V1QueryRoundtripBenchmark.java +++ b/test/sdk-benchmarks/src/main/java/software/amazon/awssdk/benchmark/apicall/protocol/V1QueryRoundtripBenchmark.java @@ -35,6 +35,7 @@ import org.openjdk.jmh.annotations.TearDown; import org.openjdk.jmh.annotations.Warmup; import org.openjdk.jmh.infra.Blackhole; +import software.amazon.awssdk.benchmark.utils.MockHttpServer; /** * V1 roundtrip benchmark for Query protocol using STS AssumeRole via HTTP servlet. @@ -47,16 +48,14 @@ @OutputTimeUnit(TimeUnit.SECONDS) public class V1QueryRoundtripBenchmark { - private ProtocolRoundtripServer server; + private MockHttpServer server; private AWSSecurityTokenService client; @Setup(Level.Trial) public void setup() throws Exception { - byte[] response = ProtocolRoundtripServer.loadFixture("query-protocol/assumerole-response.xml"); + byte[] response = MockHttpServer.loadFixture("query-protocol/assumerole-response.xml"); - ProtocolRoundtripServlet servlet = new ProtocolRoundtripServlet(response, "text/xml"); - - server = new ProtocolRoundtripServer(servlet); + server = new MockHttpServer(response, "text/xml"); server.start(); client = AWSSecurityTokenServiceClientBuilder.standard() diff --git a/test/sdk-benchmarks/src/main/java/software/amazon/awssdk/benchmark/apicall/protocol/V1RestJsonRoundtripBenchmark.java b/test/sdk-benchmarks/src/main/java/software/amazon/awssdk/benchmark/apicall/protocol/V1RestJsonRoundtripBenchmark.java index 5bc6d9ca080f..f0f532b430ea 100644 --- a/test/sdk-benchmarks/src/main/java/software/amazon/awssdk/benchmark/apicall/protocol/V1RestJsonRoundtripBenchmark.java +++ b/test/sdk-benchmarks/src/main/java/software/amazon/awssdk/benchmark/apicall/protocol/V1RestJsonRoundtripBenchmark.java @@ -42,6 +42,7 @@ import org.openjdk.jmh.annotations.TearDown; import org.openjdk.jmh.annotations.Warmup; import org.openjdk.jmh.infra.Blackhole; +import software.amazon.awssdk.benchmark.utils.MockHttpServer; /** * V1 roundtrip benchmark for REST-JSON protocol using Lambda CreateFunction via HTTP servlet. @@ -54,16 +55,14 @@ @OutputTimeUnit(TimeUnit.SECONDS) public class V1RestJsonRoundtripBenchmark { - private ProtocolRoundtripServer server; + private MockHttpServer server; private AWSLambda client; @Setup(Level.Trial) public void setup() throws Exception { - byte[] response = ProtocolRoundtripServer.loadFixture("rest-json-protocol/createfunction-response.json"); + byte[] response = MockHttpServer.loadFixture("rest-json-protocol/createfunction-response.json"); - ProtocolRoundtripServlet servlet = new ProtocolRoundtripServlet(response, "application/json"); - - server = new ProtocolRoundtripServer(servlet); + server = new MockHttpServer(response, "application/json"); server.start(); client = AWSLambdaClientBuilder.standard() diff --git a/test/sdk-benchmarks/src/main/java/software/amazon/awssdk/benchmark/apicall/protocol/V1RestXmlRoundtripBenchmark.java b/test/sdk-benchmarks/src/main/java/software/amazon/awssdk/benchmark/apicall/protocol/V1RestXmlRoundtripBenchmark.java index ff42e19ada68..91d4376c26ae 100644 --- a/test/sdk-benchmarks/src/main/java/software/amazon/awssdk/benchmark/apicall/protocol/V1RestXmlRoundtripBenchmark.java +++ b/test/sdk-benchmarks/src/main/java/software/amazon/awssdk/benchmark/apicall/protocol/V1RestXmlRoundtripBenchmark.java @@ -45,6 +45,7 @@ import org.openjdk.jmh.annotations.TearDown; import org.openjdk.jmh.annotations.Warmup; import org.openjdk.jmh.infra.Blackhole; +import software.amazon.awssdk.benchmark.utils.MockHttpServer; /** * V1 roundtrip benchmark for REST-XML protocol using CloudFront CreateDistribution via HTTP servlet. @@ -57,16 +58,14 @@ @OutputTimeUnit(TimeUnit.SECONDS) public class V1RestXmlRoundtripBenchmark { - private ProtocolRoundtripServer server; + private MockHttpServer server; private AmazonCloudFront client; @Setup(Level.Trial) public void setup() throws Exception { - byte[] response = ProtocolRoundtripServer.loadFixture("rest-xml-protocol/create-distribution-response.xml"); + byte[] response = MockHttpServer.loadFixture("rest-xml-protocol/create-distribution-response.xml"); - ProtocolRoundtripServlet servlet = new ProtocolRoundtripServlet(response, "text/xml"); - - server = new ProtocolRoundtripServer(servlet); + server = new MockHttpServer(response, "text/xml"); server.start(); client = AmazonCloudFrontClientBuilder.standard() diff --git a/test/sdk-benchmarks/src/main/java/software/amazon/awssdk/benchmark/apicall/protocol/V2CborRoundtripBenchmark.java b/test/sdk-benchmarks/src/main/java/software/amazon/awssdk/benchmark/apicall/protocol/V2CborRoundtripBenchmark.java index f2f3e60a740b..a7b0cf78b3a0 100644 --- a/test/sdk-benchmarks/src/main/java/software/amazon/awssdk/benchmark/apicall/protocol/V2CborRoundtripBenchmark.java +++ b/test/sdk-benchmarks/src/main/java/software/amazon/awssdk/benchmark/apicall/protocol/V2CborRoundtripBenchmark.java @@ -30,6 +30,7 @@ import org.openjdk.jmh.annotations.TearDown; import org.openjdk.jmh.annotations.Warmup; import org.openjdk.jmh.infra.Blackhole; +import software.amazon.awssdk.benchmark.utils.MockHttpServer; import software.amazon.awssdk.auth.credentials.AwsBasicCredentials; import software.amazon.awssdk.auth.credentials.StaticCredentialsProvider; import software.amazon.awssdk.http.apache5.Apache5HttpClient; @@ -53,16 +54,14 @@ @OutputTimeUnit(TimeUnit.SECONDS) public class V2CborRoundtripBenchmark { - private ProtocolRoundtripServer server; + private MockHttpServer server; private CloudWatchClient client; @Setup(Level.Trial) public void setup() throws Exception { byte[] response = createCborResponseFixture(); - ProtocolRoundtripServlet servlet = new ProtocolRoundtripServlet(response, "application/cbor"); - - server = new ProtocolRoundtripServer(servlet); + server = new MockHttpServer(response, "application/cbor"); server.start(); client = CloudWatchClient.builder() diff --git a/test/sdk-benchmarks/src/main/java/software/amazon/awssdk/benchmark/apicall/protocol/V2Ec2RoundtripBenchmark.java b/test/sdk-benchmarks/src/main/java/software/amazon/awssdk/benchmark/apicall/protocol/V2Ec2RoundtripBenchmark.java index 3dfdd9cd74d2..9b23335c9818 100644 --- a/test/sdk-benchmarks/src/main/java/software/amazon/awssdk/benchmark/apicall/protocol/V2Ec2RoundtripBenchmark.java +++ b/test/sdk-benchmarks/src/main/java/software/amazon/awssdk/benchmark/apicall/protocol/V2Ec2RoundtripBenchmark.java @@ -29,6 +29,7 @@ import org.openjdk.jmh.annotations.TearDown; import org.openjdk.jmh.annotations.Warmup; import org.openjdk.jmh.infra.Blackhole; +import software.amazon.awssdk.benchmark.utils.MockHttpServer; import software.amazon.awssdk.auth.credentials.AwsBasicCredentials; import software.amazon.awssdk.auth.credentials.StaticCredentialsProvider; import software.amazon.awssdk.http.apache5.Apache5HttpClient; @@ -48,16 +49,14 @@ @OutputTimeUnit(TimeUnit.SECONDS) public class V2Ec2RoundtripBenchmark { - private ProtocolRoundtripServer server; + private MockHttpServer server; private Ec2Client client; @Setup(Level.Trial) public void setup() throws Exception { - byte[] response = ProtocolRoundtripServer.loadFixture("ec2-protocol/describe-instances-response.xml"); + byte[] response = MockHttpServer.loadFixture("ec2-protocol/describe-instances-response.xml"); - ProtocolRoundtripServlet servlet = new ProtocolRoundtripServlet(response, "text/xml"); - - server = new ProtocolRoundtripServer(servlet); + server = new MockHttpServer(response, "text/xml"); server.start(); client = Ec2Client.builder() diff --git a/test/sdk-benchmarks/src/main/java/software/amazon/awssdk/benchmark/apicall/protocol/V2JsonRoundtripBenchmark.java b/test/sdk-benchmarks/src/main/java/software/amazon/awssdk/benchmark/apicall/protocol/V2JsonRoundtripBenchmark.java index 6d23fc581f6e..f96c8a44a42f 100644 --- a/test/sdk-benchmarks/src/main/java/software/amazon/awssdk/benchmark/apicall/protocol/V2JsonRoundtripBenchmark.java +++ b/test/sdk-benchmarks/src/main/java/software/amazon/awssdk/benchmark/apicall/protocol/V2JsonRoundtripBenchmark.java @@ -31,6 +31,7 @@ import org.openjdk.jmh.annotations.TearDown; import org.openjdk.jmh.annotations.Warmup; import org.openjdk.jmh.infra.Blackhole; +import software.amazon.awssdk.benchmark.utils.MockHttpServer; import software.amazon.awssdk.auth.credentials.AwsBasicCredentials; import software.amazon.awssdk.auth.credentials.StaticCredentialsProvider; import software.amazon.awssdk.core.SdkBytes; @@ -51,16 +52,14 @@ @OutputTimeUnit(TimeUnit.SECONDS) public class V2JsonRoundtripBenchmark { - private ProtocolRoundtripServer server; + private MockHttpServer server; private DynamoDbClient client; @Setup(Level.Trial) public void setup() throws Exception { - byte[] response = ProtocolRoundtripServer.loadFixture("json-protocol/putitem-response.json"); + byte[] response = MockHttpServer.loadFixture("json-protocol/putitem-response.json"); - ProtocolRoundtripServlet servlet = new ProtocolRoundtripServlet(response, "application/x-amz-json-1.0"); - - server = new ProtocolRoundtripServer(servlet); + server = new MockHttpServer(response, "application/x-amz-json-1.0"); server.start(); client = DynamoDbClient.builder() diff --git a/test/sdk-benchmarks/src/main/java/software/amazon/awssdk/benchmark/apicall/protocol/V2QueryRoundtripBenchmark.java b/test/sdk-benchmarks/src/main/java/software/amazon/awssdk/benchmark/apicall/protocol/V2QueryRoundtripBenchmark.java index aff4d9c23cd2..a03e94f2808b 100644 --- a/test/sdk-benchmarks/src/main/java/software/amazon/awssdk/benchmark/apicall/protocol/V2QueryRoundtripBenchmark.java +++ b/test/sdk-benchmarks/src/main/java/software/amazon/awssdk/benchmark/apicall/protocol/V2QueryRoundtripBenchmark.java @@ -29,6 +29,7 @@ import org.openjdk.jmh.annotations.TearDown; import org.openjdk.jmh.annotations.Warmup; import org.openjdk.jmh.infra.Blackhole; +import software.amazon.awssdk.benchmark.utils.MockHttpServer; import software.amazon.awssdk.auth.credentials.AwsBasicCredentials; import software.amazon.awssdk.auth.credentials.StaticCredentialsProvider; import software.amazon.awssdk.http.apache5.Apache5HttpClient; @@ -47,16 +48,14 @@ @OutputTimeUnit(TimeUnit.SECONDS) public class V2QueryRoundtripBenchmark { - private ProtocolRoundtripServer server; + private MockHttpServer server; private StsClient client; @Setup(Level.Trial) public void setup() throws Exception { - byte[] response = ProtocolRoundtripServer.loadFixture("query-protocol/assumerole-response.xml"); + byte[] response = MockHttpServer.loadFixture("query-protocol/assumerole-response.xml"); - ProtocolRoundtripServlet servlet = new ProtocolRoundtripServlet(response, "text/xml"); - - server = new ProtocolRoundtripServer(servlet); + server = new MockHttpServer(response, "text/xml"); server.start(); client = StsClient.builder() diff --git a/test/sdk-benchmarks/src/main/java/software/amazon/awssdk/benchmark/apicall/protocol/V2RestJsonRoundtripBenchmark.java b/test/sdk-benchmarks/src/main/java/software/amazon/awssdk/benchmark/apicall/protocol/V2RestJsonRoundtripBenchmark.java index 563ad7fd1dd3..c2d48a398ae1 100644 --- a/test/sdk-benchmarks/src/main/java/software/amazon/awssdk/benchmark/apicall/protocol/V2RestJsonRoundtripBenchmark.java +++ b/test/sdk-benchmarks/src/main/java/software/amazon/awssdk/benchmark/apicall/protocol/V2RestJsonRoundtripBenchmark.java @@ -31,6 +31,7 @@ import org.openjdk.jmh.annotations.TearDown; import org.openjdk.jmh.annotations.Warmup; import org.openjdk.jmh.infra.Blackhole; +import software.amazon.awssdk.benchmark.utils.MockHttpServer; import software.amazon.awssdk.auth.credentials.AwsBasicCredentials; import software.amazon.awssdk.auth.credentials.StaticCredentialsProvider; import software.amazon.awssdk.http.apache5.Apache5HttpClient; @@ -54,16 +55,14 @@ @OutputTimeUnit(TimeUnit.SECONDS) public class V2RestJsonRoundtripBenchmark { - private ProtocolRoundtripServer server; + private MockHttpServer server; private LambdaClient client; @Setup(Level.Trial) public void setup() throws Exception { - byte[] response = ProtocolRoundtripServer.loadFixture("rest-json-protocol/createfunction-response.json"); + byte[] response = MockHttpServer.loadFixture("rest-json-protocol/createfunction-response.json"); - ProtocolRoundtripServlet servlet = new ProtocolRoundtripServlet(response, "application/json"); - - server = new ProtocolRoundtripServer(servlet); + server = new MockHttpServer(response, "application/json"); server.start(); client = LambdaClient.builder() diff --git a/test/sdk-benchmarks/src/main/java/software/amazon/awssdk/benchmark/apicall/protocol/V2RestXmlRoundtripBenchmark.java b/test/sdk-benchmarks/src/main/java/software/amazon/awssdk/benchmark/apicall/protocol/V2RestXmlRoundtripBenchmark.java index 04cdea1b56ef..61af837142b4 100644 --- a/test/sdk-benchmarks/src/main/java/software/amazon/awssdk/benchmark/apicall/protocol/V2RestXmlRoundtripBenchmark.java +++ b/test/sdk-benchmarks/src/main/java/software/amazon/awssdk/benchmark/apicall/protocol/V2RestXmlRoundtripBenchmark.java @@ -29,6 +29,7 @@ import org.openjdk.jmh.annotations.TearDown; import org.openjdk.jmh.annotations.Warmup; import org.openjdk.jmh.infra.Blackhole; +import software.amazon.awssdk.benchmark.utils.MockHttpServer; import software.amazon.awssdk.auth.credentials.AwsBasicCredentials; import software.amazon.awssdk.auth.credentials.StaticCredentialsProvider; import software.amazon.awssdk.http.apache5.Apache5HttpClient; @@ -57,16 +58,14 @@ @OutputTimeUnit(TimeUnit.SECONDS) public class V2RestXmlRoundtripBenchmark { - private ProtocolRoundtripServer server; + private MockHttpServer server; private CloudFrontClient client; @Setup(Level.Trial) public void setup() throws Exception { - byte[] response = ProtocolRoundtripServer.loadFixture("rest-xml-protocol/create-distribution-response.xml"); + byte[] response = MockHttpServer.loadFixture("rest-xml-protocol/create-distribution-response.xml"); - ProtocolRoundtripServlet servlet = new ProtocolRoundtripServlet(response, "text/xml"); - - server = new ProtocolRoundtripServer(servlet); + server = new MockHttpServer(response, "text/xml"); server.start(); client = CloudFrontClient.builder() diff --git a/test/sdk-benchmarks/src/main/java/software/amazon/awssdk/benchmark/coldstart/V2ColdStartAfterWarmUpBenchmark.java b/test/sdk-benchmarks/src/main/java/software/amazon/awssdk/benchmark/coldstart/V2ColdStartAfterWarmUpBenchmark.java new file mode 100644 index 000000000000..eddfd677c98b --- /dev/null +++ b/test/sdk-benchmarks/src/main/java/software/amazon/awssdk/benchmark/coldstart/V2ColdStartAfterWarmUpBenchmark.java @@ -0,0 +1,144 @@ +/* + * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"). + * You may not use this file except in compliance with the License. + * A copy of the License is located at + * + * http://aws.amazon.com/apache2.0 + * + * or in the "license" file accompanying this file. This file is distributed + * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing + * permissions and limitations under the License. + */ + +package software.amazon.awssdk.benchmark.coldstart; + +import java.util.Collection; +import java.util.HashMap; +import java.util.Map; +import java.util.concurrent.TimeUnit; +import org.openjdk.jmh.annotations.Benchmark; +import org.openjdk.jmh.annotations.BenchmarkMode; +import org.openjdk.jmh.annotations.Fork; +import org.openjdk.jmh.annotations.Level; +import org.openjdk.jmh.annotations.Measurement; +import org.openjdk.jmh.annotations.Mode; +import org.openjdk.jmh.annotations.OutputTimeUnit; +import org.openjdk.jmh.annotations.Scope; +import org.openjdk.jmh.annotations.Setup; +import org.openjdk.jmh.annotations.State; +import org.openjdk.jmh.annotations.TearDown; +import org.openjdk.jmh.annotations.Warmup; +import org.openjdk.jmh.infra.Blackhole; +import org.openjdk.jmh.profile.StackProfiler; +import org.openjdk.jmh.results.RunResult; +import org.openjdk.jmh.runner.Runner; +import org.openjdk.jmh.runner.RunnerException; +import org.openjdk.jmh.runner.options.CommandLineOptionException; +import org.openjdk.jmh.runner.options.CommandLineOptions; +import org.openjdk.jmh.runner.options.Options; +import org.openjdk.jmh.runner.options.OptionsBuilder; +import software.amazon.awssdk.auth.credentials.AwsBasicCredentials; +import software.amazon.awssdk.auth.credentials.StaticCredentialsProvider; +import software.amazon.awssdk.benchmark.utils.MockHttpServer; +import software.amazon.awssdk.core.SdkBytes; +import software.amazon.awssdk.core.crac.SdkWarmUp; +import software.amazon.awssdk.http.apache5.Apache5HttpClient; +import software.amazon.awssdk.regions.Region; +import software.amazon.awssdk.services.dynamodb.DynamoDbClient; +import software.amazon.awssdk.services.dynamodb.model.AttributeValue; +import software.amazon.awssdk.services.dynamodb.model.PutItemRequest; + +/** + * Same measurement as {@link V2ColdStartNoWarmUpBenchmark}, but {@link SdkWarmUp#prime(Class[])} runs in {@code @Setup} + * (untimed, needs network for the STS warm-up GET). The score difference between the two is the first-call work that + * warm-up front-loads. See {@link V2SdkWarmUpExecutionTimeBenchmark} for how long prime() itself takes. + */ +@State(Scope.Benchmark) +@BenchmarkMode(Mode.SingleShotTime) +@OutputTimeUnit(TimeUnit.MILLISECONDS) +@Warmup(iterations = 0) +@Measurement(iterations = 1) +@Fork(20) +public class V2ColdStartAfterWarmUpBenchmark { + + private static final String FIXTURE = "json-protocol/putitem-response.json"; + private static final String CONTENT_TYPE = "application/x-amz-json-1.0"; + + private MockHttpServer server; + private DynamoDbClient client; + + @Setup(Level.Trial) + public void setup() throws Exception { + server = new MockHttpServer(MockHttpServer.loadFixture(FIXTURE), CONTENT_TYPE); + server.start(); + + SdkWarmUp.prime(DynamoDbClient.class); + } + @Benchmark + public void coldFirstCall(Blackhole blackhole) throws Exception { + client = DynamoDbClient.builder() + .endpointOverride(server.getHttpUri()) + .region(Region.US_EAST_1) + .credentialsProvider(StaticCredentialsProvider.create( + AwsBasicCredentials.create("test", "test"))) + .httpClient(Apache5HttpClient.create()) + .endpointDiscoveryEnabled(false) + .build(); + + blackhole.consume(client.putItem(putItemRequest())); + } + + @TearDown(Level.Trial) + public void tearDown() throws Exception { + if (client != null) { + client.close(); + } + if (server != null) { + server.stop(); + } + } + + public static void main(String... args) throws RunnerException, CommandLineOptionException { + Options opt = new OptionsBuilder() + .parent(new CommandLineOptions()) + .include(V2ColdStartAfterWarmUpBenchmark.class.getSimpleName()) + .addProfiler(StackProfiler.class) + .build(); + Collection run = new Runner(opt).run(); + } + + private static PutItemRequest putItemRequest() { + return PutItemRequest.builder() + .tableName("benchmark-table") + .item(itemMap()) + .build(); + } + + private static Map itemMap() { + Map item = new HashMap<>(); + item.put("pk", AttributeValue.fromS("benchmark-key")); + item.put("sk", AttributeValue.fromN("100")); + item.put("stringField", AttributeValue.fromS("test-value")); + item.put("numberField", AttributeValue.fromN("123.456")); + item.put("binaryField", AttributeValue.fromB(SdkBytes.fromUtf8String("hello world"))); + item.put("stringSetField", AttributeValue.builder().ss("value1", "value2", "value3").build()); + item.put("numberSetField", AttributeValue.builder().ns("1.1", "2.2", "3.3").build()); + item.put("boolField", AttributeValue.fromBool(false)); + item.put("nullField", AttributeValue.builder().nul(true).build()); + Map deep = new HashMap<>(); + deep.put("level2", AttributeValue.fromN("999")); + Map nested = new HashMap<>(); + nested.put("nested", AttributeValue.fromS("nested-value")); + nested.put("deepNested", AttributeValue.fromM(deep)); + item.put("mapField", AttributeValue.fromM(nested)); + item.put("listField", AttributeValue.builder().l( + AttributeValue.fromS("item1"), + AttributeValue.fromN("42"), + AttributeValue.fromBool(true), + AttributeValue.builder().nul(true).build()).build()); + return item; + } +} diff --git a/test/sdk-benchmarks/src/main/java/software/amazon/awssdk/benchmark/coldstart/V2ColdStartNoWarmUpBenchmark.java b/test/sdk-benchmarks/src/main/java/software/amazon/awssdk/benchmark/coldstart/V2ColdStartNoWarmUpBenchmark.java new file mode 100644 index 000000000000..eded601fa2e3 --- /dev/null +++ b/test/sdk-benchmarks/src/main/java/software/amazon/awssdk/benchmark/coldstart/V2ColdStartNoWarmUpBenchmark.java @@ -0,0 +1,142 @@ +/* + * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"). + * You may not use this file except in compliance with the License. + * A copy of the License is located at + * + * http://aws.amazon.com/apache2.0 + * + * or in the "license" file accompanying this file. This file is distributed + * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing + * permissions and limitations under the License. + */ + +package software.amazon.awssdk.benchmark.coldstart; + +import java.util.Collection; +import java.util.HashMap; +import java.util.Map; +import java.util.concurrent.TimeUnit; +import org.openjdk.jmh.annotations.Benchmark; +import org.openjdk.jmh.annotations.BenchmarkMode; +import org.openjdk.jmh.annotations.Fork; +import org.openjdk.jmh.annotations.Level; +import org.openjdk.jmh.annotations.Measurement; +import org.openjdk.jmh.annotations.Mode; +import org.openjdk.jmh.annotations.OutputTimeUnit; +import org.openjdk.jmh.annotations.Scope; +import org.openjdk.jmh.annotations.Setup; +import org.openjdk.jmh.annotations.State; +import org.openjdk.jmh.annotations.TearDown; +import org.openjdk.jmh.annotations.Warmup; +import org.openjdk.jmh.infra.Blackhole; +import org.openjdk.jmh.profile.StackProfiler; +import org.openjdk.jmh.results.RunResult; +import org.openjdk.jmh.runner.Runner; +import org.openjdk.jmh.runner.RunnerException; +import org.openjdk.jmh.runner.options.CommandLineOptionException; +import org.openjdk.jmh.runner.options.CommandLineOptions; +import org.openjdk.jmh.runner.options.Options; +import org.openjdk.jmh.runner.options.OptionsBuilder; +import software.amazon.awssdk.auth.credentials.AwsBasicCredentials; +import software.amazon.awssdk.auth.credentials.StaticCredentialsProvider; +import software.amazon.awssdk.benchmark.utils.MockHttpServer; +import software.amazon.awssdk.core.SdkBytes; +import software.amazon.awssdk.http.apache5.Apache5HttpClient; +import software.amazon.awssdk.regions.Region; +import software.amazon.awssdk.services.dynamodb.DynamoDbClient; +import software.amazon.awssdk.services.dynamodb.model.AttributeValue; +import software.amazon.awssdk.services.dynamodb.model.PutItemRequest; + +/** + * Baseline: build a {@link DynamoDbClient} and make its first call (to a local mock server) with no warm-up. + * Compare against {@link V2ColdStartAfterWarmUpBenchmark}. Single-shot, zero warmup, high fork count: only the first + * invocation per JVM is cold. Do not override these JMH parameters from the CLI, or both benchmarks converge. + */ +@State(Scope.Benchmark) +@BenchmarkMode(Mode.SingleShotTime) +@OutputTimeUnit(TimeUnit.MILLISECONDS) +@Warmup(iterations = 0) +@Measurement(iterations = 1) +@Fork(20) +public class V2ColdStartNoWarmUpBenchmark { + + private static final String FIXTURE = "json-protocol/putitem-response.json"; + private static final String CONTENT_TYPE = "application/x-amz-json-1.0"; + + private MockHttpServer server; + private DynamoDbClient client; + + @Setup(Level.Trial) + public void setup() throws Exception { + server = new MockHttpServer(MockHttpServer.loadFixture(FIXTURE), CONTENT_TYPE); + server.start(); + } + + @Benchmark + public void coldFirstCall(Blackhole blackhole) throws Exception { + client = DynamoDbClient.builder() + .endpointOverride(server.getHttpUri()) + .region(Region.US_EAST_1) + .credentialsProvider(StaticCredentialsProvider.create( + AwsBasicCredentials.create("test", "test"))) + .httpClient(Apache5HttpClient.create()) + .endpointDiscoveryEnabled(false) + .build(); + + blackhole.consume(client.putItem(putItemRequest())); + } + + @TearDown(Level.Trial) + public void tearDown() throws Exception { + if (client != null) { + client.close(); + } + if (server != null) { + server.stop(); + } + } + + public static void main(String... args) throws RunnerException, CommandLineOptionException { + Options opt = new OptionsBuilder() + .parent(new CommandLineOptions()) + .include(V2ColdStartNoWarmUpBenchmark.class.getSimpleName()) + .addProfiler(StackProfiler.class) + .build(); + Collection run = new Runner(opt).run(); + } + + private static PutItemRequest putItemRequest() { + return PutItemRequest.builder() + .tableName("benchmark-table") + .item(itemMap()) + .build(); + } + + private static Map itemMap() { + Map item = new HashMap<>(); + item.put("pk", AttributeValue.fromS("benchmark-key")); + item.put("sk", AttributeValue.fromN("100")); + item.put("stringField", AttributeValue.fromS("test-value")); + item.put("numberField", AttributeValue.fromN("123.456")); + item.put("binaryField", AttributeValue.fromB(SdkBytes.fromUtf8String("hello world"))); + item.put("stringSetField", AttributeValue.builder().ss("value1", "value2", "value3").build()); + item.put("numberSetField", AttributeValue.builder().ns("1.1", "2.2", "3.3").build()); + item.put("boolField", AttributeValue.fromBool(false)); + item.put("nullField", AttributeValue.builder().nul(true).build()); + Map deep = new HashMap<>(); + deep.put("level2", AttributeValue.fromN("999")); + Map nested = new HashMap<>(); + nested.put("nested", AttributeValue.fromS("nested-value")); + nested.put("deepNested", AttributeValue.fromM(deep)); + item.put("mapField", AttributeValue.fromM(nested)); + item.put("listField", AttributeValue.builder().l( + AttributeValue.fromS("item1"), + AttributeValue.fromN("42"), + AttributeValue.fromBool(true), + AttributeValue.builder().nul(true).build()).build()); + return item; + } +} diff --git a/test/sdk-benchmarks/src/main/java/software/amazon/awssdk/benchmark/coldstart/V2SdkWarmUpExecutionTimeBenchmark.java b/test/sdk-benchmarks/src/main/java/software/amazon/awssdk/benchmark/coldstart/V2SdkWarmUpExecutionTimeBenchmark.java new file mode 100644 index 000000000000..7904b539e0d8 --- /dev/null +++ b/test/sdk-benchmarks/src/main/java/software/amazon/awssdk/benchmark/coldstart/V2SdkWarmUpExecutionTimeBenchmark.java @@ -0,0 +1,66 @@ +/* + * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"). + * You may not use this file except in compliance with the License. + * A copy of the License is located at + * + * http://aws.amazon.com/apache2.0 + * + * or in the "license" file accompanying this file. This file is distributed + * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing + * permissions and limitations under the License. + */ + +package software.amazon.awssdk.benchmark.coldstart; + +import java.util.Collection; +import java.util.concurrent.TimeUnit; +import org.openjdk.jmh.annotations.Benchmark; +import org.openjdk.jmh.annotations.BenchmarkMode; +import org.openjdk.jmh.annotations.Fork; +import org.openjdk.jmh.annotations.Measurement; +import org.openjdk.jmh.annotations.Mode; +import org.openjdk.jmh.annotations.OutputTimeUnit; +import org.openjdk.jmh.annotations.Scope; +import org.openjdk.jmh.annotations.State; +import org.openjdk.jmh.annotations.Warmup; +import org.openjdk.jmh.profile.StackProfiler; +import org.openjdk.jmh.results.RunResult; +import org.openjdk.jmh.runner.Runner; +import org.openjdk.jmh.runner.RunnerException; +import org.openjdk.jmh.runner.options.CommandLineOptionException; +import org.openjdk.jmh.runner.options.CommandLineOptions; +import org.openjdk.jmh.runner.options.Options; +import org.openjdk.jmh.runner.options.OptionsBuilder; +import software.amazon.awssdk.core.crac.SdkWarmUp; +import software.amazon.awssdk.services.dynamodb.DynamoDbClient; + +/** + * Measures how long {@link SdkWarmUp#prime(Class[])} takes: the upfront price for the first-call saving shown by + * {@link V2ColdStartAfterWarmUpBenchmark}. The score includes real network time (STS GET per HTTP client on the + * classpath), so it varies by host — do not add it to {@code baseline.json}. + */ +@State(Scope.Benchmark) +@BenchmarkMode(Mode.SingleShotTime) +@OutputTimeUnit(TimeUnit.MILLISECONDS) +@Warmup(iterations = 0) +@Measurement(iterations = 1) +@Fork(20) +public class V2SdkWarmUpExecutionTimeBenchmark { + + @Benchmark + public void prime() throws Exception { + SdkWarmUp.prime(DynamoDbClient.class); + } + + public static void main(String... args) throws RunnerException, CommandLineOptionException { + Options opt = new OptionsBuilder() + .parent(new CommandLineOptions()) + .include(V2SdkWarmUpExecutionTimeBenchmark.class.getSimpleName()) + .addProfiler(StackProfiler.class) + .build(); + Collection run = new Runner(opt).run(); + } +} diff --git a/test/sdk-benchmarks/src/main/java/software/amazon/awssdk/benchmark/utils/MockHttpServer.java b/test/sdk-benchmarks/src/main/java/software/amazon/awssdk/benchmark/utils/MockHttpServer.java new file mode 100644 index 000000000000..17ae56221a34 --- /dev/null +++ b/test/sdk-benchmarks/src/main/java/software/amazon/awssdk/benchmark/utils/MockHttpServer.java @@ -0,0 +1,94 @@ +/* + * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"). + * You may not use this file except in compliance with the License. + * A copy of the License is located at + * + * http://aws.amazon.com/apache2.0 + * + * or in the "license" file accompanying this file. This file is distributed + * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing + * permissions and limitations under the License. + */ + +package software.amazon.awssdk.benchmark.utils; + +import java.io.IOException; +import java.io.InputStream; +import java.net.URI; +import javax.servlet.http.HttpServlet; +import javax.servlet.http.HttpServletRequest; +import javax.servlet.http.HttpServletResponse; +import org.eclipse.jetty.server.Connector; +import org.eclipse.jetty.server.Server; +import org.eclipse.jetty.server.ServerConnector; +import org.eclipse.jetty.servlet.ServletContextHandler; +import org.eclipse.jetty.servlet.ServletHolder; +import software.amazon.awssdk.utils.IoUtils; + +/** + * Lightweight plain-HTTP Jetty server that answers every request with a fixed 200 status, body and content type, so a + * real service client can complete an operation without leaving the machine. Used by the protocol roundtrip and + * cold-start benchmarks. + */ +public final class MockHttpServer { + + private final Server server; + private final int port; + + public MockHttpServer(byte[] responseBody, String contentType) throws IOException { + port = BenchmarkUtils.getUnusedPort(); + server = new Server(); + ServerConnector connector = new ServerConnector(server); + connector.setPort(port); + server.setConnectors(new Connector[] {connector}); + + ServletContextHandler context = new ServletContextHandler(server, "/", ServletContextHandler.SESSIONS); + context.addServlet(new ServletHolder(new FixedResponseServlet(responseBody, contentType)), "/*"); + server.setHandler(context); + } + + public void start() throws Exception { + server.start(); + } + + public void stop() throws Exception { + server.stop(); + } + + public URI getHttpUri() { + return URI.create("http://localhost:" + port); + } + + public static byte[] loadFixture(String path) throws IOException { + try (InputStream is = MockHttpServer.class.getClassLoader().getResourceAsStream("fixtures/" + path)) { + if (is == null) { + throw new IOException("Fixture not found: fixtures/" + path); + } + return IoUtils.toByteArray(is); + } + } + + /** + * Stateless: returns the same response for every method, path and body, and does not validate the request. + */ + private static final class FixedResponseServlet extends HttpServlet { + private final byte[] body; + private final String contentType; + + private FixedResponseServlet(byte[] body, String contentType) { + this.body = body; + this.contentType = contentType; + } + + @Override + protected void service(HttpServletRequest req, HttpServletResponse resp) throws IOException { + resp.setStatus(200); + resp.setContentLength(body.length); + resp.setContentType(contentType); + resp.getOutputStream().write(body); + } + } +}