diff --git a/build_tools/services.rb b/build_tools/services.rb index 45fc3228d10..74ef1b8fdc3 100644 --- a/build_tools/services.rb +++ b/build_tools/services.rb @@ -9,10 +9,10 @@ class ServiceEnumerator MANIFEST_PATH = File.expand_path('../../services.json', __FILE__) # Minimum `aws-sdk-core` version for new gem builds - MINIMUM_CORE_VERSION = "3.265.0" + MINIMUM_CORE_VERSION = "3.266.0" # Minimum `aws-sdk-core` version for new S3 gem builds - MINIMUM_CORE_VERSION_S3 = "3.265.0" + MINIMUM_CORE_VERSION_S3 = "3.266.0" EVENTSTREAM_PLUGIN = "Aws::Plugins::EventStreamConfiguration" diff --git a/gems/aws-sdk-core/CHANGELOG.md b/gems/aws-sdk-core/CHANGELOG.md index f815aff5271..95b509cf473 100644 --- a/gems/aws-sdk-core/CHANGELOG.md +++ b/gems/aws-sdk-core/CHANGELOG.md @@ -1,6 +1,8 @@ Unreleased Changes ------------------ +* Feature - Reduce memory allocations when streaming request bodies with checksum trailers while preserving compatibility with custom IO bodies. + 3.265.0 (2026-11-06) ------------------ diff --git a/gems/aws-sdk-core/lib/aws-sdk-core/plugins/checksum_algorithm.rb b/gems/aws-sdk-core/lib/aws-sdk-core/plugins/checksum_algorithm.rb index fe7aeafa8c5..2eacecbf542 100644 --- a/gems/aws-sdk-core/lib/aws-sdk-core/plugins/checksum_algorithm.rb +++ b/gems/aws-sdk-core/lib/aws-sdk-core/plugins/checksum_algorithm.rb @@ -335,7 +335,8 @@ def calculate_request_checksum(context, checksum_properties) when 'header' apply_request_checksum(context, headers, checksum_properties) when 'trailer' - apply_request_trailer_checksum(context, headers, checksum_properties) + buffer_enabled = context.http_request.buffer_supported? + apply_request_trailer_checksum(context, headers, checksum_properties, buffer_enabled) else # nothing end @@ -385,7 +386,7 @@ def update_in_chunks(digest, io) io.rewind end - def apply_request_trailer_checksum(context, headers, checksum_properties) + def apply_request_trailer_checksum(context, headers, checksum_properties, buffer_enabled) location_name = checksum_properties[:name] # set required headers @@ -410,6 +411,7 @@ def apply_request_trailer_checksum(context, headers, checksum_properties) context.http_request.body = AwsChunkedTrailerDigestIO.new( io: context.http_request.body, + buffer_enabled: buffer_enabled, algorithm: checksum_properties[:algorithm], location_name: location_name ) @@ -484,6 +486,7 @@ class AwsChunkedTrailerDigestIO HEX_BASE = 16 def initialize(options = {}) + buffer_enabled = options.delete(:buffer_enabled) || false @io = options.delete(:io) @io.rewind if @io.respond_to?(:rewind) @location_name = options.delete(:location_name) @@ -492,6 +495,8 @@ def initialize(options = {}) @chunk_size = Thread.current[:net_http_override_body_stream_chunk] || DEFAULT_TRAILER_CHUNK_SIZE @overhead_bytes = calculate_overhead(@chunk_size) @base_chunk_size = @chunk_size - @overhead_bytes + @chunk = ''.b + @read_chunk = method(buffer_enabled ? :read_chunk_with_buffer : :read_chunk_without_buffer) @encoded_buffer = +'' @eof = false end @@ -527,8 +532,9 @@ def read(length = nil, buf = nil) fill_encoded_buffer(length) - if length - output_buffer << @encoded_buffer.slice!(0, length) + if length && length < @encoded_buffer.bytesize + output_buffer << @encoded_buffer.byteslice(0, length) + @encoded_buffer = @encoded_buffer.byteslice(length, @encoded_buffer.bytesize - length) else output_buffer << @encoded_buffer @encoded_buffer.clear @@ -547,16 +553,24 @@ def calculate_overhead(chunk_size) chunk_size.to_s(HEX_BASE).size + CHUNK_OVERHEAD end + def read_chunk_with_buffer + @io.read(@base_chunk_size, @chunk) + end + + def read_chunk_without_buffer + @io.read(@base_chunk_size) + end + def fill_encoded_buffer(required_length) return if required_length && @encoded_buffer.bytesize >= required_length while !@eof && fill_data?(required_length) - chunk = @io.read(@base_chunk_size) + chunk = @read_chunk.call if chunk && !chunk.empty? @digest.update(chunk) - @encoded_buffer << "#{chunk.bytesize.to_s(HEX_BASE)}\r\n#{chunk}\r\n" + @encoded_buffer << chunk.bytesize.to_s(HEX_BASE) << "\r\n" << chunk << "\r\n" else - @encoded_buffer << "0\r\n#{trailer_string}\r\n\r\n" + @encoded_buffer << "0\r\n" << trailer_string << "\r\n\r\n" @eof = true end end diff --git a/gems/aws-sdk-core/lib/seahorse/client/http/request.rb b/gems/aws-sdk-core/lib/seahorse/client/http/request.rb index e1f1fdda76b..eb00c319ad8 100644 --- a/gems/aws-sdk-core/lib/seahorse/client/http/request.rb +++ b/gems/aws-sdk-core/lib/seahorse/client/http/request.rb @@ -55,8 +55,16 @@ def body_contents contents end + # @api private + def buffer_supported? + return @buffer_enabled unless @buffer_enabled.nil? + + @buffer_enabled = detect_read_buffer_support + end + # @param [#read, #size, #rewind] io def body=(io) + @buffer_enabled = nil @body = case io when nil then StringIO.new('') when String then StringIO.new(io) @@ -64,6 +72,17 @@ def body=(io) end end + private + + # Only probe ArgumentError falls back. Payload errors must propagate. + def detect_read_buffer_support + buffer = "\0".b + @body.read(0, buffer) + buffer.empty? + rescue ArgumentError + false + end + end end end diff --git a/gems/aws-sdk-core/spec/aws/plugins/checksum_algorithm_spec.rb b/gems/aws-sdk-core/spec/aws/plugins/checksum_algorithm_spec.rb index ca514f18c0c..03a5cec9aa7 100644 --- a/gems/aws-sdk-core/spec/aws/plugins/checksum_algorithm_spec.rb +++ b/gems/aws-sdk-core/spec/aws/plugins/checksum_algorithm_spec.rb @@ -2,6 +2,7 @@ require_relative '../../spec_helper' require 'aws-sdk-core/plugins/checksum_algorithm' +require 'delegate' module Aws module Plugins @@ -284,6 +285,91 @@ module Plugins end end + context 'IO-like request body compatibility' do + let(:payload) { "\x00\xff".b * 11 } + let(:expected_checksum) do + Base64.strict_encode64([Zlib.crc32(payload)].pack('N')) + end + let(:one_argument_reader) do + Class.new(StringIO) do + def read(length) + super(length) + end + end + end + + def send_checksum_request(io) + operation = placement == 'header' ? :http_checksum_operation : :http_checksum_streaming_operation + client.stub_responses(operation, lambda do |context| + body = context.http_request.body + output = body.read(body.size) + headers = context.http_request.headers + expect(headers['content-length'].to_i).to eq(output.bytesize) + if placement == 'header' + expect(output).to eq(payload) + expect(headers['x-amz-checksum-crc32']).to eq(expected_checksum) + else + expected = "#{payload.bytesize.to_s(16)}\r\n#{payload}\r\n" \ + "0\r\nx-amz-checksum-crc32:#{expected_checksum}\r\n\r\n" + expect(output).to eq(expected) + end + {} + end) + client.public_send(operation, body: io, checksum_algorithm: 'CRC32') + end + + %w[trailer].each do |location| + context "checksum in #{location}", + skip: location == 'trailer' && defined?(JRUBY_VERSION) && 'Trailer checksums are disabled on JRuby' do + let(:placement) { location } + + it 'supports a one-argument reader behind a delegator' do + send_checksum_request(SimpleDelegator.new(one_argument_reader.new(payload))) + end + + it 'supports a reader that ignores the optional output buffer' do + klass = Class.new(one_argument_reader) do + def read(length, _buffer = nil) + super(length) + end + end + send_checksum_request(klass.new(payload)) + end + + it 'supports a reader that returns a frozen String instead of filling the buffer' do + klass = Class.new(StringIO) do + def read(length, buffer = nil) + return super if length.zero? || @returned_frozen + + @returned_frozen = true + super(length).freeze + end + end + send_checksum_request(klass.new(payload)) + end + + it 'propagates payload read errors without retrying' do + klass = Class.new(StringIO) do + attr_reader :payload_reads + + def read(length, buffer = nil) + return super if length.zero? + + @payload_reads = (@payload_reads || 0) + 1 + super(1, buffer) + raise ArgumentError, 'reader failed after consuming a byte' + end + end + io = klass.new(payload) + expect { send_checksum_request(io) }.to raise_error( + ArgumentError, 'reader failed after consuming a byte' + ) + expect(io.payload_reads).to eq(1) + end + end + end + end + context 'response checksum calculation' do it 'computes a validation_list' do resp = client.http_checksum_operation @@ -481,12 +567,55 @@ def stub_client(client) end describe 'AwsChunkedTrailerDigestIO' do + let(:payload) { "\x00\xff".b * 11 } + let(:io) { StringIO.new(payload) } + let(:body) do + ChecksumAlgorithm::AwsChunkedTrailerDigestIO.new( + buffer_enabled: true, + io: io, + algorithm: 'CRC32', + location_name: 'x-amz-checksum-crc32' + ) + end + let(:encoded_payload) do + checksum = Base64.strict_encode64([Zlib.crc32(payload)].pack('N')) + "a\r\n#{payload.byteslice(0, 10)}\r\n" \ + "a\r\n#{payload.byteslice(10, 10)}\r\n" \ + "2\r\n#{payload.byteslice(20, 2)}\r\n" \ + "0\r\nx-amz-checksum-crc32:#{checksum}\r\n\r\n" + end + + around do |example| + previous_chunk_size = Thread.current[:net_http_override_body_stream_chunk] + Thread.current[:net_http_override_body_stream_chunk] = 16 + example.run + ensure + Thread.current[:net_http_override_body_stream_chunk] = previous_chunk_size + end + + it 'supports a one-argument reader when buffer_enabled is omitted' do + klass = Class.new(StringIO) do + def read(length) + super(length) + end + end + body = ChecksumAlgorithm::AwsChunkedTrailerDigestIO.new( + io: klass.new(payload), + algorithm: 'CRC32', + location_name: 'x-amz-checksum-crc32' + ) + + expect(body.read).to eq(encoded_payload) + expect(body.read).to be_nil + end + it 'rewinds IO on initialization' do io = StringIO.new('hello world') io.read # move to EOF expect(io.pos).to eq(11) ChecksumAlgorithm::AwsChunkedTrailerDigestIO.new( + buffer_enabled: true, io: io, algorithm: 'CRC32', location_name: 'x-amz-checksum-crc32' @@ -494,6 +623,57 @@ def stub_client(client) expect(io.pos).to eq(0) end + + it 'reuses the input buffer without changing encoded binary chunks' do + buffers = [] + allow(io).to receive(:read).and_wrap_original do |read, length, buffer| + buffers << buffer unless length.zero? + read.call(length, buffer) + end + + output = String.new + buffer = String.new + output << buffer while body.read(7, buffer) + + expect(output).to eq(encoded_payload) + expect(buffers.size).to eq(4) # three data chunks and EOF + expect(buffers.first).to be_a(String) + expect(buffers).to all(equal(buffers.first)) + expect(body).to be_eof + end + + it 'replays encoded content when rewound after EOF' do + expect(body.read).to eq(encoded_payload) + expect(body.read).to be_nil + body.rewind + + expect(io).to receive(:read).with(10, an_instance_of(String)) + .at_least(:once).and_call_original + expect(body.read).to eq(encoded_payload) + expect(body).to be_eof + end + + [16, 17, 64].each do |length| + it "preserves chunk boundaries and the trailer with #{length}-byte reads" do + output = String.new + buffer = String.new + while (chunk = body.read(length, buffer)) + expect(chunk).to equal(buffer) + expect(chunk.bytesize).to be <= length + output << chunk + end + + expect(output).to eq(encoded_payload) + expect(body).to be_eof + end + end + + it 'writes the checksum trailer for an empty input' do + io.truncate(0) + + expect(body.read).to eq("0\r\nx-amz-checksum-crc32:AAAAAA==\r\n\r\n") + expect(body.read).to be_nil + end end end end diff --git a/gems/aws-sdk-core/spec/seahorse/client/http/request_spec.rb b/gems/aws-sdk-core/spec/seahorse/client/http/request_spec.rb index df785137701..2438bf04dcf 100644 --- a/gems/aws-sdk-core/spec/seahorse/client/http/request_spec.rb +++ b/gems/aws-sdk-core/spec/seahorse/client/http/request_spec.rb @@ -10,6 +10,59 @@ module Http def request(*args) Request.new(*args) end + describe '#buffer_supported?' do + let(:reader_class) do + Class.new(StringIO) do + attr_reader :probes + + def initialize(supported) + super('payload') + @supported = supported + @probes = 0 + end + + def read(length = nil, buffer = nil) + @probes += 1 if length == 0 + @supported ? super(length, buffer) : super(length) + end + end + end + + [true, false].each do |supported| + it "caches #{supported} without consuming body bytes" do + io = reader_class.new(supported) + req = request(body: io) + 3.times { expect(req.buffer_supported?).to be(supported) } + expect(io.probes).to eq(1) + expect(io.pos).to eq(0) + end + end + + it 'checks the new body after assignment' do + first = reader_class.new(false) + second = reader_class.new(true) + req = request(body: first) + expect(req.buffer_supported?).to be(false) + req.body = second + 2.times { expect(req.buffer_supported?).to be(true) } + expect(first.probes).to eq(1) + expect(second.probes).to eq(1) + end + + it 'keeps results separate for different requests' do + first = reader_class.new(true) + second = reader_class.new(false) + first_request = request(body: first) + second_request = request(body: second) + 2.times do + expect(first_request.buffer_supported?).to be(true) + expect(second_request.buffer_supported?).to be(false) + end + expect(first.probes).to eq(1) + expect(second.probes).to eq(1) + end + end + describe '#endpoint' do it 'defaults to nil' do diff --git a/gems/aws-sdk-s3/spec/plugins/checksum_algorithm_spec.rb b/gems/aws-sdk-s3/spec/plugins/checksum_algorithm_spec.rb index 08a0b03f674..b545423592b 100644 --- a/gems/aws-sdk-s3/spec/plugins/checksum_algorithm_spec.rb +++ b/gems/aws-sdk-s3/spec/plugins/checksum_algorithm_spec.rb @@ -151,6 +151,7 @@ module Plugins context 'AwsChunkedTrailerDigestIO' do let(:subject) do ChecksumAlgorithm::AwsChunkedTrailerDigestIO.new( + buffer_enabled: true, io: StringIO.new('x' * (32 * 1024)), algorithm: 'CRC32', location_name: 'x-amz-checksum-crc32' @@ -165,6 +166,7 @@ module Plugins it 'returns zero bytes for empty content' do expected_empty_trailer = "0\r\nx-amz-checksum-crc32:AAAAAA==\r\n\r\n".bytesize subject = ChecksumAlgorithm::AwsChunkedTrailerDigestIO.new( + buffer_enabled: true, io: StringIO.new(''), algorithm: 'CRC32', location_name: 'x-amz-checksum-crc32'