diff --git a/gems/aws-sdk-core/CHANGELOG.md b/gems/aws-sdk-core/CHANGELOG.md index 95b509cf473..a89888a0dd7 100644 --- a/gems/aws-sdk-core/CHANGELOG.md +++ b/gems/aws-sdk-core/CHANGELOG.md @@ -1,6 +1,8 @@ Unreleased Changes ------------------ +* Feature - Reuse read buffers to reduce allocations when calculating request checksums sent in headers. + * Feature - Reduce memory allocations when streaming request bodies with checksum trailers while preserving compatibility with custom IO bodies. 3.265.0 (2026-11-06) diff --git a/gems/aws-sdk-core/lib/aws-sdk-core/plugins/checksum_algorithm.rb b/gems/aws-sdk-core/lib/aws-sdk-core/plugins/checksum_algorithm.rb index 2eacecbf542..cac6455b7db 100644 --- a/gems/aws-sdk-core/lib/aws-sdk-core/plugins/checksum_algorithm.rb +++ b/gems/aws-sdk-core/lib/aws-sdk-core/plugins/checksum_algorithm.rb @@ -331,14 +331,15 @@ def calculate_request_checksum(context, checksum_properties) headers[algorithm_header] = checksum_properties[:algorithm] end + return unless %w[header trailer].include?(checksum_properties[:in]) + + body = context.http_request.body + buffer_enabled = context.http_request.buffer_supported? if body.respond_to?(:read) case checksum_properties[:in] when 'header' - apply_request_checksum(context, headers, checksum_properties) + apply_request_checksum(context, headers, checksum_properties, buffer_enabled) when 'trailer' - buffer_enabled = context.http_request.buffer_supported? apply_request_trailer_checksum(context, headers, checksum_properties, buffer_enabled) - else - # nothing end end @@ -356,19 +357,20 @@ def should_fallback_to_header?(context) context[:skip_trailer_checksums] end - def apply_request_checksum(context, headers, checksum_properties) + def apply_request_checksum(context, headers, checksum_properties, buffer_enabled) header_name = checksum_properties[:name] headers[header_name] = calculate_checksum( checksum_properties[:algorithm], - context.http_request.body + context.http_request.body, + buffer_enabled ) end - def calculate_checksum(algorithm, body) + def calculate_checksum(algorithm, body, buffer_enabled) digest = ChecksumAlgorithm.digest_for_algorithm(algorithm) if body.respond_to?(:read) body.rewind - update_in_chunks(digest, body) + update_in_chunks(digest, body, buffer_enabled) body.rewind else digest.update(body) @@ -376,14 +378,26 @@ def calculate_checksum(algorithm, body) digest.base64digest end - def update_in_chunks(digest, io) - loop do - chunk = io.read(CHECKSUM_CHUNK_SIZE) - break unless chunk + def update_in_chunks(digest, io, buffer_enabled) + if buffer_enabled + update_in_chunks_with_buffer(digest, io) + else + update_in_chunks_without_buffer(digest, io) + end + io.rewind + end + def update_in_chunks_with_buffer(digest, io) + buffer = ''.b + while (chunk = io.read(CHECKSUM_CHUNK_SIZE, buffer)) + digest.update(chunk) + end + end + + def update_in_chunks_without_buffer(digest, io) + while (chunk = io.read(CHECKSUM_CHUNK_SIZE)) digest.update(chunk) end - io.rewind end def apply_request_trailer_checksum(context, headers, checksum_properties, buffer_enabled) diff --git a/gems/aws-sdk-core/spec/aws/plugins/checksum_algorithm_spec.rb b/gems/aws-sdk-core/spec/aws/plugins/checksum_algorithm_spec.rb index 03a5cec9aa7..9ddd6cb717b 100644 --- a/gems/aws-sdk-core/spec/aws/plugins/checksum_algorithm_spec.rb +++ b/gems/aws-sdk-core/spec/aws/plugins/checksum_algorithm_spec.rb @@ -215,6 +215,23 @@ module Plugins expect(resp.context.http_request.headers['x-amz-checksum-crc32']).to eq(original_checksum) end + it 'computes the checksum of a multi-chunk body with a single read buffer' do + stub_const('Aws::Plugins::ChecksumAlgorithm::CHECKSUM_CHUNK_SIZE', 4) + payload = 'hello world' + io = StringIO.new(payload) + buffers = [] + allow(io).to receive(:read).and_wrap_original do |read, length, buffer| + buffers << buffer unless length.zero? + read.call(length, buffer) + end + + resp = client.http_checksum_operation(body: io) + expected = Base64.strict_encode64([Zlib.crc32(payload)].pack('N')) + expect(resp.context.http_request.headers['x-amz-checksum-crc32']).to eq(expected) + expect(buffers.first).to be_a(String) + expect(buffers.uniq(&:object_id).size).to eq(1) + end + file = File.expand_path('checksum_request.json', __dir__) test_cases = JSON.load_file(file) @@ -318,7 +335,7 @@ def send_checksum_request(io) client.public_send(operation, body: io, checksum_algorithm: 'CRC32') end - %w[trailer].each do |location| + %w[header trailer].each do |location| context "checksum in #{location}", skip: location == 'trailer' && defined?(JRUBY_VERSION) && 'Trailer checksums are disabled on JRuby' do let(:placement) { location }