From 3db5d32ba01e7009c85bee2a53f0301a24c9b2b0 Mon Sep 17 00:00:00 2001 From: Dmytro Shteflyuk Date: Wed, 14 Oct 2026 18:55:18 -0400 Subject: [PATCH 1/7] Reduce memory allocations in chunked checksum uploads Reuse the input buffer, append chunk and trailer framing directly, and avoid temporary substrings when draining the encoded buffer. Cover binary payloads, partial reads, EOF, and rewind behavior. --- gems/aws-sdk-core/CHANGELOG.md | 2 + .../plugins/checksum_algorithm.rb | 17 +++-- .../aws/plugins/checksum_algorithm_spec.rb | 76 +++++++++++++++++++ 3 files changed, 88 insertions(+), 7 deletions(-) diff --git a/gems/aws-sdk-core/CHANGELOG.md b/gems/aws-sdk-core/CHANGELOG.md index 58df0e488ba..20132bf1f76 100644 --- a/gems/aws-sdk-core/CHANGELOG.md +++ b/gems/aws-sdk-core/CHANGELOG.md @@ -3,6 +3,8 @@ Unreleased Changes * Feature - Improve RBS support for service-specific error handling by making modeled errors inherit from each service's `ServiceError`. +* Issue - Reduce memory allocations when streaming request bodies with checksum trailers. + 3.257.0 (2026-09-14) ------------------ diff --git a/gems/aws-sdk-core/lib/aws-sdk-core/plugins/checksum_algorithm.rb b/gems/aws-sdk-core/lib/aws-sdk-core/plugins/checksum_algorithm.rb index fe7aeafa8c5..7b57c50ce36 100644 --- a/gems/aws-sdk-core/lib/aws-sdk-core/plugins/checksum_algorithm.rb +++ b/gems/aws-sdk-core/lib/aws-sdk-core/plugins/checksum_algorithm.rb @@ -492,6 +492,7 @@ def initialize(options = {}) @chunk_size = Thread.current[:net_http_override_body_stream_chunk] || DEFAULT_TRAILER_CHUNK_SIZE @overhead_bytes = calculate_overhead(@chunk_size) @base_chunk_size = @chunk_size - @overhead_bytes + @chunk = ''.b @encoded_buffer = +'' @eof = false end @@ -513,6 +514,7 @@ def size def rewind @io.rewind + @chunk = ''.b @encoded_buffer = +'' @eof = false @digest = ChecksumAlgorithm.digest_for_algorithm(@algorithm) @@ -527,8 +529,9 @@ def read(length = nil, buf = nil) fill_encoded_buffer(length) - if length - output_buffer << @encoded_buffer.slice!(0, length) + if length && length < @encoded_buffer.bytesize + output_buffer << @encoded_buffer.byteslice(0, length) + @encoded_buffer = @encoded_buffer.byteslice(length, @encoded_buffer.bytesize - length) else output_buffer << @encoded_buffer @encoded_buffer.clear @@ -551,12 +554,12 @@ def fill_encoded_buffer(required_length) return if required_length && @encoded_buffer.bytesize >= required_length while !@eof && fill_data?(required_length) - chunk = @io.read(@base_chunk_size) - if chunk && !chunk.empty? - @digest.update(chunk) - @encoded_buffer << "#{chunk.bytesize.to_s(HEX_BASE)}\r\n#{chunk}\r\n" + @chunk = @io.read(@base_chunk_size, @chunk) + if @chunk && !@chunk.empty? + @digest.update(@chunk) + @encoded_buffer << @chunk.bytesize.to_s(HEX_BASE) << "\r\n" << @chunk << "\r\n" else - @encoded_buffer << "0\r\n#{trailer_string}\r\n\r\n" + @encoded_buffer << "0\r\n" << trailer_string << "\r\n\r\n" @eof = true end end diff --git a/gems/aws-sdk-core/spec/aws/plugins/checksum_algorithm_spec.rb b/gems/aws-sdk-core/spec/aws/plugins/checksum_algorithm_spec.rb index ca514f18c0c..6685f9631d3 100644 --- a/gems/aws-sdk-core/spec/aws/plugins/checksum_algorithm_spec.rb +++ b/gems/aws-sdk-core/spec/aws/plugins/checksum_algorithm_spec.rb @@ -481,6 +481,31 @@ def stub_client(client) end describe 'AwsChunkedTrailerDigestIO' do + let(:payload) { "\x00\xff".b * 11 } + let(:io) { StringIO.new(payload) } + let(:body) do + ChecksumAlgorithm::AwsChunkedTrailerDigestIO.new( + io: io, + algorithm: 'CRC32', + location_name: 'x-amz-checksum-crc32' + ) + end + let(:encoded_payload) do + checksum = Base64.strict_encode64([Zlib.crc32(payload)].pack('N')) + "a\r\n#{payload.byteslice(0, 10)}\r\n" \ + "a\r\n#{payload.byteslice(10, 10)}\r\n" \ + "2\r\n#{payload.byteslice(20, 2)}\r\n" \ + "0\r\nx-amz-checksum-crc32:#{checksum}\r\n\r\n" + end + + around do |example| + previous_chunk_size = Thread.current[:net_http_override_body_stream_chunk] + Thread.current[:net_http_override_body_stream_chunk] = 16 + example.run + ensure + Thread.current[:net_http_override_body_stream_chunk] = previous_chunk_size + end + it 'rewinds IO on initialization' do io = StringIO.new('hello world') io.read # move to EOF @@ -494,6 +519,57 @@ def stub_client(client) expect(io.pos).to eq(0) end + + it 'reuses the input buffer without changing encoded binary chunks' do + buffers = [] + allow(io).to receive(:read).and_wrap_original do |read, length, buffer| + buffers << buffer + read.call(length, buffer) + end + + output = String.new + buffer = String.new + output << buffer while body.read(7, buffer) + + expect(output).to eq(encoded_payload) + expect(buffers.size).to eq(4) # three data chunks and EOF + expect(buffers.first).to be_a(String) + expect(buffers).to all(equal(buffers.first)) + expect(body).to be_eof + end + + it 'reinitializes the input buffer when rewound after EOF' do + expect(body.read).to eq(encoded_payload) + expect(body.read).to be_nil + body.rewind + + expect(io).to receive(:read).with(10, an_instance_of(String)) + .at_least(:once).and_call_original + expect(body.read).to eq(encoded_payload) + expect(body).to be_eof + end + + [16, 17, 64].each do |length| + it "preserves chunk boundaries and the trailer with #{length}-byte reads" do + output = String.new + buffer = String.new + while (chunk = body.read(length, buffer)) + expect(chunk).to equal(buffer) + expect(chunk.bytesize).to be <= length + output << chunk + end + + expect(output).to eq(encoded_payload) + expect(body).to be_eof + end + end + + it 'writes the checksum trailer for an empty input' do + io.truncate(0) + + expect(body.read).to eq("0\r\nx-amz-checksum-crc32:AAAAAA==\r\n\r\n") + expect(body.read).to be_nil + end end end end From b708773d3cffa63256a7807f86fca85c503c4940 Mon Sep 17 00:00:00 2001 From: terajul Date: Thu, 5 Nov 2026 22:08:54 +0000 Subject: [PATCH 2/7] Preserve custom body compatibility for trailer checksums --- .../plugins/checksum_algorithm.rb | 18 ++++- .../lib/seahorse/client/http/request.rb | 19 +++++ .../aws/plugins/checksum_algorithm_spec.rb | 77 ++++++++++++++++++- .../spec/seahorse/client/http/request_spec.rb | 53 +++++++++++++ .../spec/plugins/checksum_algorithm_spec.rb | 2 + 5 files changed, 165 insertions(+), 4 deletions(-) diff --git a/gems/aws-sdk-core/lib/aws-sdk-core/plugins/checksum_algorithm.rb b/gems/aws-sdk-core/lib/aws-sdk-core/plugins/checksum_algorithm.rb index 7b57c50ce36..a40c236cdcf 100644 --- a/gems/aws-sdk-core/lib/aws-sdk-core/plugins/checksum_algorithm.rb +++ b/gems/aws-sdk-core/lib/aws-sdk-core/plugins/checksum_algorithm.rb @@ -335,7 +335,8 @@ def calculate_request_checksum(context, checksum_properties) when 'header' apply_request_checksum(context, headers, checksum_properties) when 'trailer' - apply_request_trailer_checksum(context, headers, checksum_properties) + buffer_enabled = context.http_request.buffer_supported? + apply_request_trailer_checksum(context, headers, checksum_properties, buffer_enabled) else # nothing end @@ -385,7 +386,7 @@ def update_in_chunks(digest, io) io.rewind end - def apply_request_trailer_checksum(context, headers, checksum_properties) + def apply_request_trailer_checksum(context, headers, checksum_properties, buffer_enabled) location_name = checksum_properties[:name] # set required headers @@ -410,6 +411,7 @@ def apply_request_trailer_checksum(context, headers, checksum_properties) context.http_request.body = AwsChunkedTrailerDigestIO.new( io: context.http_request.body, + buffer_enabled: buffer_enabled, algorithm: checksum_properties[:algorithm], location_name: location_name ) @@ -484,6 +486,7 @@ class AwsChunkedTrailerDigestIO HEX_BASE = 16 def initialize(options = {}) + buffer_enabled = options.fetch(:buffer_enabled) @io = options.delete(:io) @io.rewind if @io.respond_to?(:rewind) @location_name = options.delete(:location_name) @@ -493,6 +496,7 @@ def initialize(options = {}) @overhead_bytes = calculate_overhead(@chunk_size) @base_chunk_size = @chunk_size - @overhead_bytes @chunk = ''.b + @read_chunk = method(buffer_enabled ? :read_chunk_with_buffer : :read_chunk_without_buffer) @encoded_buffer = +'' @eof = false end @@ -550,11 +554,19 @@ def calculate_overhead(chunk_size) chunk_size.to_s(HEX_BASE).size + CHUNK_OVERHEAD end + def read_chunk_with_buffer + @io.read(@base_chunk_size, @chunk) + end + + def read_chunk_without_buffer + @io.read(@base_chunk_size) + end + def fill_encoded_buffer(required_length) return if required_length && @encoded_buffer.bytesize >= required_length while !@eof && fill_data?(required_length) - @chunk = @io.read(@base_chunk_size, @chunk) + @chunk = @read_chunk.call if @chunk && !@chunk.empty? @digest.update(@chunk) @encoded_buffer << @chunk.bytesize.to_s(HEX_BASE) << "\r\n" << @chunk << "\r\n" diff --git a/gems/aws-sdk-core/lib/seahorse/client/http/request.rb b/gems/aws-sdk-core/lib/seahorse/client/http/request.rb index e1f1fdda76b..eb00c319ad8 100644 --- a/gems/aws-sdk-core/lib/seahorse/client/http/request.rb +++ b/gems/aws-sdk-core/lib/seahorse/client/http/request.rb @@ -55,8 +55,16 @@ def body_contents contents end + # @api private + def buffer_supported? + return @buffer_enabled unless @buffer_enabled.nil? + + @buffer_enabled = detect_read_buffer_support + end + # @param [#read, #size, #rewind] io def body=(io) + @buffer_enabled = nil @body = case io when nil then StringIO.new('') when String then StringIO.new(io) @@ -64,6 +72,17 @@ def body=(io) end end + private + + # Only probe ArgumentError falls back. Payload errors must propagate. + def detect_read_buffer_support + buffer = "\0".b + @body.read(0, buffer) + buffer.empty? + rescue ArgumentError + false + end + end end end diff --git a/gems/aws-sdk-core/spec/aws/plugins/checksum_algorithm_spec.rb b/gems/aws-sdk-core/spec/aws/plugins/checksum_algorithm_spec.rb index 6685f9631d3..e477485904b 100644 --- a/gems/aws-sdk-core/spec/aws/plugins/checksum_algorithm_spec.rb +++ b/gems/aws-sdk-core/spec/aws/plugins/checksum_algorithm_spec.rb @@ -2,6 +2,7 @@ require_relative '../../spec_helper' require 'aws-sdk-core/plugins/checksum_algorithm' +require 'delegate' module Aws module Plugins @@ -284,6 +285,78 @@ module Plugins end end + context 'IO-like request body compatibility' do + let(:payload) { "\x00\xff".b * 11 } + let(:expected_checksum) do + Base64.strict_encode64([Zlib.crc32(payload)].pack('N')) + end + let(:one_argument_reader) do + Class.new(StringIO) do + def read(length) + super(length) + end + end + end + + def send_checksum_request(io) + operation = placement == 'header' ? :http_checksum_operation : :http_checksum_streaming_operation + client.stub_responses(operation, lambda do |context| + body = context.http_request.body + output = body.read(body.size) + headers = context.http_request.headers + expect(headers['content-length'].to_i).to eq(output.bytesize) + if placement == 'header' + expect(output).to eq(payload) + expect(headers['x-amz-checksum-crc32']).to eq(expected_checksum) + else + expected = "#{payload.bytesize.to_s(16)}\r\n#{payload}\r\n" \ + "0\r\nx-amz-checksum-crc32:#{expected_checksum}\r\n\r\n" + expect(output).to eq(expected) + end + {} + end) + client.public_send(operation, body: io, checksum_algorithm: 'CRC32') + end + + %w[trailer].each do |location| + context "checksum in #{location}" do + let(:placement) { location } + + it 'supports a one-argument reader behind a delegator' do + send_checksum_request(SimpleDelegator.new(one_argument_reader.new(payload))) + end + + it 'supports a reader that ignores the optional output buffer' do + klass = Class.new(one_argument_reader) do + def read(length, _buffer = nil) + super(length) + end + end + send_checksum_request(klass.new(payload)) + end + + it 'propagates payload read errors without retrying' do + klass = Class.new(StringIO) do + attr_reader :payload_reads + + def read(length, buffer = nil) + return super if length.zero? + + @payload_reads = (@payload_reads || 0) + 1 + super(1, buffer) + raise ArgumentError, 'reader failed after consuming a byte' + end + end + io = klass.new(payload) + expect { send_checksum_request(io) }.to raise_error( + ArgumentError, 'reader failed after consuming a byte' + ) + expect(io.payload_reads).to eq(1) + end + end + end + end + context 'response checksum calculation' do it 'computes a validation_list' do resp = client.http_checksum_operation @@ -485,6 +558,7 @@ def stub_client(client) let(:io) { StringIO.new(payload) } let(:body) do ChecksumAlgorithm::AwsChunkedTrailerDigestIO.new( + buffer_enabled: true, io: io, algorithm: 'CRC32', location_name: 'x-amz-checksum-crc32' @@ -512,6 +586,7 @@ def stub_client(client) expect(io.pos).to eq(11) ChecksumAlgorithm::AwsChunkedTrailerDigestIO.new( + buffer_enabled: true, io: io, algorithm: 'CRC32', location_name: 'x-amz-checksum-crc32' @@ -523,7 +598,7 @@ def stub_client(client) it 'reuses the input buffer without changing encoded binary chunks' do buffers = [] allow(io).to receive(:read).and_wrap_original do |read, length, buffer| - buffers << buffer + buffers << buffer unless length.zero? read.call(length, buffer) end diff --git a/gems/aws-sdk-core/spec/seahorse/client/http/request_spec.rb b/gems/aws-sdk-core/spec/seahorse/client/http/request_spec.rb index df785137701..2438bf04dcf 100644 --- a/gems/aws-sdk-core/spec/seahorse/client/http/request_spec.rb +++ b/gems/aws-sdk-core/spec/seahorse/client/http/request_spec.rb @@ -10,6 +10,59 @@ module Http def request(*args) Request.new(*args) end + describe '#buffer_supported?' do + let(:reader_class) do + Class.new(StringIO) do + attr_reader :probes + + def initialize(supported) + super('payload') + @supported = supported + @probes = 0 + end + + def read(length = nil, buffer = nil) + @probes += 1 if length == 0 + @supported ? super(length, buffer) : super(length) + end + end + end + + [true, false].each do |supported| + it "caches #{supported} without consuming body bytes" do + io = reader_class.new(supported) + req = request(body: io) + 3.times { expect(req.buffer_supported?).to be(supported) } + expect(io.probes).to eq(1) + expect(io.pos).to eq(0) + end + end + + it 'checks the new body after assignment' do + first = reader_class.new(false) + second = reader_class.new(true) + req = request(body: first) + expect(req.buffer_supported?).to be(false) + req.body = second + 2.times { expect(req.buffer_supported?).to be(true) } + expect(first.probes).to eq(1) + expect(second.probes).to eq(1) + end + + it 'keeps results separate for different requests' do + first = reader_class.new(true) + second = reader_class.new(false) + first_request = request(body: first) + second_request = request(body: second) + 2.times do + expect(first_request.buffer_supported?).to be(true) + expect(second_request.buffer_supported?).to be(false) + end + expect(first.probes).to eq(1) + expect(second.probes).to eq(1) + end + end + describe '#endpoint' do it 'defaults to nil' do diff --git a/gems/aws-sdk-s3/spec/plugins/checksum_algorithm_spec.rb b/gems/aws-sdk-s3/spec/plugins/checksum_algorithm_spec.rb index 08a0b03f674..b545423592b 100644 --- a/gems/aws-sdk-s3/spec/plugins/checksum_algorithm_spec.rb +++ b/gems/aws-sdk-s3/spec/plugins/checksum_algorithm_spec.rb @@ -151,6 +151,7 @@ module Plugins context 'AwsChunkedTrailerDigestIO' do let(:subject) do ChecksumAlgorithm::AwsChunkedTrailerDigestIO.new( + buffer_enabled: true, io: StringIO.new('x' * (32 * 1024)), algorithm: 'CRC32', location_name: 'x-amz-checksum-crc32' @@ -165,6 +166,7 @@ module Plugins it 'returns zero bytes for empty content' do expected_empty_trailer = "0\r\nx-amz-checksum-crc32:AAAAAA==\r\n\r\n".bytesize subject = ChecksumAlgorithm::AwsChunkedTrailerDigestIO.new( + buffer_enabled: true, io: StringIO.new(''), algorithm: 'CRC32', location_name: 'x-amz-checksum-crc32' From cbc11ab5dbdaeecd2c9a33c962bff3415524a931 Mon Sep 17 00:00:00 2001 From: terajul Date: Thu, 5 Nov 2026 22:14:56 +0000 Subject: [PATCH 3/7] Classify checksum allocation improvements as a core feature --- gems/aws-sdk-core/CHANGELOG.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/gems/aws-sdk-core/CHANGELOG.md b/gems/aws-sdk-core/CHANGELOG.md index 1949fa41112..34bd5e42409 100644 --- a/gems/aws-sdk-core/CHANGELOG.md +++ b/gems/aws-sdk-core/CHANGELOG.md @@ -1,7 +1,7 @@ Unreleased Changes ------------------ -* Issue - Reduce memory allocations when streaming request bodies with checksum trailers. +* Feature - Reduce memory allocations when streaming request bodies with checksum trailers while preserving compatibility with custom IO bodies. 3.264.0 (2026-11-04) ------------------ From e403f9052c3c476614ef3c6b1fa6125fc77a618b Mon Sep 17 00:00:00 2001 From: terajul Date: Thu, 5 Nov 2026 22:28:33 +0000 Subject: [PATCH 4/7] Temporarily include ruby-ll unpin from PR 3467 for CI --- Gemfile | 1 - 1 file changed, 1 deletion(-) diff --git a/Gemfile b/Gemfile index 8c0e23dee70..6290c2eb0d6 100644 --- a/Gemfile +++ b/Gemfile @@ -26,7 +26,6 @@ gem 'json' gem 'nokogiri' gem 'oga' gem 'rexml' -gem 'ruby-ll', '2.1.3' # temporary unless defined?(JRUBY_VERSION) gem 'libxml-ruby' gem 'oj' From ac343ffe4f120e26ce375f87cf36af5ada0a50f2 Mon Sep 17 00:00:00 2001 From: terajul Date: Fri, 6 Nov 2026 16:15:07 +0000 Subject: [PATCH 5/7] Preserve trailer read buffers and fix compatibility defaults --- .../plugins/checksum_algorithm.rb | 11 +++---- .../aws/plugins/checksum_algorithm_spec.rb | 33 +++++++++++++++++-- 2 files changed, 36 insertions(+), 8 deletions(-) diff --git a/gems/aws-sdk-core/lib/aws-sdk-core/plugins/checksum_algorithm.rb b/gems/aws-sdk-core/lib/aws-sdk-core/plugins/checksum_algorithm.rb index a40c236cdcf..2eacecbf542 100644 --- a/gems/aws-sdk-core/lib/aws-sdk-core/plugins/checksum_algorithm.rb +++ b/gems/aws-sdk-core/lib/aws-sdk-core/plugins/checksum_algorithm.rb @@ -486,7 +486,7 @@ class AwsChunkedTrailerDigestIO HEX_BASE = 16 def initialize(options = {}) - buffer_enabled = options.fetch(:buffer_enabled) + buffer_enabled = options.delete(:buffer_enabled) || false @io = options.delete(:io) @io.rewind if @io.respond_to?(:rewind) @location_name = options.delete(:location_name) @@ -518,7 +518,6 @@ def size def rewind @io.rewind - @chunk = ''.b @encoded_buffer = +'' @eof = false @digest = ChecksumAlgorithm.digest_for_algorithm(@algorithm) @@ -566,10 +565,10 @@ def fill_encoded_buffer(required_length) return if required_length && @encoded_buffer.bytesize >= required_length while !@eof && fill_data?(required_length) - @chunk = @read_chunk.call - if @chunk && !@chunk.empty? - @digest.update(@chunk) - @encoded_buffer << @chunk.bytesize.to_s(HEX_BASE) << "\r\n" << @chunk << "\r\n" + chunk = @read_chunk.call + if chunk && !chunk.empty? + @digest.update(chunk) + @encoded_buffer << chunk.bytesize.to_s(HEX_BASE) << "\r\n" << chunk << "\r\n" else @encoded_buffer << "0\r\n" << trailer_string << "\r\n\r\n" @eof = true diff --git a/gems/aws-sdk-core/spec/aws/plugins/checksum_algorithm_spec.rb b/gems/aws-sdk-core/spec/aws/plugins/checksum_algorithm_spec.rb index e477485904b..03a5cec9aa7 100644 --- a/gems/aws-sdk-core/spec/aws/plugins/checksum_algorithm_spec.rb +++ b/gems/aws-sdk-core/spec/aws/plugins/checksum_algorithm_spec.rb @@ -319,7 +319,8 @@ def send_checksum_request(io) end %w[trailer].each do |location| - context "checksum in #{location}" do + context "checksum in #{location}", + skip: location == 'trailer' && defined?(JRUBY_VERSION) && 'Trailer checksums are disabled on JRuby' do let(:placement) { location } it 'supports a one-argument reader behind a delegator' do @@ -335,6 +336,18 @@ def read(length, _buffer = nil) send_checksum_request(klass.new(payload)) end + it 'supports a reader that returns a frozen String instead of filling the buffer' do + klass = Class.new(StringIO) do + def read(length, buffer = nil) + return super if length.zero? || @returned_frozen + + @returned_frozen = true + super(length).freeze + end + end + send_checksum_request(klass.new(payload)) + end + it 'propagates payload read errors without retrying' do klass = Class.new(StringIO) do attr_reader :payload_reads @@ -580,6 +593,22 @@ def stub_client(client) Thread.current[:net_http_override_body_stream_chunk] = previous_chunk_size end + it 'supports a one-argument reader when buffer_enabled is omitted' do + klass = Class.new(StringIO) do + def read(length) + super(length) + end + end + body = ChecksumAlgorithm::AwsChunkedTrailerDigestIO.new( + io: klass.new(payload), + algorithm: 'CRC32', + location_name: 'x-amz-checksum-crc32' + ) + + expect(body.read).to eq(encoded_payload) + expect(body.read).to be_nil + end + it 'rewinds IO on initialization' do io = StringIO.new('hello world') io.read # move to EOF @@ -613,7 +642,7 @@ def stub_client(client) expect(body).to be_eof end - it 'reinitializes the input buffer when rewound after EOF' do + it 'replays encoded content when rewound after EOF' do expect(body.read).to eq(encoded_payload) expect(body.read).to be_nil body.rewind From 0e464148a22e41dd2e0fe47dc9e2581f1e620e48 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Cl=C3=A9ment=20Joubert?= Date: Fri, 23 Oct 2026 18:28:28 +0300 Subject: [PATCH 6/7] Reuse a single read buffer when computing request checksums in a header --- gems/aws-sdk-core/CHANGELOG.md | 2 ++ .../aws-sdk-core/plugins/checksum_algorithm.rb | 8 ++------ .../spec/aws/plugins/checksum_algorithm_spec.rb | 17 +++++++++++++++++ 3 files changed, 21 insertions(+), 6 deletions(-) diff --git a/gems/aws-sdk-core/CHANGELOG.md b/gems/aws-sdk-core/CHANGELOG.md index 34bd5e42409..8a8afd16dc6 100644 --- a/gems/aws-sdk-core/CHANGELOG.md +++ b/gems/aws-sdk-core/CHANGELOG.md @@ -1,6 +1,8 @@ Unreleased Changes ------------------ +* Issue - Reuse a single read buffer when computing request checksums in a header. + * Feature - Reduce memory allocations when streaming request bodies with checksum trailers while preserving compatibility with custom IO bodies. 3.264.0 (2026-11-04) diff --git a/gems/aws-sdk-core/lib/aws-sdk-core/plugins/checksum_algorithm.rb b/gems/aws-sdk-core/lib/aws-sdk-core/plugins/checksum_algorithm.rb index 2eacecbf542..18580c7f9ba 100644 --- a/gems/aws-sdk-core/lib/aws-sdk-core/plugins/checksum_algorithm.rb +++ b/gems/aws-sdk-core/lib/aws-sdk-core/plugins/checksum_algorithm.rb @@ -377,12 +377,8 @@ def calculate_checksum(algorithm, body) end def update_in_chunks(digest, io) - loop do - chunk = io.read(CHECKSUM_CHUNK_SIZE) - break unless chunk - - digest.update(chunk) - end + chunk = ''.b + digest.update(chunk) while io.read(CHECKSUM_CHUNK_SIZE, chunk) io.rewind end diff --git a/gems/aws-sdk-core/spec/aws/plugins/checksum_algorithm_spec.rb b/gems/aws-sdk-core/spec/aws/plugins/checksum_algorithm_spec.rb index 03a5cec9aa7..a047ef87b6b 100644 --- a/gems/aws-sdk-core/spec/aws/plugins/checksum_algorithm_spec.rb +++ b/gems/aws-sdk-core/spec/aws/plugins/checksum_algorithm_spec.rb @@ -215,6 +215,23 @@ module Plugins expect(resp.context.http_request.headers['x-amz-checksum-crc32']).to eq(original_checksum) end + it 'computes the checksum of a multi-chunk body with a single read buffer' do + stub_const('Aws::Plugins::ChecksumAlgorithm::CHECKSUM_CHUNK_SIZE', 4) + payload = 'hello world' + io = StringIO.new(payload) + buffers = [] + allow(io).to receive(:read).and_wrap_original do |read, length, buffer| + buffers << buffer + read.call(length, buffer) + end + + resp = client.http_checksum_operation(body: io) + expected = Base64.strict_encode64([Zlib.crc32(payload)].pack('N')) + expect(resp.context.http_request.headers['x-amz-checksum-crc32']).to eq(expected) + expect(buffers.first).to be_a(String) + expect(buffers.uniq(&:object_id).size).to eq(1) + end + file = File.expand_path('checksum_request.json', __dir__) test_cases = JSON.load_file(file) From 3a88a090294a59b7a9b0f192fe881124a6479fe3 Mon Sep 17 00:00:00 2001 From: terajul Date: Thu, 5 Nov 2026 22:33:47 +0000 Subject: [PATCH 7/7] Preserve custom body compatibility for header checksums --- gems/aws-sdk-core/CHANGELOG.md | 2 +- .../plugins/checksum_algorithm.rb | 40 ++++++++++++++----- .../aws/plugins/checksum_algorithm_spec.rb | 4 +- 3 files changed, 32 insertions(+), 14 deletions(-) diff --git a/gems/aws-sdk-core/CHANGELOG.md b/gems/aws-sdk-core/CHANGELOG.md index 8a8afd16dc6..50f55f8459d 100644 --- a/gems/aws-sdk-core/CHANGELOG.md +++ b/gems/aws-sdk-core/CHANGELOG.md @@ -1,7 +1,7 @@ Unreleased Changes ------------------ -* Issue - Reuse a single read buffer when computing request checksums in a header. +* Feature - Reduce memory allocations when computing request checksums in a header. * Feature - Reduce memory allocations when streaming request bodies with checksum trailers while preserving compatibility with custom IO bodies. diff --git a/gems/aws-sdk-core/lib/aws-sdk-core/plugins/checksum_algorithm.rb b/gems/aws-sdk-core/lib/aws-sdk-core/plugins/checksum_algorithm.rb index 18580c7f9ba..cac6455b7db 100644 --- a/gems/aws-sdk-core/lib/aws-sdk-core/plugins/checksum_algorithm.rb +++ b/gems/aws-sdk-core/lib/aws-sdk-core/plugins/checksum_algorithm.rb @@ -331,14 +331,15 @@ def calculate_request_checksum(context, checksum_properties) headers[algorithm_header] = checksum_properties[:algorithm] end + return unless %w[header trailer].include?(checksum_properties[:in]) + + body = context.http_request.body + buffer_enabled = context.http_request.buffer_supported? if body.respond_to?(:read) case checksum_properties[:in] when 'header' - apply_request_checksum(context, headers, checksum_properties) + apply_request_checksum(context, headers, checksum_properties, buffer_enabled) when 'trailer' - buffer_enabled = context.http_request.buffer_supported? apply_request_trailer_checksum(context, headers, checksum_properties, buffer_enabled) - else - # nothing end end @@ -356,19 +357,20 @@ def should_fallback_to_header?(context) context[:skip_trailer_checksums] end - def apply_request_checksum(context, headers, checksum_properties) + def apply_request_checksum(context, headers, checksum_properties, buffer_enabled) header_name = checksum_properties[:name] headers[header_name] = calculate_checksum( checksum_properties[:algorithm], - context.http_request.body + context.http_request.body, + buffer_enabled ) end - def calculate_checksum(algorithm, body) + def calculate_checksum(algorithm, body, buffer_enabled) digest = ChecksumAlgorithm.digest_for_algorithm(algorithm) if body.respond_to?(:read) body.rewind - update_in_chunks(digest, body) + update_in_chunks(digest, body, buffer_enabled) body.rewind else digest.update(body) @@ -376,12 +378,28 @@ def calculate_checksum(algorithm, body) digest.base64digest end - def update_in_chunks(digest, io) - chunk = ''.b - digest.update(chunk) while io.read(CHECKSUM_CHUNK_SIZE, chunk) + def update_in_chunks(digest, io, buffer_enabled) + if buffer_enabled + update_in_chunks_with_buffer(digest, io) + else + update_in_chunks_without_buffer(digest, io) + end io.rewind end + def update_in_chunks_with_buffer(digest, io) + buffer = ''.b + while (chunk = io.read(CHECKSUM_CHUNK_SIZE, buffer)) + digest.update(chunk) + end + end + + def update_in_chunks_without_buffer(digest, io) + while (chunk = io.read(CHECKSUM_CHUNK_SIZE)) + digest.update(chunk) + end + end + def apply_request_trailer_checksum(context, headers, checksum_properties, buffer_enabled) location_name = checksum_properties[:name] diff --git a/gems/aws-sdk-core/spec/aws/plugins/checksum_algorithm_spec.rb b/gems/aws-sdk-core/spec/aws/plugins/checksum_algorithm_spec.rb index a047ef87b6b..9ddd6cb717b 100644 --- a/gems/aws-sdk-core/spec/aws/plugins/checksum_algorithm_spec.rb +++ b/gems/aws-sdk-core/spec/aws/plugins/checksum_algorithm_spec.rb @@ -221,7 +221,7 @@ module Plugins io = StringIO.new(payload) buffers = [] allow(io).to receive(:read).and_wrap_original do |read, length, buffer| - buffers << buffer + buffers << buffer unless length.zero? read.call(length, buffer) end @@ -335,7 +335,7 @@ def send_checksum_request(io) client.public_send(operation, body: io, checksum_algorithm: 'CRC32') end - %w[trailer].each do |location| + %w[header trailer].each do |location| context "checksum in #{location}", skip: location == 'trailer' && defined?(JRUBY_VERSION) && 'Trailer checksums are disabled on JRuby' do let(:placement) { location }