From 3db5d32ba01e7009c85bee2a53f0301a24c9b2b0 Mon Sep 17 00:00:00 2001 From: Dmytro Shteflyuk Date: Wed, 14 Oct 2026 18:55:18 -0400 Subject: [PATCH 1/5] Reduce memory allocations in chunked checksum uploads Reuse the input buffer, append chunk and trailer framing directly, and avoid temporary substrings when draining the encoded buffer. Cover binary payloads, partial reads, EOF, and rewind behavior. --- gems/aws-sdk-core/CHANGELOG.md | 2 + .../plugins/checksum_algorithm.rb | 17 +++-- .../aws/plugins/checksum_algorithm_spec.rb | 76 +++++++++++++++++++ 3 files changed, 88 insertions(+), 7 deletions(-) diff --git a/gems/aws-sdk-core/CHANGELOG.md b/gems/aws-sdk-core/CHANGELOG.md index 58df0e488ba..20132bf1f76 100644 --- a/gems/aws-sdk-core/CHANGELOG.md +++ b/gems/aws-sdk-core/CHANGELOG.md @@ -3,6 +3,8 @@ Unreleased Changes * Feature - Improve RBS support for service-specific error handling by making modeled errors inherit from each service's `ServiceError`. +* Issue - Reduce memory allocations when streaming request bodies with checksum trailers. + 3.257.0 (2026-09-14) ------------------ diff --git a/gems/aws-sdk-core/lib/aws-sdk-core/plugins/checksum_algorithm.rb b/gems/aws-sdk-core/lib/aws-sdk-core/plugins/checksum_algorithm.rb index fe7aeafa8c5..7b57c50ce36 100644 --- a/gems/aws-sdk-core/lib/aws-sdk-core/plugins/checksum_algorithm.rb +++ b/gems/aws-sdk-core/lib/aws-sdk-core/plugins/checksum_algorithm.rb @@ -492,6 +492,7 @@ def initialize(options = {}) @chunk_size = Thread.current[:net_http_override_body_stream_chunk] || DEFAULT_TRAILER_CHUNK_SIZE @overhead_bytes = calculate_overhead(@chunk_size) @base_chunk_size = @chunk_size - @overhead_bytes + @chunk = ''.b @encoded_buffer = +'' @eof = false end @@ -513,6 +514,7 @@ def size def rewind @io.rewind + @chunk = ''.b @encoded_buffer = +'' @eof = false @digest = ChecksumAlgorithm.digest_for_algorithm(@algorithm) @@ -527,8 +529,9 @@ def read(length = nil, buf = nil) fill_encoded_buffer(length) - if length - output_buffer << @encoded_buffer.slice!(0, length) + if length && length < @encoded_buffer.bytesize + output_buffer << @encoded_buffer.byteslice(0, length) + @encoded_buffer = @encoded_buffer.byteslice(length, @encoded_buffer.bytesize - length) else output_buffer << @encoded_buffer @encoded_buffer.clear @@ -551,12 +554,12 @@ def fill_encoded_buffer(required_length) return if required_length && @encoded_buffer.bytesize >= required_length while !@eof && fill_data?(required_length) - chunk = @io.read(@base_chunk_size) - if chunk && !chunk.empty? - @digest.update(chunk) - @encoded_buffer << "#{chunk.bytesize.to_s(HEX_BASE)}\r\n#{chunk}\r\n" + @chunk = @io.read(@base_chunk_size, @chunk) + if @chunk && !@chunk.empty? + @digest.update(@chunk) + @encoded_buffer << @chunk.bytesize.to_s(HEX_BASE) << "\r\n" << @chunk << "\r\n" else - @encoded_buffer << "0\r\n#{trailer_string}\r\n\r\n" + @encoded_buffer << "0\r\n" << trailer_string << "\r\n\r\n" @eof = true end end diff --git a/gems/aws-sdk-core/spec/aws/plugins/checksum_algorithm_spec.rb b/gems/aws-sdk-core/spec/aws/plugins/checksum_algorithm_spec.rb index ca514f18c0c..6685f9631d3 100644 --- a/gems/aws-sdk-core/spec/aws/plugins/checksum_algorithm_spec.rb +++ b/gems/aws-sdk-core/spec/aws/plugins/checksum_algorithm_spec.rb @@ -481,6 +481,31 @@ def stub_client(client) end describe 'AwsChunkedTrailerDigestIO' do + let(:payload) { "\x00\xff".b * 11 } + let(:io) { StringIO.new(payload) } + let(:body) do + ChecksumAlgorithm::AwsChunkedTrailerDigestIO.new( + io: io, + algorithm: 'CRC32', + location_name: 'x-amz-checksum-crc32' + ) + end + let(:encoded_payload) do + checksum = Base64.strict_encode64([Zlib.crc32(payload)].pack('N')) + "a\r\n#{payload.byteslice(0, 10)}\r\n" \ + "a\r\n#{payload.byteslice(10, 10)}\r\n" \ + "2\r\n#{payload.byteslice(20, 2)}\r\n" \ + "0\r\nx-amz-checksum-crc32:#{checksum}\r\n\r\n" + end + + around do |example| + previous_chunk_size = Thread.current[:net_http_override_body_stream_chunk] + Thread.current[:net_http_override_body_stream_chunk] = 16 + example.run + ensure + Thread.current[:net_http_override_body_stream_chunk] = previous_chunk_size + end + it 'rewinds IO on initialization' do io = StringIO.new('hello world') io.read # move to EOF @@ -494,6 +519,57 @@ def stub_client(client) expect(io.pos).to eq(0) end + + it 'reuses the input buffer without changing encoded binary chunks' do + buffers = [] + allow(io).to receive(:read).and_wrap_original do |read, length, buffer| + buffers << buffer + read.call(length, buffer) + end + + output = String.new + buffer = String.new + output << buffer while body.read(7, buffer) + + expect(output).to eq(encoded_payload) + expect(buffers.size).to eq(4) # three data chunks and EOF + expect(buffers.first).to be_a(String) + expect(buffers).to all(equal(buffers.first)) + expect(body).to be_eof + end + + it 'reinitializes the input buffer when rewound after EOF' do + expect(body.read).to eq(encoded_payload) + expect(body.read).to be_nil + body.rewind + + expect(io).to receive(:read).with(10, an_instance_of(String)) + .at_least(:once).and_call_original + expect(body.read).to eq(encoded_payload) + expect(body).to be_eof + end + + [16, 17, 64].each do |length| + it "preserves chunk boundaries and the trailer with #{length}-byte reads" do + output = String.new + buffer = String.new + while (chunk = body.read(length, buffer)) + expect(chunk).to equal(buffer) + expect(chunk.bytesize).to be <= length + output << chunk + end + + expect(output).to eq(encoded_payload) + expect(body).to be_eof + end + end + + it 'writes the checksum trailer for an empty input' do + io.truncate(0) + + expect(body.read).to eq("0\r\nx-amz-checksum-crc32:AAAAAA==\r\n\r\n") + expect(body.read).to be_nil + end end end end From b708773d3cffa63256a7807f86fca85c503c4940 Mon Sep 17 00:00:00 2001 From: terajul Date: Thu, 5 Nov 2026 22:08:54 +0000 Subject: [PATCH 2/5] Preserve custom body compatibility for trailer checksums --- .../plugins/checksum_algorithm.rb | 18 ++++- .../lib/seahorse/client/http/request.rb | 19 +++++ .../aws/plugins/checksum_algorithm_spec.rb | 77 ++++++++++++++++++- .../spec/seahorse/client/http/request_spec.rb | 53 +++++++++++++ .../spec/plugins/checksum_algorithm_spec.rb | 2 + 5 files changed, 165 insertions(+), 4 deletions(-) diff --git a/gems/aws-sdk-core/lib/aws-sdk-core/plugins/checksum_algorithm.rb b/gems/aws-sdk-core/lib/aws-sdk-core/plugins/checksum_algorithm.rb index 7b57c50ce36..a40c236cdcf 100644 --- a/gems/aws-sdk-core/lib/aws-sdk-core/plugins/checksum_algorithm.rb +++ b/gems/aws-sdk-core/lib/aws-sdk-core/plugins/checksum_algorithm.rb @@ -335,7 +335,8 @@ def calculate_request_checksum(context, checksum_properties) when 'header' apply_request_checksum(context, headers, checksum_properties) when 'trailer' - apply_request_trailer_checksum(context, headers, checksum_properties) + buffer_enabled = context.http_request.buffer_supported? + apply_request_trailer_checksum(context, headers, checksum_properties, buffer_enabled) else # nothing end @@ -385,7 +386,7 @@ def update_in_chunks(digest, io) io.rewind end - def apply_request_trailer_checksum(context, headers, checksum_properties) + def apply_request_trailer_checksum(context, headers, checksum_properties, buffer_enabled) location_name = checksum_properties[:name] # set required headers @@ -410,6 +411,7 @@ def apply_request_trailer_checksum(context, headers, checksum_properties) context.http_request.body = AwsChunkedTrailerDigestIO.new( io: context.http_request.body, + buffer_enabled: buffer_enabled, algorithm: checksum_properties[:algorithm], location_name: location_name ) @@ -484,6 +486,7 @@ class AwsChunkedTrailerDigestIO HEX_BASE = 16 def initialize(options = {}) + buffer_enabled = options.fetch(:buffer_enabled) @io = options.delete(:io) @io.rewind if @io.respond_to?(:rewind) @location_name = options.delete(:location_name) @@ -493,6 +496,7 @@ def initialize(options = {}) @overhead_bytes = calculate_overhead(@chunk_size) @base_chunk_size = @chunk_size - @overhead_bytes @chunk = ''.b + @read_chunk = method(buffer_enabled ? :read_chunk_with_buffer : :read_chunk_without_buffer) @encoded_buffer = +'' @eof = false end @@ -550,11 +554,19 @@ def calculate_overhead(chunk_size) chunk_size.to_s(HEX_BASE).size + CHUNK_OVERHEAD end + def read_chunk_with_buffer + @io.read(@base_chunk_size, @chunk) + end + + def read_chunk_without_buffer + @io.read(@base_chunk_size) + end + def fill_encoded_buffer(required_length) return if required_length && @encoded_buffer.bytesize >= required_length while !@eof && fill_data?(required_length) - @chunk = @io.read(@base_chunk_size, @chunk) + @chunk = @read_chunk.call if @chunk && !@chunk.empty? @digest.update(@chunk) @encoded_buffer << @chunk.bytesize.to_s(HEX_BASE) << "\r\n" << @chunk << "\r\n" diff --git a/gems/aws-sdk-core/lib/seahorse/client/http/request.rb b/gems/aws-sdk-core/lib/seahorse/client/http/request.rb index e1f1fdda76b..eb00c319ad8 100644 --- a/gems/aws-sdk-core/lib/seahorse/client/http/request.rb +++ b/gems/aws-sdk-core/lib/seahorse/client/http/request.rb @@ -55,8 +55,16 @@ def body_contents contents end + # @api private + def buffer_supported? + return @buffer_enabled unless @buffer_enabled.nil? + + @buffer_enabled = detect_read_buffer_support + end + # @param [#read, #size, #rewind] io def body=(io) + @buffer_enabled = nil @body = case io when nil then StringIO.new('') when String then StringIO.new(io) @@ -64,6 +72,17 @@ def body=(io) end end + private + + # Only probe ArgumentError falls back. Payload errors must propagate. + def detect_read_buffer_support + buffer = "\0".b + @body.read(0, buffer) + buffer.empty? + rescue ArgumentError + false + end + end end end diff --git a/gems/aws-sdk-core/spec/aws/plugins/checksum_algorithm_spec.rb b/gems/aws-sdk-core/spec/aws/plugins/checksum_algorithm_spec.rb index 6685f9631d3..e477485904b 100644 --- a/gems/aws-sdk-core/spec/aws/plugins/checksum_algorithm_spec.rb +++ b/gems/aws-sdk-core/spec/aws/plugins/checksum_algorithm_spec.rb @@ -2,6 +2,7 @@ require_relative '../../spec_helper' require 'aws-sdk-core/plugins/checksum_algorithm' +require 'delegate' module Aws module Plugins @@ -284,6 +285,78 @@ module Plugins end end + context 'IO-like request body compatibility' do + let(:payload) { "\x00\xff".b * 11 } + let(:expected_checksum) do + Base64.strict_encode64([Zlib.crc32(payload)].pack('N')) + end + let(:one_argument_reader) do + Class.new(StringIO) do + def read(length) + super(length) + end + end + end + + def send_checksum_request(io) + operation = placement == 'header' ? :http_checksum_operation : :http_checksum_streaming_operation + client.stub_responses(operation, lambda do |context| + body = context.http_request.body + output = body.read(body.size) + headers = context.http_request.headers + expect(headers['content-length'].to_i).to eq(output.bytesize) + if placement == 'header' + expect(output).to eq(payload) + expect(headers['x-amz-checksum-crc32']).to eq(expected_checksum) + else + expected = "#{payload.bytesize.to_s(16)}\r\n#{payload}\r\n" \ + "0\r\nx-amz-checksum-crc32:#{expected_checksum}\r\n\r\n" + expect(output).to eq(expected) + end + {} + end) + client.public_send(operation, body: io, checksum_algorithm: 'CRC32') + end + + %w[trailer].each do |location| + context "checksum in #{location}" do + let(:placement) { location } + + it 'supports a one-argument reader behind a delegator' do + send_checksum_request(SimpleDelegator.new(one_argument_reader.new(payload))) + end + + it 'supports a reader that ignores the optional output buffer' do + klass = Class.new(one_argument_reader) do + def read(length, _buffer = nil) + super(length) + end + end + send_checksum_request(klass.new(payload)) + end + + it 'propagates payload read errors without retrying' do + klass = Class.new(StringIO) do + attr_reader :payload_reads + + def read(length, buffer = nil) + return super if length.zero? + + @payload_reads = (@payload_reads || 0) + 1 + super(1, buffer) + raise ArgumentError, 'reader failed after consuming a byte' + end + end + io = klass.new(payload) + expect { send_checksum_request(io) }.to raise_error( + ArgumentError, 'reader failed after consuming a byte' + ) + expect(io.payload_reads).to eq(1) + end + end + end + end + context 'response checksum calculation' do it 'computes a validation_list' do resp = client.http_checksum_operation @@ -485,6 +558,7 @@ def stub_client(client) let(:io) { StringIO.new(payload) } let(:body) do ChecksumAlgorithm::AwsChunkedTrailerDigestIO.new( + buffer_enabled: true, io: io, algorithm: 'CRC32', location_name: 'x-amz-checksum-crc32' @@ -512,6 +586,7 @@ def stub_client(client) expect(io.pos).to eq(11) ChecksumAlgorithm::AwsChunkedTrailerDigestIO.new( + buffer_enabled: true, io: io, algorithm: 'CRC32', location_name: 'x-amz-checksum-crc32' @@ -523,7 +598,7 @@ def stub_client(client) it 'reuses the input buffer without changing encoded binary chunks' do buffers = [] allow(io).to receive(:read).and_wrap_original do |read, length, buffer| - buffers << buffer + buffers << buffer unless length.zero? read.call(length, buffer) end diff --git a/gems/aws-sdk-core/spec/seahorse/client/http/request_spec.rb b/gems/aws-sdk-core/spec/seahorse/client/http/request_spec.rb index df785137701..2438bf04dcf 100644 --- a/gems/aws-sdk-core/spec/seahorse/client/http/request_spec.rb +++ b/gems/aws-sdk-core/spec/seahorse/client/http/request_spec.rb @@ -10,6 +10,59 @@ module Http def request(*args) Request.new(*args) end + describe '#buffer_supported?' do + let(:reader_class) do + Class.new(StringIO) do + attr_reader :probes + + def initialize(supported) + super('payload') + @supported = supported + @probes = 0 + end + + def read(length = nil, buffer = nil) + @probes += 1 if length == 0 + @supported ? super(length, buffer) : super(length) + end + end + end + + [true, false].each do |supported| + it "caches #{supported} without consuming body bytes" do + io = reader_class.new(supported) + req = request(body: io) + 3.times { expect(req.buffer_supported?).to be(supported) } + expect(io.probes).to eq(1) + expect(io.pos).to eq(0) + end + end + + it 'checks the new body after assignment' do + first = reader_class.new(false) + second = reader_class.new(true) + req = request(body: first) + expect(req.buffer_supported?).to be(false) + req.body = second + 2.times { expect(req.buffer_supported?).to be(true) } + expect(first.probes).to eq(1) + expect(second.probes).to eq(1) + end + + it 'keeps results separate for different requests' do + first = reader_class.new(true) + second = reader_class.new(false) + first_request = request(body: first) + second_request = request(body: second) + 2.times do + expect(first_request.buffer_supported?).to be(true) + expect(second_request.buffer_supported?).to be(false) + end + expect(first.probes).to eq(1) + expect(second.probes).to eq(1) + end + end + describe '#endpoint' do it 'defaults to nil' do diff --git a/gems/aws-sdk-s3/spec/plugins/checksum_algorithm_spec.rb b/gems/aws-sdk-s3/spec/plugins/checksum_algorithm_spec.rb index 08a0b03f674..b545423592b 100644 --- a/gems/aws-sdk-s3/spec/plugins/checksum_algorithm_spec.rb +++ b/gems/aws-sdk-s3/spec/plugins/checksum_algorithm_spec.rb @@ -151,6 +151,7 @@ module Plugins context 'AwsChunkedTrailerDigestIO' do let(:subject) do ChecksumAlgorithm::AwsChunkedTrailerDigestIO.new( + buffer_enabled: true, io: StringIO.new('x' * (32 * 1024)), algorithm: 'CRC32', location_name: 'x-amz-checksum-crc32' @@ -165,6 +166,7 @@ module Plugins it 'returns zero bytes for empty content' do expected_empty_trailer = "0\r\nx-amz-checksum-crc32:AAAAAA==\r\n\r\n".bytesize subject = ChecksumAlgorithm::AwsChunkedTrailerDigestIO.new( + buffer_enabled: true, io: StringIO.new(''), algorithm: 'CRC32', location_name: 'x-amz-checksum-crc32' From cbc11ab5dbdaeecd2c9a33c962bff3415524a931 Mon Sep 17 00:00:00 2001 From: terajul Date: Thu, 5 Nov 2026 22:14:56 +0000 Subject: [PATCH 3/5] Classify checksum allocation improvements as a core feature --- gems/aws-sdk-core/CHANGELOG.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/gems/aws-sdk-core/CHANGELOG.md b/gems/aws-sdk-core/CHANGELOG.md index 1949fa41112..34bd5e42409 100644 --- a/gems/aws-sdk-core/CHANGELOG.md +++ b/gems/aws-sdk-core/CHANGELOG.md @@ -1,7 +1,7 @@ Unreleased Changes ------------------ -* Issue - Reduce memory allocations when streaming request bodies with checksum trailers. +* Feature - Reduce memory allocations when streaming request bodies with checksum trailers while preserving compatibility with custom IO bodies. 3.264.0 (2026-11-04) ------------------ From e403f9052c3c476614ef3c6b1fa6125fc77a618b Mon Sep 17 00:00:00 2001 From: terajul Date: Thu, 5 Nov 2026 22:28:33 +0000 Subject: [PATCH 4/5] Temporarily include ruby-ll unpin from PR 3467 for CI --- Gemfile | 1 - 1 file changed, 1 deletion(-) diff --git a/Gemfile b/Gemfile index 8c0e23dee70..6290c2eb0d6 100644 --- a/Gemfile +++ b/Gemfile @@ -26,7 +26,6 @@ gem 'json' gem 'nokogiri' gem 'oga' gem 'rexml' -gem 'ruby-ll', '2.1.3' # temporary unless defined?(JRUBY_VERSION) gem 'libxml-ruby' gem 'oj' From ac343ffe4f120e26ce375f87cf36af5ada0a50f2 Mon Sep 17 00:00:00 2001 From: terajul Date: Fri, 6 Nov 2026 16:15:07 +0000 Subject: [PATCH 5/5] Preserve trailer read buffers and fix compatibility defaults --- .../plugins/checksum_algorithm.rb | 11 +++---- .../aws/plugins/checksum_algorithm_spec.rb | 33 +++++++++++++++++-- 2 files changed, 36 insertions(+), 8 deletions(-) diff --git a/gems/aws-sdk-core/lib/aws-sdk-core/plugins/checksum_algorithm.rb b/gems/aws-sdk-core/lib/aws-sdk-core/plugins/checksum_algorithm.rb index a40c236cdcf..2eacecbf542 100644 --- a/gems/aws-sdk-core/lib/aws-sdk-core/plugins/checksum_algorithm.rb +++ b/gems/aws-sdk-core/lib/aws-sdk-core/plugins/checksum_algorithm.rb @@ -486,7 +486,7 @@ class AwsChunkedTrailerDigestIO HEX_BASE = 16 def initialize(options = {}) - buffer_enabled = options.fetch(:buffer_enabled) + buffer_enabled = options.delete(:buffer_enabled) || false @io = options.delete(:io) @io.rewind if @io.respond_to?(:rewind) @location_name = options.delete(:location_name) @@ -518,7 +518,6 @@ def size def rewind @io.rewind - @chunk = ''.b @encoded_buffer = +'' @eof = false @digest = ChecksumAlgorithm.digest_for_algorithm(@algorithm) @@ -566,10 +565,10 @@ def fill_encoded_buffer(required_length) return if required_length && @encoded_buffer.bytesize >= required_length while !@eof && fill_data?(required_length) - @chunk = @read_chunk.call - if @chunk && !@chunk.empty? - @digest.update(@chunk) - @encoded_buffer << @chunk.bytesize.to_s(HEX_BASE) << "\r\n" << @chunk << "\r\n" + chunk = @read_chunk.call + if chunk && !chunk.empty? + @digest.update(chunk) + @encoded_buffer << chunk.bytesize.to_s(HEX_BASE) << "\r\n" << chunk << "\r\n" else @encoded_buffer << "0\r\n" << trailer_string << "\r\n\r\n" @eof = true diff --git a/gems/aws-sdk-core/spec/aws/plugins/checksum_algorithm_spec.rb b/gems/aws-sdk-core/spec/aws/plugins/checksum_algorithm_spec.rb index e477485904b..03a5cec9aa7 100644 --- a/gems/aws-sdk-core/spec/aws/plugins/checksum_algorithm_spec.rb +++ b/gems/aws-sdk-core/spec/aws/plugins/checksum_algorithm_spec.rb @@ -319,7 +319,8 @@ def send_checksum_request(io) end %w[trailer].each do |location| - context "checksum in #{location}" do + context "checksum in #{location}", + skip: location == 'trailer' && defined?(JRUBY_VERSION) && 'Trailer checksums are disabled on JRuby' do let(:placement) { location } it 'supports a one-argument reader behind a delegator' do @@ -335,6 +336,18 @@ def read(length, _buffer = nil) send_checksum_request(klass.new(payload)) end + it 'supports a reader that returns a frozen String instead of filling the buffer' do + klass = Class.new(StringIO) do + def read(length, buffer = nil) + return super if length.zero? || @returned_frozen + + @returned_frozen = true + super(length).freeze + end + end + send_checksum_request(klass.new(payload)) + end + it 'propagates payload read errors without retrying' do klass = Class.new(StringIO) do attr_reader :payload_reads @@ -580,6 +593,22 @@ def stub_client(client) Thread.current[:net_http_override_body_stream_chunk] = previous_chunk_size end + it 'supports a one-argument reader when buffer_enabled is omitted' do + klass = Class.new(StringIO) do + def read(length) + super(length) + end + end + body = ChecksumAlgorithm::AwsChunkedTrailerDigestIO.new( + io: klass.new(payload), + algorithm: 'CRC32', + location_name: 'x-amz-checksum-crc32' + ) + + expect(body.read).to eq(encoded_payload) + expect(body.read).to be_nil + end + it 'rewinds IO on initialization' do io = StringIO.new('hello world') io.read # move to EOF @@ -613,7 +642,7 @@ def stub_client(client) expect(body).to be_eof end - it 'reinitializes the input buffer when rewound after EOF' do + it 'replays encoded content when rewound after EOF' do expect(body.read).to eq(encoded_payload) expect(body.read).to be_nil body.rewind