Reduce memory allocations in chunked checksum uploads

build_tools/services.rb

@@ -9,10 +9,10 @@ class ServiceEnumerator
     MANIFEST_PATH = File.expand_path('../../services.json', __FILE__)
 
     # Minimum `aws-sdk-core` version for new gem builds
-    MINIMUM_CORE_VERSION = "3.265.0"
+    MINIMUM_CORE_VERSION = "3.266.0"
 
     # Minimum `aws-sdk-core` version for new S3 gem builds
-    MINIMUM_CORE_VERSION_S3 = "3.265.0"
+    MINIMUM_CORE_VERSION_S3 = "3.266.0"
 
     EVENTSTREAM_PLUGIN = "Aws::Plugins::EventStreamConfiguration"
 

gems/aws-sdk-core/CHANGELOG.md

@@ -1,6 +1,8 @@
 Unreleased Changes
 ------------------
 
+* Feature - Reduce memory allocations when streaming request bodies with checksum trailers while preserving compatibility with custom IO bodies.
+
 3.265.0 (2026-11-06)
 ------------------
 

gems/aws-sdk-core/lib/aws-sdk-core/plugins/checksum_algorithm.rb

@@ -335,7 +335,8 @@ def calculate_request_checksum(context, checksum_properties)
           when 'header'
             apply_request_checksum(context, headers, checksum_properties)
           when 'trailer'
-            apply_request_trailer_checksum(context, headers, checksum_properties)
+            buffer_enabled = context.http_request.buffer_supported?
+            apply_request_trailer_checksum(context, headers, checksum_properties, buffer_enabled)
           else
             # nothing
           end
@@ -385,7 +386,7 @@ def update_in_chunks(digest, io)
           io.rewind
         end
 
-        def apply_request_trailer_checksum(context, headers, checksum_properties)
+        def apply_request_trailer_checksum(context, headers, checksum_properties, buffer_enabled)
           location_name = checksum_properties[:name]
 
           # set required headers
@@ -410,6 +411,7 @@ def apply_request_trailer_checksum(context, headers, checksum_properties)
           context.http_request.body =
             AwsChunkedTrailerDigestIO.new(
               io: context.http_request.body,
+              buffer_enabled: buffer_enabled,
               algorithm: checksum_properties[:algorithm],
               location_name: location_name
             )
@@ -484,6 +486,7 @@ class AwsChunkedTrailerDigestIO
         HEX_BASE = 16
 
         def initialize(options = {})
+          buffer_enabled = options.delete(:buffer_enabled) || false
           @io = options.delete(:io)
           @io.rewind if @io.respond_to?(:rewind)
           @location_name = options.delete(:location_name)
@@ -492,6 +495,8 @@ def initialize(options = {})
           @chunk_size = Thread.current[:net_http_override_body_stream_chunk] || DEFAULT_TRAILER_CHUNK_SIZE
           @overhead_bytes = calculate_overhead(@chunk_size)
           @base_chunk_size = @chunk_size - @overhead_bytes
+          @chunk = ''.b
+          @read_chunk = method(buffer_enabled ? :read_chunk_with_buffer : :read_chunk_without_buffer)
           @encoded_buffer = +''
           @eof = false
         end
@@ -527,8 +532,9 @@ def read(length = nil, buf = nil)
 
           fill_encoded_buffer(length)
 
-          if length
-            output_buffer << @encoded_buffer.slice!(0, length)
+          if length && length < @encoded_buffer.bytesize
+            output_buffer << @encoded_buffer.byteslice(0, length)
+            @encoded_buffer = @encoded_buffer.byteslice(length, @encoded_buffer.bytesize - length)
           else
             output_buffer << @encoded_buffer
             @encoded_buffer.clear
@@ -547,16 +553,24 @@ def calculate_overhead(chunk_size)
           chunk_size.to_s(HEX_BASE).size + CHUNK_OVERHEAD
         end
 
+        def read_chunk_with_buffer
+          @io.read(@base_chunk_size, @chunk)
+        end
+
+        def read_chunk_without_buffer
+          @io.read(@base_chunk_size)
+        end
+
         def fill_encoded_buffer(required_length)
           return if required_length && @encoded_buffer.bytesize >= required_length
 
           while !@eof && fill_data?(required_length)
-            chunk = @io.read(@base_chunk_size)
+            chunk = @read_chunk.call
             if chunk && !chunk.empty?
               @digest.update(chunk)
-              @encoded_buffer << "#{chunk.bytesize.to_s(HEX_BASE)}\r\n#{chunk}\r\n"
+              @encoded_buffer << chunk.bytesize.to_s(HEX_BASE) << "\r\n" << chunk << "\r\n"
             else
-              @encoded_buffer << "0\r\n#{trailer_string}\r\n\r\n"
+              @encoded_buffer << "0\r\n" << trailer_string << "\r\n\r\n"
               @eof = true
             end
           end

gems/aws-sdk-core/lib/seahorse/client/http/request.rb

@@ -55,15 +55,34 @@ def body_contents
           contents
         end
 
+        # @api private
+        def buffer_supported?
+          return @buffer_enabled unless @buffer_enabled.nil?
+
+          @buffer_enabled = detect_read_buffer_support
+        end
+
         # @param [#read, #size, #rewind] io
         def body=(io)
+          @buffer_enabled = nil
           @body = case io
             when nil then StringIO.new('')
             when String then StringIO.new(io)
             else io
           end
         end
 
+        private
+
+        # Only probe ArgumentError falls back. Payload errors must propagate.
+        def detect_read_buffer_support
+          buffer = "\0".b
+          @body.read(0, buffer)
+          buffer.empty?
+        rescue ArgumentError
+          false
+        end
+
       end
     end
   end

gems/aws-sdk-core/spec/aws/plugins/checksum_algorithm_spec.rb

@@ -2,6 +2,7 @@
 
 require_relative '../../spec_helper'
 require 'aws-sdk-core/plugins/checksum_algorithm'
+require 'delegate'
 
 module Aws
   module Plugins
@@ -284,6 +285,91 @@ module Plugins
         end
       end
 
+      context 'IO-like request body compatibility' do
+        let(:payload) { "\x00\xff".b * 11 }
+        let(:expected_checksum) do
+          Base64.strict_encode64([Zlib.crc32(payload)].pack('N'))
+        end
+        let(:one_argument_reader) do
+          Class.new(StringIO) do
+            def read(length)
+              super(length)
+            end
+          end
+        end
+
+        def send_checksum_request(io)
+          operation = placement == 'header' ? :http_checksum_operation : :http_checksum_streaming_operation
+          client.stub_responses(operation, lambda do |context|
+            body = context.http_request.body
+            output = body.read(body.size)
+            headers = context.http_request.headers
+            expect(headers['content-length'].to_i).to eq(output.bytesize)
+            if placement == 'header'
+              expect(output).to eq(payload)
+              expect(headers['x-amz-checksum-crc32']).to eq(expected_checksum)
+            else
+              expected = "#{payload.bytesize.to_s(16)}\r\n#{payload}\r\n" \
+                         "0\r\nx-amz-checksum-crc32:#{expected_checksum}\r\n\r\n"
+              expect(output).to eq(expected)
+            end
+            {}
+          end)
+          client.public_send(operation, body: io, checksum_algorithm: 'CRC32')
+        end
+
+        %w[trailer].each do |location|
+          context "checksum in #{location}",
+                  skip: location == 'trailer' && defined?(JRUBY_VERSION) && 'Trailer checksums are disabled on JRuby' do
+            let(:placement) { location }
+
+            it 'supports a one-argument reader behind a delegator' do
+              send_checksum_request(SimpleDelegator.new(one_argument_reader.new(payload)))
+            end
+
+            it 'supports a reader that ignores the optional output buffer' do
+              klass = Class.new(one_argument_reader) do
+                def read(length, _buffer = nil)
+                  super(length)
+                end
+              end
+              send_checksum_request(klass.new(payload))
+            end
+
+            it 'supports a reader that returns a frozen String instead of filling the buffer' do
+              klass = Class.new(StringIO) do
+                def read(length, buffer = nil)
+                  return super if length.zero? || @returned_frozen
+
+                  @returned_frozen = true
+                  super(length).freeze
+                end
+              end
+              send_checksum_request(klass.new(payload))
+            end
+
+            it 'propagates payload read errors without retrying' do
+              klass = Class.new(StringIO) do
+                attr_reader :payload_reads
+
+                def read(length, buffer = nil)
+                  return super if length.zero?
+
+                  @payload_reads = (@payload_reads || 0) + 1
+                  super(1, buffer)
+                  raise ArgumentError, 'reader failed after consuming a byte'
+                end
+              end
+              io = klass.new(payload)
+              expect { send_checksum_request(io) }.to raise_error(
+                ArgumentError, 'reader failed after consuming a byte'
+              )
+              expect(io.payload_reads).to eq(1)
+            end
+          end
+        end
+      end
+
       context 'response checksum calculation' do
         it 'computes a validation_list' do
           resp = client.http_checksum_operation
@@ -481,19 +567,113 @@ def stub_client(client)
       end
 
       describe 'AwsChunkedTrailerDigestIO' do
+        let(:payload) { "\x00\xff".b * 11 }
+        let(:io) { StringIO.new(payload) }
+        let(:body) do
+          ChecksumAlgorithm::AwsChunkedTrailerDigestIO.new(
+            buffer_enabled: true,
+            io: io,
+            algorithm: 'CRC32',
+            location_name: 'x-amz-checksum-crc32'
+          )
+        end
+        let(:encoded_payload) do
+          checksum = Base64.strict_encode64([Zlib.crc32(payload)].pack('N'))
+          "a\r\n#{payload.byteslice(0, 10)}\r\n" \
+            "a\r\n#{payload.byteslice(10, 10)}\r\n" \
+            "2\r\n#{payload.byteslice(20, 2)}\r\n" \
+            "0\r\nx-amz-checksum-crc32:#{checksum}\r\n\r\n"
+        end
+
+        around do |example|
+          previous_chunk_size = Thread.current[:net_http_override_body_stream_chunk]
+          Thread.current[:net_http_override_body_stream_chunk] = 16
+          example.run
+        ensure
+          Thread.current[:net_http_override_body_stream_chunk] = previous_chunk_size
+        end
+
+        it 'supports a one-argument reader when buffer_enabled is omitted' do
+          klass = Class.new(StringIO) do
+            def read(length)
+              super(length)
+            end
+          end
+          body = ChecksumAlgorithm::AwsChunkedTrailerDigestIO.new(
+            io: klass.new(payload),
+            algorithm: 'CRC32',
+            location_name: 'x-amz-checksum-crc32'
+          )
+
+          expect(body.read).to eq(encoded_payload)
+          expect(body.read).to be_nil
+        end
+
         it 'rewinds IO on initialization' do
           io = StringIO.new('hello world')
           io.read # move to EOF
           expect(io.pos).to eq(11)
 
           ChecksumAlgorithm::AwsChunkedTrailerDigestIO.new(
+            buffer_enabled: true,
             io: io,
             algorithm: 'CRC32',
             location_name: 'x-amz-checksum-crc32'
           )
 
           expect(io.pos).to eq(0)
         end
+
+        it 'reuses the input buffer without changing encoded binary chunks' do
+          buffers = []
+          allow(io).to receive(:read).and_wrap_original do |read, length, buffer|
+            buffers << buffer unless length.zero?
+            read.call(length, buffer)
+          end
+
+          output = String.new
+          buffer = String.new
+          output << buffer while body.read(7, buffer)
+
+          expect(output).to eq(encoded_payload)
+          expect(buffers.size).to eq(4) # three data chunks and EOF
+          expect(buffers.first).to be_a(String)
+          expect(buffers).to all(equal(buffers.first))
+          expect(body).to be_eof
+        end
+
+        it 'replays encoded content when rewound after EOF' do
+          expect(body.read).to eq(encoded_payload)
+          expect(body.read).to be_nil
+          body.rewind
+
+          expect(io).to receive(:read).with(10, an_instance_of(String))
+            .at_least(:once).and_call_original
+          expect(body.read).to eq(encoded_payload)
+          expect(body).to be_eof
+        end
+
+        [16, 17, 64].each do |length|
+          it "preserves chunk boundaries and the trailer with #{length}-byte reads" do
+            output = String.new
+            buffer = String.new
+            while (chunk = body.read(length, buffer))
+              expect(chunk).to equal(buffer)
+              expect(chunk.bytesize).to be <= length
+              output << chunk
+            end
+
+            expect(output).to eq(encoded_payload)
+            expect(body).to be_eof
+          end
+        end
+
+        it 'writes the checksum trailer for an empty input' do
+          io.truncate(0)
+
+          expect(body.read).to eq("0\r\nx-amz-checksum-crc32:AAAAAA==\r\n\r\n")
+          expect(body.read).to be_nil
+        end
       end
     end
   end