Upload with a single 1 MiB buffer instead of one per chunk when checksumming in a header (−60 MB RSS on a 21 MB PUT)

gems/aws-sdk-core/CHANGELOG.md

@@ -1,6 +1,8 @@
 Unreleased Changes
 ------------------
 
+* Feature - Reuse read buffers to reduce allocations when calculating request checksums sent in headers.
+
 * Feature - Reduce memory allocations when streaming request bodies with checksum trailers while preserving compatibility with custom IO bodies.
 
 3.265.0 (2026-11-06)

gems/aws-sdk-core/lib/aws-sdk-core/plugins/checksum_algorithm.rb

@@ -331,14 +331,15 @@ def calculate_request_checksum(context, checksum_properties)
             headers[algorithm_header] = checksum_properties[:algorithm]
           end
 
+          return unless %w[header trailer].include?(checksum_properties[:in])
+
+          body = context.http_request.body
+          buffer_enabled = context.http_request.buffer_supported? if body.respond_to?(:read)
           case checksum_properties[:in]
           when 'header'
-            apply_request_checksum(context, headers, checksum_properties)
+            apply_request_checksum(context, headers, checksum_properties, buffer_enabled)
           when 'trailer'
-            buffer_enabled = context.http_request.buffer_supported?
             apply_request_trailer_checksum(context, headers, checksum_properties, buffer_enabled)
-          else
-            # nothing
           end
         end
 
@@ -356,34 +357,47 @@ def should_fallback_to_header?(context)
           context[:skip_trailer_checksums]
         end
 
-        def apply_request_checksum(context, headers, checksum_properties)
+        def apply_request_checksum(context, headers, checksum_properties, buffer_enabled)
           header_name = checksum_properties[:name]
           headers[header_name] = calculate_checksum(
             checksum_properties[:algorithm],
-            context.http_request.body
+            context.http_request.body,
+            buffer_enabled
           )
         end
 
-        def calculate_checksum(algorithm, body)
+        def calculate_checksum(algorithm, body, buffer_enabled)
           digest = ChecksumAlgorithm.digest_for_algorithm(algorithm)
           if body.respond_to?(:read)
             body.rewind
-            update_in_chunks(digest, body)
+            update_in_chunks(digest, body, buffer_enabled)
             body.rewind
           else
             digest.update(body)
           end
           digest.base64digest
         end
 
-        def update_in_chunks(digest, io)
-          loop do
-            chunk = io.read(CHECKSUM_CHUNK_SIZE)
-            break unless chunk
+        def update_in_chunks(digest, io, buffer_enabled)
+          if buffer_enabled
+            update_in_chunks_with_buffer(digest, io)
+          else
+            update_in_chunks_without_buffer(digest, io)
+          end
+          io.rewind
+        end
 
+        def update_in_chunks_with_buffer(digest, io)
+          buffer = ''.b
+          while (chunk = io.read(CHECKSUM_CHUNK_SIZE, buffer))
+            digest.update(chunk)
+          end
+        end
+
+        def update_in_chunks_without_buffer(digest, io)
+          while (chunk = io.read(CHECKSUM_CHUNK_SIZE))
             digest.update(chunk)
           end
-          io.rewind
         end
 
         def apply_request_trailer_checksum(context, headers, checksum_properties, buffer_enabled)

gems/aws-sdk-core/spec/aws/plugins/checksum_algorithm_spec.rb

@@ -215,6 +215,23 @@ module Plugins
           expect(resp.context.http_request.headers['x-amz-checksum-crc32']).to eq(original_checksum)
         end
 
+        it 'computes the checksum of a multi-chunk body with a single read buffer' do
+          stub_const('Aws::Plugins::ChecksumAlgorithm::CHECKSUM_CHUNK_SIZE', 4)
+          payload = 'hello world'
+          io = StringIO.new(payload)
+          buffers = []
+          allow(io).to receive(:read).and_wrap_original do |read, length, buffer|
+            buffers << buffer unless length.zero?
+            read.call(length, buffer)
+          end
+
+          resp = client.http_checksum_operation(body: io)
+          expected = Base64.strict_encode64([Zlib.crc32(payload)].pack('N'))
+          expect(resp.context.http_request.headers['x-amz-checksum-crc32']).to eq(expected)
+          expect(buffers.first).to be_a(String)
+          expect(buffers.uniq(&:object_id).size).to eq(1)
+        end
+
         file = File.expand_path('checksum_request.json', __dir__)
         test_cases = JSON.load_file(file)
 
@@ -318,7 +335,7 @@ def send_checksum_request(io)
           client.public_send(operation, body: io, checksum_algorithm: 'CRC32')
         end
 
-        %w[trailer].each do |location|
+        %w[header trailer].each do |location|
           context "checksum in #{location}",
                   skip: location == 'trailer' && defined?(JRUBY_VERSION) && 'Trailer checksums are disabled on JRuby' do
             let(:placement) { location }