diff --git a/build_tools/aws-sdk-code-generator/lib/aws-sdk-code-generator/helper.rb b/build_tools/aws-sdk-code-generator/lib/aws-sdk-code-generator/helper.rb index a4a39b1a2ad..b1006981211 100644 --- a/build_tools/aws-sdk-code-generator/lib/aws-sdk-code-generator/helper.rb +++ b/build_tools/aws-sdk-code-generator/lib/aws-sdk-code-generator/helper.rb @@ -178,6 +178,31 @@ def wrap_string(str, width, indent = '') str.gsub(/(.{1,#{width}})(\s+|\Z)/, "#{indent}\\1\n").chomp end + # Returns true if the member reference or its target shape is sensitive. + # Also checks nested list members and map keys and values. + # + # Stops at structures because each structure has its own SENSITIVE list. + def sensitive_shape_ref?(ref, api, visited = Set.new) + return true if ref['sensitive'] + + name = ref['shape'] + return false if visited.include?(name) + + visited << name + target = Api.shape(ref, api) + return true if target['sensitive'] + + case target['type'] + when 'list' + sensitive_shape_ref?(target['member'], api, visited) + when 'map' + sensitive_shape_ref?(target['key'], api, visited) || + sensitive_shape_ref?(target['value'], api, visited) + else + false + end + end + def gem_lib_path(gem_name) File.expand_path( File.join( diff --git a/build_tools/aws-sdk-code-generator/lib/aws-sdk-code-generator/views/rbs/types_module.rb b/build_tools/aws-sdk-code-generator/lib/aws-sdk-code-generator/views/rbs/types_module.rb index 9d89d69ac05..e8ca27e2672 100644 --- a/build_tools/aws-sdk-code-generator/lib/aws-sdk-code-generator/views/rbs/types_module.rb +++ b/build_tools/aws-sdk-code-generator/lib/aws-sdk-code-generator/views/rbs/types_module.rb @@ -82,8 +82,7 @@ def eventstreams def struct_members(shape) return if shape['members'].nil? members = shape['members'].map do |member_name, member_ref| - sensitive = !!(member_ref['sensitive'] || - @api['shapes'][member_ref['shape']]['sensitive']) + sensitive = sensitive_shape_ref?(member_ref, @api) StructMember.new( member_name: underscore(member_name), sensitive: sensitive, diff --git a/build_tools/aws-sdk-code-generator/lib/aws-sdk-code-generator/views/types_module.rb b/build_tools/aws-sdk-code-generator/lib/aws-sdk-code-generator/views/types_module.rb index 134294d663f..37714f1b93b 100644 --- a/build_tools/aws-sdk-code-generator/lib/aws-sdk-code-generator/views/types_module.rb +++ b/build_tools/aws-sdk-code-generator/lib/aws-sdk-code-generator/views/types_module.rb @@ -97,19 +97,7 @@ def gem_name def struct_members(shape) return if shape['members'].nil? members = shape['members'].map do |member_name, member_ref| - member_target = @api['shapes'][member_ref['shape']] - sensitive = !!(member_ref['sensitive'] || - member_target['sensitive']) - - case member_target["type"] - when 'map' - key_shape = @api['shapes'][member_target['key']['shape']] - value_shape = @api['shapes'][member_target['value']['shape']] - sensitive ||= !!(key_shape['sensitive'] || value_shape['sensitive']) - when 'list' - list_member = @api['shapes'][member_target['member']['shape']] - sensitive ||= !!(list_member['sensitive']) - end + sensitive = sensitive_shape_ref?(member_ref, @api) StructMember.new( member_name: underscore(member_name), diff --git a/build_tools/aws-sdk-code-generator/spec/fixtures/interfaces/sensitive/api.json b/build_tools/aws-sdk-code-generator/spec/fixtures/interfaces/sensitive/api.json index d8fae730483..262094e5a73 100644 --- a/build_tools/aws-sdk-code-generator/spec/fixtures/interfaces/sensitive/api.json +++ b/build_tools/aws-sdk-code-generator/spec/fixtures/interfaces/sensitive/api.json @@ -13,9 +13,160 @@ }, "input":{"shape":"KitchenSink"}, "output":{"shape":"KitchenSink"} + }, + "NestedCollectionsOperation": { + "name":"NestedCollectionsOperation", + "http":{ + "method":"POST", + "requestUri":"/NestedCollectionsOperation" + }, + "input":{"shape":"NestedCollections"}, + "output":{"shape":"NestedCollections"} + }, + "NestedStructuresOperation": { + "name":"NestedStructuresOperation", + "http":{ + "method":"POST", + "requestUri":"/NestedStructuresOperation" + }, + "input":{"shape":"NestedStructures"}, + "output":{"shape":"NestedStructures"} } }, "shapes": { + "NestedCollections": { + "type": "structure", + "members": { + "ListOfListOfStrings": { + "shape": "ListOfListOfStrings" + }, + "MapOfMapsOfLists": { + "shape": "MapOfMapsOfLists" + }, + "SensitiveKeyMap": { + "shape": "SensitiveKeyMap" + }, + "PlainList": { + "shape": "ListOfPlainStrings" + }, + "PlainMapOfLists": { + "shape": "MapOfListsOfPlainStrings" + }, + "MemberSensitiveLists": { + "shape": "ListOfMemberSensitiveLists" + } + } + }, + "MapOfMapsOfLists": { + "type": "map", + "key": { + "shape": "PlainString" + }, + "value": { + "shape": "MapOfListsOfStringsWithPlainKeys" + } + }, + "MapOfListsOfStringsWithPlainKeys": { + "type": "map", + "key": { + "shape": "PlainString" + }, + "value": { + "shape": "ListOfStrings" + } + }, + "ListOfMemberSensitiveLists": { + "type": "list", + "member": { + "shape": "MemberSensitiveList" + } + }, + "MemberSensitiveList": { + "type": "list", + "member": { + "shape": "PlainString", + "sensitive": true + } + }, + "NestedStructures": { + "type": "structure", + "members": { + "Inner": { + "shape": "CollectionMemberStruct" + }, + "InnerList": { + "shape": "ListOfCollectionMemberStructs" + }, + "InnerMap": { + "shape": "MapOfCollectionMemberStructs" + } + } + }, + "CollectionMemberStruct": { + "type": "structure", + "members": { + "Values": { + "shape": "ListOfListOfStrings" + }, + "PlainValue": { + "shape": "PlainString" + }, + "ClientSecret": { + "shape": "PlainString" + }, + "Password": { + "shape": "PlainString" + }, + "PrivateKey": { + "shape": "PlainString" + }, + "RefreshToken": { + "shape": "PlainString" + } + } + }, + "ListOfCollectionMemberStructs": { + "type": "list", + "member": { + "shape": "CollectionMemberStruct" + } + }, + "MapOfCollectionMemberStructs": { + "type": "map", + "key": { + "shape": "PlainString" + }, + "value": { + "shape": "CollectionMemberStruct" + } + }, + "SensitiveKeyMap": { + "type": "map", + "key": { + "shape": "String" + }, + "value": { + "shape": "PlainString" + } + }, + "ListOfPlainStrings": { + "type": "list", + "member": { + "shape": "PlainString" + } + }, + "MapOfListsOfPlainStrings": { + "type": "map", + "key": { + "shape": "PlainString" + }, + "value": { + "shape": "ListOfPlainStrings" + } + }, + "PlainString": { + "type": "string" + }, "KitchenSink": { "type": "structure", "members": { diff --git a/build_tools/aws-sdk-code-generator/spec/interfaces/sensitive_spec.rb b/build_tools/aws-sdk-code-generator/spec/interfaces/sensitive_spec.rb index 2f792de5057..3b155fe0c17 100644 --- a/build_tools/aws-sdk-code-generator/spec/interfaces/sensitive_spec.rb +++ b/build_tools/aws-sdk-code-generator/spec/interfaces/sensitive_spec.rb @@ -41,5 +41,110 @@ expect(resp.to_s).to eq(expected.to_s) end end + + describe '#nested_collections' do + it 'filters members that reach a sensitive shape through lists and maps' do + expect(Sensitive::Types::NestedCollections::SENSITIVE).to contain_exactly( + :list_of_list_of_strings, :map_of_maps_of_lists, :sensitive_key_map, + :member_sensitive_lists + ) + + client.stub_responses( + :nested_collections_operation, + list_of_list_of_strings: [['secret']], + map_of_maps_of_lists: { 'a' => { 'b' => ['secret'] } }, + sensitive_key_map: { 'secret' => 'value' }, + plain_list: ['visible'], + plain_map_of_lists: { 'c' => ['visible'] }, + member_sensitive_lists: [['member-secret']] + ) + resp = client.nested_collections_operation + expected = { + list_of_list_of_strings: '[FILTERED]', + map_of_maps_of_lists: '[FILTERED]', + sensitive_key_map: '[FILTERED]', + plain_list: ['visible'], + plain_map_of_lists: { 'c' => ['visible'] }, + member_sensitive_lists: '[FILTERED]' + } + + expect(resp.to_s).to eq(expected.to_s) + end + + it 'filters collections inside structures while keeping ordinary members visible' do + expect(Sensitive::Types::CollectionMemberStruct::SENSITIVE).to eq([:values]) + expect(Sensitive::Types::NestedStructures::SENSITIVE).to be_empty + + inner = { values: [['nested-secret']], plain_value: 'visible' } + params = { + inner: inner, + inner_list: [inner], + inner_map: { 'a' => inner } + } + client.stub_responses(:nested_structures_operation, params) + resp = client.nested_structures_operation(params) + filtered_inner = { + values: '[FILTERED]', plain_value: 'visible', + client_secret: nil, password: nil, private_key: nil, refresh_token: nil + } + expected = { + inner: filtered_inner, + inner_list: [filtered_inner], + inner_map: { 'a' => filtered_inner } + } + + expect(resp.to_s).to eq(expected.to_s) + + log = Aws::Log::Formatter.new(':request_params').format(resp) + logged_inner = '{values:"[FILTERED]",plain_value:"visible"}' + expect(log).to include("inner:#{logged_inner}") + expect(log).to include("inner_list:[#{logged_inner}]") + expect(log).to include("inner_map:{\"a\"=>#{logged_inner}}") + expect(log).not_to include('nested-secret') + end + + it 'filters globally sensitive names missing model traits from request logs' do + expect(Sensitive::Types::CollectionMemberStruct::SENSITIVE).to eq([:values]) + + inner = { + values: [['modeled-secret']], plain_value: 'visible', + client_secret: 'oidc-secret', password: 'mq-secret', + private_key: 'apns-secret', refresh_token: 'salesforce-secret' + } + resp = client.nested_structures_operation( + inner: inner, + inner_list: [inner], + inner_map: { 'a' => inner } + ) + log = Aws::Log::Formatter.new(':request_params').format(resp) + logged_inner = '{values:"[FILTERED]",plain_value:"visible",' \ + 'client_secret:"[FILTERED]",password:"[FILTERED]",' \ + 'private_key:"[FILTERED]",refresh_token:"[FILTERED]"}' + expect(log).to eq( + "inner:#{logged_inner}," \ + "inner_list:[#{logged_inner}]," \ + "inner_map:{\"a\"=>#{logged_inner}}" + ) + end + + it 'uses the same sensitive members in the RBS types' do + api = JSON.parse(File.read(File.expand_path( + '../fixtures/interfaces/sensitive/api.json', __dir__ + ))) + service = AwsSdkCodeGenerator::Service.new( + name: 'Sensitive', module_name: 'Sensitive', api: api, + gem_version: '1.0.0' + ) + view = AwsSdkCodeGenerator::Views::RBS::TypesModule.new(service: service) + structs = view.structures.select do |struct| + %w[KitchenSink NestedCollections NestedStructures CollectionMemberStruct].include?(struct.class_name) + end + expect(structs.size).to eq(4) + structs.each do |struct| + expected = Sensitive::Types.const_get(struct.class_name)::SENSITIVE + expect(struct.sensitive_params).to match_array(expected) + end + end + end end end diff --git a/build_tools/services.rb b/build_tools/services.rb index dd39e571b8c..45fc3228d10 100644 --- a/build_tools/services.rb +++ b/build_tools/services.rb @@ -9,10 +9,10 @@ class ServiceEnumerator MANIFEST_PATH = File.expand_path('../../services.json', __FILE__) # Minimum `aws-sdk-core` version for new gem builds - MINIMUM_CORE_VERSION = "3.264.0" + MINIMUM_CORE_VERSION = "3.265.0" # Minimum `aws-sdk-core` version for new S3 gem builds - MINIMUM_CORE_VERSION_S3 = "3.264.0" + MINIMUM_CORE_VERSION_S3 = "3.265.0" EVENTSTREAM_PLUGIN = "Aws::Plugins::EventStreamConfiguration" diff --git a/gems/aws-sdk-core/CHANGELOG.md b/gems/aws-sdk-core/CHANGELOG.md index 718762649a4..7d6b7f1bb00 100644 --- a/gems/aws-sdk-core/CHANGELOG.md +++ b/gems/aws-sdk-core/CHANGELOG.md @@ -1,6 +1,8 @@ Unreleased Changes ------------------ +* Feature - Improve model-aware filtering of nested request parameters and sensitivity detection for nested collections. + 3.264.0 (2026-11-04) ------------------ diff --git a/gems/aws-sdk-core/lib/aws-sdk-core/log/formatter.rb b/gems/aws-sdk-core/lib/aws-sdk-core/log/formatter.rb index dd1e2dfef7f..3dfe6ee9e74 100644 --- a/gems/aws-sdk-core/lib/aws-sdk-core/log/formatter.rb +++ b/gems/aws-sdk-core/lib/aws-sdk-core/log/formatter.rb @@ -132,8 +132,9 @@ def _operation(response) def _request_params(response) params = response.context.params - type = response.context.operation.input.shape.struct_class - @param_formatter.summarize(@param_filter.filter(params, type)) + operation = response.context.operation + ref = operation && operation.input + @param_formatter.summarize(@param_filter.filter(params, ref)) end def _time(response) diff --git a/gems/aws-sdk-core/lib/aws-sdk-core/log/param_filter.rb b/gems/aws-sdk-core/lib/aws-sdk-core/log/param_filter.rb index d3f26b5061f..2490af119cd 100644 --- a/gems/aws-sdk-core/lib/aws-sdk-core/log/param_filter.rb +++ b/gems/aws-sdk-core/lib/aws-sdk-core/log/param_filter.rb @@ -24,7 +24,68 @@ def initialize(options = {}) @additional_filters = options[:filter] || [] end + # Returns a copy with sensitive values replaced by "[FILTERED]". + # + # Uses generated SENSITIVE lists to choose which members to redact. + # Nested request hashes also use the global list for backwards compatibility. + # A ShapeRef preserves model types while filtering nested request parameters. + # Structure classes are also accepted for Structure#to_s and older callers. + # + # @param [Hash, Struct, Array, Object] values + # @param [Seahorse::Model::Shapes::ShapeRef, Class, nil] type def filter(values, type) + return filter_ref(values, type, root: true) if type.is_a?(Seahorse::Model::Shapes::ShapeRef) + + filter_by_class(values, type) + end + + private + + def filter_ref(value, ref, root: false) + return filter_by_class(value, value.class) unless ref + + shape = ref.shape + case shape + when Seahorse::Model::Shapes::StructureShape + if value.is_a?(Hash) || value.is_a?(Struct) + return filter_struct( + value, shape.struct_class, shape, + use_global_filters: !root && value.is_a?(Hash) + ) + end + when Seahorse::Model::Shapes::ListShape + return value.map { |v| filter_ref(v, shape.member) } if value.is_a?(Array) + when Seahorse::Model::Shapes::MapShape + return filter_map(value, shape) if value.is_a?(Hash) + when Seahorse::Model::Shapes::DocumentShape, nil + return filter_by_class(value, value.class) + else + return value + end + + filter_by_class(value, value.class) + end + + def filter_map(values, shape) + filtered = {} + values.each_pair do |key, value| + filtered[key] = + if @enabled && @additional_filters.include?(key) + '[FILTERED]' + else + filter_ref(value, shape.value) + end + end + filtered + end + + def member_ref(shape, key) + return unless key.is_a?(Symbol) || key.is_a?(String) + + shape.member(key) if shape.member?(key) + end + + def filter_by_class(values, type) case values when Struct then filter_struct(values, type) when Hash then filter_hash(values, type) @@ -33,18 +94,19 @@ def filter(values, type) end end - private - - def filter_struct(values, type) - if values.class.include? Aws::Structure::Union + def filter_struct(values, type, shape = nil, use_global_filters: false) + if values.is_a?(Struct) && values.class.include?(Aws::Structure::Union) values = { values.member => values.value } end - filter_hash(values, type) + filter_hash(values, type, shape, use_global_filters: use_global_filters) end - def filter_hash(values, type) - if type.const_defined?('SENSITIVE') + def filter_hash(values, type, shape = nil, use_global_filters: false) + if type && type.const_defined?(:SENSITIVE) filters = type::SENSITIVE + @additional_filters + # Nested hashes previously used the global list. Keep that protection + # alongside modeled sensitivity when following their shape references. + filters += SENSITIVE if use_global_filters else # Support backwards compatibility (new core + old service) filters = SENSITIVE + @additional_filters @@ -54,15 +116,17 @@ def filter_hash(values, type) values.each_pair do |key, value| filtered[key] = if @enabled && filters.include?(key) '[FILTERED]' + elsif shape + filter_ref(value, member_ref(shape, key)) else - filter(value, value.class) + filter_by_class(value, value.class) end end filtered end def filter_array(values, type) - values.map { |value| filter(value, value.class) } + values.map { |value| filter_by_class(value, value.class) } end end diff --git a/gems/aws-sdk-core/spec/aws/log/formatter_spec.rb b/gems/aws-sdk-core/spec/aws/log/formatter_spec.rb index 7ea773cd1fc..f99e7643e89 100644 --- a/gems/aws-sdk-core/spec/aws/log/formatter_spec.rb +++ b/gems/aws-sdk-core/spec/aws/log/formatter_spec.rb @@ -65,6 +65,15 @@ def format(pattern, options = {}) file.close end + it 'filters nested sensitive request parameters' do + response.context.params = { + nested: { sensitive_string: 'secret', string: 'visible' } + } + expect(format(':request_params')).to eq( + 'nested:{sensitive_string:"[FILTERED]",string:"visible"}' + ) + end + it 'provides a :time replacement' do now = Time.now response.context[:logging_started_at] = now - 1.54321 diff --git a/gems/aws-sdk-core/spec/aws/log/param_filter_spec.rb b/gems/aws-sdk-core/spec/aws/log/param_filter_spec.rb index ae1648e0bcd..bea3c265975 100644 --- a/gems/aws-sdk-core/spec/aws/log/param_filter_spec.rb +++ b/gems/aws-sdk-core/spec/aws/log/param_filter_spec.rb @@ -136,6 +136,157 @@ class ComplexSensitiveType < Struct.new( expect(filtered).to eq(password: '[FILTERED]', peccy_id: 'peccy-id') end end + + context 'with a ShapeRef' do + let(:input_ref) do + ApiHelper.sample_api.operation(:example_operation).input + end + + let(:inner_params) do + { sensitive_string: 'secret', string: 'visible' } + end + + let(:filtered_inner) do + { sensitive_string: '[FILTERED]', string: 'visible' } + end + + let(:params) do + { + sensitive_string: 'secret', + nested: inner_params, + nested_list: [inner_params], + nested_map: { 'a' => { nested_list: [inner_params] } } + } + end + + it 'filters sensitive members through nested structures, lists, and maps' do + expect(subject.filter(params, input_ref)).to eq( + sensitive_string: '[FILTERED]', + nested: filtered_inner, + nested_list: [filtered_inner], + nested_map: { 'a' => { nested_list: [filtered_inner] } } + ) + end + + it 'does not apply the global name list to the root structure' do + shapes = ApiHelper.sample_shapes + shapes['StructureShape']['members']['ClientId'] = { 'shape' => 'StringShape' } + ref = ApiHelper.sample_api(shapes: shapes).operation(:example_operation).input + expect(subject.filter({ client_id: 'visible' }, ref)) + .to eq(client_id: 'visible') + end + + context 'when nested members lack sensitivity traits' do + let(:input_ref) do + shapes = ApiHelper.sample_shapes + %w[ClientSecret Password PrivateKey RefreshToken ClientId].each do |name| + shapes['StructureShape']['members'][name] = { 'shape' => 'StringShape' } + end + ApiHelper.sample_api(shapes: shapes).operation(:example_operation).input + end + + let(:inner_params) do + { + client_secret: 'oidc-secret', password: 'mq-secret', + private_key: 'apns-secret', refresh_token: 'salesforce-secret', + sensitive_string: 'modeled-secret', string: 'visible' + } + end + + let(:filtered_inner) do + { + client_secret: '[FILTERED]', password: '[FILTERED]', + private_key: '[FILTERED]', refresh_token: '[FILTERED]', + sensitive_string: '[FILTERED]', string: 'visible' + } + end + + it 'combines global and modeled redactions without mutating nested hashes' do + params = { nested: inner_params.dup } + expect(subject.filter(params, input_ref)).to eq(nested: filtered_inner) + expect(params).to eq(nested: inner_params) + end + + it 'retains the global fallback through nested lists' do + expect(subject.filter({ nested_list: [inner_params] }, input_ref)) + .to eq(nested_list: [filtered_inner]) + end + + it 'retains the global fallback in structure map values' do + params = { nested_map: { 'password' => inner_params } } + expect(subject.filter(params, input_ref)) + .to eq(nested_map: { 'password' => filtered_inner }) + end + + it 'keeps typed nested structures on their generated sensitive list' do + struct = input_ref.shape.struct_class.new( + client_id: 'visible', sensitive_string: 'modeled-secret' + ) + filtered = subject.filter({ nested: struct }, input_ref) + expect(filtered[:nested]).to include( + client_id: 'visible', sensitive_string: '[FILTERED]' + ) + end + + context 'when disabled' do + subject do + Aws::Log::ParamFilter.new(filter_sensitive_params: false, filter: [:string]) + end + + it 'keeps global, modeled, and custom-filter values visible' do + request_params = params.merge( + string_map: { string: 'secret', other: 'visible' } + ) + expect(subject.filter(request_params, input_ref)).to eq(request_params) + end + end + + context 'with additional filters' do + subject { Aws::Log::ParamFilter.new(filter: [:string]) } + + it 'combines global and modeled redaction with custom filters in structures and maps' do + params = { + nested: inner_params, + string_map: { string: 'secret', other: 'visible' } + } + expect(subject.filter(params, input_ref)).to eq( + nested: filtered_inner.merge(string: '[FILTERED]'), + string_map: { string: '[FILTERED]', other: 'visible' } + ) + end + end + end + + it 'does not filter ordinary map keys using the global name list' do + params = { string_map: { password: 'visible', name: 'visible' } } + expect(subject.filter(params, input_ref)).to eq(params) + end + + it 'filters a typed union using its member model' do + shape = Seahorse::Model::Shapes::StructureShape.new(struct_class: UnionType) + shape.union = true + shape.add_member(:member_1, input_ref) + ref = Seahorse::Model::Shapes::ShapeRef.new(shape: shape) + params = UnionType.new(member_1: inner_params) + expect(subject.filter(params, ref)).to eq(member_1: filtered_inner) + end + + it 'uses class-based filtering for documents and unknown target shapes' do + [Seahorse::Model::Shapes::DocumentShape.new, nil].each do |shape| + ref = Seahorse::Model::Shapes::ShapeRef.new(shape: shape) + params = { password: 'secret', other: 'visible' } + expect(subject.filter(params, ref)) + .to eq(password: '[FILTERED]', other: 'visible') + end + end + end + + it 'uses the global list when the type is nil' do + filtered = subject.filter( + { password: 'secret', peccy_id: 'peccy-id' }, nil + ) + expect(filtered).to eq(password: '[FILTERED]', peccy_id: 'peccy-id') + end end end end