Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Original file line number Diff line number Diff line change
Expand Up @@ -178,6 +178,31 @@ def wrap_string(str, width, indent = '')
str.gsub(/(.{1,#{width}})(\s+|\Z)/, "#{indent}\\1\n").chomp
end

# Returns true if the member reference or its target shape is sensitive.
# Also checks nested list members and map keys and values.
#
# Stops at structures because each structure has its own SENSITIVE list.
def sensitive_shape_ref?(ref, api, visited = Set.new)
return true if ref['sensitive']

name = ref['shape']
return false if visited.include?(name)

visited << name
target = Api.shape(ref, api)
return true if target['sensitive']

case target['type']
when 'list'
sensitive_shape_ref?(target['member'], api, visited)
when 'map'
sensitive_shape_ref?(target['key'], api, visited) ||
sensitive_shape_ref?(target['value'], api, visited)
else
false
end
end

def gem_lib_path(gem_name)
File.expand_path(
File.join(
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -82,8 +82,7 @@ def eventstreams
def struct_members(shape)
return if shape['members'].nil?
members = shape['members'].map do |member_name, member_ref|
sensitive = !!(member_ref['sensitive'] ||
@api['shapes'][member_ref['shape']]['sensitive'])
sensitive = sensitive_shape_ref?(member_ref, @api)
StructMember.new(
member_name: underscore(member_name),
sensitive: sensitive,
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -97,19 +97,7 @@ def gem_name
def struct_members(shape)
return if shape['members'].nil?
members = shape['members'].map do |member_name, member_ref|
member_target = @api['shapes'][member_ref['shape']]
sensitive = !!(member_ref['sensitive'] ||
member_target['sensitive'])

case member_target["type"]
when 'map'
key_shape = @api['shapes'][member_target['key']['shape']]
value_shape = @api['shapes'][member_target['value']['shape']]
sensitive ||= !!(key_shape['sensitive'] || value_shape['sensitive'])
when 'list'
list_member = @api['shapes'][member_target['member']['shape']]
sensitive ||= !!(list_member['sensitive'])
end
sensitive = sensitive_shape_ref?(member_ref, @api)

StructMember.new(
member_name: underscore(member_name),
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -13,9 +13,160 @@
},
"input":{"shape":"KitchenSink"},
"output":{"shape":"KitchenSink"}
},
"NestedCollectionsOperation": {
"name":"NestedCollectionsOperation",
"http":{
"method":"POST",
"requestUri":"/NestedCollectionsOperation"
},
"input":{"shape":"NestedCollections"},
"output":{"shape":"NestedCollections"}
},
"NestedStructuresOperation": {
"name":"NestedStructuresOperation",
"http":{
"method":"POST",
"requestUri":"/NestedStructuresOperation"
},
"input":{"shape":"NestedStructures"},
"output":{"shape":"NestedStructures"}
}
},
"shapes": {
"NestedCollections": {
"type": "structure",
"members": {
"ListOfListOfStrings": {
"shape": "ListOfListOfStrings"
},
"MapOfMapsOfLists": {
"shape": "MapOfMapsOfLists"
},
"SensitiveKeyMap": {
"shape": "SensitiveKeyMap"
},
"PlainList": {
"shape": "ListOfPlainStrings"
},
"PlainMapOfLists": {
"shape": "MapOfListsOfPlainStrings"
},
"MemberSensitiveLists": {
"shape": "ListOfMemberSensitiveLists"
}
}
},
"MapOfMapsOfLists": {
"type": "map",
"key": {
"shape": "PlainString"
},
"value": {
"shape": "MapOfListsOfStringsWithPlainKeys"
}
},
"MapOfListsOfStringsWithPlainKeys": {
"type": "map",
"key": {
"shape": "PlainString"
},
"value": {
"shape": "ListOfStrings"
}
},
"ListOfMemberSensitiveLists": {
"type": "list",
"member": {
"shape": "MemberSensitiveList"
}
},
"MemberSensitiveList": {
"type": "list",
"member": {
"shape": "PlainString",
"sensitive": true
}
},
"NestedStructures": {
"type": "structure",
"members": {
"Inner": {
"shape": "CollectionMemberStruct"
},
"InnerList": {
"shape": "ListOfCollectionMemberStructs"
},
"InnerMap": {
"shape": "MapOfCollectionMemberStructs"
}
}
},
"CollectionMemberStruct": {
"type": "structure",
"members": {
"Values": {
"shape": "ListOfListOfStrings"
},
"PlainValue": {
"shape": "PlainString"
},
"ClientSecret": {
"shape": "PlainString"
},
"Password": {
"shape": "PlainString"
},
"PrivateKey": {
"shape": "PlainString"
},
"RefreshToken": {
"shape": "PlainString"
}
}
},
"ListOfCollectionMemberStructs": {
"type": "list",
"member": {
"shape": "CollectionMemberStruct"
}
},
"MapOfCollectionMemberStructs": {
"type": "map",
"key": {
"shape": "PlainString"
},
"value": {
"shape": "CollectionMemberStruct"
}
},
"SensitiveKeyMap": {
"type": "map",
"key": {
"shape": "String"
},
"value": {
"shape": "PlainString"
}
},
"ListOfPlainStrings": {
"type": "list",
"member": {
"shape": "PlainString"
}
},
"MapOfListsOfPlainStrings": {
"type": "map",
"key": {
"shape": "PlainString"
},
"value": {
"shape": "ListOfPlainStrings"
}
},
"PlainString": {
"type": "string"
},
"KitchenSink": {
"type": "structure",
"members": {
Expand Down
105 changes: 105 additions & 0 deletions build_tools/aws-sdk-code-generator/spec/interfaces/sensitive_spec.rb
Original file line number Diff line number Diff line change
Expand Up @@ -41,5 +41,110 @@
expect(resp.to_s).to eq(expected.to_s)
end
end

describe '#nested_collections' do
it 'filters members that reach a sensitive shape through lists and maps' do
expect(Sensitive::Types::NestedCollections::SENSITIVE).to contain_exactly(
:list_of_list_of_strings, :map_of_maps_of_lists, :sensitive_key_map,
:member_sensitive_lists
)

client.stub_responses(
:nested_collections_operation,
list_of_list_of_strings: [['secret']],
map_of_maps_of_lists: { 'a' => { 'b' => ['secret'] } },
sensitive_key_map: { 'secret' => 'value' },
plain_list: ['visible'],
plain_map_of_lists: { 'c' => ['visible'] },
member_sensitive_lists: [['member-secret']]
)
resp = client.nested_collections_operation
expected = {
list_of_list_of_strings: '[FILTERED]',
map_of_maps_of_lists: '[FILTERED]',
sensitive_key_map: '[FILTERED]',
plain_list: ['visible'],
plain_map_of_lists: { 'c' => ['visible'] },
member_sensitive_lists: '[FILTERED]'
}

expect(resp.to_s).to eq(expected.to_s)
end

it 'filters collections inside structures while keeping ordinary members visible' do
expect(Sensitive::Types::CollectionMemberStruct::SENSITIVE).to eq([:values])
expect(Sensitive::Types::NestedStructures::SENSITIVE).to be_empty

inner = { values: [['nested-secret']], plain_value: 'visible' }
params = {
inner: inner,
inner_list: [inner],
inner_map: { 'a' => inner }
}
client.stub_responses(:nested_structures_operation, params)
resp = client.nested_structures_operation(params)
filtered_inner = {
values: '[FILTERED]', plain_value: 'visible',
client_secret: nil, password: nil, private_key: nil, refresh_token: nil
}
expected = {
inner: filtered_inner,
inner_list: [filtered_inner],
inner_map: { 'a' => filtered_inner }
}

expect(resp.to_s).to eq(expected.to_s)

log = Aws::Log::Formatter.new(':request_params').format(resp)
logged_inner = '{values:"[FILTERED]",plain_value:"visible"}'
expect(log).to include("inner:#{logged_inner}")
expect(log).to include("inner_list:[#{logged_inner}]")
expect(log).to include("inner_map:{\"a\"=>#{logged_inner}}")
expect(log).not_to include('nested-secret')
end

it 'filters globally sensitive names missing model traits from request logs' do
expect(Sensitive::Types::CollectionMemberStruct::SENSITIVE).to eq([:values])

inner = {
values: [['modeled-secret']], plain_value: 'visible',
client_secret: 'oidc-secret', password: 'mq-secret',
private_key: 'apns-secret', refresh_token: 'salesforce-secret'
}
resp = client.nested_structures_operation(
inner: inner,
inner_list: [inner],
inner_map: { 'a' => inner }
)
log = Aws::Log::Formatter.new(':request_params').format(resp)
logged_inner = '{values:"[FILTERED]",plain_value:"visible",' \
'client_secret:"[FILTERED]",password:"[FILTERED]",' \
'private_key:"[FILTERED]",refresh_token:"[FILTERED]"}'
expect(log).to eq(
"inner:#{logged_inner}," \
"inner_list:[#{logged_inner}]," \
"inner_map:{\"a\"=>#{logged_inner}}"
)
end

it 'uses the same sensitive members in the RBS types' do
api = JSON.parse(File.read(File.expand_path(
'../fixtures/interfaces/sensitive/api.json', __dir__
)))
service = AwsSdkCodeGenerator::Service.new(
name: 'Sensitive', module_name: 'Sensitive', api: api,
gem_version: '1.0.0'
)
view = AwsSdkCodeGenerator::Views::RBS::TypesModule.new(service: service)
structs = view.structures.select do |struct|
%w[KitchenSink NestedCollections NestedStructures CollectionMemberStruct].include?(struct.class_name)
end
expect(structs.size).to eq(4)
structs.each do |struct|
expected = Sensitive::Types.const_get(struct.class_name)::SENSITIVE
expect(struct.sensitive_params).to match_array(expected)
end
end
end
end
end
4 changes: 2 additions & 2 deletions build_tools/services.rb
Original file line number Diff line number Diff line change
Expand Up @@ -9,10 +9,10 @@ class ServiceEnumerator
MANIFEST_PATH = File.expand_path('../../services.json', __FILE__)

# Minimum `aws-sdk-core` version for new gem builds
MINIMUM_CORE_VERSION = "3.264.0"
MINIMUM_CORE_VERSION = "3.265.0"

# Minimum `aws-sdk-core` version for new S3 gem builds
MINIMUM_CORE_VERSION_S3 = "3.264.0"
MINIMUM_CORE_VERSION_S3 = "3.265.0"

EVENTSTREAM_PLUGIN = "Aws::Plugins::EventStreamConfiguration"

Expand Down
2 changes: 2 additions & 0 deletions gems/aws-sdk-core/CHANGELOG.md
Original file line number Diff line number Diff line change
@@ -1,6 +1,8 @@
Unreleased Changes
------------------

* Feature - Improve model-aware filtering of nested request parameters and sensitivity detection for nested collections.

3.264.0 (2026-11-04)
------------------

Expand Down
5 changes: 3 additions & 2 deletions gems/aws-sdk-core/lib/aws-sdk-core/log/formatter.rb
Original file line number Diff line number Diff line change
Expand Up @@ -132,8 +132,9 @@ def _operation(response)

def _request_params(response)
params = response.context.params
type = response.context.operation.input.shape.struct_class
@param_formatter.summarize(@param_filter.filter(params, type))
operation = response.context.operation
ref = operation && operation.input
@param_formatter.summarize(@param_filter.filter(params, ref))
end

def _time(response)
Expand Down
Loading
Loading