diff --git a/.gitignore b/.gitignore index 8c496b9064be..5d3d2b876b93 100644 --- a/.gitignore +++ b/.gitignore @@ -95,3 +95,6 @@ src/aws-cpp-sdk-core/include/aws/core/SDKConfig.h #temp codegen files tools/code-generation/smithy/codegen/cpp-smoke-tests/codegen_output/ + +# FuzzTest reproducers (run_fuzz.sh) +tests/fuzz/reproducers/ diff --git a/CMakeLists.txt b/CMakeLists.txt index 63aac896235c..337984eccaca 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -74,6 +74,7 @@ if (LEGACY_BUILD) option(DISABLE_DNS_REQUIRED_TESTS "Disable unit tests that require DNS lookup to succeed, useful when using a http client that does not perform DNS lookup" OFF) option(AWS_APPSTORE_SAFE "Remove reference to private Apple APIs for AES GCM in Common Crypto. If set to OFF you application will get rejected from the apple app store." OFF) option(AWS_ENABLE_CORE_INTEGRATION_TEST "Enables the core integration tests to be built which contains dependencies on other clients for setup and tear down" OFF) + option(ENABLE_SCHEMA_FUZZ_TESTS "Build google/fuzztest fuzz tests for the schema serde (Linux + Clang, static libs)" OFF) set(AWS_USER_AGENT_CUSTOMIZATION "" CACHE STRING "User agent extension") @@ -230,6 +231,11 @@ if (LEGACY_BUILD) MinSizeRel # Like Release, but optimized for memory rather than speed. ) + if (ENABLE_SCHEMA_FUZZ_TESTS) + include(schema_fuzz_tests) + setup_schema_fuzz_tests() + endif () + # build third-party targets if (BUILD_DEPS) set(CMAKE_INSTALL_RPATH "$ORIGIN") @@ -346,6 +352,9 @@ if (LEGACY_BUILD) add_sdks() include(tests) + if (ENABLE_SCHEMA_FUZZ_TESTS) + add_schema_fuzz_tests() + endif () # Performance tests for services if (BUILD_PERFORMANCE_TESTS) diff --git a/cmake/schema_fuzz_tests.cmake b/cmake/schema_fuzz_tests.cmake new file mode 100644 index 000000000000..be5823c89891 --- /dev/null +++ b/cmake/schema_fuzz_tests.cmake @@ -0,0 +1,45 @@ +# Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +# SPDX-License-Identifier: Apache-2.0. +# + +set(SCHEMA_FUZZTEST_GIT_TAG "305251260059201f842e5de6bb10e7b73efb1a6e" CACHE STRING "google/fuzztest commit used by ENABLE_SCHEMA_FUZZ_TESTS") +option(SCHEMA_FUZZ_UNIT_SANITIZERS "Build the non-fuzzing (unit mode) schema fuzz tests with ASan and UBSan" ON) + +# Must run before the CRT and core are defined so fuzzing mode instruments them. +macro(setup_schema_fuzz_tests) + if (NOT CMAKE_SYSTEM_NAME STREQUAL "Linux") + message(FATAL_ERROR "ENABLE_SCHEMA_FUZZ_TESTS requires Linux (use tools/scripts/run_fuzz.sh)") + endif () + if (NOT CMAKE_CXX_COMPILER_ID STREQUAL "Clang") + message(FATAL_ERROR "ENABLE_SCHEMA_FUZZ_TESTS requires Clang; got ${CMAKE_CXX_COMPILER_ID}") + endif () + if (BUILD_SHARED_LIBS) + message(FATAL_ERROR "ENABLE_SCHEMA_FUZZ_TESTS requires -DBUILD_SHARED_LIBS=OFF") + endif () + if (ENABLE_TESTING) + message(FATAL_ERROR "ENABLE_SCHEMA_FUZZ_TESTS cannot be combined with ENABLE_TESTING: the vendored gtest and FuzzTest's gtest collide. Use -DENABLE_TESTING=OFF") + endif () + if (CMAKE_VERSION VERSION_LESS 3.25) + message(FATAL_ERROR "ENABLE_SCHEMA_FUZZ_TESTS requires CMake >= 3.25 (FuzzTest, FetchContent SYSTEM)") + endif () + + include(FetchContent) + FetchContent_Declare(fuzztest + GIT_REPOSITORY https://github.com/google/fuzztest.git + GIT_TAG ${SCHEMA_FUZZTEST_GIT_TAG} + # Keeps FuzzTest and Abseil headers out of the SDK warning set. + SYSTEM) + FetchContent_MakeAvailable(fuzztest) + + fuzztest_setup_fuzzing_flags() + if (FUZZTEST_FUZZING_MODE) + # -fno-jump-tables: on AArch64, switch lookup tables of __sancov_cntrs pointers built after SanitizerCoverage + # land outside COMDAT groups and fail to link with "defined in discarded section". + set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -fno-omit-frame-pointer -fno-jump-tables") + set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -fno-omit-frame-pointer -fno-jump-tables") + endif () +endmacro() + +function(add_schema_fuzz_tests) + add_subdirectory(${CMAKE_SOURCE_DIR}/tests/fuzz ${CMAKE_BINARY_DIR}/tests/fuzz) +endfunction() diff --git a/src/aws-cpp-sdk-core/include/aws/core/utils/StringUtils.h b/src/aws-cpp-sdk-core/include/aws/core/utils/StringUtils.h index 7f91b614e30c..bc809f71598d 100644 --- a/src/aws-cpp-sdk-core/include/aws/core/utils/StringUtils.h +++ b/src/aws-cpp-sdk-core/include/aws/core/utils/StringUtils.h @@ -13,6 +13,8 @@ #include #include +#include + namespace Aws { @@ -176,8 +178,7 @@ namespace Aws /** - * Parses the entire string as a base-10 int64. Returns empty if the string is empty or has - * characters after the number. Out-of-range values saturate, matching ConvertToInt64. + * Parses the entire string as a base-10 int64. Returns empty if it is malformed or out of range. */ static Aws::Crt::Optional ParseInt64(const Aws::String& source); @@ -188,6 +189,18 @@ namespace Aws */ static Aws::Crt::Optional ParseDouble(const Aws::String& source); + /** + * Formats a double with 15 significant digits, or 17 when 15 does not round-trip. + */ + static Aws::String FormatRoundTripDouble(double value); + + + /** + * Formats a finite float with 6 significant digits, or 9 when 6 does not round-trip. + */ + static Aws::String FormatRoundTripFloat(float value); + + #ifdef _WIN32 /** @@ -202,12 +215,13 @@ namespace Aws #endif /** - * not all platforms (Android) have std::to_string + * not all platforms (Android) have std::to_string. Formats in the classic locale. */ template< typename T > static Aws::String to_string(T value) { Aws::OStringStream os; + os.imbue(std::locale::classic()); os << value; return os.str(); } diff --git a/src/aws-cpp-sdk-core/include/smithy/client/schema/CborShapeDeserializer.h b/src/aws-cpp-sdk-core/include/smithy/client/schema/CborShapeDeserializer.h index 59c0b6663a73..96804d5b9767 100644 --- a/src/aws-cpp-sdk-core/include/smithy/client/schema/CborShapeDeserializer.h +++ b/src/aws-cpp-sdk-core/include/smithy/client/schema/CborShapeDeserializer.h @@ -28,6 +28,7 @@ class SMITHY_API CborShapeDeserializer final : public ShapeDeserializer { std::shared_ptr ReadDocument(const Schema& schema) override; bool IsNull() override; + Aws::Crt::Optional GetError() const override; private: class Impl; diff --git a/src/aws-cpp-sdk-core/include/smithy/client/schema/ClientProtocol.h b/src/aws-cpp-sdk-core/include/smithy/client/schema/ClientProtocol.h index 7b91351ec3fa..ec90e7532b3f 100644 --- a/src/aws-cpp-sdk-core/include/smithy/client/schema/ClientProtocol.h +++ b/src/aws-cpp-sdk-core/include/smithy/client/schema/ClientProtocol.h @@ -28,8 +28,8 @@ class SMITHY_API ClientProtocol { virtual SerializerOutcome SerializeRequest(const OperationRequestContext& operation, const Schema& inputSchema, const SerializableStruct& input) const; - virtual void DeserializeResponse(const OperationRequestContext& operation, Aws::Crt::ByteCursor data, - SerializableStruct& output) const; + virtual Aws::Crt::Optional DeserializeResponse(const OperationRequestContext& operation, Aws::Crt::ByteCursor data, + SerializableStruct& output) const; }; class SMITHY_API RestJsonProtocol final : public ClientProtocol { @@ -95,8 +95,8 @@ class SMITHY_API AwsQueryProtocol final : public ClientProtocol { Aws::UniquePtr CreateOutputDeserializer(Aws::Crt::ByteCursor data) const override; SerializerOutcome SerializeRequest(const OperationRequestContext& operation, const Schema& inputSchema, const SerializableStruct& input) const override; - void DeserializeResponse(const OperationRequestContext& operation, Aws::Crt::ByteCursor data, - SerializableStruct& output) const override; + Aws::Crt::Optional DeserializeResponse(const OperationRequestContext& operation, Aws::Crt::ByteCursor data, + SerializableStruct& output) const override; }; class SMITHY_API Ec2QueryProtocol final : public ClientProtocol { diff --git a/src/aws-cpp-sdk-core/include/smithy/client/schema/Codec.h b/src/aws-cpp-sdk-core/include/smithy/client/schema/Codec.h index 61060e422ff2..a4213e2732f3 100644 --- a/src/aws-cpp-sdk-core/include/smithy/client/schema/Codec.h +++ b/src/aws-cpp-sdk-core/include/smithy/client/schema/Codec.h @@ -43,8 +43,10 @@ class SMITHY_API Codec { virtual Aws::UniquePtr CreateDeserializer(Aws::Crt::ByteCursor data) const = 0; - void DeserializeShape(Aws::Crt::ByteCursor data, SerializableStruct& shape) const { - shape.Deserialize(*CreateDeserializer(data)); + Aws::Crt::Optional DeserializeShape(Aws::Crt::ByteCursor data, SerializableStruct& shape) const { + auto deserializer = CreateDeserializer(data); + shape.Deserialize(*deserializer); + return deserializer->GetError(); } }; diff --git a/src/aws-cpp-sdk-core/include/smithy/client/schema/JsonShapeDeserializer.h b/src/aws-cpp-sdk-core/include/smithy/client/schema/JsonShapeDeserializer.h index 21e94ed37054..bd8bae9fdafa 100644 --- a/src/aws-cpp-sdk-core/include/smithy/client/schema/JsonShapeDeserializer.h +++ b/src/aws-cpp-sdk-core/include/smithy/client/schema/JsonShapeDeserializer.h @@ -30,6 +30,7 @@ class SMITHY_API JsonShapeDeserializer final : public ShapeDeserializer { std::shared_ptr ReadDocument(const Schema& schema) override; bool IsNull() override; + Aws::Crt::Optional GetError() const override; private: class Impl; diff --git a/src/aws-cpp-sdk-core/include/smithy/client/schema/Schema.h b/src/aws-cpp-sdk-core/include/smithy/client/schema/Schema.h index a2b1363cc32b..e76f61e5408f 100644 --- a/src/aws-cpp-sdk-core/include/smithy/client/schema/Schema.h +++ b/src/aws-cpp-sdk-core/include/smithy/client/schema/Schema.h @@ -12,6 +12,9 @@ namespace smithy { namespace schema { +// Container nesting limit, root included, shared by every serializer and deserializer. +constexpr int MAX_NESTING_DEPTH = 1000; + class SchemaBuilder; enum class ShapeType : uint8_t { diff --git a/src/aws-cpp-sdk-core/include/smithy/client/schema/SerdeTraits.h b/src/aws-cpp-sdk-core/include/smithy/client/schema/SerdeTraits.h index de3484515da4..0c2cc873e0aa 100644 --- a/src/aws-cpp-sdk-core/include/smithy/client/schema/SerdeTraits.h +++ b/src/aws-cpp-sdk-core/include/smithy/client/schema/SerdeTraits.h @@ -7,10 +7,13 @@ #include #include #include +#include #include #include #include +#include +#include #include namespace smithy { @@ -64,6 +67,16 @@ inline Aws::String EpochSecondsText(const Aws::Utils::DateTime& value) { return out; } +// Decodes as DateTime(double) does, for parity with legacy clients. Empty where DateTime(double) would overflow. +inline Aws::Crt::Optional EpochSecondsToDateTime(double seconds) { + static const double maxSeconds = static_cast(std::chrono::duration_cast( + (std::chrono::system_clock::duration::max)()).count()) / 1000.0; + if (!std::isfinite(seconds) || std::fabs(seconds) >= maxSeconds) { + return {}; + } + return Aws::Utils::DateTime(seconds); +} + inline Aws::String FormatTimestampText(const Aws::Utils::DateTime& value, TimestampFormatTrait::Format format) { switch (format) { case TimestampFormatTrait::Format::HTTP_DATE: diff --git a/src/aws-cpp-sdk-core/include/smithy/client/schema/ShapeDeserializer.h b/src/aws-cpp-sdk-core/include/smithy/client/schema/ShapeDeserializer.h index 22f735fc92b5..0d5d7517eae4 100644 --- a/src/aws-cpp-sdk-core/include/smithy/client/schema/ShapeDeserializer.h +++ b/src/aws-cpp-sdk-core/include/smithy/client/schema/ShapeDeserializer.h @@ -1,20 +1,28 @@ #pragma once +#include +#include #include #include +#include #include #include #include #include #include +#include +#include #include #include +#include #include namespace smithy { namespace schema { +using DeserializerError = Aws::Client::AWSError; + class SMITHY_API ShapeDeserializer { public: virtual ~ShapeDeserializer() = default; @@ -37,10 +45,117 @@ class SMITHY_API ShapeDeserializer { virtual Aws::Crt::Optional ReadString(const Schema& schema) = 0; virtual Aws::Crt::Optional ReadTimestamp(const Schema& schema) = 0; virtual Aws::Crt::Optional ReadBlob(const Schema& schema) = 0; - // Null when the member is absent, unparsable, or the protocol does not support documents. virtual std::shared_ptr ReadDocument(const Schema& schema) = 0; virtual bool IsNull() = 0; + + // First failure wins; once set, reads return empty. + virtual Aws::Crt::Optional GetError() const { return m_error; } + + protected: + void Fail(const Aws::String& message) { + if (!m_error.has_value()) { + m_error = DeserializerError(Aws::Client::CoreErrors::INTERNAL_FAILURE, "SerializationException", message, false); + } + } + bool HasFailed() const { return m_error.has_value(); } + bool AtRoot() const { return m_depth == 0; } + + void FailUnrepresentable(const char* type) { Fail(Aws::String("Value out of range for ") + type); } + + // Call before delivering each non-null modeled member; a second member of a union fails the read. + bool AdmitMember(const Schema& schema, bool& delivered) { + if (delivered && schema.GetType() == ShapeType::Union) { + Fail("Union has more than one member set"); + return false; + } + delivered = true; + return true; + } + + Aws::Crt::Optional NarrowToInt(int64_t value) { + if (value < (std::numeric_limits::min)() || value > (std::numeric_limits::max)()) { + FailUnrepresentable("integer"); + return {}; + } + return static_cast(value); + } + + // Float text printed to 9 digits can parse just above FLT_MAX, so values below the rounding point to infinity clamp. + Aws::Crt::Optional NarrowToFloat(double value) { + const float floatMax = (std::numeric_limits::max)(); + const double roundsToInfinity = static_cast(floatMax) + std::ldexp(1.0, 103); + if (std::isfinite(value) && std::fabs(value) > static_cast(floatMax)) { + if (std::fabs(value) >= roundsToInfinity) { + FailUnrepresentable("float"); + return {}; + } + return value < 0 ? -floatMax : floatMax; + } + return static_cast(value); + } + + // Out-of-range integer text fails the read; other non-integer text just returns empty. + Aws::Crt::Optional ParseLong(const Aws::String& text) { + const auto value = Aws::Utils::StringUtils::ParseInt64(text); + if (!value.has_value() && IsPlainInteger(text)) { + FailUnrepresentable("long"); + } + return value; + } + + // Must accept what strtoll does. + static bool IsPlainInteger(const Aws::String& text) { + size_t start = text.find_first_not_of(" \t\n\v\f\r"); + if (start == Aws::String::npos) { + return false; + } + if (text[start] == '-' || text[start] == '+') { + ++start; + } + return text.size() > start && text.find_first_not_of("0123456789", start) == Aws::String::npos; + } + + Aws::Crt::Optional EpochSecondsToTimestamp(double seconds) { + auto parsed = EpochSecondsToDateTime(seconds); + if (!parsed.has_value()) { + FailUnrepresentable("timestamp"); + } + return parsed; + } + + class ContainerScope final { + public: + explicit ContainerScope(ShapeDeserializer& owner) : m_owner(owner), m_entered(owner.EnterContainer()) {} + ~ContainerScope() { + if (m_entered) { + --m_owner.m_depth; + } + } + ContainerScope(const ContainerScope&) = delete; + ContainerScope& operator=(const ContainerScope&) = delete; + bool Entered() const { return m_entered; } + + private: + ShapeDeserializer& m_owner; + bool m_entered; + }; + + private: + bool EnterContainer() { + if (HasFailed()) { + return false; + } + if (m_depth >= MAX_NESTING_DEPTH) { + Fail("Maximum nesting depth exceeded"); + return false; + } + ++m_depth; + return true; + } + + Aws::Crt::Optional m_error; + int m_depth = 0; }; } // namespace schema diff --git a/src/aws-cpp-sdk-core/include/smithy/client/schema/ShapeSerializer.h b/src/aws-cpp-sdk-core/include/smithy/client/schema/ShapeSerializer.h index f09509ee2eeb..7d0b2f144f95 100644 --- a/src/aws-cpp-sdk-core/include/smithy/client/schema/ShapeSerializer.h +++ b/src/aws-cpp-sdk-core/include/smithy/client/schema/ShapeSerializer.h @@ -16,6 +16,26 @@ namespace schema { class SerializableStruct; class MapSerializer; +class NestingDepth final { + public: + bool Enter(Aws::String& errorMessage) { + if (!errorMessage.empty()) { + return false; + } + if (m_depth >= MAX_NESTING_DEPTH) { + errorMessage = "Maximum nesting depth exceeded"; + return false; + } + ++m_depth; + return true; + } + void Exit() { --m_depth; } + int Get() const { return m_depth; } + + private: + int m_depth = 0; +}; + // Push-based serializer: the model drives serialization and the serializer owns // all wire-format structure. Aggregates hand control back through a consumer. class SMITHY_API ShapeSerializer { diff --git a/src/aws-cpp-sdk-core/include/smithy/client/schema/XmlShapeDeserializer.h b/src/aws-cpp-sdk-core/include/smithy/client/schema/XmlShapeDeserializer.h index 1447e8c8fba2..ec0bfa2f5c0c 100644 --- a/src/aws-cpp-sdk-core/include/smithy/client/schema/XmlShapeDeserializer.h +++ b/src/aws-cpp-sdk-core/include/smithy/client/schema/XmlShapeDeserializer.h @@ -31,6 +31,7 @@ class SMITHY_API XmlShapeDeserializer final : public ShapeDeserializer { std::shared_ptr ReadDocument(const Schema& schema) override; bool IsNull() override; + Aws::Crt::Optional GetError() const override; private: class Impl; diff --git a/src/aws-cpp-sdk-core/source/smithy/client/schema/CborShapeDeserializer.cpp b/src/aws-cpp-sdk-core/source/smithy/client/schema/CborShapeDeserializer.cpp index 55c84193a09e..687acf287d05 100644 --- a/src/aws-cpp-sdk-core/source/smithy/client/schema/CborShapeDeserializer.cpp +++ b/src/aws-cpp-sdk-core/source/smithy/client/schema/CborShapeDeserializer.cpp @@ -7,33 +7,24 @@ #include #include +#include +#include +#include + using namespace smithy::schema; using namespace Aws::Utils; using namespace Aws::Crt::Cbor; class CborShapeDeserializer::Impl final : public ShapeDeserializer { public: - explicit Impl(Aws::Crt::ByteCursor data) : m_decoder(data) {} + explicit Impl(Aws::Crt::ByteCursor data) : m_decoder(data), m_inputSize(data.len) {} void ReadStruct(const Schema& schema, const StructMemberConsumer& consumer) override { - bool indefinite = false; - size_t count = 0; - if (!OpenMap(indefinite, count)) { - return; - } - for (size_t i = 0; MoreEntries(indefinite, count, i); ++i) { - auto key = ReadText(); - if (!key.has_value()) { - break; - } - const auto member = schema.GetMember(key->c_str()); - if (member.has_value() && *member && !IsNull()) { - consumer(**member, *this); - } else { - SkipValue(); - } + const bool root = AtRoot(); + ReadStructBody(schema, consumer); + if (root && !HasFailed() && m_decoder.GetRemainingLength() != 0) { + Fail("Malformed CBOR: trailing data after root item"); } - CloseIndefinite(indefinite); } void ReadList(const Schema&, const ListElementConsumer& consumer) override { @@ -42,12 +33,19 @@ class CborShapeDeserializer::Impl final : public ShapeDeserializer { if (!OpenArray(indefinite, count)) { return; } + const ContainerScope scope(*this); + if (!scope.Entered()) { + return; + } for (size_t i = 0; MoreEntries(indefinite, count, i); ++i) { + if (!HasNextItem()) { + break; + } if (IsNull()) { SkipValue(); - continue; + } else { + consumer(*this); } - consumer(*this); } CloseIndefinite(indefinite); } @@ -58,23 +56,35 @@ class CborShapeDeserializer::Impl final : public ShapeDeserializer { if (!OpenMap(indefinite, count)) { return; } + const ContainerScope scope(*this); + if (!scope.Entered()) { + return; + } for (size_t i = 0; MoreEntries(indefinite, count, i); ++i) { auto key = ReadText(); if (!key.has_value()) { + Fail("Malformed CBOR: expected text key"); + break; + } + if (!HasNextItem()) { break; } if (IsNull()) { SkipValue(); - continue; + } else { + consumer(*key, *this); } - consumer(*key, *this); } CloseIndefinite(indefinite); } Aws::Crt::Optional ReadBoolean(const Schema&) override { + if (!Expect({CborType::Bool})) { + return {}; + } auto val = m_decoder.PopNextBooleanVal(); if (!val.has_value()) { + FailMalformed(); return {}; } return val.value(); @@ -85,29 +95,36 @@ class CborShapeDeserializer::Impl final : public ShapeDeserializer { if (!val.has_value()) { return {}; } - return static_cast(val.value()); + return NarrowToInt(val.value()); } Aws::Crt::Optional ReadLong(const Schema&) override { - auto type = m_decoder.PeekType(); - if (!type.has_value()) { + if (!Expect({CborType::UInt, CborType::NegInt})) { return {}; } - if (*type == CborType::UInt) { + constexpr uint64_t INT64_MAX_AS_UINT64 = static_cast((std::numeric_limits::max)()); + if (*m_decoder.PeekType() == CborType::UInt) { auto val = m_decoder.PopNextUnsignedIntVal(); if (!val.has_value()) { + FailMalformed(); return {}; } - return static_cast(val.value()); - } - if (*type == CborType::NegInt) { - auto val = m_decoder.PopNextNegativeIntVal(); - if (!val.has_value()) { + if (val.value() > INT64_MAX_AS_UINT64) { + FailUnrepresentable("long"); return {}; } - return static_cast(-(static_cast(val.value()) + 1)); + return static_cast(val.value()); + } + auto val = m_decoder.PopNextNegativeIntVal(); + if (!val.has_value()) { + FailMalformed(); + return {}; + } + if (val.value() > INT64_MAX_AS_UINT64) { + FailUnrepresentable("long"); + return {}; } - return {}; + return -1 - static_cast(val.value()); } Aws::Crt::Optional ReadFloat(const Schema& schema) override { @@ -115,65 +132,74 @@ class CborShapeDeserializer::Impl final : public ShapeDeserializer { if (!val.has_value()) { return {}; } - return static_cast(val.value()); + return NarrowToFloat(val.value()); } - Aws::Crt::Optional ReadDouble(const Schema& schema) override { - auto type = m_decoder.PeekType(); - if (!type.has_value()) { + Aws::Crt::Optional ReadDouble(const Schema&) override { + if (!Expect({CborType::Float, CborType::UInt, CborType::NegInt})) { return {}; } - if (*type == CborType::Float) { + if (*m_decoder.PeekType() == CborType::Float) { auto val = m_decoder.PopNextFloatVal(); if (!val.has_value()) { + FailMalformed(); return {}; } return val.value(); } - auto longVal = ReadLong(schema); - if (!longVal.has_value()) { + if (*m_decoder.PeekType() == CborType::UInt) { + auto val = m_decoder.PopNextUnsignedIntVal(); + if (!val.has_value()) { + FailMalformed(); + return {}; + } + return static_cast(val.value()); + } + auto val = m_decoder.PopNextNegativeIntVal(); + if (!val.has_value()) { + FailMalformed(); return {}; } - return static_cast(longVal.value()); + // Negate in integer space so the cast is the only rounding; n + 1 wraps at UINT64_MAX. + constexpr double MINUS_TWO_POW_64 = -18446744073709551616.0; + return val.value() == UINT64_MAX ? MINUS_TWO_POW_64 : -static_cast(val.value() + 1); } Aws::Crt::Optional ReadString(const Schema&) override { return ReadText(); } Aws::Crt::Optional ReadTimestamp(const Schema& schema) override { - auto tag = m_decoder.PopNextTagVal(); - if (!tag.has_value() || tag.value() != 1) { + if (!Expect({CborType::Tag})) { return {}; } - auto type = m_decoder.PeekType(); - if (!type.has_value()) { + auto tag = m_decoder.PopNextTagVal(); + if (!tag.has_value()) { + FailMalformed(); return {}; } - if (*type == CborType::Float) { - auto val = m_decoder.PopNextFloatVal(); - if (!val.has_value() || val.value() < 0.0) { - return {}; - } - return DateTime(val.value()); - } - if (*type == CborType::NegInt) { + if (tag.value() != 1) { + SkipValue(); return {}; } - auto val = ReadLong(schema); - if (!val.has_value()) { + auto seconds = ReadDouble(schema); + // Pre-epoch values are rejected deliberately (see TimestampNegative*ReturnsEmpty). + if (!seconds.has_value() || seconds.value() < 0.0) { return {}; } - return DateTime(static_cast(val.value())); + return EpochSecondsToTimestamp(seconds.value()); } Aws::Crt::Optional ReadBlob(const Schema&) override { - auto val = m_decoder.PopNextBytesVal(); - if (!val.has_value()) { + const auto bytes = ReadChunkable(CborType::Bytes, CborType::IndefBytesStart, [this] { return m_decoder.PopNextBytesVal(); }); + if (!bytes.has_value()) { return {}; } - return ByteBuffer(val->ptr, val->len); + return ByteBuffer(reinterpret_cast(bytes->data()), bytes->size()); } std::shared_ptr ReadDocument(const Schema&) override { + if (HasFailed()) { + return nullptr; + } AWS_LOGSTREAM_WARN("CborShapeDeserializer", "document type is not yet supported by the RPCv2 CBOR protocol"); SkipValue(); // consume the whole data item so sibling members stay aligned return nullptr; @@ -185,9 +211,49 @@ class CborShapeDeserializer::Impl final : public ShapeDeserializer { } private: + void ReadStructBody(const Schema& schema, const StructMemberConsumer& consumer) { + bool indefinite = false; + size_t count = 0; + if (!OpenMap(indefinite, count)) { + return; + } + const ContainerScope scope(*this); + if (!scope.Entered()) { + return; + } + bool delivered = false; + for (size_t i = 0; MoreEntries(indefinite, count, i); ++i) { + auto key = ReadText(); + if (!key.has_value()) { + Fail("Malformed CBOR: expected text key"); + break; + } + if (!HasNextItem()) { + break; + } + const auto member = schema.GetMember(key->c_str()); + if (member.has_value() && *member && !IsNull()) { + if (!AdmitMember(schema, delivered)) { + break; + } + consumer(**member, *this); + } else { + SkipValue(); + } + } + CloseIndefinite(indefinite); + } + + // Only the root can fail to decode here. Empty input and a null root are an empty struct, as in JSON. bool OpenMap(bool& indefinite, size_t& count) { + if (HasFailed()) { + return false; + } auto type = m_decoder.PeekType(); if (!type.has_value()) { + if (m_inputSize != 0) { + FailMalformed(); + } return false; } if (*type == CborType::IndefMapStart) { @@ -196,8 +262,19 @@ class CborShapeDeserializer::Impl final : public ShapeDeserializer { count = 0; return true; } + if (*type != CborType::MapStart) { + if (AtRoot() && *type == CborType::Null) { + SkipValue(); + } else if (AtRoot()) { + FailMalformed(); + } else { + SkipValue(); + } + return false; + } auto size = m_decoder.PopNextMapStart(); if (!size.has_value()) { + FailMalformed(); return false; } indefinite = false; @@ -206,8 +283,14 @@ class CborShapeDeserializer::Impl final : public ShapeDeserializer { } bool OpenArray(bool& indefinite, size_t& count) { + if (HasFailed()) { + return false; + } auto type = m_decoder.PeekType(); if (!type.has_value()) { + if (m_inputSize != 0) { + FailMalformed(); + } return false; } if (*type == CborType::IndefArrayStart) { @@ -216,8 +299,13 @@ class CborShapeDeserializer::Impl final : public ShapeDeserializer { count = 0; return true; } + if (*type != CborType::ArrayStart) { + SkipValue(); + return false; + } auto size = m_decoder.PopNextArrayStart(); if (!size.has_value()) { + FailMalformed(); return false; } indefinite = false; @@ -225,31 +313,91 @@ class CborShapeDeserializer::Impl final : public ShapeDeserializer { return true; } + void FailMalformed() { Fail("Malformed CBOR: truncated or invalid data item"); } + + bool HasNextItem() { + if (HasFailed()) { + return false; + } + if (m_decoder.PeekType().has_value()) { + return true; + } + FailMalformed(); + return false; + } + bool MoreEntries(bool indefinite, size_t count, size_t index) { + if (HasFailed()) { + return false; + } if (indefinite) { - auto type = m_decoder.PeekType(); - return type.has_value() && *type != CborType::Break; + return HasNextItem() && *m_decoder.PeekType() != CborType::Break; } return index < count; } void CloseIndefinite(bool indefinite) { - if (indefinite) { + if (indefinite && !HasFailed()) { m_decoder.ConsumeNextSingleElement(); } } + // A mismatched item is consumed so later reads stay aligned. + bool Expect(std::initializer_list accepted) { + if (!HasNextItem()) { + return false; + } + const CborType type = *m_decoder.PeekType(); + if (std::find(accepted.begin(), accepted.end(), type) != accepted.end()) { + return true; + } + SkipValue(); + return false; + } + Aws::Crt::Optional ReadText() { - auto val = m_decoder.PopNextTextVal(); - if (!val.has_value()) { + return ReadChunkable(CborType::Text, CborType::IndefTextStart, [this] { return m_decoder.PopNextTextVal(); }); + } + + template + Aws::Crt::Optional ReadChunkable(CborType definite, CborType indefinite, const PopChunk& popChunk) { + if (!Expect({definite, indefinite})) { + return {}; + } + const bool chunked = *m_decoder.PeekType() == indefinite; + if (chunked) { + m_decoder.ConsumeNextSingleElement(); + } + Aws::String joined; + bool more = true; + while (more && HasNextItem()) { + const CborType type = *m_decoder.PeekType(); + if (chunked && type == CborType::Break) { + m_decoder.ConsumeNextSingleElement(); + return joined; + } + const auto chunk = type == definite ? popChunk() : Aws::Crt::Optional(); + if (!chunk.has_value()) { + FailMalformed(); + return {}; + } + joined.append(reinterpret_cast(chunk->ptr), chunk->len); + more = chunked; + } + if (HasFailed()) { return {}; } - return Aws::String(reinterpret_cast(val->ptr), val->len); + return joined; } - void SkipValue() { m_decoder.ConsumeNextWholeDataItem(); } + void SkipValue() { + if (!m_decoder.ConsumeNextWholeDataItem()) { + FailMalformed(); + } + } CborDecoder m_decoder; + size_t m_inputSize; }; CborShapeDeserializer::CborShapeDeserializer(Aws::Crt::ByteCursor data) @@ -271,3 +419,4 @@ std::shared_ptr CborShapeDeserializer::ReadDocum return m_impl->ReadDocument(schema); } bool CborShapeDeserializer::IsNull() { return m_impl->IsNull(); } +Aws::Crt::Optional CborShapeDeserializer::GetError() const { return m_impl->GetError(); } diff --git a/src/aws-cpp-sdk-core/source/smithy/client/schema/CborShapeSerializer.cpp b/src/aws-cpp-sdk-core/source/smithy/client/schema/CborShapeSerializer.cpp index a76038845b9d..1de50967e697 100644 --- a/src/aws-cpp-sdk-core/source/smithy/client/schema/CborShapeSerializer.cpp +++ b/src/aws-cpp-sdk-core/source/smithy/client/schema/CborShapeSerializer.cpp @@ -17,22 +17,34 @@ using SerializerOutcome = Aws::Utils::Outcome& consumer) override { + if (!EnterContainer()) { + return; + } m_encoder.WriteArrayStart(size); ListContext ctx(this); consumer(ctx); + ExitContainer(); } void WriteMap(const Schema&, size_t size, const std::function& consumer) override { + if (!EnterContainer()) { + return; + } m_encoder.WriteMapStart(size); MapContext ctx(this); consumer(ctx); + ExitContainer(); } void WriteBoolean(const Schema&, bool value) override { m_encoder.WriteBool(value); } @@ -173,7 +185,11 @@ class CborShapeSerializer::Impl final : public ShapeSerializer { } } + bool EnterContainer() { return m_depth.Enter(m_errorMessage); } + void ExitContainer() { m_depth.Exit(); } + Aws::Crt::Cbor::CborEncoder m_encoder; + NestingDepth m_depth; bool m_finalized = false; Aws::String m_errorMessage; }; diff --git a/src/aws-cpp-sdk-core/source/smithy/client/schema/ClientProtocol.cpp b/src/aws-cpp-sdk-core/source/smithy/client/schema/ClientProtocol.cpp index 21353c5f211a..64034c37b2dd 100644 --- a/src/aws-cpp-sdk-core/source/smithy/client/schema/ClientProtocol.cpp +++ b/src/aws-cpp-sdk-core/source/smithy/client/schema/ClientProtocol.cpp @@ -41,10 +41,11 @@ ClientProtocol::SerializerOutcome ClientProtocol::SerializeRequest(const Operati return SerializeInput(inputSchema, input); } -void ClientProtocol::DeserializeResponse(const OperationRequestContext&, Aws::Crt::ByteCursor data, - SerializableStruct& output) const { +Aws::Crt::Optional ClientProtocol::DeserializeResponse(const OperationRequestContext&, Aws::Crt::ByteCursor data, + SerializableStruct& output) const { auto deserializer = CreateOutputDeserializer(data); output.Deserialize(*deserializer); + return deserializer->GetError(); } Aws::String RestJsonProtocol::GetProtocolId() const { return "aws.protocols#restJson1"; } @@ -105,11 +106,12 @@ ClientProtocol::SerializerOutcome AwsQueryProtocol::SerializeRequest(const Opera const SerializableStruct& input) const { return WrapQueryEnvelope(operation, SerializeInput(inputSchema, input)); } -void AwsQueryProtocol::DeserializeResponse(const OperationRequestContext& operation, Aws::Crt::ByteCursor data, - SerializableStruct& output) const { +Aws::Crt::Optional AwsQueryProtocol::DeserializeResponse(const OperationRequestContext& operation, + Aws::Crt::ByteCursor data, SerializableStruct& output) const { XmlShapeDeserializer deserializer(data); deserializer.EnterWrapperElement(operation.action + "Result"); output.Deserialize(deserializer); + return deserializer.GetError(); } Aws::String Ec2QueryProtocol::GetProtocolId() const { return "aws.protocols#ec2Query"; } diff --git a/src/aws-cpp-sdk-core/source/smithy/client/schema/Document.cpp b/src/aws-cpp-sdk-core/source/smithy/client/schema/Document.cpp index 10c69292eb0f..faed490a4091 100644 --- a/src/aws-cpp-sdk-core/source/smithy/client/schema/Document.cpp +++ b/src/aws-cpp-sdk-core/source/smithy/client/schema/Document.cpp @@ -76,14 +76,14 @@ class JsonDocument final : public AbstractDocument { if (!numericFormat || !value.has_value()) { return {}; } - return Aws::Utils::DateTime(static_cast(*value)); + return EpochSecondsToDateTime(static_cast(*value)); } case ShapeType::Double: { const auto value = AsDouble(); if (!numericFormat || !value.has_value()) { return {}; } - return Aws::Utils::DateTime(*value); + return EpochSecondsToDateTime(*value); } case ShapeType::String: { const auto text = AsString(); diff --git a/src/aws-cpp-sdk-core/source/smithy/client/schema/JsonShapeDeserializer.cpp b/src/aws-cpp-sdk-core/source/smithy/client/schema/JsonShapeDeserializer.cpp index 74213f9cf93b..47e0cc63c655 100644 --- a/src/aws-cpp-sdk-core/source/smithy/client/schema/JsonShapeDeserializer.cpp +++ b/src/aws-cpp-sdk-core/source/smithy/client/schema/JsonShapeDeserializer.cpp @@ -14,6 +14,7 @@ #include #include +#include #include #include #include @@ -23,97 +24,91 @@ using namespace Aws::Utils; class JsonShapeDeserializer::Impl final : public ShapeDeserializer { public: + // A leading byte-order mark is skipped, as cJSON does. explicit Impl(Aws::Crt::ByteCursor data, CodecSettings settings) - : m_bytes(reinterpret_cast(data.ptr), data.len), m_pos(0), m_settings(settings) {} + : m_bytes(reinterpret_cast(data.ptr), data.len), + m_pos(m_bytes.compare(0, 3, "\xEF\xBB\xBF") == 0 ? 3 : 0), + m_settings(settings) {} void ReadStruct(const Schema& schema, const StructMemberConsumer& consumer) override { - if (PeekNonWs() != '{') { - SkipValue(); - return; - } - ++m_pos; - if (PeekNonWs() == '}') { - ++m_pos; - return; - } - while (true) { - auto key = ParseString(); - if (!key.has_value()) { - return; - } - if (PeekNonWs() == ':') { - ++m_pos; - } - PeekNonWs(); - const auto member = ResolveMember(schema, *key); - if (member.has_value() && *member && !IsNull()) { - ConsumeOne([&] { consumer(**member, *this); }); - } else { - SkipValue(); - } - const auto more = NextInContainer('}'); - if (!more.has_value() || !*more) { - break; - } + const bool root = AtRoot(); + ReadStructBody(schema, consumer); + if (root && !HasFailed() && !AtEnd()) { + Fail("Malformed JSON: trailing data after root value"); } } void ReadList(const Schema&, const ListElementConsumer& consumer) override { + if (HasFailed()) { + return; + } if (PeekNonWs() != '[') { SkipValue(); return; } + const ContainerScope scope(*this); + if (!scope.Entered()) { + return; + } ++m_pos; if (PeekNonWs() == ']') { ++m_pos; return; } - while (true) { + while (!HasFailed()) { PeekNonWs(); if (IsNull()) { SkipValue(); } else { ConsumeOne([&] { consumer(*this); }); } - const auto more = NextInContainer(']'); - if (!more.has_value() || !*more) { + if (!NextOrClose(']')) { break; } } } void ReadMap(const Schema&, const MapEntryConsumer& consumer) override { + if (HasFailed()) { + return; + } if (PeekNonWs() != '{') { SkipValue(); return; } + const ContainerScope scope(*this); + if (!scope.Entered()) { + return; + } ++m_pos; if (PeekNonWs() == '}') { ++m_pos; return; } - while (true) { + while (!HasFailed()) { auto key = ParseString(); if (!key.has_value()) { + Fail("Malformed JSON: expected object key"); return; } - if (PeekNonWs() == ':') { - ++m_pos; + if (!ExpectColon()) { + return; } - PeekNonWs(); if (IsNull()) { SkipValue(); } else { ConsumeOne([&] { consumer(*key, *this); }); } - const auto more = NextInContainer('}'); - if (!more.has_value() || !*more) { + if (!NextOrClose('}')) { break; } } } Aws::Crt::Optional ReadBoolean(const Schema&) override { + if (HasFailed()) { + return {}; + } if (Match("true")) { return true; } @@ -129,22 +124,30 @@ class JsonShapeDeserializer::Impl final : public ShapeDeserializer { if (!val.has_value()) { return {}; } - return static_cast(val.value()); + return NarrowToInt(val.value()); } Aws::Crt::Optional ReadLong(const Schema&) override { + if (HasFailed()) { + return {}; + } if (!IsNumberStart(PeekNonWs())) { SkipValue(); return {}; } const Aws::String token = ReadNumberToken(); - // ParseInt64 saturates, so clamping out-of-range plain integers is intentional. - const auto value = StringUtils::ParseInt64(token); - if (value.has_value()) { + const auto value = ParseLong(token); + if (value.has_value() || HasFailed()) { return value; } + // Fraction/exponent forms are accepted only when they denote an exact int64. const auto d = StringUtils::ParseDouble(token); - if (!d.has_value() || !IsRepresentableAsInt64(*d)) { + if (!d.has_value()) { + Fail("Malformed JSON: invalid number"); + return {}; + } + if (!IsRepresentableAsInt64(*d) || std::trunc(*d) != *d) { + FailUnrepresentable("long"); return {}; } return static_cast(*d); @@ -155,10 +158,13 @@ class JsonShapeDeserializer::Impl final : public ShapeDeserializer { if (!val.has_value()) { return {}; } - return static_cast(val.value()); + return NarrowToFloat(val.value()); } Aws::Crt::Optional ReadDouble(const Schema&) override { + if (HasFailed()) { + return {}; + } const char c = PeekNonWs(); if (c == '"') { auto token = ParseString(); @@ -180,10 +186,22 @@ class JsonShapeDeserializer::Impl final : public ShapeDeserializer { SkipValue(); return {}; } - return StringUtils::ParseDouble(ReadNumberToken()); + const auto value = StringUtils::ParseDouble(ReadNumberToken()); + if (!value.has_value()) { + Fail("Malformed JSON: invalid number"); + return {}; + } + if (!std::isfinite(*value)) { + FailUnrepresentable("double"); + return {}; + } + return value; } Aws::Crt::Optional ReadString(const Schema&) override { + if (HasFailed()) { + return {}; + } if (PeekNonWs() != '"') { SkipValue(); return {}; @@ -192,13 +210,17 @@ class JsonShapeDeserializer::Impl final : public ShapeDeserializer { } Aws::Crt::Optional ReadTimestamp(const Schema& schema) override { + if (HasFailed()) { + return {}; + } const char c = PeekNonWs(); if (IsNumberStart(c)) { const auto seconds = StringUtils::ParseDouble(ReadNumberToken()); if (!seconds.has_value()) { + Fail("Malformed JSON: invalid number"); return {}; } - return DateTime(*seconds); + return EpochSecondsToTimestamp(*seconds); } if (c == '"') { auto token = ParseString(); @@ -218,6 +240,9 @@ class JsonShapeDeserializer::Impl final : public ShapeDeserializer { } Aws::Crt::Optional ReadBlob(const Schema&) override { + if (HasFailed()) { + return {}; + } if (PeekNonWs() != '"') { SkipValue(); return {}; @@ -230,10 +255,13 @@ class JsonShapeDeserializer::Impl final : public ShapeDeserializer { } std::shared_ptr ReadDocument(const Schema&) override { + if (HasFailed()) { + return nullptr; + } const size_t start = m_pos; - auto doc = ReadDocumentValue(0); - if (!doc) { - // A failed parse can stop part-way into the value; rewind and skip it whole so sibling members stay aligned. + auto doc = ReadDocumentValue(); + if (!doc && !HasFailed()) { + // Rewind past a partly parsed unrecognised literal so siblings stay aligned. m_pos = start; SkipValue(); } @@ -249,9 +277,51 @@ class JsonShapeDeserializer::Impl final : public ShapeDeserializer { } private: - // Below the serializer's MAX_DEPTH so a document that parses can always be written back, and far - // enough below cJSON's 1000 to keep the recursive descent inside a 1 MB stack. - static constexpr int MAX_DOCUMENT_DEPTH = 256; + void ReadStructBody(const Schema& schema, const StructMemberConsumer& consumer) { + if (HasFailed()) { + return; + } + if (PeekNonWs() != '{') { + if (!AtRoot()) { + SkipValue(); + } else if (!AtEnd() && !Match("null")) { + Fail("Malformed JSON: expected object"); + } + return; + } + const ContainerScope scope(*this); + if (!scope.Entered()) { + return; + } + ++m_pos; + if (PeekNonWs() == '}') { + ++m_pos; + return; + } + bool delivered = false; + while (!HasFailed()) { + auto key = ParseString(); + if (!key.has_value()) { + Fail("Malformed JSON: expected object key"); + return; + } + if (!ExpectColon()) { + return; + } + const auto member = ResolveMember(schema, *key); + if (member.has_value() && *member && !IsNull()) { + if (!AdmitMember(schema, delivered)) { + return; + } + ConsumeOne([&] { consumer(**member, *this); }); + } else { + SkipValue(); + } + if (!NextOrClose('}')) { + break; + } + } + } // JSON-flavored document nodes: carries this deserializer's timestamp format so AsBlob/AsTimestamp // on the resulting Document apply JSON coercion (base64 strings, epoch/ISO-8601 timestamps). @@ -259,12 +329,13 @@ class JsonShapeDeserializer::Impl final : public ShapeDeserializer { return NewJsonDocument(m_settings.GetDefaultTimestampFormat()); } - std::shared_ptr ReadDocumentValue(int depth) { - if (depth >= MAX_DOCUMENT_DEPTH) { - return nullptr; - } + std::shared_ptr ReadDocumentValue() { const char c = PeekNonWs(); if (c == '{') { + const ContainerScope scope(*this); + if (!scope.Entered()) { + return nullptr; + } ++m_pos; Aws::Map> object; if (PeekNonWs() == '}') { @@ -276,22 +347,21 @@ class JsonShapeDeserializer::Impl final : public ShapeDeserializer { while (true) { auto key = ParseString(); if (!key.has_value()) { + Fail("Malformed JSON: expected object key"); return nullptr; } - if (PeekNonWs() != ':') { + if (!ExpectColon()) { return nullptr; } - ++m_pos; - auto value = ReadDocumentValue(depth + 1); + auto value = ReadDocumentValue(); if (!value) { return nullptr; } object.emplace(std::move(*key), std::move(value)); - const auto more = NextInContainer('}'); - if (!more.has_value()) { - return nullptr; - } - if (!*more) { + if (!NextOrClose('}')) { + if (HasFailed()) { + return nullptr; + } break; } } @@ -300,6 +370,10 @@ class JsonShapeDeserializer::Impl final : public ShapeDeserializer { return node; } if (c == '[') { + const ContainerScope scope(*this); + if (!scope.Entered()) { + return nullptr; + } ++m_pos; Aws::Vector> list; if (PeekNonWs() == ']') { @@ -309,16 +383,15 @@ class JsonShapeDeserializer::Impl final : public ShapeDeserializer { return node; } while (true) { - auto value = ReadDocumentValue(depth + 1); + auto value = ReadDocumentValue(); if (!value) { return nullptr; } list.push_back(std::move(value)); - const auto more = NextInContainer(']'); - if (!more.has_value()) { - return nullptr; - } - if (!*more) { + if (!NextOrClose(']')) { + if (HasFailed()) { + return nullptr; + } break; } } @@ -351,7 +424,7 @@ class JsonShapeDeserializer::Impl final : public ShapeDeserializer { if (IsNumberStart(c)) { const Aws::String token = ReadNumberToken(); const auto v = StringUtils::ParseInt64(token); - if (v.has_value() && !SaturatedInt64(*v, token)) { + if (v.has_value()) { auto node = NewJsonNode(); node->SetInteger(*v); return node; @@ -359,6 +432,11 @@ class JsonShapeDeserializer::Impl final : public ShapeDeserializer { // Fraction/exponent forms, and integers out of int64 range (magnitude preserved rather than clamped). const auto d = StringUtils::ParseDouble(token); if (!d.has_value()) { + Fail("Malformed JSON: invalid number"); + return nullptr; + } + if (!std::isfinite(*d)) { + FailUnrepresentable("document number"); return nullptr; } auto node = NewJsonNode(); @@ -369,15 +447,6 @@ class JsonShapeDeserializer::Impl final : public ShapeDeserializer { return nullptr; } - // True when ParseInt64 clamped an out-of-range integer, detected via its double magnitude. - static bool SaturatedInt64(int64_t value, const Aws::String& token) { - if (value != (std::numeric_limits::max)() && value != (std::numeric_limits::min)()) { - return false; - } - const auto d = StringUtils::ParseDouble(token); - return d.has_value() && (*d > LLONG_MAX_PLUS_ONE || *d < LLONG_MIN_AS_DOUBLE); - } - static Aws::String JsonName(const Schema& member) { const auto trait = member.GetTrait(JsonNameTrait::KEY()); return trait ? trait->GetValue() : member.GetMemberName(); @@ -396,27 +465,34 @@ class JsonShapeDeserializer::Impl final : public ShapeDeserializer { void ConsumeOne(const Fn& fn) { const size_t before = m_pos; fn(); - if (m_pos == before) { + if (m_pos == before && !HasFailed()) { SkipValue(); } } - // True after a separator, false at the closing bracket, absent when the next token is neither. Modeled - // readers treat absent as a break; the document parser rejects, so a document never comes back - // silently truncated. - Aws::Crt::Optional NextInContainer(char close) { + bool NextOrClose(char close) { const char c = PeekNonWs(); if (c == ',') { ++m_pos; return true; } if (c != close) { - return {}; + Fail("Malformed JSON: expected ',' or closing bracket"); + return false; } ++m_pos; return false; } + bool ExpectColon() { + if (PeekNonWs() != ':') { + Fail("Malformed JSON: expected ':' after object key"); + return false; + } + ++m_pos; + return true; + } + static bool IsWs(char c) { return c == ' ' || c == '\t' || c == '\n' || c == '\r'; } static bool IsNumberStart(char c) { return c == '-' || (c >= '0' && c <= '9'); } char Peek() const { return m_pos < m_bytes.size() ? m_bytes[m_pos] : '\0'; } @@ -429,6 +505,10 @@ class JsonShapeDeserializer::Impl final : public ShapeDeserializer { SkipWs(); return Peek(); } + bool AtEnd() { + SkipWs(); + return m_pos >= m_bytes.size(); + } bool Match(const char* literal) { SkipWs(); const size_t n = std::strlen(literal); @@ -497,71 +577,78 @@ class JsonShapeDeserializer::Impl final : public ShapeDeserializer { out += '\t'; break; case 'u': - AppendUnicodeEscape(out); + if (!AppendUnicodeEscape(out)) { + Fail("Malformed JSON: invalid \\u escape"); + return {}; + } break; default: - out += esc; - break; + Fail("Malformed JSON: invalid escape"); + return {}; } } + Fail("Malformed JSON: unterminated string"); return {}; } + // Held to the same grammar as a value that is read, but numbers are never out of range. void SkipValue() { const char c = PeekNonWs(); if (c == '"') { ParseString(); - return; + } else if (c == '{' || c == '[') { + SkipContainer(c == '{' ? '}' : ']'); + } else if (IsNumberStart(c)) { + if (!StringUtils::ParseDouble(ReadNumberToken()).has_value()) { + Fail("Malformed JSON: invalid number"); + } + } else if (!Match("true") && !Match("false") && !Match("null")) { + Fail("Malformed JSON: unexpected token"); } - if (c == '{' || c == '[') { - SkipContainer(); + } + + void SkipContainer(char close) { + const ContainerScope scope(*this); + if (!scope.Entered()) { return; } - while (m_pos < m_bytes.size()) { - const char ch = m_bytes[m_pos]; - if (ch == ',' || ch == '}' || ch == ']' || IsWs(ch)) { - break; - } + ++m_pos; + if (PeekNonWs() == close) { ++m_pos; + return; } - } - - void SkipContainer() { - int depth = 0; - while (m_pos < m_bytes.size()) { - const char ch = m_bytes[m_pos]; - if (ch == '"') { - ParseString(); - continue; - } - if (ch == '{' || ch == '[') { - ++depth; - ++m_pos; - } else if (ch == '}' || ch == ']') { - --depth; - ++m_pos; - if (depth == 0) { + bool more = true; + while (more && !HasFailed()) { + if (close == '}') { + if (PeekNonWs() != '"') { + Fail("Malformed JSON: expected object key"); + return; + } + if (!ParseString().has_value() || !ExpectColon()) { return; } - } else { - ++m_pos; } + SkipValue(); + more = !HasFailed() && NextOrClose(close); } } - void AppendUnicodeEscape(Aws::String& out) { + bool AppendUnicodeEscape(Aws::String& out) { uint32_t cp = 0; - if (!ReadHex4(cp)) { - return; + if (!ReadHex4(cp) || (cp >= 0xDC00 && cp <= 0xDFFF)) { + return false; } - if (cp >= 0xD800 && cp <= 0xDBFF && m_pos + 1 < m_bytes.size() && m_bytes[m_pos] == '\\' && m_bytes[m_pos + 1] == 'u') { - m_pos += 2; + if (cp >= 0xD800 && cp <= 0xDBFF) { uint32_t low = 0; - if (ReadHex4(low) && low >= 0xDC00 && low <= 0xDFFF) { - cp = 0x10000 + ((cp - 0xD800) << 10) + (low - 0xDC00); + const bool escape = m_pos + 1 < m_bytes.size() && m_bytes[m_pos] == '\\' && m_bytes[m_pos + 1] == 'u'; + m_pos += escape ? 2 : 0; + if (!escape || !ReadHex4(low) || low < 0xDC00 || low > 0xDFFF) { + return false; } + cp = 0x10000 + ((cp - 0xD800) << 10) + (low - 0xDC00); } AppendUtf8(out, cp); + return true; } bool ReadHex4(uint32_t& out) { if (m_pos + 4 > m_bytes.size()) { @@ -626,3 +713,4 @@ std::shared_ptr JsonShapeDeserializer::ReadDocum return m_impl->ReadDocument(schema); } bool JsonShapeDeserializer::IsNull() { return m_impl->IsNull(); } +Aws::Crt::Optional JsonShapeDeserializer::GetError() const { return m_impl->GetError(); } diff --git a/src/aws-cpp-sdk-core/source/smithy/client/schema/JsonShapeSerializer.cpp b/src/aws-cpp-sdk-core/source/smithy/client/schema/JsonShapeSerializer.cpp index 25f72255d5fe..6ffe2fb575b4 100644 --- a/src/aws-cpp-sdk-core/source/smithy/client/schema/JsonShapeSerializer.cpp +++ b/src/aws-cpp-sdk-core/source/smithy/client/schema/JsonShapeSerializer.cpp @@ -4,7 +4,6 @@ */ #include #include -#include #include #include #include @@ -14,8 +13,6 @@ #include #include -#include -#include #include "aws/core/client/AWSClient.h" #include "aws/core/utils/Outcome.h" @@ -25,8 +22,6 @@ using namespace smithy::schema; using namespace Aws::Utils; using SerializerOutcome = Aws::Utils::Outcome>; -static constexpr int MAX_DEPTH = 500; - class JsonShapeSerializer::Impl final : public ShapeSerializer { public: explicit Impl(CodecSettings settings) : m_settings(settings) { m_buf.reserve(8192); } @@ -88,42 +83,18 @@ class JsonShapeSerializer::Impl final : public ShapeSerializer { void WriteNull(const Schema&) override { m_buf += "null"; } void WriteDocument(const Schema& schema, const Document& value) override { value.Serialize(*this, schema); } - // Shortest decimal that round-trips (matches the SDK cJSON writer): 15 significant digits, - // falling back to 17 (max_digits10, always round-trips) when 15 does not reparse equal. - static Aws::String RoundTripDouble(double value) { - Aws::OStringStream s15; - s15 << std::setprecision(15) << value; - Aws::String t15 = s15.str(); - if (StringUtils::ConvertToDouble(t15.c_str()) == value) { - return t15; - } - Aws::OStringStream s17; - s17 << std::setprecision(17) << value; - return s17.str(); - } - // Preserve the double type through a JSON round-trip: an integral-valued double formats as "2", // which the document deserializer reclassifies as Long. Append a ".0" marker when absent. static Aws::String FormatDouble(double value) { - Aws::String text = RoundTripDouble(value); + Aws::String text = StringUtils::FormatRoundTripDouble(value); if (text.find_first_of(".eE") == Aws::String::npos) { text += ".0"; } return text; } - // Float counterpart of FormatDouble: reparse at float precision (strtof) so a float emits the - // shortest decimal that round-trips the float ("3.14"), not the double expansion of its imprecision - // ("3.1400001049041748"). 6 significant digits, falling back to 9 (max_digits10 for float). static Aws::String FormatFloat(float value) { - Aws::OStringStream s6; - s6 << std::setprecision(6) << value; - Aws::String text = s6.str(); - if (std::strtof(text.c_str(), nullptr) != value) { - Aws::OStringStream s9; - s9 << std::setprecision(9) << value; - text = s9.str(); - } + Aws::String text = StringUtils::FormatRoundTripFloat(value); if (text.find_first_of(".eE") == Aws::String::npos) { text += ".0"; } @@ -142,10 +113,10 @@ class JsonShapeSerializer::Impl final : public ShapeSerializer { } void WriteCommaIfNeeded() { - if (m_needsComma[m_depth]) { + if (m_needsComma[m_depth.Get()]) { m_buf += ','; } else { - m_needsComma[m_depth] = true; + m_needsComma[m_depth.Get()] = true; } } @@ -308,27 +279,22 @@ class JsonShapeSerializer::Impl final : public ShapeSerializer { }; bool OpenContainer(char open) { - if (!m_errorMessage.empty()) { - return false; - } - if (m_depth + 1 >= MAX_DEPTH) { - m_errorMessage = "Maximum nesting depth exceeded"; + if (!m_depth.Enter(m_errorMessage)) { return false; } m_buf += open; - ++m_depth; - m_needsComma[m_depth] = false; + m_needsComma[m_depth.Get()] = false; return true; } void CloseContainer(char close) { - --m_depth; + m_depth.Exit(); m_buf += close; } Aws::String m_buf; - int m_depth = 0; - Aws::Array m_needsComma{}; + NestingDepth m_depth; + Aws::Array m_needsComma{}; bool m_finalized = false; Aws::String m_errorMessage; CodecSettings m_settings; diff --git a/src/aws-cpp-sdk-core/source/smithy/client/schema/QueryShapeSerializer.cpp b/src/aws-cpp-sdk-core/source/smithy/client/schema/QueryShapeSerializer.cpp index 7a3725accae0..d712261bc064 100644 --- a/src/aws-cpp-sdk-core/source/smithy/client/schema/QueryShapeSerializer.cpp +++ b/src/aws-cpp-sdk-core/source/smithy/client/schema/QueryShapeSerializer.cpp @@ -21,8 +21,6 @@ using namespace smithy::schema; using namespace Aws::Utils; using SerializerOutcome = Aws::Utils::Outcome>; -static constexpr int MAX_DEPTH = 500; - class QueryShapeSerializer::Impl final : public ShapeSerializer { public: explicit Impl(bool ec2, CodecSettings settings) : m_ec2(ec2), m_settings(settings) { m_buf.reserve(1024); } @@ -44,6 +42,10 @@ class QueryShapeSerializer::Impl final : public ShapeSerializer { const bool flattened = m_ec2 || IsFlattened(schema); ListContext ctx(this, m_prefix, flattened, GetListItemName(schema)); consumer(ctx); + // awsQuery sends an empty list as a bare key; ec2Query omits it. + if (!m_ec2 && ctx.IsEmpty()) { + EmitField(""); + } ExitContainer(); } @@ -59,8 +61,11 @@ class QueryShapeSerializer::Impl final : public ShapeSerializer { void WriteBoolean(const Schema&, bool value) override { EmitField(value ? "true" : "false"); } void WriteInteger(const Schema&, int value) override { EmitField(StringUtils::to_string(value)); } void WriteLong(const Schema&, int64_t value) override { EmitField(StringUtils::to_string(value)); } - void WriteFloat(const Schema&, float value) override { EmitField(FormatFloatingPoint(value)); } - void WriteDouble(const Schema&, double value) override { EmitField(FormatFloatingPoint(value)); } + void WriteFloat(const Schema&, float value) override { + EmitField(StringUtils::URLEncode( + (std::isfinite(value) ? StringUtils::FormatRoundTripFloat(value) : FormatFloatingPoint(static_cast(value))).c_str())); + } + void WriteDouble(const Schema&, double value) override { EmitField(StringUtils::URLEncode(FormatFloatingPoint(value).c_str())); } void WriteString(const Schema&, const Aws::String& value) override { EmitField(StringUtils::URLEncode(value.c_str())); } void WriteTimestamp(const Schema& schema, const DateTime& value) override { EmitField(StringUtils::URLEncode( @@ -197,6 +202,8 @@ class QueryShapeSerializer::Impl final : public ShapeSerializer { void WriteNull(const Schema&) override { ++m_index; } void WriteDocument(const Schema& s, const Document& v) override { m_outer->WriteDocument(s, v); } + bool IsEmpty() const { return m_index == 0; } + private: Aws::String ItemKey() { Aws::String key = m_base; @@ -263,18 +270,8 @@ class QueryShapeSerializer::Impl final : public ShapeSerializer { m_buf += value; } - bool EnterContainer() { - if (!m_errorMessage.empty()) { - return false; - } - if (m_depth + 1 > MAX_DEPTH) { - m_errorMessage = "Maximum serialization depth exceeded"; - return false; - } - ++m_depth; - return true; - } - void ExitContainer() { --m_depth; } + bool EnterContainer() { return m_depth.Enter(m_errorMessage); } + void ExitContainer() { m_depth.Exit(); } static Aws::String JoinKey(const Aws::String& prefix, const Aws::String& name) { if (prefix.empty()) { @@ -288,7 +285,7 @@ class QueryShapeSerializer::Impl final : public ShapeSerializer { static Aws::String FormatFloatingPoint(double value) { if (std::isfinite(value)) { - return StringUtils::to_string(value); + return StringUtils::FormatRoundTripDouble(value); } if (std::isnan(value)) { return "NaN"; @@ -330,7 +327,7 @@ class QueryShapeSerializer::Impl final : public ShapeSerializer { Aws::String m_buf; Aws::String m_prefix; - int m_depth = 0; + NestingDepth m_depth; bool m_ec2 = false; CodecSettings m_settings; bool m_hasContent = false; diff --git a/src/aws-cpp-sdk-core/source/smithy/client/schema/XmlShapeDeserializer.cpp b/src/aws-cpp-sdk-core/source/smithy/client/schema/XmlShapeDeserializer.cpp index 8bcd77b23230..177c9bc7b341 100644 --- a/src/aws-cpp-sdk-core/source/smithy/client/schema/XmlShapeDeserializer.cpp +++ b/src/aws-cpp-sdk-core/source/smithy/client/schema/XmlShapeDeserializer.cpp @@ -5,13 +5,18 @@ #include #include #include -#include +#include #include #include #include #include +#include +#include +#include +#include #include +#include using namespace smithy::schema; using namespace Aws::Utils; @@ -20,19 +25,32 @@ class XmlShapeDeserializer::Impl final : public ShapeDeserializer { public: explicit Impl(Aws::Crt::ByteCursor data, CodecSettings settings) : m_xml(reinterpret_cast(data.ptr), data.len), m_settings(settings) { + const auto malformed = CheckWellFormed(); + if (malformed.has_value()) { + Fail(*malformed); + return; + } const Element root = FindRoot(); - m_tagBegin = root.tagBegin; - m_tagEnd = root.tagEnd; - m_contentBegin = root.contentBegin; - m_contentEnd = root.contentEnd; - m_valid = root.found; + if (!root.found) { + if (m_xml.find_first_not_of(" \t\r\n") != Aws::String::npos) { + Fail("Malformed XML: no root element"); + } + return; + } + EnterElement(root); + m_valid = true; } void ReadStruct(const Schema& schema, const StructMemberConsumer& consumer) override { - if (!m_valid) { + if (!m_valid || HasFailed()) { + return; + } + const ContainerScope scope(*this); + if (!scope.Entered()) { return; } - for (uint16_t i = 0; i < schema.GetMemberCount(); ++i) { + bool delivered = false; + for (uint16_t i = 0; i < schema.GetMemberCount() && !HasFailed(); ++i) { const auto member = schema.GetMember(static_cast(i)); if (!member.has_value() || !*member) { continue; @@ -43,9 +61,9 @@ class XmlShapeDeserializer::Impl final : public ShapeDeserializer { bool deliver = false; if (IsAttribute(memberSchema)) { - m_attr = name; + m_attrValue = AttributeValue(name); m_flattened = false; - deliver = true; + deliver = m_attrValue.has_value(); } else { const ShapeType type = memberSchema.GetType(); const bool flattened = IsFlattened(memberSchema) && (type == ShapeType::List || type == ShapeType::Map); @@ -53,7 +71,7 @@ class XmlShapeDeserializer::Impl final : public ShapeDeserializer { if (FindChild(m_contentBegin, m_contentEnd, name, m_contentBegin).found) { m_flattened = true; m_flatName = name; - m_attr.clear(); + m_attrValue.reset(); deliver = true; } } else { @@ -61,12 +79,12 @@ class XmlShapeDeserializer::Impl final : public ShapeDeserializer { if (child.found) { EnterElement(child); m_flattened = false; - m_attr.clear(); + m_attrValue.reset(); deliver = true; } } } - if (deliver) { + if (deliver && AdmitMember(schema, delivered)) { consumer(memberSchema, *this); } RestoreCursor(saved); @@ -74,25 +92,34 @@ class XmlShapeDeserializer::Impl final : public ShapeDeserializer { } void ReadList(const Schema& schema, const ListElementConsumer& consumer) override { - if (!m_valid) { + if (!m_valid || HasFailed()) { + return; + } + const ContainerScope scope(*this); + if (!scope.Entered()) { return; } const Aws::String itemName = m_flattened ? m_flatName : GetListItemName(schema); const size_t begin = m_contentBegin; const size_t end = m_contentEnd; - for (Element item = FindChild(begin, end, itemName, begin); item.found; item = FindChild(begin, end, itemName, item.nodeEnd)) { + for (Element item = FindChild(begin, end, itemName, begin); item.found && !HasFailed(); + item = FindChild(begin, end, itemName, item.nodeEnd)) { const Cursor saved = SaveCursor(); EnterElement(item); m_flattened = false; - m_attr.clear(); + m_attrValue.reset(); consumer(*this); RestoreCursor(saved); } } void ReadMap(const Schema& schema, const MapEntryConsumer& consumer) override { - if (!m_valid) { + if (!m_valid || HasFailed()) { + return; + } + const ContainerScope scope(*this); + if (!scope.Entered()) { return; } const Aws::String entryName = m_flattened ? m_flatName : GetMapEntryName(schema); @@ -100,7 +127,8 @@ class XmlShapeDeserializer::Impl final : public ShapeDeserializer { const Aws::String valueName = GetMapValueName(schema); const size_t begin = m_contentBegin; const size_t end = m_contentEnd; - for (Element entry = FindChild(begin, end, entryName, begin); entry.found; entry = FindChild(begin, end, entryName, entry.nodeEnd)) { + for (Element entry = FindChild(begin, end, entryName, begin); entry.found && !HasFailed(); + entry = FindChild(begin, end, entryName, entry.nodeEnd)) { const Element valueNode = FindChild(entry.contentBegin, entry.contentEnd, valueName, entry.contentBegin); if (!valueNode.found) { continue; @@ -110,14 +138,19 @@ class XmlShapeDeserializer::Impl final : public ShapeDeserializer { const Cursor saved = SaveCursor(); EnterElement(valueNode); m_flattened = false; - m_attr.clear(); + m_attrValue.reset(); consumer(key, *this); RestoreCursor(saved); } } Aws::Crt::Optional ReadBoolean(const Schema&) override { - const Aws::String text = CurrentText(); + if (HasFailed()) { + return {}; + } + Aws::String text = ScalarText(); + std::transform(text.begin(), text.end(), text.begin(), + [](char c) { return c >= 'A' && c <= 'Z' ? static_cast(c - 'A' + 'a') : c; }); if (text == "true") { return true; } @@ -132,11 +165,14 @@ class XmlShapeDeserializer::Impl final : public ShapeDeserializer { if (!val.has_value()) { return {}; } - return static_cast(val.value()); + return NarrowToInt(val.value()); } Aws::Crt::Optional ReadLong(const Schema&) override { - return StringUtils::ParseInt64(CurrentText()); + if (HasFailed()) { + return {}; + } + return ParseLong(ScalarText()); } Aws::Crt::Optional ReadFloat(const Schema& schema) override { @@ -144,11 +180,14 @@ class XmlShapeDeserializer::Impl final : public ShapeDeserializer { if (!val.has_value()) { return {}; } - return static_cast(val.value()); + return NarrowToFloat(val.value()); } Aws::Crt::Optional ReadDouble(const Schema&) override { - const Aws::String text = CurrentText(); + if (HasFailed()) { + return {}; + } + const Aws::String text = ScalarText(); if (text == "NaN") { return std::numeric_limits::quiet_NaN(); } @@ -158,30 +197,48 @@ class XmlShapeDeserializer::Impl final : public ShapeDeserializer { if (text == "-Infinity") { return -std::numeric_limits::infinity(); } - return StringUtils::ParseDouble(text); + const auto value = StringUtils::ParseDouble(text); + if (value.has_value() && !std::isfinite(*value)) { + FailUnrepresentable("double"); + return {}; + } + return value; } - Aws::Crt::Optional ReadString(const Schema&) override { return CurrentText(); } + Aws::Crt::Optional ReadString(const Schema&) override { + if (HasFailed()) { + return {}; + } + return CurrentText(); + } Aws::Crt::Optional ReadTimestamp(const Schema& schema) override { + if (HasFailed()) { + return {}; + } const auto format = ResolveTimestampFormat(schema, m_settings.GetDefaultTimestampFormat()); if (format == TimestampFormatTrait::Format::EPOCH_SECONDS) { - const auto seconds = StringUtils::ParseDouble(CurrentText()); + const auto seconds = StringUtils::ParseDouble(ScalarText()); if (!seconds.has_value()) { return {}; } - return DateTime(*seconds); + return EpochSecondsToTimestamp(*seconds); } const DateFormat df = format == TimestampFormatTrait::Format::HTTP_DATE ? DateFormat::RFC822 : DateFormat::ISO_8601; - DateTime parsed(CurrentText(), df); + DateTime parsed(ScalarText(), df); if (!parsed.WasParseSuccessful()) { return {}; } return parsed; } - Aws::Crt::Optional ReadBlob(const Schema&) override { return HashingUtils::Base64Decode(CurrentText()); } + Aws::Crt::Optional ReadBlob(const Schema&) override { + if (HasFailed()) { + return {}; + } + return HashingUtils::Base64Decode(CurrentText()); + } std::shared_ptr ReadDocument(const Schema&) override { AWS_LOGSTREAM_WARN("XmlShapeDeserializer", "document type is not supported by the REST XML protocol"); @@ -203,9 +260,9 @@ class XmlShapeDeserializer::Impl final : public ShapeDeserializer { size_t tagBegin, tagEnd, contentBegin, contentEnd; bool flattened; Aws::String flatName; - Aws::String attr; + Aws::Crt::Optional attrValue; }; - Cursor SaveCursor() const { return Cursor{m_tagBegin, m_tagEnd, m_contentBegin, m_contentEnd, m_flattened, m_flatName, m_attr}; } + Cursor SaveCursor() const { return Cursor{m_tagBegin, m_tagEnd, m_contentBegin, m_contentEnd, m_flattened, m_flatName, m_attrValue}; } void RestoreCursor(const Cursor& c) { m_tagBegin = c.tagBegin; m_tagEnd = c.tagEnd; @@ -213,7 +270,7 @@ class XmlShapeDeserializer::Impl final : public ShapeDeserializer { m_contentEnd = c.contentEnd; m_flattened = c.flattened; m_flatName = c.flatName; - m_attr = c.attr; + m_attrValue = c.attrValue; } void EnterElement(const Element& e) { m_tagBegin = e.tagBegin; @@ -232,7 +289,7 @@ class XmlShapeDeserializer::Impl final : public ShapeDeserializer { } EnterElement(child); m_flattened = false; - m_attr.clear(); + m_attrValue.reset(); return true; } @@ -245,6 +302,18 @@ class XmlShapeDeserializer::Impl final : public ShapeDeserializer { } size_t Npos() const { return Aws::String::npos; } + // Every scanner classifies markup here so they agree on the tree. npos if unterminated; lt if lt starts a tag. + size_t SkipNonElementMarkup(size_t lt) const { + static const std::pair MARKUP[] = {{""}, {""}, {""}, {""}}; + for (const auto& markup : MARKUP) { + if (StartsWith(lt, markup.first)) { + const size_t e = m_xml.find(markup.second, lt + std::strlen(markup.first)); + return e == Npos() ? Npos() : e + std::strlen(markup.second); + } + } + return lt; + } + size_t FindTagEnd(size_t lt) const { size_t p = lt + 1; while (p < m_xml.size()) { @@ -284,19 +353,9 @@ class XmlShapeDeserializer::Impl final : public ShapeDeserializer { if (lt == Aws::String::npos || lt >= limit) { return limit; } - if (StartsWith(lt, "", lt); - p = (e == Aws::String::npos) ? limit : e + 3; - continue; - } - if (StartsWith(lt, "", lt); - p = (e == Aws::String::npos) ? limit : e + 3; - continue; - } - if (StartsWith(lt, "", lt); - p = (e == Aws::String::npos) ? limit : e + 2; + const size_t skip = SkipNonElementMarkup(lt); + if (skip != lt) { + p = (skip == Npos()) ? limit : skip; continue; } if (m_xml[lt + 1] == '/') { @@ -324,19 +383,9 @@ class XmlShapeDeserializer::Impl final : public ShapeDeserializer { if (lt == Aws::String::npos || lt >= end) { break; } - if (StartsWith(lt, "", lt); - p = (e == Aws::String::npos) ? end : e + 3; - continue; - } - if (StartsWith(lt, "", lt); - p = (e == Aws::String::npos) ? end : e + 3; - continue; - } - if (StartsWith(lt, "", lt); - p = (e == Aws::String::npos) ? end : e + 2; + const size_t skip = SkipNonElementMarkup(lt); + if (skip != lt) { + p = (skip == Npos()) ? end : skip; continue; } if (m_xml[lt + 1] == '/') { @@ -368,6 +417,117 @@ class XmlShapeDeserializer::Impl final : public ShapeDeserializer { return Element{}; } + // Navigation assumes the balanced, single-root tree this checks for. + Aws::Crt::Optional CheckWellFormed() const { + Aws::Vector open; + bool rootSeen = false; + bool instructionsOnly = true; + size_t p = StartsWith(0, "\xEF\xBB\xBF") ? 3 : 0; + while (p < m_xml.size()) { + const size_t lt = m_xml.find('<', p); + const size_t textEnd = (lt == Aws::String::npos) ? m_xml.size() : lt; + if (open.empty()) { + const size_t text = m_xml.find_first_not_of(" \t\r\n", p); + if (text != Aws::String::npos && text < textEnd) { + return Aws::String("Malformed XML: text outside the root element"); + } + } + if (lt == Aws::String::npos) { + break; + } + if (open.empty() && StartsWith(lt, "= m_xml.size()) { + return Aws::String("Malformed XML: unterminated tag"); + } + if (m_xml[lt + 1] == '/') { + const Aws::String name = ReadName(lt + 2); + if (open.empty() || open.back() != name) { + return Aws::String("Malformed XML: mismatched closing tag"); + } + // Navigation ends a closing tag at the first '>', so nothing but whitespace may follow the name. + if (m_xml.find_first_not_of(" \t\r\n", lt + 2 + name.size()) != gt) { + return Aws::String("Malformed XML: invalid closing tag"); + } + open.pop_back(); + } else { + if (open.empty() && rootSeen) { + return Aws::String("Malformed XML: multiple root elements"); + } + Aws::String name = ReadName(lt + 1); + if (name.empty()) { + return Aws::String("Malformed XML: missing element name"); + } + if (!IsXmlName(name)) { + return Aws::String("Malformed XML: invalid element name"); + } + auto attributes = CheckAttributes(lt + 1 + name.size(), gt); + if (attributes.has_value()) { + return attributes; + } + rootSeen = true; + if (m_xml[gt - 1] != '/') { + open.push_back(std::move(name)); + } + } + p = gt + 1; + } + if (!open.empty()) { + return Aws::String("Malformed XML: unclosed element"); + } + return {}; + } + + // Names and attributes follow tinyxml2, which legacy clients used. + static bool IsNameStartChar(unsigned char c) { + return c >= 0x80 || (c >= 'a' && c <= 'z') || (c >= 'A' && c <= 'Z') || c == ':' || c == '_'; + } + static bool IsNameChar(unsigned char c) { return IsNameStartChar(c) || (c >= '0' && c <= '9') || c == '.' || c == '-'; } + static bool IsXmlName(const Aws::String& name) { + return !name.empty() && IsNameStartChar(name[0]) && std::all_of(name.begin() + 1, name.end(), [](char c) { return IsNameChar(c); }); + } + + Aws::Crt::Optional CheckAttributes(size_t p, size_t gt) const { + const size_t limit = m_xml[gt - 1] == '/' ? gt - 1 : gt; + Aws::Vector names; + for (size_t nameBegin = m_xml.find_first_not_of(" \t\r\n", p); nameBegin < limit; + nameBegin = m_xml.find_first_not_of(" \t\r\n", p)) { + size_t nameEnd = nameBegin; + while (nameEnd < limit && IsNameChar(m_xml[nameEnd])) { + ++nameEnd; + } + const size_t eq = m_xml.find_first_not_of(" \t\r\n", nameEnd); + const size_t quote = eq < limit && m_xml[eq] == '=' ? m_xml.find_first_not_of(" \t\r\n", eq + 1) : Npos(); + const bool quoted = quote < limit && (m_xml[quote] == '"' || m_xml[quote] == '\''); + const size_t close = quoted ? m_xml.find(m_xml[quote], quote + 1) : Npos(); + if (nameEnd == nameBegin || !IsNameStartChar(m_xml[nameBegin]) || close >= limit) { + return Aws::String("Malformed XML: invalid attribute"); + } + Aws::String name = m_xml.substr(nameBegin, nameEnd - nameBegin); + if (std::find(names.begin(), names.end(), name) != names.end()) { + return Aws::String("Malformed XML: duplicate attribute"); + } + names.push_back(std::move(name)); + p = close + 1; + } + return {}; + } + Element FindRoot() const { size_t p = 0; while (p < m_xml.size()) { @@ -375,14 +535,9 @@ class XmlShapeDeserializer::Impl final : public ShapeDeserializer { if (lt == Aws::String::npos) { break; } - if (StartsWith(lt, "", lt); - p = (e == Aws::String::npos) ? m_xml.size() : e + 3; - } else { - const size_t gt = m_xml.find('>', lt); - p = (gt == Aws::String::npos) ? m_xml.size() : gt + 1; - } + const size_t skip = SkipNonElementMarkup(lt); + if (skip != lt) { + p = (skip == Npos()) ? m_xml.size() : skip; continue; } const size_t gt = FindTagEnd(lt); @@ -406,49 +561,174 @@ class XmlShapeDeserializer::Impl final : public ShapeDeserializer { return Element{}; } + // Decodes as tinyxml2 did for legacy clients; an unrecognised reference is kept as written. + static void AppendCharacterData(const Aws::String& text, size_t begin, size_t end, bool decodeReferences, Aws::String& out) { + size_t p = begin; + while (p < end) { + const char c = text[p]; + if (c == '\r' || c == '\n') { + const char pair = c == '\r' ? '\n' : '\r'; + out += '\n'; + p += (p + 1 < end && text[p + 1] == pair) ? 2 : 1; + } else if (c == '&' && decodeReferences) { + p = AppendReference(text, p, end, out); + } else { + out += c; + ++p; + } + } + } + + static size_t AppendReference(const Aws::String& text, size_t amp, size_t end, Aws::String& out) { + static const std::pair PREDEFINED[] = {{"<", '<'}, {">", '>'}, {"&", '&'}, {""", '"'}, {"'", '\''}}; + for (const auto& entity : PREDEFINED) { + const size_t length = std::strlen(entity.first); + if (amp + length <= end && text.compare(amp, length, entity.first) == 0) { + out += entity.second; + return amp + length; + } + } + uint32_t codePoint = 0; + const size_t after = ReadCharacterReference(text, amp, end, codePoint); + if (after == amp) { + out += '&'; + return amp + 1; + } + AppendUtf8(codePoint, out); + return after; + } + + static size_t ReadCharacterReference(const Aws::String& text, size_t amp, size_t end, uint32_t& codePoint) { + if (amp + 2 >= end || text[amp + 1] != '#') { + return amp; + } + const bool hex = text[amp + 2] == 'x'; + const size_t first = amp + (hex ? 3 : 2); + uint32_t value = 0; + size_t p = first; + for (int digit = DigitValue(text, p, end, hex); digit >= 0; digit = DigitValue(text, ++p, end, hex)) { + value = (std::min)(value * (hex ? 16u : 10u) + static_cast(digit), 0x110000u); + } + if (p == first || p >= end || text[p] != ';' || value > 0x10FFFF) { + return amp; + } + codePoint = value; + return p + 1; + } + + static int DigitValue(const Aws::String& text, size_t p, size_t end, bool hex) { + const char c = p < end ? text[p] : '\0'; + if (c >= '0' && c <= '9') { + return c - '0'; + } + if (hex && c >= 'a' && c <= 'f') { + return c - 'a' + 10; + } + return hex && c >= 'A' && c <= 'F' ? c - 'A' + 10 : -1; + } + + static void AppendUtf8(uint32_t codePoint, Aws::String& out) { + if (codePoint < 0x80) { + out += static_cast(codePoint); + } else if (codePoint < 0x800) { + out += static_cast(0xC0 | (codePoint >> 6)); + out += static_cast(0x80 | (codePoint & 0x3F)); + } else if (codePoint < 0x10000) { + out += static_cast(0xE0 | (codePoint >> 12)); + out += static_cast(0x80 | ((codePoint >> 6) & 0x3F)); + out += static_cast(0x80 | (codePoint & 0x3F)); + } else { + out += static_cast(0xF0 | (codePoint >> 18)); + out += static_cast(0x80 | ((codePoint >> 12) & 0x3F)); + out += static_cast(0x80 | ((codePoint >> 6) & 0x3F)); + out += static_cast(0x80 | (codePoint & 0x3F)); + } + } + Aws::String DecodeText(size_t begin, size_t end) const { - if (end <= begin) { - return Aws::String(); + Aws::String out; + size_t p = begin; + while (p < end) { + const size_t lt = (std::min)(m_xml.find('<', p), end); + AppendCharacterData(m_xml, p, lt, true, out); + p = lt < end ? AppendMarkup(lt, end, out) : end; } - return Aws::Utils::Xml::DecodeEscapedXmlText(m_xml.substr(begin, end - begin)); + return out; } - Aws::String AttributeValue(const Aws::String& attr) const { - size_t p = m_tagBegin; + size_t AppendMarkup(size_t lt, size_t end, Aws::String& out) const { + const size_t markupEnd = (std::min)(SkipNonElementMarkup(lt), end); + if (markupEnd == lt) { + out += '<'; + return lt + 1; + } + if (StartsWith(lt, "")), false, out); + } + return markupEnd; + } + + Aws::Crt::Optional AttributeValue(const Aws::String& attr) const { + if (attr.empty()) { + return Aws::Crt::Optional(); + } + size_t p = m_tagBegin + 1; + while (p < m_tagEnd && !IsWs(m_xml[p]) && m_xml[p] != '/') { + ++p; + } while (p < m_tagEnd) { - const size_t f = m_xml.find(attr, p); - if (f == Aws::String::npos || f >= m_tagEnd) { - break; + while (p < m_tagEnd && IsWs(m_xml[p])) { + ++p; } - const char prev = (f > m_tagBegin) ? m_xml[f - 1] : ' '; - size_t a = f + attr.size(); - while (a < m_tagEnd && IsWs(m_xml[a])) { - ++a; + const size_t nameBegin = p; + while (p < m_tagEnd && !IsWs(m_xml[p]) && m_xml[p] != '=' && m_xml[p] != '/') { + ++p; } - if ((IsWs(prev) || prev == '<') && a < m_tagEnd && m_xml[a] == '=') { - size_t q = a + 1; - while (q < m_tagEnd && IsWs(m_xml[q])) { - ++q; - } - if (q < m_tagEnd && (m_xml[q] == '"' || m_xml[q] == '\'')) { - const char quote = m_xml[q]; - const size_t vb = q + 1; - const size_t ve = m_xml.find(quote, vb); - if (ve != Aws::String::npos) { - return m_xml.substr(vb, ve - vb); - } - } + const size_t nameEnd = p; + while (p < m_tagEnd && IsWs(m_xml[p])) { + ++p; + } + if (nameEnd == nameBegin || p >= m_tagEnd || m_xml[p] != '=') { + return Aws::Crt::Optional(); + } + ++p; + while (p < m_tagEnd && IsWs(m_xml[p])) { + ++p; + } + if (p >= m_tagEnd || (m_xml[p] != '"' && m_xml[p] != '\'')) { + return Aws::Crt::Optional(); + } + const size_t valueBegin = p + 1; + const auto tagLimit = m_xml.begin() + static_cast(m_tagEnd); + const auto quoteAt = std::find(m_xml.begin() + static_cast(valueBegin), tagLimit, m_xml[p]); + if (quoteAt == tagLimit) { + return Aws::Crt::Optional(); } - p = f + attr.size(); + const size_t valueEnd = static_cast(quoteAt - m_xml.begin()); + if (m_xml.compare(nameBegin, nameEnd - nameBegin, attr) == 0) { + return Aws::Crt::Optional(m_xml.substr(valueBegin, valueEnd - valueBegin)); + } + p = valueEnd + 1; } - return Aws::String(); + return Aws::Crt::Optional(); } Aws::String CurrentText() const { - if (m_attr.empty()) { - return DecodeText(m_contentBegin, m_contentEnd); + if (m_attrValue.has_value()) { + Aws::String out; + AppendCharacterData(*m_attrValue, 0, m_attrValue->size(), true, out); + return out; } - return Aws::Utils::Xml::DecodeEscapedXmlText(AttributeValue(m_attr)); + return DecodeText(m_contentBegin, m_contentEnd); + } + + // Legacy clients trim number, boolean and timestamp text before parsing it. + Aws::String ScalarText() const { + const Aws::String text = CurrentText(); + const char* whitespace = " \t\n\v\f\r"; + const size_t begin = text.find_first_not_of(whitespace); + return begin == Aws::String::npos ? Aws::String() : text.substr(begin, text.find_last_not_of(whitespace) - begin + 1); } static Aws::String XmlName(const Schema& schema) { @@ -482,7 +762,7 @@ class XmlShapeDeserializer::Impl final : public ShapeDeserializer { bool m_valid = false; bool m_flattened = false; Aws::String m_flatName; - Aws::String m_attr; + Aws::Crt::Optional m_attrValue; CodecSettings m_settings; }; @@ -506,3 +786,4 @@ std::shared_ptr XmlShapeDeserializer::ReadDocume return m_impl->ReadDocument(schema); } bool XmlShapeDeserializer::IsNull() { return m_impl->IsNull(); } +Aws::Crt::Optional XmlShapeDeserializer::GetError() const { return m_impl->GetError(); } diff --git a/src/aws-cpp-sdk-core/source/smithy/client/schema/XmlShapeSerializer.cpp b/src/aws-cpp-sdk-core/source/smithy/client/schema/XmlShapeSerializer.cpp index 218975888356..5471ba7ca419 100644 --- a/src/aws-cpp-sdk-core/source/smithy/client/schema/XmlShapeSerializer.cpp +++ b/src/aws-cpp-sdk-core/source/smithy/client/schema/XmlShapeSerializer.cpp @@ -21,17 +21,26 @@ using namespace smithy::schema; using namespace Aws::Utils; using SerializerOutcome = Aws::Utils::Outcome>; -static constexpr int MAX_DEPTH = 500; - namespace { +Aws::String NonFiniteText(double value) { + if (std::isnan(value)) { + return "NaN"; + } + return value > 0 ? "Infinity" : "-Infinity"; +} + +Aws::String FloatText(float value) { return std::isfinite(value) ? StringUtils::FormatRoundTripFloat(value) : NonFiniteText(value); } + +Aws::String DoubleText(double value) { return std::isfinite(value) ? StringUtils::FormatRoundTripDouble(value) : NonFiniteText(value); } + void AppendXmlEscaped(Aws::String& buf, const Aws::String& value) { const char* data = value.data(); const size_t len = value.size(); size_t i = 0; while (i < len) { size_t start = i; - while (i < len && data[i] != '&' && data[i] != '<' && data[i] != '>' && data[i] != '"' && data[i] != '\'') { + while (i < len && data[i] != '&' && data[i] != '<' && data[i] != '>' && data[i] != '"' && data[i] != '\'' && data[i] != '\r') { i++; } if (i > start) { @@ -54,6 +63,10 @@ void AppendXmlEscaped(Aws::String& buf, const Aws::String& value) { case '\'': buf += "'"; break; + case '\r': + // Readers turn a literal CR into LF. + buf += " "; + break; } i++; } @@ -177,18 +190,8 @@ class XmlShapeSerializer::Impl final : public InterceptingSerializer { } } - bool EnterContainer() { - if (!m_errorMessage.empty()) { - return false; - } - if (m_depth + 1 > MAX_DEPTH) { - m_errorMessage = "Maximum serialization depth exceeded"; - return false; - } - ++m_depth; - return true; - } - void ExitContainer() { --m_depth; } + bool EnterContainer() { return m_depth.Enter(m_errorMessage); } + void ExitContainer() { m_depth.Exit(); } void ClosePendingTag() { if (m_pendingClose) { @@ -285,7 +288,7 @@ class XmlShapeSerializer::Impl final : public InterceptingSerializer { private: Aws::String m_buf; - int m_depth = 0; + NestingDepth m_depth; bool m_pendingClose = false; bool m_finalized = false; Aws::String m_errorMessage; @@ -345,23 +348,11 @@ void XmlShapeSerializer::Impl::ValueSerializer::WriteLong(const Schema&, int64_t } void XmlShapeSerializer::Impl::ValueSerializer::WriteFloat(const Schema&, float value) { m_outer->ClosePendingTag(); - if (std::isfinite(value)) { - m_outer->AppendRaw(StringUtils::to_string(value)); - } else if (std::isnan(value)) { - m_outer->AppendRaw("NaN"); - } else { - m_outer->AppendRaw(value > 0 ? "Infinity" : "-Infinity"); - } + m_outer->AppendRaw(FloatText(value)); } void XmlShapeSerializer::Impl::ValueSerializer::WriteDouble(const Schema&, double value) { m_outer->ClosePendingTag(); - if (std::isfinite(value)) { - m_outer->AppendRaw(StringUtils::to_string(value)); - } else if (std::isnan(value)) { - m_outer->AppendRaw("NaN"); - } else { - m_outer->AppendRaw(value > 0 ? "Infinity" : "-Infinity"); - } + m_outer->AppendRaw(DoubleText(value)); } void XmlShapeSerializer::Impl::ValueSerializer::WriteString(const Schema&, const Aws::String& value) { m_outer->ClosePendingTag(); @@ -422,10 +413,10 @@ void XmlShapeSerializer::Impl::InlineAttributeSerializer::WriteLong(const Schema WriteAttr(schema, StringUtils::to_string(value)); } void XmlShapeSerializer::Impl::InlineAttributeSerializer::WriteFloat(const Schema& schema, float value) { - WriteAttr(schema, StringUtils::to_string(value)); + WriteAttr(schema, FloatText(value)); } void XmlShapeSerializer::Impl::InlineAttributeSerializer::WriteDouble(const Schema& schema, double value) { - WriteAttr(schema, StringUtils::to_string(value)); + WriteAttr(schema, DoubleText(value)); } void XmlShapeSerializer::Impl::InlineAttributeSerializer::WriteString(const Schema& schema, const Aws::String& value) { WriteAttr(schema, value); diff --git a/src/aws-cpp-sdk-core/source/utils/StringUtils.cpp b/src/aws-cpp-sdk-core/source/utils/StringUtils.cpp index b66ce0e1cd95..ed7ef84a7f52 100644 --- a/src/aws-cpp-sdk-core/source/utils/StringUtils.cpp +++ b/src/aws-cpp-sdk-core/source/utils/StringUtils.cpp @@ -9,6 +9,8 @@ #include #include #include +#include +#include #include #include #include @@ -390,29 +392,100 @@ Aws::Crt::Optional StringUtils::ParseInt64(const Aws::String& source) return {}; } char* end = nullptr; + errno = 0; const long long value = std::strtoll(source.c_str(), &end, 10); - if (end != source.c_str() + source.size()) + if (end != source.c_str() + source.size() || errno == ERANGE) { return {}; } return static_cast(value); } +namespace +{ + // strtod and strtof use the C locale's decimal point; wire numbers always use '.'. + Aws::Crt::Optional InCLocaleForm(const Aws::String& source) + { + const Aws::String point = std::localeconv()->decimal_point; + if (point == ".") + { + return source; + } + if (source.find(point) != Aws::String::npos) + { + return {}; + } + Aws::String text; + for (const char c : source) + { + if (c == '.') + { + text += point; + } + else + { + text += c; + } + } + return text; + } + + bool ReparsesAsFloat(const Aws::String& text, float value) + { + const auto localized = InCLocaleForm(text); + return localized.has_value() && std::strtof(localized->c_str(), nullptr) == value; + } +} + Aws::Crt::Optional StringUtils::ParseDouble(const Aws::String& source) { - if (source.empty()) + const auto text = InCLocaleForm(source); + if (source.empty() || !text.has_value()) { return {}; } char* end = nullptr; - const double value = std::strtod(source.c_str(), &end); - if (end != source.c_str() + source.size()) + const double value = std::strtod(text->c_str(), &end); + if (end != text->c_str() + text->size()) { return {}; } return value; } +Aws::String StringUtils::FormatRoundTripDouble(double value) +{ + Aws::OStringStream s15; + s15.imbue(std::locale::classic()); + s15 << std::setprecision(15) << value; + Aws::String text = s15.str(); + const auto reparsed = ParseDouble(text); + if (reparsed.has_value() && *reparsed == value) + { + return text; + } + Aws::OStringStream s17; + s17.imbue(std::locale::classic()); + s17 << std::setprecision(17) << value; + return s17.str(); +} + +Aws::String StringUtils::FormatRoundTripFloat(float value) +{ + Aws::OStringStream s6; + s6.imbue(std::locale::classic()); + s6 << std::setprecision(6) << value; + Aws::String text = s6.str(); + if (!ReparsesAsFloat(text, value)) + { + Aws::OStringStream s9; + s9.imbue(std::locale::classic()); + s9 << std::setprecision(9) << value; + text = s9.str(); + } + return text; +} + Aws::String StringUtils::Join(const Aws::Vector& elements, char delimiter) { return std::accumulate(std::begin(elements), std::end(elements), Aws::String{}, diff --git a/tests/aws-cpp-sdk-core-tests/smithy/client/schema/CborShapeDeserializerTest.cpp b/tests/aws-cpp-sdk-core-tests/smithy/client/schema/CborShapeDeserializerTest.cpp index e4d5db052e5f..36638eaa9b1f 100644 --- a/tests/aws-cpp-sdk-core-tests/smithy/client/schema/CborShapeDeserializerTest.cpp +++ b/tests/aws-cpp-sdk-core-tests/smithy/client/schema/CborShapeDeserializerTest.cpp @@ -13,6 +13,7 @@ #include #include +#include #include "SchemaSerializerTestHelpers.h" @@ -433,6 +434,17 @@ TEST_F(CborShapeDeserializerTest, TimestampFromFloat) { EXPECT_DOUBLE_EQ(ts.value().SecondsWithMSPrecision(), 1234567890.5); } +TEST_F(CborShapeDeserializerTest, TimestampFloatMatchesLegacyDateTimeDouble) { + auto scalar = Schema::CreateTimestamp("T"); + // Tag 1, float64 1.9996 (0x3FFFFE5C91D14E3C) + const unsigned char data[] = {0xC1, 0xFB, 0x3F, 0xFF, 0xFE, 0x5C, 0x91, 0xD1, 0x4E, 0x3C}; + CborShapeDeserializer d(Aws::Crt::ByteCursorFromArray(data, sizeof(data))); + auto ts = d.ReadTimestamp(*scalar); + ASSERT_TRUE(ts.has_value()); + EXPECT_EQ(ts.value().Millis(), 1999); + EXPECT_EQ(ts.value().Millis(), Aws::Utils::DateTime(1.9996).Millis()); +} + TEST_F(CborShapeDeserializerTest, TimestampNegativeIntReturnsEmpty) { auto scalar = Schema::CreateTimestamp("T"); const unsigned char data[] = {0xC1, 0x20}; @@ -555,3 +567,388 @@ TEST_F(CborShapeDeserializerTest, ReadDocumentConsumesValueSoTrailingMemberSurvi }); EXPECT_EQ(trailing, "x"); } + +TEST_F(CborShapeDeserializerTest, DefiniteMapShorterThanHeaderSetsError) { + auto root = Schema::StructureBuilder("Root").PutMember("a", Schema::CreateInteger("I")).Build(); + const unsigned char bytes[] = {0xA2, 0x61, 'a', 0x01}; // map(2) holding one entry + CborShapeDeserializer d(Aws::Crt::ByteCursorFromArray(bytes, sizeof(bytes))); + Aws::Crt::Optional got; + d.ReadStruct(*root, [&](const Schema& m, ShapeDeserializer& de) { got = de.ReadInteger(m); }); + ASSERT_TRUE(got.has_value()); + EXPECT_EQ(got.value(), 1); + EXPECT_TRUE(d.GetError().has_value()); +} + +namespace { +struct ChunkedRead { + Aws::Crt::Optional text; + Aws::Crt::Optional blob; + bool failed = false; +}; + +ChunkedRead ReadChunked(const Aws::Vector& bytes) { + auto root = Schema::StructureBuilder("Root").PutMember("s", Schema::CreateString("S")).PutMember("b", Schema::CreateBlob("B")).Build(); + CborShapeDeserializer d(Aws::Crt::ByteCursorFromArray(bytes.data(), bytes.size())); + ChunkedRead got; + d.ReadStruct(*root, [&](const Schema& m, ShapeDeserializer& de) { + if (m.GetMemberName() == "s") { + got.text = de.ReadString(m); + } else { + got.blob = de.ReadBlob(m); + } + }); + got.failed = d.GetError().has_value(); + return got; +} +} // namespace + +TEST_F(CborShapeDeserializerTest, IndefiniteTextIsJoined) { + // {"s": (_ "ab", "", "c")} + const auto got = ReadChunked({0xA1, 0x61, 's', 0x7F, 0x62, 'a', 'b', 0x60, 0x61, 'c', 0xFF}); + EXPECT_FALSE(got.failed); + ASSERT_TRUE(got.text.has_value()); + EXPECT_EQ(got.text.value(), "abc"); +} + +TEST_F(CborShapeDeserializerTest, EmptyIndefiniteTextIsEmptyString) { + const auto got = ReadChunked({0xA1, 0x61, 's', 0x7F, 0xFF}); + EXPECT_FALSE(got.failed); + ASSERT_TRUE(got.text.has_value()); + EXPECT_EQ(got.text.value(), ""); +} + +TEST_F(CborShapeDeserializerTest, IndefiniteBytesAreJoined) { + // {"b": (_ h'0102', h'03'), "s": "x"} + const auto got = ReadChunked({0xA2, 0x61, 'b', 0x5F, 0x42, 0x01, 0x02, 0x41, 0x03, 0xFF, 0x61, 's', 0x61, 'x'}); + EXPECT_FALSE(got.failed); + ASSERT_TRUE(got.blob.has_value()); + const unsigned char expected[] = {0x01, 0x02, 0x03}; + EXPECT_EQ(got.blob.value(), Aws::Utils::ByteBuffer(expected, sizeof(expected))); + ASSERT_TRUE(got.text.has_value()); + EXPECT_EQ(got.text.value(), "x"); +} + +TEST_F(CborShapeDeserializerTest, IndefiniteTextKeyIsJoined) { + // {(_ "", "s"): "x"} + const auto got = ReadChunked({0xA1, 0x7F, 0x60, 0x61, 's', 0xFF, 0x61, 'x'}); + EXPECT_FALSE(got.failed); + ASSERT_TRUE(got.text.has_value()); + EXPECT_EQ(got.text.value(), "x"); +} + +TEST_F(CborShapeDeserializerTest, MalformedIndefiniteStringRecordsError) { + for (const auto& bytes : Aws::Vector>{ + {0xA1, 0x61, 's', 0x7F, 0x01, 0xFF}, // integer chunk + {0xA1, 0x61, 's', 0x7F, 0x41, 0x01, 0xFF}, // byte-string chunk in a text string + {0xA1, 0x61, 'b', 0x5F, 0x61, 'a', 0xFF}, // text chunk in a byte string + {0xA1, 0x61, 's', 0x7F, 0x7F, 0xFF, 0xFF}, // nested indefinite chunk + {0xA1, 0x61, 's', 0x7F, 0x61, 'a'}}) { // no break + EXPECT_TRUE(ReadChunked(bytes).failed); + } +} + +TEST_F(CborShapeDeserializerTest, EmptyInputIsNotAnError) { + auto root = Schema::StructureBuilder("Root").PutMember("a", Schema::CreateInteger("I")).Build(); + const unsigned char empty[1] = {0}; + CborShapeDeserializer d(Aws::Crt::ByteCursorFromArray(empty, 0)); + d.ReadStruct(*root, [](const Schema&, ShapeDeserializer&) { FAIL() << "no members expected"; }); + EXPECT_FALSE(d.GetError().has_value()); +} + +TEST_F(CborShapeDeserializerTest, GarbageRootSetsError) { + auto root = Schema::StructureBuilder("Root").PutMember("a", Schema::CreateInteger("I")).Build(); + const unsigned char bytes[] = {0xBA, 0x00}; // map with 4-byte length, truncated header + CborShapeDeserializer d(Aws::Crt::ByteCursorFromArray(bytes, sizeof(bytes))); + d.ReadStruct(*root, [](const Schema&, ShapeDeserializer&) {}); + EXPECT_TRUE(d.GetError().has_value()); +} + +TEST_F(CborShapeDeserializerTest, RecursiveStructAtMaxDepthSucceeds) { + RecursiveNodeSchema node; + const Aws::String cbor = NestedCborNodes(MAX_NESTING_DEPTH); + CborShapeDeserializer d(Aws::Crt::ByteCursorFromArray(reinterpret_cast(cbor.data()), cbor.size())); + EXPECT_EQ(DescendRecursiveNode(d, node.Get()), MAX_NESTING_DEPTH); + EXPECT_FALSE(d.GetError().has_value()); +} + +TEST_F(CborShapeDeserializerTest, RecursiveStructPastMaxDepthSetsError) { + RecursiveNodeSchema node; + const Aws::String cbor = NestedCborNodes(MAX_NESTING_DEPTH + 1); + CborShapeDeserializer d(Aws::Crt::ByteCursorFromArray(reinterpret_cast(cbor.data()), cbor.size())); + DescendRecursiveNode(d, node.Get()); + ASSERT_TRUE(d.GetError().has_value()); + EXPECT_NE(d.GetError()->GetMessage().find("depth"), Aws::String::npos); +} + +namespace { +void ExpectOutOfRange(const ShapeDeserializer& d) { + ASSERT_TRUE(d.GetError().has_value()); + EXPECT_NE(d.GetError()->GetMessage().find("out of range"), Aws::String::npos); +} +} // namespace + +TEST_F(CborShapeDeserializerTest, NegIntBeyondInt64RecordsError) { + const unsigned char bytes[] = {0x3B, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF}; + CborShapeDeserializer d(Aws::Crt::ByteCursorFromArray(bytes, sizeof(bytes))); + auto schema = Schema::CreateLong("L"); + EXPECT_FALSE(d.ReadLong(*schema).has_value()); + ExpectOutOfRange(d); +} + +TEST_F(CborShapeDeserializerTest, NegIntAtInt64MinIsExact) { + const unsigned char bytes[] = {0x3B, 0x7F, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF}; + CborShapeDeserializer d(Aws::Crt::ByteCursorFromArray(bytes, sizeof(bytes))); + auto schema = Schema::CreateLong("L"); + auto v = d.ReadLong(*schema); + ASSERT_TRUE(v.has_value()); + EXPECT_EQ(v.value(), (std::numeric_limits::min)()); +} + +namespace { +double ReadDoubleFromInteger(uint8_t majorTypeByte, uint64_t n) { + unsigned char bytes[9] = {majorTypeByte}; + for (int i = 0; i < 8; ++i) { + bytes[1 + i] = static_cast(n >> (8 * (7 - i))); + } + CborShapeDeserializer d(Aws::Crt::ByteCursorFromArray(bytes, sizeof(bytes))); + auto schema = Schema::CreateDouble("D"); + auto v = d.ReadDouble(*schema); + EXPECT_TRUE(v.has_value()); + return v.has_value() ? v.value() : 0.0; +} +} // namespace + +TEST_F(CborShapeDeserializerTest, NegIntDoubleRoundsOnce) { + EXPECT_EQ(ReadDoubleFromInteger(0x3B, UINT64_C(11943773121102237)), static_cast(INT64_C(-11943773121102238))); +} + +TEST_F(CborShapeDeserializerTest, NegIntDoubleAtUint64Max) { EXPECT_EQ(ReadDoubleFromInteger(0x3B, UINT64_MAX), -18446744073709551616.0); } + +TEST_F(CborShapeDeserializerTest, NegIntDoubleZeroIsMinusOne) { EXPECT_EQ(ReadDoubleFromInteger(0x3B, 0), -1.0); } + +TEST_F(CborShapeDeserializerTest, NegIntDoubleAboveTwoPow53) { + const uint64_t n = UINT64_C(1) << 53; + EXPECT_EQ(ReadDoubleFromInteger(0x3B, n), static_cast(-static_cast(n) - 1)); +} + +TEST_F(CborShapeDeserializerTest, UIntDoubleAtUint64MaxRoundsOnce) { + EXPECT_EQ(ReadDoubleFromInteger(0x1B, UINT64_MAX), static_cast(UINT64_MAX)); +} + +TEST_F(CborShapeDeserializerTest, UIntBeyondInt64RecordsError) { + const unsigned char bytes[] = {0x1B, 0x80, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00}; // 2^63 + CborShapeDeserializer d(Aws::Crt::ByteCursorFromArray(bytes, sizeof(bytes))); + auto schema = Schema::CreateLong("L"); + EXPECT_FALSE(d.ReadLong(*schema).has_value()); + ExpectOutOfRange(d); +} + +TEST_F(CborShapeDeserializerTest, IntegerBeyondInt32RecordsError) { + const unsigned char bytes[] = {0x1B, 0x00, 0x00, 0x01, 0x00, 0x00, 0x00, 0x00, 0x00}; // 2^40 + CborShapeDeserializer d(Aws::Crt::ByteCursorFromArray(bytes, sizeof(bytes))); + auto schema = Schema::CreateInteger("I"); + EXPECT_FALSE(d.ReadInteger(*schema).has_value()); + ExpectOutOfRange(d); +} + +TEST_F(CborShapeDeserializerTest, FloatBeyondFloatRangeRecordsError) { + const unsigned char bytes[] = {0xFB, 0x7E, 0x37, 0xE4, 0x3C, 0x88, 0x00, 0x75, 0x9C}; // 1e300 + CborShapeDeserializer d(Aws::Crt::ByteCursorFromArray(bytes, sizeof(bytes))); + auto schema = Schema::CreateFloat("F"); + EXPECT_FALSE(d.ReadFloat(*schema).has_value()); + ExpectOutOfRange(d); +} + +TEST_F(CborShapeDeserializerTest, UnrepresentableTimestampRecordsError) { + const unsigned char bytes[] = {0xC1, 0xFB, 0x7E, 0x37, 0xE4, 0x3C, 0x88, 0x00, 0x75, 0x9C}; // 1(1e300) + CborShapeDeserializer d(Aws::Crt::ByteCursorFromArray(bytes, sizeof(bytes))); + auto schema = Schema::CreateTimestamp("T"); + EXPECT_FALSE(d.ReadTimestamp(*schema).has_value()); + ExpectOutOfRange(d); +} + +TEST_F(CborShapeDeserializerTest, MismatchedListElementsAreConsumed) { + auto listBuilder = Schema::ListBuilder("L"); + listBuilder.PutMember("member", Schema::CreateInteger("I")); + auto list = listBuilder.Build(); + const unsigned char bytes[] = {0x9F, 0x61, 'a', 0x61, 'b', 0xFF}; // [_ "a", "b"] + CborShapeDeserializer d(Aws::Crt::ByteCursorFromArray(bytes, sizeof(bytes))); + int calls = 0; + d.ReadList(*list, [&](ShapeDeserializer& de) { + ++calls; + EXPECT_FALSE(de.ReadInteger(*list->GetMember("member").value()).has_value()); + }); + EXPECT_EQ(calls, 2); + EXPECT_FALSE(d.GetError().has_value()); +} + +TEST_F(CborShapeDeserializerTest, MismatchedMemberKeepsSiblingsAligned) { + auto root = + Schema::StructureBuilder("Root").PutMember("a", Schema::CreateInteger("I")).PutMember("b", Schema::CreateInteger("I")).Build(); + const unsigned char bytes[] = {0xA2, 0x61, 'a', 0x63, 'x', 'y', 'z', 0x61, 'b', 0x02}; // {"a":"xyz","b":2} + CborShapeDeserializer d(Aws::Crt::ByteCursorFromArray(bytes, sizeof(bytes))); + Aws::Crt::Optional b; + d.ReadStruct(*root, [&](const Schema& m, ShapeDeserializer& de) { + auto v = de.ReadInteger(m); + if (m.GetMemberName() == "b") { + b = v; + } + }); + ASSERT_TRUE(b.has_value()); + EXPECT_EQ(b.value(), 2); + EXPECT_FALSE(d.GetError().has_value()); +} + +TEST_F(CborShapeDeserializerTest, ScalarWhereStructExpectedKeepsSiblingsAligned) { + auto inner = Schema::StructureBuilder("Inner").Build(); + auto root = Schema::StructureBuilder("Root").PutMember("child", inner).PutMember("x", Schema::CreateInteger("I")).Build(); + const unsigned char bytes[] = {0xA2, 0x65, 'c', 'h', 'i', 'l', 'd', 0x05, 0x61, 'x', 0x01}; // {"child":5,"x":1} + CborShapeDeserializer d(Aws::Crt::ByteCursorFromArray(bytes, sizeof(bytes))); + Aws::Crt::Optional x; + d.ReadStruct(*root, [&](const Schema& m, ShapeDeserializer& de) { + if (m.GetMemberName() == "x") { + x = de.ReadInteger(m); + } else { + de.ReadStruct(*m.GetMemberTarget().value(), [](const Schema&, ShapeDeserializer&) {}); + } + }); + ASSERT_TRUE(x.has_value()); + EXPECT_EQ(x.value(), 1); + EXPECT_FALSE(d.GetError().has_value()); +} + +TEST_F(CborShapeDeserializerTest, HugeDefiniteArrayHeaderTerminatesWithError) { + auto listBuilder = Schema::ListBuilder("L"); + listBuilder.PutMember("member", Schema::CreateInteger("I")); + auto list = listBuilder.Build(); + const unsigned char bytes[] = {0x9B, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF}; // array(2^64-1), no items + CborShapeDeserializer d(Aws::Crt::ByteCursorFromArray(bytes, sizeof(bytes))); + d.ReadList(*list, [&](ShapeDeserializer& de) { de.ReadInteger(*list->GetMember("member").value()); }); + EXPECT_TRUE(d.GetError().has_value()); +} + +TEST_F(CborShapeDeserializerTest, NonMapRootRecordsError) { + auto root = Schema::StructureBuilder("Root").PutMember("a", Schema::CreateInteger("I")).Build(); + const Aws::Vector> roots = {{0x05}, {0x81, 0x01}, {0x61, 'x'}, {0xF5}}; + for (const auto& bytes : roots) { + CborShapeDeserializer d(Aws::Crt::ByteCursorFromArray(bytes.data(), bytes.size())); + int delivered = 0; + d.ReadStruct(*root, [&](const Schema&, ShapeDeserializer&) { ++delivered; }); + EXPECT_EQ(delivered, 0); + ASSERT_TRUE(d.GetError().has_value()) << static_cast(bytes[0]); + } +} + +TEST_F(CborShapeDeserializerTest, NullRootIsEmptyStructWithoutError) { + auto root = Schema::StructureBuilder("Root").PutMember("a", Schema::CreateInteger("I")).Build(); + const unsigned char bytes[] = {0xF6}; + CborShapeDeserializer d(Aws::Crt::ByteCursorFromArray(bytes, sizeof(bytes))); + int delivered = 0; + d.ReadStruct(*root, [&](const Schema&, ShapeDeserializer&) { ++delivered; }); + EXPECT_EQ(delivered, 0); + EXPECT_FALSE(d.GetError().has_value()); +} + +TEST_F(CborShapeDeserializerTest, TrailingDataAfterRootRecordsError) { + auto root = Schema::StructureBuilder("Root").PutMember("a", Schema::CreateInteger("I")).Build(); + // map(1){"a": 1} then a stray item; map(0) then a stray item; null then a stray item. + const Aws::Vector> payloads = {{0xA1, 0x61, 'a', 0x01, 0x05}, {0xA0, 0xA0}, {0xF6, 0x00}}; + for (const auto& bytes : payloads) { + CborShapeDeserializer d(Aws::Crt::ByteCursorFromArray(bytes.data(), bytes.size())); + d.ReadStruct(*root, [&](const Schema& m, ShapeDeserializer& de) { de.ReadInteger(m); }); + ASSERT_TRUE(d.GetError().has_value()) << bytes.size(); + EXPECT_NE(d.GetError()->GetMessage().find("trailing data"), Aws::String::npos); + } +} + +TEST_F(CborShapeDeserializerTest, EmptyRootIsEmptyStructWithoutError) { + auto root = Schema::StructureBuilder("Root").PutMember("a", Schema::CreateInteger("I")).Build(); + const unsigned char none[1] = {0}; + CborShapeDeserializer d(Aws::Crt::ByteCursorFromArray(none, 0)); + int delivered = 0; + d.ReadStruct(*root, [&](const Schema&, ShapeDeserializer&) { ++delivered; }); + EXPECT_EQ(delivered, 0); + EXPECT_FALSE(d.GetError().has_value()); +} + +TEST_F(CborShapeDeserializerTest, ScalarWhereNestedStructExpectedIsNotAnError) { + auto inner = Schema::StructureBuilder("Inner").PutMember("x", Schema::CreateInteger("I")).Build(); + auto root = Schema::StructureBuilder("Root").PutMember("inner", inner).Build(); + const unsigned char bytes[] = {0xA1, 0x65, 'i', 'n', 'n', 'e', 'r', 0x05}; + CborShapeDeserializer d(Aws::Crt::ByteCursorFromArray(bytes, sizeof(bytes))); + int innerDelivered = 0; + d.ReadStruct(*root, [&](const Schema& m, ShapeDeserializer& de) { + de.ReadStruct(m, [&](const Schema&, ShapeDeserializer&) { ++innerDelivered; }); + }); + EXPECT_EQ(innerDelivered, 0); + EXPECT_FALSE(d.GetError().has_value()); +} + +TEST_F(CborShapeDeserializerTest, UIntAboveInt64MaxOnDoubleConvertsDirectly) { + const unsigned char bytes[] = {0x1B, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF}; + CborShapeDeserializer d(Aws::Crt::ByteCursorFromArray(bytes, sizeof(bytes))); + auto v = d.ReadDouble(*Schema::CreateDouble("D")); + ASSERT_TRUE(v.has_value()); + EXPECT_EQ(v.value(), 1.8446744073709552e19); + EXPECT_FALSE(d.GetError().has_value()); +} + +TEST_F(CborShapeDeserializerTest, NegIntBeyondInt64OnDoubleConvertsDirectly) { + const unsigned char bytes[] = {0x3B, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF}; + CborShapeDeserializer d(Aws::Crt::ByteCursorFromArray(bytes, sizeof(bytes))); + auto v = d.ReadDouble(*Schema::CreateDouble("D")); + ASSERT_TRUE(v.has_value()); + EXPECT_EQ(v.value(), -1.8446744073709552e19); + EXPECT_FALSE(d.GetError().has_value()); +} + +TEST_F(CborShapeDeserializerTest, SmallIntegersOnDoubleAndFloat) { + const unsigned char neg[] = {0x38, 0x63}; // -100 + CborShapeDeserializer d(Aws::Crt::ByteCursorFromArray(neg, sizeof(neg))); + EXPECT_EQ(d.ReadDouble(*Schema::CreateDouble("D")).value(), -100.0); + CborShapeDeserializer f(Aws::Crt::ByteCursorFromArray(neg, sizeof(neg))); + EXPECT_EQ(f.ReadFloat(*Schema::CreateFloat("F")).value(), -100.0f); +} + +namespace { + +int ReadUnionVariants(CborShapeDeserializer& d) { + auto variants = Schema::UnionBuilder("U").PutMember("a", Schema::CreateString("S")).PutMember("b", Schema::CreateInteger("I")).Build(); + auto root = Schema::StructureBuilder("Root").PutMember("u", variants).Build(); + int delivered = 0; + d.ReadStruct(*root, [&](const Schema& m, ShapeDeserializer& de) { + de.ReadStruct(**m.GetMemberTarget(), [&](const Schema& variant, ShapeDeserializer& vde) { + ++delivered; + if (variant.GetMemberName() == Aws::String("a")) { + vde.ReadString(variant); + } else { + vde.ReadInteger(variant); + } + }); + }); + return delivered; +} + +} // namespace + +TEST_F(CborShapeDeserializerTest, UnionWithTwoMembersSetRecordsError) { + // {"u": {"a": "x", "b": 1}} + const unsigned char bytes[] = {0xA1, 0x61, 'u', 0xA2, 0x61, 'a', 0x61, 'x', 0x61, 'b', 0x01}; + CborShapeDeserializer d(Aws::Crt::ByteCursorFromArray(bytes, sizeof(bytes))); + EXPECT_EQ(ReadUnionVariants(d), 1); + ASSERT_TRUE(d.GetError().has_value()); + EXPECT_NE(d.GetError()->GetMessage().find("more than one member"), Aws::String::npos); +} + +TEST_F(CborShapeDeserializerTest, UnionIgnoresNullAndUnmodeledMembers) { + // {"u": {"a": "x", "b": null}} + const unsigned char withNull[] = {0xA1, 0x61, 'u', 0xA2, 0x61, 'a', 0x61, 'x', 0x61, 'b', 0xF6}; + // {"u": {"z": 2, "b": 1}} + const unsigned char withUnmodeled[] = {0xA1, 0x61, 'u', 0xA2, 0x61, 'z', 0x02, 0x61, 'b', 0x01}; + CborShapeDeserializer nullVariant(Aws::Crt::ByteCursorFromArray(withNull, sizeof(withNull))); + EXPECT_EQ(ReadUnionVariants(nullVariant), 1); + EXPECT_FALSE(nullVariant.GetError().has_value()); + CborShapeDeserializer unmodeled(Aws::Crt::ByteCursorFromArray(withUnmodeled, sizeof(withUnmodeled))); + EXPECT_EQ(ReadUnionVariants(unmodeled), 1); + EXPECT_FALSE(unmodeled.GetError().has_value()); +} diff --git a/tests/aws-cpp-sdk-core-tests/smithy/client/schema/CborShapeSerializerTest.cpp b/tests/aws-cpp-sdk-core-tests/smithy/client/schema/CborShapeSerializerTest.cpp index bf26953b8445..40b93914d62b 100644 --- a/tests/aws-cpp-sdk-core-tests/smithy/client/schema/CborShapeSerializerTest.cpp +++ b/tests/aws-cpp-sdk-core-tests/smithy/client/schema/CborShapeSerializerTest.cpp @@ -966,6 +966,31 @@ TEST_F(CborShapeSerializerTest, FloatWholeNumberEncodedAsInt) { EXPECT_EQ(payload, expected); } +TEST_F(CborShapeSerializerTest, NegativeZeroFloatIsEncodedAsIntegerZero) { + // CRT encodes any double that converts losslessly to int64 as an integer, so -0.0 loses its sign + const Aws::String expected = Aws::String("\xBF\x61\x66", 3) + '\x00' + '\xFF'; + { + CborShapeSerializer s; + auto root = Schema::StructureBuilder("Root").Build(); + auto member = Schema::CreateMember("f", ShapeType::Float); + LambdaStruct rootStruct(*root, [&](ShapeSerializer& ser) { ser.WriteFloat(*member, -0.0f); }); + s.WriteStruct(*root, rootStruct); + auto outcome = s.GetPayload(); + ASSERT_TRUE(outcome.IsSuccess()); + EXPECT_EQ(outcome.GetResult(), expected); + } + { + CborShapeSerializer s; + auto root = Schema::StructureBuilder("Root").Build(); + auto member = Schema::CreateMember("f", ShapeType::Double); + LambdaStruct rootStruct(*root, [&](ShapeSerializer& ser) { ser.WriteDouble(*member, -0.0); }); + s.WriteStruct(*root, rootStruct); + auto outcome = s.GetPayload(); + ASSERT_TRUE(outcome.IsSuccess()); + EXPECT_EQ(outcome.GetResult(), expected); + } +} + TEST_F(CborShapeSerializerTest, TimestampEpochZero) { CborShapeSerializer s; auto root = Schema::StructureBuilder("Root").Build(); @@ -1070,3 +1095,19 @@ TEST_F(CborShapeSerializerTest, TimestampIgnoresTimestampFormatTrait) { ASSERT_TRUE(bOut.IsSuccess()); EXPECT_EQ(aOut.GetResult(), bOut.GetResult()); // trait ignored -> identical bytes } + +TEST_F(CborShapeSerializerTest, NestingAtMaxDepthSucceeds) { + CborShapeSerializer s; + auto root = Schema::StructureBuilder("Root").Build(); + WriteNestedStructs(s, *root, MAX_NESTING_DEPTH); + EXPECT_TRUE(s.GetPayload().IsSuccess()); +} + +TEST_F(CborShapeSerializerTest, NestingPastMaxDepthFails) { + CborShapeSerializer s; + auto root = Schema::StructureBuilder("Root").Build(); + WriteNestedStructs(s, *root, MAX_NESTING_DEPTH + 1); + auto outcome = s.GetPayload(); + ASSERT_FALSE(outcome.IsSuccess()); + EXPECT_NE(outcome.GetError().GetMessage().find("depth"), Aws::String::npos); +} diff --git a/tests/aws-cpp-sdk-core-tests/smithy/client/schema/ClientProtocolTest.cpp b/tests/aws-cpp-sdk-core-tests/smithy/client/schema/ClientProtocolTest.cpp index 74be4ecf74aa..e1741a1fceba 100644 --- a/tests/aws-cpp-sdk-core-tests/smithy/client/schema/ClientProtocolTest.cpp +++ b/tests/aws-cpp-sdk-core-tests/smithy/client/schema/ClientProtocolTest.cpp @@ -185,7 +185,10 @@ TEST_F(ClientProtocolTest, AwsQueryDeserializeResponsePeelsResultWrapper) { "req-1"; CaptureResult result(CallerIdentityResultSchema()); OperationRequestContext ctx{"GetCallerIdentity", "2011-06-15"}; - AwsQueryProtocol().DeserializeResponse(ctx, Aws::Crt::ByteCursorFromArray(reinterpret_cast(xml.data()), xml.size()), result); + ASSERT_FALSE( + AwsQueryProtocol() + .DeserializeResponse(ctx, Aws::Crt::ByteCursorFromArray(reinterpret_cast(xml.data()), xml.size()), result) + .has_value()); EXPECT_EQ(result.arn, "arn:aws:iam::123456789012:user/x"); EXPECT_EQ(result.userId, "AIDAEXAMPLE"); EXPECT_EQ(result.account, "123456789012"); @@ -217,3 +220,17 @@ TEST_F(ClientProtocolTest, QueryResponseParsedAsXml) { ASSERT_TRUE(got.has_value()); EXPECT_EQ(got.value(), "ok"); } + +TEST_F(ClientProtocolTest, DeserializeResponseReportsMalformedBody) { + CaptureResult result(CallerIdentityResultSchema()); + OperationRequestContext ctx{"GetCallerIdentity", "2011-06-15"}; + const Aws::String json = R"({"Arn":"unterminated)"; + EXPECT_TRUE( + RestJsonProtocol() + .DeserializeResponse(ctx, Aws::Crt::ByteCursorFromArray(reinterpret_cast(json.data()), json.size()), result) + .has_value()); + const Aws::String xml = ""; + EXPECT_TRUE(AwsQueryProtocol() + .DeserializeResponse(ctx, Aws::Crt::ByteCursorFromArray(reinterpret_cast(xml.data()), xml.size()), result) + .has_value()); +} diff --git a/tests/aws-cpp-sdk-core-tests/smithy/client/schema/CodecTest.cpp b/tests/aws-cpp-sdk-core-tests/smithy/client/schema/CodecTest.cpp index aac69ec6e0de..5a94133ed165 100644 --- a/tests/aws-cpp-sdk-core-tests/smithy/client/schema/CodecTest.cpp +++ b/tests/aws-cpp-sdk-core-tests/smithy/client/schema/CodecTest.cpp @@ -137,6 +137,15 @@ TEST_F(CodecTest, JsonDeserializeShapeReturnsTypedObject) { EXPECT_EQ(p.count, 7); } +TEST_F(CodecTest, JsonDeserializeShapeReportsNonObjectRoot) { + const Aws::String payload = "hello"; + JsonCodec codec; + Person p; + const auto error = + codec.DeserializeShape(Aws::Crt::ByteCursorFromArray(reinterpret_cast(payload.data()), payload.size()), p); + EXPECT_TRUE(error.has_value()); +} + TEST_F(CodecTest, CborDeserializeShapeRoundTrip) { Person source; source.name = "Bob"; @@ -456,15 +465,81 @@ TEST_F(CodecTest, JsonDocumentInt64BoundariesStayExactOnLongRung) { } TEST_F(CodecTest, JsonDocumentAtReaderDepthLimitStillSerializes) { - // The reader's cap must stay under the serializer's: a document nested inside a struct spends an - // extra writer level on the wrapper, so an equal pair rejects on write what it just accepted on read. - for (int depth : {254, 255}) { + // The wrapper struct is one container, so the document may nest MAX_NESTING_DEPTH - 1 more. + for (int depth : {MAX_NESTING_DEPTH - 2, MAX_NESTING_DEPTH - 1}) { Aws::String body = "{\"payload\":" + Aws::String(depth, '[') + "1" + Aws::String(depth, ']') + "}"; JsonCodec codec; DocHolder in; - codec.DeserializeShape(Aws::Crt::ByteCursorFromArray(reinterpret_cast(body.c_str()), body.size()), in); + ASSERT_FALSE( + codec.DeserializeShape(Aws::Crt::ByteCursorFromArray(reinterpret_cast(body.c_str()), body.size()), in).has_value()) + << "parse failed at depth " << depth; ASSERT_TRUE(in.HasPayload()) << "parse failed at depth " << depth; auto reserialized = codec.Serialize(in.GetSchema(), in); EXPECT_TRUE(reserialized.IsSuccess()) << "write failed at depth " << depth; } } + +TEST_F(CodecTest, JsonDocumentPastReaderDepthLimitFails) { + const int depth = MAX_NESTING_DEPTH; + Aws::String body = "{\"payload\":" + Aws::String(depth, '[') + "1" + Aws::String(depth, ']') + "}"; + DocHolder in; + auto error = JsonCodec().DeserializeShape(Aws::Crt::ByteCursorFromArray(reinterpret_cast(body.c_str()), body.size()), in); + ASSERT_TRUE(error.has_value()); + EXPECT_NE(Aws::String(error->GetMessage()).find("depth"), Aws::String::npos); +} + +namespace { +Aws::Crt::ByteCursor CursorOf(const Aws::String& s) { + return Aws::Crt::ByteCursorFromArray(reinterpret_cast(s.data()), s.size()); +} +} // namespace + +TEST_F(CodecTest, DeserializeShapeSucceedsOnWellFormedInput) { + Person p; + EXPECT_FALSE(JsonCodec().DeserializeShape(CursorOf("{\"name\":\"Alice\",\"count\":7}"), p).has_value()); +} + +TEST_F(CodecTest, DeserializeShapeTreatsEmptyPayloadAsEmptyStruct) { + const Aws::String empty; + Person json; + Person cbor; + Person xml; + EXPECT_FALSE(JsonCodec().DeserializeShape(CursorOf(empty), json).has_value()); + EXPECT_FALSE(CborCodec().DeserializeShape(CursorOf(empty), cbor).has_value()); + EXPECT_FALSE(XmlCodec().DeserializeShape(CursorOf(empty), xml).has_value()); +} + +TEST_F(CodecTest, JsonDeserializeShapeReportsUnterminatedString) { + Person p; + auto error = JsonCodec().DeserializeShape(CursorOf("{\"name\":\"Ali"), p); + ASSERT_TRUE(error.has_value()); + EXPECT_EQ(error->GetExceptionName(), "SerializationException"); + EXPECT_NE(error->GetMessage().find("unterminated"), Aws::String::npos); +} + +TEST_F(CodecTest, JsonDeserializeShapeTypeMismatchIsNotAnError) { + Person p; + EXPECT_FALSE(JsonCodec().DeserializeShape(CursorOf("{\"name\":5,\"count\":7}"), p).has_value()); + EXPECT_EQ(p.count, 7); +} + +TEST_F(CodecTest, CborDeserializeShapeReportsTruncatedInput) { + Person source; + source.name = "Bob"; + source.count = 9; + CborCodec codec; + Aws::String encoded = codec.Serialize(source.GetSchema(), source).GetResult(); + encoded.resize(encoded.size() - 2); + Person p; + EXPECT_TRUE(codec.DeserializeShape(CursorOf(encoded), p).has_value()); +} + +TEST_F(CodecTest, XmlDeserializeShapeReportsNonXmlInput) { + Person p; + EXPECT_TRUE(XmlCodec().DeserializeShape(CursorOf("not xml at all"), p).has_value()); +} + +TEST_F(CodecTest, XmlDeserializeShapeReportsUnclosedRoot) { + Person p; + EXPECT_TRUE(XmlCodec().DeserializeShape(CursorOf("Bob"), p).has_value()); +} diff --git a/tests/aws-cpp-sdk-core-tests/smithy/client/schema/DocumentTest.cpp b/tests/aws-cpp-sdk-core-tests/smithy/client/schema/DocumentTest.cpp index d64eac43becb..df75943e43e2 100644 --- a/tests/aws-cpp-sdk-core-tests/smithy/client/schema/DocumentTest.cpp +++ b/tests/aws-cpp-sdk-core-tests/smithy/client/schema/DocumentTest.cpp @@ -14,6 +14,7 @@ #include #include +#include using namespace smithy::schema; @@ -289,6 +290,32 @@ TEST_F(SchemaDocumentTest, ContentEqualDocumentsCoerceDifferentlyByCodec) { EXPECT_EQ(jsonNumber->AsTimestamp()->Seconds(), 1234567890); } +TEST_F(SchemaDocumentTest, JsonEpochNumberMatchesLegacyDateTimeDouble) { + auto schema = Schema::CreateDocument("smithy.api#Document"); + // DateTime(double) truncates, so 4307758798.296 (stored just below) reads back as ...295. + const Aws::Vector> cases = { + {"-4307758798.296", -4307758798295}, {"4307758798.296", 4307758798295}, {"1.296", 1296}, {"-1.296", -1296}, {"-0.001", -1}}; + for (const auto& c : cases) { + JsonShapeDeserializer d(Aws::Crt::ByteCursorFromArray(reinterpret_cast(c.first.data()), c.first.size())); + auto doc = d.ReadDocument(*schema); + ASSERT_NE(doc, nullptr) << c.first; + auto ts = doc->AsTimestamp(); + ASSERT_TRUE(ts.has_value()) << c.first; + EXPECT_EQ(ts->Millis(), c.second) << c.first; + } +} + +TEST_F(SchemaDocumentTest, JsonEpochNumberBeyondClockRangeIsAbsent) { + auto schema = Schema::CreateDocument("smithy.api#Document"); + for (const char* wire : {"1e300", "-1e300", "9223372036854775807"}) { + const Aws::String json = wire; + JsonShapeDeserializer d(Aws::Crt::ByteCursorFromArray(reinterpret_cast(json.data()), json.size())); + auto doc = d.ReadDocument(*schema); + ASSERT_NE(doc, nullptr) << wire; + EXPECT_FALSE(doc->AsTimestamp().has_value()) << wire; + } +} + TEST_F(SchemaDocumentTest, EqualityIsSymmetricAcrossImplementations) { const Aws::Utils::ByteBuffer hi(reinterpret_cast("hi"), 2); const Aws::Utils::ByteBuffer ho(reinterpret_cast("ho"), 2); diff --git a/tests/aws-cpp-sdk-core-tests/smithy/client/schema/JsonShapeDeserializerTest.cpp b/tests/aws-cpp-sdk-core-tests/smithy/client/schema/JsonShapeDeserializerTest.cpp index 7097cc3b7268..508d64417a32 100644 --- a/tests/aws-cpp-sdk-core-tests/smithy/client/schema/JsonShapeDeserializerTest.cpp +++ b/tests/aws-cpp-sdk-core-tests/smithy/client/schema/JsonShapeDeserializerTest.cpp @@ -3,6 +3,7 @@ * SPDX-License-Identifier: Apache-2.0. */ #include +#include #include #include #include @@ -17,10 +18,12 @@ #include #include +#include #include #include #include #include +#include #include "SchemaSerializerTestHelpers.h" @@ -30,6 +33,14 @@ class JsonShapeDeserializerTest : public Aws::Testing::AwsCppSdkGTestSuite {}; namespace { +Aws::String EpochText(int64_t millis) { + const bool negative = millis < 0; + const int64_t abs = negative ? -millis : millis; + Aws::String frac = Aws::Utils::StringUtils::to_string(abs % 1000); + frac.insert(0, 3 - frac.size(), '0'); + return Aws::String(negative ? "-" : "") + Aws::Utils::StringUtils::to_string(abs / 1000) + "." + frac; +} + Aws::String Encode(const std::shared_ptr& root, const std::function& writeMembers) { JsonShapeSerializer s; LambdaStruct rootStruct(*root, writeMembers); @@ -162,6 +173,30 @@ TEST_F(JsonShapeDeserializerTest, TimestampEpochSeconds) { EXPECT_EQ(got.value().Seconds(), 1234567890); } +TEST_F(JsonShapeDeserializerTest, EpochTimestampMatchesLegacyDateTimeDouble) { + auto root = Schema::StructureBuilder("Root").PutMember("ts", Schema::CreateTimestamp("T")).Build(); + auto member = root->GetMember("ts").value(); + for (const int64_t millis : {int64_t(-4307758798296), int64_t(4307758798296), int64_t(1296), int64_t(-1296), int64_t(1700000000123)}) { + auto payload = Encode(root, [&](ShapeSerializer& ser) { ser.WriteTimestamp(*member, Aws::Utils::DateTime(millis)); }); + JsonShapeDeserializer d(Aws::Crt::ByteCursorFromArray(reinterpret_cast(payload.data()), payload.size())); + Aws::Crt::Optional got; + d.ReadStruct(*root, [&](const Schema& m, ShapeDeserializer& de) { got = de.ReadTimestamp(m); }); + ASSERT_TRUE(got.has_value()) << payload; + EXPECT_EQ(got.value().Millis(), LegacyEpochMillis(EpochSecondsText(Aws::Utils::DateTime(millis)))) << payload; + } +} + +TEST_F(JsonShapeDeserializerTest, EpochTimestampSweepMatchesLegacyDateTimeDouble) { + auto schema = Schema::CreateTimestamp("T"); + for (int64_t millis = -3000; millis <= 3000; ++millis) { + const Aws::String wire = EpochText(millis); + JsonShapeDeserializer d(Aws::Crt::ByteCursorFromArray(reinterpret_cast(wire.data()), wire.size())); + auto got = d.ReadTimestamp(*schema); + ASSERT_TRUE(got.has_value()) << wire; + EXPECT_EQ(got.value().Millis(), LegacyEpochMillis(wire)) << wire; + } +} + TEST_F(JsonShapeDeserializerTest, TimestampDateTimeString) { auto root = Schema::StructureBuilder("Root").PutMember("ts", Schema::CreateTimestamp("T")).Build(); const Aws::String payload = "{\"ts\":\"2009-02-13T23:31:30Z\"}"; @@ -310,6 +345,55 @@ TEST_F(JsonShapeDeserializerTest, SkipsUnknownField) { EXPECT_EQ(got["known"], 5); } +namespace { + +std::pair ReadName(const Aws::String& json) { + auto root = Schema::StructureBuilder("Root").PutMember("name", Schema::CreateString("S")).Build(); + JsonShapeDeserializer d(Aws::Crt::ByteCursorFromArray(reinterpret_cast(json.data()), json.size())); + Aws::String name; + d.ReadStruct(*root, [&](const Schema& m, ShapeDeserializer& de) { + auto v = de.ReadString(m); + if (v.has_value()) { + name = v.value(); + } + }); + return {name, d.GetError().has_value()}; +} + +} // namespace + +TEST_F(JsonShapeDeserializerTest, SkippedValidValuesAreConsumed) { + for (const Aws::String json : {R"({"unknown":{"a":[1,2.5e3,-0,true,false,null,"x\"y"],"b":{}},"name":"bob"})", + R"({"unknown":1e400,"name":"bob"})", R"({"unknown":[],"name":"bob"})", R"({"unknown" : "\ud83d\ude00" ,"name":"bob"})"}) { + const auto result = ReadName(json); + EXPECT_FALSE(result.second) << json; + EXPECT_EQ(result.first, "bob") << json; + } +} + +TEST_F(JsonShapeDeserializerTest, SkippedMalformedValueRecordsError) { + for (const Aws::String json : {R"({"unknown":[1 2],"name":"bob"})", R"({"unknown":{"a" 1},"name":"bob"})", + R"({"unknown":[1},"name":"bob"})", R"({"unknown":[1,],"name":"bob"})", + R"({"unknown":{"a":1,},"name":"bob"})", R"({"unknown":{1:2},"name":"bob"})", + R"({"unknown":tru,"name":"bob"})", R"({"unknown":1e,"name":"bob"})", R"({"unknown":-,"name":"bob"})", + R"({"unknown":[[[)", R"({"name":[1 2]})", R"({"unknown":"\q","name":"bob"})"}) { + EXPECT_TRUE(ReadName(json).second) << json; + } +} + +TEST_F(JsonShapeDeserializerTest, SurrogatePairDecodes) { + const auto result = ReadName(R"({"name":"\ud83d\ude00"})"); + EXPECT_FALSE(result.second); + EXPECT_EQ(result.first, "\xF0\x9F\x98\x80"); +} + +TEST_F(JsonShapeDeserializerTest, InvalidEscapeRecordsError) { + for (const Aws::String json : {R"({"name":"\ud800"})", R"({"name":"\ude00"})", R"({"name":"\ud800x"})", R"({"name":"\ud800\u0041"})", + R"({"name":"\u12G4"})", R"({"name":"\u12"})", R"({"name":"\q"})"}) { + EXPECT_TRUE(ReadName(json).second) << json; + } +} + TEST_F(JsonShapeDeserializerTest, EmptyOptionalOnTypeMismatch) { auto root = Schema::StructureBuilder("Root").PutMember("val", Schema::CreateString("S")).Build(); auto member = root->GetMember("val").value(); @@ -551,23 +635,24 @@ TEST_F(JsonShapeDeserializerTest, ReadDocumentIntegerVsDouble) { } TEST_F(JsonShapeDeserializerTest, ReadDocumentAcceptsNestingAtDepthLimit) { - // The cap admits MAX_DOCUMENT_DEPTH - 1 containers. - Aws::String json(255, '['); + Aws::String json(MAX_NESTING_DEPTH, '['); json += "1"; - json.append(255, ']'); + json.append(MAX_NESTING_DEPTH, ']'); JsonShapeDeserializer deser(Aws::Crt::ByteCursorFromArray(reinterpret_cast(json.c_str()), json.size())); auto schema = Schema::CreateDocument("smithy.api#Document"); EXPECT_NE(deser.ReadDocument(*schema), nullptr); + EXPECT_FALSE(deser.GetError().has_value()); } TEST_F(JsonShapeDeserializerTest, ReadDocumentRejectsExcessiveNesting) { - // Locks the MAX_DOCUMENT_DEPTH guard: nesting past the cap yields nullptr, not a crash. - Aws::String json(256, '['); + Aws::String json(MAX_NESTING_DEPTH + 1, '['); json += "1"; - json.append(256, ']'); + json.append(MAX_NESTING_DEPTH + 1, ']'); JsonShapeDeserializer deser(Aws::Crt::ByteCursorFromArray(reinterpret_cast(json.c_str()), json.size())); auto schema = Schema::CreateDocument("smithy.api#Document"); EXPECT_EQ(deser.ReadDocument(*schema), nullptr); + ASSERT_TRUE(deser.GetError().has_value()); + EXPECT_NE(deser.GetError()->GetMessage().find("depth"), Aws::String::npos); } TEST_F(JsonShapeDeserializerTest, ReadDocumentRequiresColonAfterKey) { @@ -582,22 +667,18 @@ TEST_F(JsonShapeDeserializerTest, ReadDocumentRequiresColonAfterKey) { const Aws::String missing = "{\"a\" 1}"; JsonShapeDeserializer bad(Aws::Crt::ByteCursorFromArray(reinterpret_cast(missing.data()), missing.size())); EXPECT_EQ(bad.ReadDocument(*schema), nullptr); + ASSERT_TRUE(bad.GetError().has_value()); + EXPECT_NE(bad.GetError()->GetMessage().find("expected ':'"), Aws::String::npos); } -TEST_F(JsonShapeDeserializerTest, ReadDocumentFailureConsumesValueSoTrailingMemberSurvives) { +TEST_F(JsonShapeDeserializerTest, ReadDocumentMalformedRecordsError) { auto root = Schema::StructureBuilder("Root") .PutMember("doc", Schema::CreateDocument("smithy.api#Document")) .PutMember("name", Schema::CreateString("S")) .Build(); - // First three fail outright; the rest would otherwise return a truncated or malformed-but-present document. const Aws::Vector payloads = { - "{\"doc\":" + Aws::String(300, '[') + "1" + Aws::String(300, ']') + ",\"name\":\"bob\"}", - "{\"doc\":[1.2.3],\"name\":\"bob\"}", - "{\"doc\":{1:2},\"name\":\"bob\"}", - "{\"doc\":[1,2 3],\"name\":\"bob\"}", - "{\"doc\":{\"a\":1 \"b\":2},\"name\":\"bob\"}", - "{\"doc\":[1:2],\"name\":\"bob\"}", - "{\"doc\":{\"a\" 1},\"name\":\"bob\"}", + "{\"doc\":[1.2.3],\"name\":\"bob\"}", "{\"doc\":{1:2},\"name\":\"bob\"}", "{\"doc\":[1,2 3],\"name\":\"bob\"}", + "{\"doc\":{\"a\":1 \"b\":2},\"name\":\"bob\"}", "{\"doc\":[1:2],\"name\":\"bob\"}", "{\"doc\":{\"a\" 1},\"name\":\"bob\"}", }; for (const auto& json : payloads) { JsonShapeDeserializer d(Aws::Crt::ByteCursorFromArray(reinterpret_cast(json.data()), json.size())); @@ -614,32 +695,78 @@ TEST_F(JsonShapeDeserializerTest, ReadDocumentFailureConsumesValueSoTrailingMemb } }); EXPECT_FALSE(docPresent) << json; - EXPECT_EQ(name, "bob") << json; + EXPECT_TRUE(name.empty()) << json; + EXPECT_TRUE(d.GetError().has_value()) << json; } } -TEST_F(JsonShapeDeserializerTest, ReadLongRejectsNonFiniteAndOutOfRange) { +TEST_F(JsonShapeDeserializerTest, ReadDocumentUnrecognisedLiteralRecordsError) { + auto root = Schema::StructureBuilder("Root") + .PutMember("doc", Schema::CreateDocument("smithy.api#Document")) + .PutMember("name", Schema::CreateString("S")) + .Build(); + for (const Aws::String json : {"{\"doc\":nul,\"name\":\"bob\"}", "{\"doc\":[1,tru],\"name\":\"bob\"}"}) { + JsonShapeDeserializer d(Aws::Crt::ByteCursorFromArray(reinterpret_cast(json.data()), json.size())); + bool docPresent = true; + Aws::String name; + d.ReadStruct(*root, [&](const Schema& m, ShapeDeserializer& de) { + if (m.GetMemberName() == "doc") { + docPresent = de.ReadDocument(m) != nullptr; + } else { + auto v = de.ReadString(m); + if (v.has_value()) { + name = v.value(); + } + } + }); + EXPECT_FALSE(docPresent) << json; + EXPECT_TRUE(name.empty()) << json; + EXPECT_TRUE(d.GetError().has_value()) << json; + } +} + +TEST_F(JsonShapeDeserializerTest, ReadLongNonIntegralOrOutOfRangeRecordsError) { auto root = Schema::StructureBuilder("Root").PutMember("n", Schema::CreateLong("L")).Build(); - const Aws::Vector wires = {"{\"n\":1e30}", "{\"n\":1e400}"}; + const Aws::Vector wires = {"{\"n\":1e30}", "{\"n\":1e400}", "{\"n\":1.5}", "{\"n\":-2.5e0}"}; for (const auto& wire : wires) { JsonShapeDeserializer d( Aws::Crt::ByteCursorFromArray(reinterpret_cast(wire.data()), wire.size())); bool present = true; d.ReadStruct(*root, [&](const Schema& m, ShapeDeserializer& de) { present = de.ReadLong(m).has_value(); }); EXPECT_FALSE(present) << wire; + ASSERT_TRUE(d.GetError().has_value()) << wire; + EXPECT_NE(d.GetError()->GetMessage().find("out of range"), Aws::String::npos) << wire; } } -TEST_F(JsonShapeDeserializerTest, ReadLongClampsPlainDigitOverflow) { +TEST_F(JsonShapeDeserializerTest, ReadLongIntegralExponentFormIsExact) { + auto schema = Schema::CreateLong("L"); + const Aws::String json = "1.5e3"; + JsonShapeDeserializer d(Aws::Crt::ByteCursorFromArray(reinterpret_cast(json.data()), json.size())); + EXPECT_EQ(d.ReadLong(*schema).value(), 1500); + EXPECT_FALSE(d.GetError().has_value()); +} + +TEST_F(JsonShapeDeserializerTest, ReadLongPlainDigitOverflowRecordsError) { auto root = Schema::StructureBuilder("Root").PutMember("n", Schema::CreateLong("L")).Build(); - const Aws::String wire = "{\"n\":99999999999999999999}"; // > INT64_MAX, no exponent - JsonShapeDeserializer d( - Aws::Crt::ByteCursorFromArray(reinterpret_cast(wire.data()), wire.size())); - Aws::Crt::Optional got; - d.ReadStruct(*root, [&](const Schema& m, ShapeDeserializer& de) { got = de.ReadLong(m); }); - // A plain-integer overflow clamps to INT64_MAX rather than dropping. - ASSERT_TRUE(got.has_value()); - EXPECT_EQ(got.value(), 9223372036854775807LL); + for (const Aws::String wire : {"{\"n\":99999999999999999999}", "{\"n\":9223372036854775808}", "{\"n\":-9223372036854775809}"}) { + JsonShapeDeserializer d(Aws::Crt::ByteCursorFromArray(reinterpret_cast(wire.data()), wire.size())); + Aws::Crt::Optional got; + d.ReadStruct(*root, [&](const Schema& m, ShapeDeserializer& de) { got = de.ReadLong(m); }); + EXPECT_FALSE(got.has_value()) << wire; + ASSERT_TRUE(d.GetError().has_value()) << wire; + EXPECT_NE(d.GetError()->GetMessage().find("out of range"), Aws::String::npos) << wire; + } + const Aws::Vector> bounds = {{"{\"n\":9223372036854775807}", (std::numeric_limits::max)()}, + {"{\"n\":-9223372036854775808}", (std::numeric_limits::min)()}}; + for (const auto& bound : bounds) { + JsonShapeDeserializer d(Aws::Crt::ByteCursorFromArray(reinterpret_cast(bound.first.data()), bound.first.size())); + Aws::Crt::Optional got; + d.ReadStruct(*root, [&](const Schema& m, ShapeDeserializer& de) { got = de.ReadLong(m); }); + ASSERT_TRUE(got.has_value()) << bound.first; + EXPECT_EQ(got.value(), bound.second); + EXPECT_FALSE(d.GetError().has_value()) << bound.first; + } } TEST_F(JsonShapeDeserializerTest, ReadDocumentOverflowIntegerBecomesDouble) { @@ -652,6 +779,37 @@ TEST_F(JsonShapeDeserializerTest, ReadDocumentOverflowIntegerBecomesDouble) { EXPECT_EQ(doc->GetType(), ShapeType::Double); // magnitude preserved as double, not clamped to Long } +TEST_F(JsonShapeDeserializerTest, ReadDocumentNonFiniteNumberRecordsError) { + auto root = Schema::StructureBuilder("Root") + .PutMember("doc", Schema::CreateDocument("smithy.api#Document")) + .PutMember("name", Schema::CreateString("S")) + .Build(); + const Aws::Vector payloads = { + "{\"doc\":1E310,\"name\":\"bob\"}", + "{\"doc\":-1E310,\"name\":\"bob\"}", + "{\"doc\":[1,1E310],\"name\":\"bob\"}", + }; + for (const auto& json : payloads) { + JsonShapeDeserializer d(Aws::Crt::ByteCursorFromArray(reinterpret_cast(json.data()), json.size())); + bool docPresent = true; + Aws::String name; + d.ReadStruct(*root, [&](const Schema& m, ShapeDeserializer& de) { + if (m.GetMemberName() == "doc") { + docPresent = de.ReadDocument(m) != nullptr; + } else { + auto v = de.ReadString(m); + if (v.has_value()) { + name = v.value(); + } + } + }); + EXPECT_FALSE(docPresent) << json; + EXPECT_TRUE(name.empty()) << json; + ASSERT_TRUE(d.GetError().has_value()) << json; + EXPECT_NE(d.GetError()->GetMessage().find("out of range"), Aws::String::npos) << json; + } +} + TEST_F(JsonShapeDeserializerTest, DocumentJsonCoercesBase64AndIsoAndRejectsMalformed) { Aws::String json = "{\"b\":\"aGk=\",\"t\":\"2023-11-14T22:13:20Z\",\"bad\":\"!!!not-base64!!!\"}"; JsonShapeDeserializer d( @@ -747,3 +905,362 @@ TEST_F(JsonShapeDeserializerTest, DocumentNumberTimestampGatedByConfiguredFormat EXPECT_FALSE(doc->AsTimestamp().has_value()); } } + +TEST_F(JsonShapeDeserializerTest, MissingSeparatorSetsErrorAndStops) { + auto root = Schema::StructureBuilder("Root") + .PutMember("a", Schema::CreateInteger("I")) + .PutMember("b", Schema::CreateInteger("I")) + .Build(); + const Aws::String json = "{\"a\":1 \"b\":2}"; + JsonShapeDeserializer d(Aws::Crt::ByteCursorFromArray(reinterpret_cast(json.data()), json.size())); + int delivered = 0; + d.ReadStruct(*root, [&](const Schema& m, ShapeDeserializer& de) { + ++delivered; + de.ReadInteger(m); + }); + EXPECT_EQ(delivered, 1); + ASSERT_TRUE(d.GetError().has_value()); + EXPECT_NE(d.GetError()->GetMessage().find("expected ','"), Aws::String::npos); +} + +TEST_F(JsonShapeDeserializerTest, ReadsReturnEmptyAfterError) { + const Aws::String json = "{\"a\":1-2,\"b\":3}"; + JsonShapeDeserializer d(Aws::Crt::ByteCursorFromArray(reinterpret_cast(json.data()), json.size())); + auto root = Schema::StructureBuilder("Root").PutMember("a", Schema::CreateLong("A")).PutMember("b", Schema::CreateLong("B")).Build(); + Aws::Vector seen; + d.ReadStruct(*root, [&](const Schema& m, ShapeDeserializer& de) { + seen.push_back(m.GetMemberName()); + de.ReadLong(m); + }); + ASSERT_EQ(seen.size(), 1u); + EXPECT_EQ(seen[0], "a"); + EXPECT_TRUE(d.GetError().has_value()); +} + +TEST_F(JsonShapeDeserializerTest, MalformedNumberSetsError) { + const Aws::String json = "1-2"; + JsonShapeDeserializer d(Aws::Crt::ByteCursorFromArray(reinterpret_cast(json.data()), json.size())); + auto schema = Schema::CreateLong("L"); + EXPECT_FALSE(d.ReadLong(*schema).has_value()); + EXPECT_TRUE(d.GetError().has_value()); +} + +TEST_F(JsonShapeDeserializerTest, BadDocumentNumberRecordsError) { + const Aws::String json = "[1.2.3]"; + JsonShapeDeserializer d(Aws::Crt::ByteCursorFromArray(reinterpret_cast(json.data()), json.size())); + auto schema = Schema::CreateDocument("smithy.api#Document"); + EXPECT_EQ(d.ReadDocument(*schema), nullptr); + ASSERT_TRUE(d.GetError().has_value()); + EXPECT_NE(d.GetError()->GetMessage().find("invalid number"), Aws::String::npos); +} + +TEST_F(JsonShapeDeserializerTest, RecursiveStructAtMaxDepthSucceeds) { + RecursiveNodeSchema node; + const Aws::String json = NestedJsonNodes(MAX_NESTING_DEPTH); + JsonShapeDeserializer d(Aws::Crt::ByteCursorFromArray(reinterpret_cast(json.data()), json.size())); + EXPECT_EQ(DescendRecursiveNode(d, node.Get()), MAX_NESTING_DEPTH); + EXPECT_FALSE(d.GetError().has_value()); +} + +TEST_F(JsonShapeDeserializerTest, RecursiveStructPastMaxDepthSetsError) { + RecursiveNodeSchema node; + const Aws::String json = NestedJsonNodes(MAX_NESTING_DEPTH + 1); + JsonShapeDeserializer d(Aws::Crt::ByteCursorFromArray(reinterpret_cast(json.data()), json.size())); + DescendRecursiveNode(d, node.Get()); + ASSERT_TRUE(d.GetError().has_value()); + EXPECT_NE(d.GetError()->GetMessage().find("depth"), Aws::String::npos); +} + +TEST_F(JsonShapeDeserializerTest, NestedListsPastMaxDepthSetErrorWithoutRecursingFurther) { + auto listBuilder = Schema::ListBuilder("L"); + listBuilder.PutMember("member", listBuilder); + auto list = listBuilder.Build(); + const Aws::String json = Aws::String(MAX_NESTING_DEPTH * 4, '[') + Aws::String(MAX_NESTING_DEPTH * 4, ']'); + JsonShapeDeserializer d(Aws::Crt::ByteCursorFromArray(reinterpret_cast(json.data()), json.size())); + size_t calls = 0; + std::function descend = [&](ShapeDeserializer& de) { + ++calls; + de.ReadList(*list, descend); + }; + descend(d); + EXPECT_TRUE(d.GetError().has_value()); + EXPECT_LE(calls, static_cast(MAX_NESTING_DEPTH) + 1); +} + +namespace { +template +void WithJson(const Aws::String& json, const Fn& fn) { + JsonShapeDeserializer d(Aws::Crt::ByteCursorFromArray(reinterpret_cast(json.data()), json.size())); + fn(d); +} + +void ExpectOutOfRange(const ShapeDeserializer& d, const Aws::String& wire) { + ASSERT_TRUE(d.GetError().has_value()) << wire; + EXPECT_NE(d.GetError()->GetMessage().find("out of range"), Aws::String::npos) << wire; +} +} // namespace + +TEST_F(JsonShapeDeserializerTest, IntegerBeyondInt32RecordsError) { + auto schema = Schema::CreateInteger("I"); + for (const char* wire : {"3000000000", "-2147483649"}) { + WithJson(wire, [&](JsonShapeDeserializer& d) { + EXPECT_FALSE(d.ReadInteger(*schema).has_value()) << wire; + ExpectOutOfRange(d, wire); + }); + } + WithJson("-2147483648", [&](JsonShapeDeserializer& d) { EXPECT_EQ(d.ReadInteger(*schema).value(), (std::numeric_limits::min)()); }); +} + +TEST_F(JsonShapeDeserializerTest, FloatBeyondFloatRangeRecordsError) { + auto schema = Schema::CreateFloat("F"); + WithJson("1e300", [&](JsonShapeDeserializer& d) { + EXPECT_FALSE(d.ReadFloat(*schema).has_value()); + ExpectOutOfRange(d, "1e300"); + }); + WithJson("\"Infinity\"", [&](JsonShapeDeserializer& d) { EXPECT_TRUE(std::isinf(d.ReadFloat(*schema).value())); }); +} + +TEST_F(JsonShapeDeserializerTest, FloatMaxRoundTrips) { + auto root = Schema::StructureBuilder("Root").PutMember("f", Schema::CreateFloat("F")).Build(); + auto member = root->GetMember("f").value(); + const float max = (std::numeric_limits::max)(); + for (const float value : {max, -max}) { + auto payload = Encode(root, [&](ShapeSerializer& ser) { ser.WriteFloat(*member, value); }); + JsonShapeDeserializer d(Aws::Crt::ByteCursorFromArray(reinterpret_cast(payload.data()), payload.size())); + Aws::Crt::Optional got; + d.ReadStruct(*root, [&](const Schema& m, ShapeDeserializer& de) { got = de.ReadFloat(m); }); + ASSERT_TRUE(got.has_value()) << payload; + EXPECT_EQ(got.value(), value); + } +} + +TEST_F(JsonShapeDeserializerTest, FloatNarrowingBoundaries) { + auto schema = Schema::CreateFloat("F"); + const auto readFloat = [&](double value) { + Aws::Crt::Optional got; + WithJson(Aws::Utils::StringUtils::FormatRoundTripDouble(value), [&](JsonShapeDeserializer& d) { got = d.ReadFloat(*schema); }); + return got; + }; + const double max = static_cast((std::numeric_limits::max)()); + const double halfUlp = std::ldexp(1.0, 103); + const float maxF = (std::numeric_limits::max)(); + + ASSERT_TRUE(readFloat(max).has_value()); + EXPECT_EQ(readFloat(max).value(), maxF); + ASSERT_TRUE(readFloat(-max).has_value()); + EXPECT_EQ(readFloat(-max).value(), -maxF); + + const double justAbove = std::nextafter(max, std::numeric_limits::infinity()); + ASSERT_TRUE(readFloat(justAbove).has_value()); + EXPECT_EQ(readFloat(justAbove).value(), maxF); + ASSERT_TRUE(readFloat(-justAbove).has_value()); + EXPECT_EQ(readFloat(-justAbove).value(), -maxF); + + const double justBelowHalf = std::nextafter(max + halfUlp, 0.0); + ASSERT_TRUE(readFloat(justBelowHalf).has_value()); + EXPECT_EQ(readFloat(justBelowHalf).value(), maxF); + + EXPECT_FALSE(readFloat(max + halfUlp).has_value()); + EXPECT_FALSE(readFloat(-(max + halfUlp)).has_value()); + EXPECT_FALSE(readFloat(1e300).has_value()); + + WithJson("\"Infinity\"", [&](JsonShapeDeserializer& d) { EXPECT_TRUE(std::isinf(d.ReadFloat(*schema).value())); }); + WithJson("\"-Infinity\"", [&](JsonShapeDeserializer& d) { EXPECT_TRUE(std::isinf(d.ReadFloat(*schema).value())); }); + WithJson("\"NaN\"", [&](JsonShapeDeserializer& d) { EXPECT_TRUE(std::isnan(d.ReadFloat(*schema).value())); }); +} + +TEST_F(JsonShapeDeserializerTest, UnrepresentableEpochTimestampRecordsError) { + auto schema = Schema::CreateTimestamp("T"); + for (const char* wire : {"1e300", "-1e300", "1e400"}) { + WithJson(wire, [&](JsonShapeDeserializer& d) { + EXPECT_FALSE(d.ReadTimestamp(*schema).has_value()) << wire; + ExpectOutOfRange(d, wire); + }); + } +} + +namespace { +void ExpectRootRecordsError(const Aws::String& json) { + auto root = Schema::StructureBuilder("Root").PutMember("a", Schema::CreateInteger("I")).Build(); + JsonShapeDeserializer d(Aws::Crt::ByteCursorFromArray(reinterpret_cast(json.data()), json.size())); + int delivered = 0; + d.ReadStruct(*root, [&](const Schema&, ShapeDeserializer&) { ++delivered; }); + EXPECT_EQ(delivered, 0) << json; + ASSERT_TRUE(d.GetError().has_value()) << json; + EXPECT_NE(d.GetError()->GetMessage().find("expected object"), Aws::String::npos) << json; +} +} // namespace + +TEST_F(JsonShapeDeserializerTest, NonObjectRootRecordsError) { + for (const char* wire : {"hello", "[1]", "\"x\"", "503", "5"}) { + ExpectRootRecordsError(wire); + } +} + +TEST_F(JsonShapeDeserializerTest, RootNullIsEmptyStructWithoutError) { + auto root = Schema::StructureBuilder("Root").PutMember("a", Schema::CreateInteger("I")).Build(); + for (const char* wire : {"null", " null ", "\nnull\t"}) { + const Aws::String json = wire; + JsonShapeDeserializer d(Aws::Crt::ByteCursorFromArray(reinterpret_cast(json.data()), json.size())); + int delivered = 0; + d.ReadStruct(*root, [&](const Schema&, ShapeDeserializer&) { ++delivered; }); + EXPECT_EQ(delivered, 0) << "[" << wire << "]"; + EXPECT_FALSE(d.GetError().has_value()) << "[" << wire << "]"; + } +} + +TEST_F(JsonShapeDeserializerTest, RootNullxRecordsError) { + auto root = Schema::StructureBuilder("Root").PutMember("a", Schema::CreateInteger("I")).Build(); + const Aws::String json = "nullx"; + JsonShapeDeserializer d(Aws::Crt::ByteCursorFromArray(reinterpret_cast(json.data()), json.size())); + int delivered = 0; + d.ReadStruct(*root, [&](const Schema&, ShapeDeserializer&) { ++delivered; }); + EXPECT_EQ(delivered, 0); + ASSERT_TRUE(d.GetError().has_value()); + EXPECT_NE(d.GetError()->GetMessage().find("trailing data"), Aws::String::npos); +} + +TEST_F(JsonShapeDeserializerTest, TrailingDataAfterRootRecordsError) { + auto root = Schema::StructureBuilder("Root").PutMember("a", Schema::CreateInteger("I")).Build(); + for (const char* wire : {"{\"a\":1}xyz", "{\"a\":1} {}", "{}]", "null null"}) { + const Aws::String json = wire; + JsonShapeDeserializer d(Aws::Crt::ByteCursorFromArray(reinterpret_cast(json.data()), json.size())); + d.ReadStruct(*root, [&](const Schema& m, ShapeDeserializer& de) { de.ReadInteger(m); }); + ASSERT_TRUE(d.GetError().has_value()) << wire; + EXPECT_NE(d.GetError()->GetMessage().find("trailing data"), Aws::String::npos) << wire; + } + const Aws::String json = "{\"a\":1} \n\t"; + JsonShapeDeserializer d(Aws::Crt::ByteCursorFromArray(reinterpret_cast(json.data()), json.size())); + d.ReadStruct(*root, [&](const Schema& m, ShapeDeserializer& de) { de.ReadInteger(m); }); + EXPECT_FALSE(d.GetError().has_value()); +} + +TEST_F(JsonShapeDeserializerTest, MissingColonAfterKeyRecordsError) { + auto mapBuilder = Schema::MapBuilder("M"); + auto root = Schema::StructureBuilder("Root").PutMember("name", Schema::CreateString("S")).PutMember("m", mapBuilder).Build(); + auto valueSchema = Schema::CreateString("V"); + for (const char* wire : {"{\"name\" \"Alice\"}", "{\"m\":{\"k\" \"v\"}}"}) { + const Aws::String json = wire; + JsonShapeDeserializer d(Aws::Crt::ByteCursorFromArray(reinterpret_cast(json.data()), json.size())); + d.ReadStruct(*root, [&](const Schema& m, ShapeDeserializer& de) { + if (m.GetType() == ShapeType::Map) { + de.ReadMap(m, [&](const Aws::String&, ShapeDeserializer& v) { v.ReadString(*valueSchema); }); + } else { + de.ReadString(m); + } + }); + ASSERT_TRUE(d.GetError().has_value()) << wire; + EXPECT_NE(d.GetError()->GetMessage().find("expected ':'"), Aws::String::npos) << wire; + } +} + +TEST_F(JsonShapeDeserializerTest, WhitespaceOnlyRootIsEmptyStructWithoutError) { + auto root = Schema::StructureBuilder("Root").PutMember("a", Schema::CreateInteger("I")).Build(); + for (const char* wire : {"", " ", " \n\t"}) { + const Aws::String json = wire; + JsonShapeDeserializer d(Aws::Crt::ByteCursorFromArray(reinterpret_cast(json.data()), json.size())); + int delivered = 0; + d.ReadStruct(*root, [&](const Schema&, ShapeDeserializer&) { ++delivered; }); + EXPECT_EQ(delivered, 0); + EXPECT_FALSE(d.GetError().has_value()) << "[" << wire << "]"; + } +} + +TEST_F(JsonShapeDeserializerTest, ScalarWhereNestedStructExpectedIsNotAnError) { + auto inner = Schema::StructureBuilder("Inner").PutMember("x", Schema::CreateInteger("I")).Build(); + auto root = Schema::StructureBuilder("Root").PutMember("inner", inner).PutMember("after", Schema::CreateInteger("I")).Build(); + const Aws::String json = "{\"inner\":5,\"after\":7}"; + JsonShapeDeserializer d(Aws::Crt::ByteCursorFromArray(reinterpret_cast(json.data()), json.size())); + Aws::Crt::Optional after; + int innerDelivered = 0; + d.ReadStruct(*root, [&](const Schema& m, ShapeDeserializer& de) { + if (m.GetMemberName() == "inner") { + de.ReadStruct(m, [&](const Schema&, ShapeDeserializer&) { ++innerDelivered; }); + } else { + after = de.ReadInteger(m); + } + }); + EXPECT_EQ(innerDelivered, 0); + EXPECT_FALSE(d.GetError().has_value()); + ASSERT_TRUE(after.has_value()); + EXPECT_EQ(after.value(), 7); +} + +TEST_F(JsonShapeDeserializerTest, OverflowingFiniteTextRecordsError) { + auto dbl = Schema::CreateDouble("D"); + auto flt = Schema::CreateFloat("F"); + for (const char* wire : {"1e400", "-1e400"}) { + WithJson(wire, [&](JsonShapeDeserializer& d) { + EXPECT_FALSE(d.ReadDouble(*dbl).has_value()) << wire; + ExpectOutOfRange(d, wire); + }); + WithJson(wire, [&](JsonShapeDeserializer& d) { + EXPECT_FALSE(d.ReadFloat(*flt).has_value()) << wire; + ExpectOutOfRange(d, wire); + }); + } + WithJson("\"Infinity\"", [&](JsonShapeDeserializer& d) { + const auto v = d.ReadDouble(*dbl); + ASSERT_TRUE(v.has_value()); + EXPECT_TRUE(std::isinf(v.value())); + }); +} + +namespace { + +std::shared_ptr UnionRoot() { + auto variants = Schema::UnionBuilder("U").PutMember("a", Schema::CreateString("S")).PutMember("b", Schema::CreateInteger("I")).Build(); + return Schema::StructureBuilder("Root").PutMember("u", variants).Build(); +} + +int ReadUnionVariants(JsonShapeDeserializer& d) { + const auto root = UnionRoot(); + int delivered = 0; + d.ReadStruct(*root, [&](const Schema& m, ShapeDeserializer& de) { + de.ReadStruct(**m.GetMemberTarget(), [&](const Schema& variant, ShapeDeserializer& vde) { + ++delivered; + if (variant.GetMemberName() == Aws::String("a")) { + vde.ReadString(variant); + } else { + vde.ReadInteger(variant); + } + }); + }); + return delivered; +} + +} // namespace + +TEST_F(JsonShapeDeserializerTest, LeadingByteOrderMarkIsSkipped) { + auto root = Schema::StructureBuilder("Root").PutMember("n", Schema::CreateInteger("I")).Build(); + const auto read = [&](const Aws::String& json, Aws::Crt::Optional& got) { + JsonShapeDeserializer d(Aws::Crt::ByteCursorFromArray(reinterpret_cast(json.data()), json.size())); + d.ReadStruct(*root, [&](const Schema& m, ShapeDeserializer& de) { got = de.ReadInteger(m); }); + return d.GetError(); + }; + Aws::Crt::Optional got; + EXPECT_FALSE(read("\xEF\xBB\xBF{\"n\":7}", got).has_value()); + ASSERT_TRUE(got.has_value()); + EXPECT_EQ(got.value(), 7); + Aws::Crt::Optional none; + EXPECT_FALSE(read("\xEF\xBB\xBF", none).has_value()); + EXPECT_FALSE(none.has_value()); + EXPECT_TRUE(read("{\"n\":7}\xEF\xBB\xBF", none).has_value()); + EXPECT_TRUE(read("\xEF\xBB{}", none).has_value()); +} + +TEST_F(JsonShapeDeserializerTest, UnionWithTwoMembersSetRecordsError) { + const Aws::String json = "{\"u\":{\"a\":\"x\",\"b\":1}}"; + JsonShapeDeserializer d(Aws::Crt::ByteCursorFromArray(reinterpret_cast(json.data()), json.size())); + EXPECT_EQ(ReadUnionVariants(d), 1); + ASSERT_TRUE(d.GetError().has_value()); + EXPECT_NE(d.GetError()->GetMessage().find("more than one member"), Aws::String::npos); +} + +TEST_F(JsonShapeDeserializerTest, UnionIgnoresNullAndUnmodeledMembers) { + for (const Aws::String json : {"{\"u\":{\"a\":\"x\",\"b\":null}}", "{\"u\":{\"__type\":\"T\",\"b\":1}}"}) { + JsonShapeDeserializer d(Aws::Crt::ByteCursorFromArray(reinterpret_cast(json.data()), json.size())); + EXPECT_EQ(ReadUnionVariants(d), 1) << json; + EXPECT_FALSE(d.GetError().has_value()) << json; + } +} diff --git a/tests/aws-cpp-sdk-core-tests/smithy/client/schema/JsonShapeSerializerTest.cpp b/tests/aws-cpp-sdk-core-tests/smithy/client/schema/JsonShapeSerializerTest.cpp index fdda488e22bc..d98811b6902e 100644 --- a/tests/aws-cpp-sdk-core-tests/smithy/client/schema/JsonShapeSerializerTest.cpp +++ b/tests/aws-cpp-sdk-core-tests/smithy/client/schema/JsonShapeSerializerTest.cpp @@ -15,6 +15,8 @@ #include #include +#include +#include #include "SchemaSerializerTestHelpers.h" @@ -464,23 +466,6 @@ TEST_F(JsonShapeSerializerTest, EscapesInMapKey) { EXPECT_NE(s.GetPayload().GetResult().find("\"key\\\"with\\\"quotes\":\"v\""), Aws::String::npos); } -TEST_F(JsonShapeSerializerTest, MaxDepthEnforcement) { - JsonShapeSerializer s; - auto root = Schema::StructureBuilder("Root").Build(); - auto nested = Schema::CreateMember("n", ShapeType::Structure); - std::function nest = [&](ShapeSerializer& ser, int remaining) { - if (remaining <= 0) { - return; - } - ser.WriteStruct(*nested, LambdaStruct(*nested, [&, remaining](ShapeSerializer& inner) { nest(inner, remaining - 1); })); - }; - LambdaStruct rootStruct(*root, [&](ShapeSerializer& ser) { nest(ser, 1000); }); - s.WriteStruct(*root, rootStruct); - auto outcome = s.GetPayload(); - ASSERT_FALSE(outcome.IsSuccess()); - EXPECT_NE(outcome.GetError().GetMessage().find("depth"), Aws::String::npos); -} - TEST_F(JsonShapeSerializerTest, JsonNameOverridesMemberName) { JsonShapeSerializer s; auto root = Schema::StructureBuilder("Root").Build(); @@ -572,6 +557,23 @@ TEST_F(JsonShapeSerializerTest, DocumentAsStructMember) { EXPECT_EQ(outcome.GetResult(), "{\"doc\":{\"a\":1}}"); } +TEST_F(JsonShapeSerializerTest, NonFiniteDocumentDoubleIsWrittenAsItsString) { + auto root = Schema::StructureBuilder("Root").Build(); + auto member = Schema::CreateMember("doc", ShapeType::Document); + const std::pair cases[] = {{std::numeric_limits::quiet_NaN(), "{\"doc\":[\"NaN\"]}"}, + {std::numeric_limits::infinity(), "{\"doc\":[\"Infinity\"]}"}, + {-std::numeric_limits::infinity(), "{\"doc\":[\"-Infinity\"]}"}}; + for (const auto& c : cases) { + JsonShapeSerializer s; + const auto doc = Document::FromList({Document::FromDouble(c.first)}); + LambdaStruct rootStruct(*root, [&](ShapeSerializer& ser) { ser.WriteDocument(*member, *doc); }); + s.WriteStruct(*root, rootStruct); + auto outcome = s.GetPayload(); + ASSERT_TRUE(outcome.IsSuccess()); + EXPECT_EQ(outcome.GetResult(), c.second); + } +} + TEST_F(JsonShapeSerializerTest, DocumentAsListElement) { JsonShapeSerializer s; auto member = Schema::CreateMember("doc", ShapeType::Document); @@ -583,3 +585,19 @@ TEST_F(JsonShapeSerializerTest, DocumentAsListElement) { ASSERT_TRUE(outcome.IsSuccess()); EXPECT_EQ(outcome.GetResult(), "[\"x\",true]"); } + +TEST_F(JsonShapeSerializerTest, NestingAtMaxDepthSucceeds) { + JsonShapeSerializer s; + auto root = Schema::StructureBuilder("Root").Build(); + WriteNestedStructs(s, *root, MAX_NESTING_DEPTH); + EXPECT_TRUE(s.GetPayload().IsSuccess()); +} + +TEST_F(JsonShapeSerializerTest, NestingPastMaxDepthFails) { + JsonShapeSerializer s; + auto root = Schema::StructureBuilder("Root").Build(); + WriteNestedStructs(s, *root, MAX_NESTING_DEPTH + 1); + auto outcome = s.GetPayload(); + ASSERT_FALSE(outcome.IsSuccess()); + EXPECT_NE(outcome.GetError().GetMessage().find("depth"), Aws::String::npos); +} diff --git a/tests/aws-cpp-sdk-core-tests/smithy/client/schema/QueryShapeSerializerTest.cpp b/tests/aws-cpp-sdk-core-tests/smithy/client/schema/QueryShapeSerializerTest.cpp index 07ecc28525c5..a4e9cfbf6392 100644 --- a/tests/aws-cpp-sdk-core-tests/smithy/client/schema/QueryShapeSerializerTest.cpp +++ b/tests/aws-cpp-sdk-core-tests/smithy/client/schema/QueryShapeSerializerTest.cpp @@ -89,6 +89,80 @@ TEST_F(QueryShapeSerializerTest, Double) { EXPECT_EQ(outcome.GetResult(), "ratio=6.5"); } +TEST_F(QueryShapeSerializerTest, LargeIntegralDoubleKeepsAllDigits) { + QueryShapeSerializer s; + auto root = Schema::StructureBuilder("Root").Build(); + auto member = Schema::CreateMember("big", ShapeType::Double); + LambdaStruct rootStruct(*root, [&](ShapeSerializer& ser) { ser.WriteDouble(*member, 2147483647.0); }); + s.WriteStruct(*root, rootStruct); + auto outcome = s.GetPayload(); + ASSERT_TRUE(outcome.IsSuccess()); + EXPECT_EQ(outcome.GetResult(), "big=2147483647"); +} + +TEST_F(QueryShapeSerializerTest, FloatKeepsRoundTripPrecision) { + QueryShapeSerializer s; + auto root = Schema::StructureBuilder("Root").Build(); + auto member = Schema::CreateMember("f", ShapeType::Float); + LambdaStruct rootStruct(*root, [&](ShapeSerializer& ser) { ser.WriteFloat(*member, 16777218.0f); }); + s.WriteStruct(*root, rootStruct); + auto outcome = s.GetPayload(); + ASSERT_TRUE(outcome.IsSuccess()); + EXPECT_EQ(outcome.GetResult(), "f=16777218"); +} + +TEST_F(QueryShapeSerializerTest, FloatIsFormattedAtFloatPrecision) { + QueryShapeSerializer s; + auto root = Schema::StructureBuilder("Root").Build(); + auto member = Schema::CreateMember("f", ShapeType::Float); + LambdaStruct rootStruct(*root, [&](ShapeSerializer& ser) { ser.WriteFloat(*member, 3.14f); }); + s.WriteStruct(*root, rootStruct); + auto outcome = s.GetPayload(); + ASSERT_TRUE(outcome.IsSuccess()); + EXPECT_EQ(outcome.GetResult(), "f=3.14"); +} + +namespace { +Aws::String SerializeFloating(QueryShapeSerializer::Flavor flavor, ShapeType type, double value) { + QueryShapeSerializer s(flavor); + auto root = Schema::StructureBuilder("Root").Build(); + auto member = Schema::CreateMember("v", type); + LambdaStruct rootStruct(*root, [&](ShapeSerializer& ser) { + if (type == ShapeType::Float) { + ser.WriteFloat(*member, static_cast(value)); + } else { + ser.WriteDouble(*member, value); + } + }); + s.WriteStruct(*root, rootStruct); + auto outcome = s.GetPayload(); + return outcome.IsSuccess() ? outcome.GetResult() : Aws::String(""); +} +} // namespace + +TEST_F(QueryShapeSerializerTest, DoubleExponentPlusIsUrlEncoded) { + const auto body = SerializeFloating(QueryShapeSerializer::Flavor::AwsQuery, ShapeType::Double, 1e21); + EXPECT_EQ(body.find('+'), Aws::String::npos) << body; + EXPECT_NE(body.find("%2B"), Aws::String::npos) << body; +} + +TEST_F(QueryShapeSerializerTest, FloatExponentPlusIsUrlEncoded) { + const auto body = SerializeFloating(QueryShapeSerializer::Flavor::AwsQuery, ShapeType::Float, + -static_cast((std::numeric_limits::max)())); + EXPECT_EQ(body.find('+'), Aws::String::npos) << body; + EXPECT_NE(body.find("%2B"), Aws::String::npos) << body; +} + +TEST_F(QueryShapeSerializerTest, Ec2FloatingPointPlusIsUrlEncoded) { + const auto body = SerializeFloating(QueryShapeSerializer::Flavor::Ec2Query, ShapeType::Double, 1e21); + EXPECT_EQ(body.find('+'), Aws::String::npos) << body; + EXPECT_NE(body.find("%2B"), Aws::String::npos) << body; +} + +TEST_F(QueryShapeSerializerTest, OrdinaryDoubleIsNotAltered) { + EXPECT_EQ(SerializeFloating(QueryShapeSerializer::Flavor::AwsQuery, ShapeType::Double, 3.14), "v=3.14"); +} + TEST_F(QueryShapeSerializerTest, NonFiniteFloats) { QueryShapeSerializer s; auto root = Schema::StructureBuilder("Root").Build(); @@ -150,6 +224,40 @@ TEST_F(QueryShapeSerializerTest, ListOfStrings) { EXPECT_EQ(outcome.GetResult(), "tags.member.1=a&tags.member.2=b&tags.member.3=c"); } +TEST_F(QueryShapeSerializerTest, EmptyListIsBareKey) { + for (const bool flattened : {false, true}) { + QueryShapeSerializer s; + auto root = Schema::StructureBuilder("Root").Build(); + auto listMember = flattened ? Schema::CreateMember("tags", ShapeType::List, + {{XmlFlattenedTrait::KEY(), Aws::MakeShared("Schema")}}) + : Schema::CreateMember("tags", ShapeType::List); + auto other = Schema::CreateMember("name", ShapeType::String); + LambdaStruct rootStruct(*root, [&](ShapeSerializer& ser) { + ser.WriteList(*listMember, 0, [](ShapeSerializer&) {}); + ser.WriteString(*other, "n"); + }); + s.WriteStruct(*root, rootStruct); + auto outcome = s.GetPayload(); + ASSERT_TRUE(outcome.IsSuccess()); + EXPECT_EQ(outcome.GetResult(), "tags=&name=n") << "flattened=" << flattened; + } +} + +TEST_F(QueryShapeSerializerTest, Ec2EmptyListIsOmitted) { + QueryShapeSerializer s(QueryShapeSerializer::Flavor::Ec2Query); + auto root = Schema::StructureBuilder("Root").Build(); + auto listMember = Schema::CreateMember("items", ShapeType::List); + auto other = Schema::CreateMember("name", ShapeType::String); + LambdaStruct rootStruct(*root, [&](ShapeSerializer& ser) { + ser.WriteList(*listMember, 0, [](ShapeSerializer&) {}); + ser.WriteString(*other, "n"); + }); + s.WriteStruct(*root, rootStruct); + auto outcome = s.GetPayload(); + ASSERT_TRUE(outcome.IsSuccess()); + EXPECT_EQ(outcome.GetResult(), "Name=n"); +} + TEST_F(QueryShapeSerializerTest, FlattenedList) { QueryShapeSerializer s; auto root = Schema::StructureBuilder("Root").Build(); @@ -220,21 +328,6 @@ TEST_F(QueryShapeSerializerTest, FlattenedMap) { EXPECT_EQ(outcome.GetResult(), "attr.1.key=k1&attr.1.value=v1"); } -TEST_F(QueryShapeSerializerTest, MaxDepthEnforcement) { - QueryShapeSerializer s; - auto root = Schema::StructureBuilder("Root").Build(); - auto nested = Schema::CreateMember("n", ShapeType::Structure); - std::function nest = [&](ShapeSerializer& ser, int remaining) { - if (remaining <= 0) { - return; - } - ser.WriteStruct(*nested, LambdaStruct(*nested, [&, remaining](ShapeSerializer& inner) { nest(inner, remaining - 1); })); - }; - LambdaStruct rootStruct(*root, [&](ShapeSerializer& ser) { nest(ser, 1000); }); - s.WriteStruct(*root, rootStruct); - EXPECT_FALSE(s.GetPayload().IsSuccess()); -} - TEST_F(QueryShapeSerializerTest, Ec2CapitalizesMemberName) { QueryShapeSerializer s(QueryShapeSerializer::Flavor::Ec2Query); auto root = Schema::StructureBuilder("Root").Build(); @@ -340,3 +433,19 @@ TEST_F(QueryShapeSerializerTest, DocumentIsUnsupported) { ASSERT_FALSE(outcome.IsSuccess()); EXPECT_EQ(outcome.GetError().GetExceptionName(), "SerializationException"); } + +TEST_F(QueryShapeSerializerTest, NestingAtMaxDepthSucceeds) { + QueryShapeSerializer s; + auto root = Schema::StructureBuilder("Root").Build(); + WriteNestedStructs(s, *root, MAX_NESTING_DEPTH); + EXPECT_TRUE(s.GetPayload().IsSuccess()); +} + +TEST_F(QueryShapeSerializerTest, NestingPastMaxDepthFails) { + QueryShapeSerializer s; + auto root = Schema::StructureBuilder("Root").Build(); + WriteNestedStructs(s, *root, MAX_NESTING_DEPTH + 1); + auto outcome = s.GetPayload(); + ASSERT_FALSE(outcome.IsSuccess()); + EXPECT_NE(outcome.GetError().GetMessage().find("depth"), Aws::String::npos); +} diff --git a/tests/aws-cpp-sdk-core-tests/smithy/client/schema/SchemaSerializerTestHelpers.h b/tests/aws-cpp-sdk-core-tests/smithy/client/schema/SchemaSerializerTestHelpers.h index 29774bc30440..9ae9f91d4ca6 100644 --- a/tests/aws-cpp-sdk-core-tests/smithy/client/schema/SchemaSerializerTestHelpers.h +++ b/tests/aws-cpp-sdk-core-tests/smithy/client/schema/SchemaSerializerTestHelpers.h @@ -4,12 +4,17 @@ */ #pragma once +#include +#include #include +#include #include #include #include +#include #include +#include #include namespace smithy { @@ -36,5 +41,74 @@ class LambdaStruct final : public SerializableStruct { std::function m_members; }; +class RecursiveNodeSchema { + public: + RecursiveNodeSchema() : m_builder(smithy::schema::Schema::StructureBuilder("Node")) { + m_builder.PutMember("child", m_builder); + m_schema = m_builder.Build(); + } + const smithy::schema::Schema& Get() const { return *m_schema; } + + private: + smithy::schema::SchemaBuilder m_builder; + std::shared_ptr m_schema; +}; + +inline int64_t LegacyEpochMillis(const Aws::String& text) { + return Aws::Utils::DateTime(Aws::Utils::StringUtils::ParseDouble(text).value()).Millis(); +} + +inline int DescendRecursiveNode(smithy::schema::ShapeDeserializer& de, const smithy::schema::Schema& schema) { + int deepest = 1; + de.ReadStruct(schema, [&](const smithy::schema::Schema& member, smithy::schema::ShapeDeserializer& d) { + deepest = (std::max)(deepest, DescendRecursiveNode(d, *member.GetMemberTarget().value()) + 1); + }); + return deepest; +} + +inline Aws::String NestedJsonNodes(int depth) { + Aws::String json; + for (int i = 1; i < depth; ++i) { + json += "{\"child\":"; + } + json += "{}"; + json.append(depth - 1, '}'); + return json; +} + +inline Aws::String NestedCborNodes(int depth) { + Aws::String cbor; + for (int i = 1; i < depth; ++i) { + cbor += '\xA1'; + cbor += + "\x65" + "child"; + } + cbor += '\xA0'; + return cbor; +} + +inline Aws::String NestedXmlNodes(int depth) { + Aws::String xml = ""; + for (int i = 1; i < depth; ++i) { + xml += ""; + } + for (int i = 1; i < depth; ++i) { + xml += ""; + } + xml += ""; + return xml; +} + +inline void WriteNestedStructs(smithy::schema::ShapeSerializer& ser, const smithy::schema::Schema& root, int depth) { + auto nested = smithy::schema::Schema::CreateMember("n", smithy::schema::ShapeType::Structure); + std::function nest = [&](smithy::schema::ShapeSerializer& s, int remaining) { + if (remaining > 0) { + s.WriteStruct(*nested, LambdaStruct(*nested, [&, remaining](smithy::schema::ShapeSerializer& inner) { nest(inner, remaining - 1); })); + } + }; + ser.WriteStruct(root, LambdaStruct(root, [&](smithy::schema::ShapeSerializer& s) { nest(s, depth - 1); })); +} + } // namespace schema } // namespace smithy diff --git a/tests/aws-cpp-sdk-core-tests/smithy/client/schema/SerdeTraitsTest.cpp b/tests/aws-cpp-sdk-core-tests/smithy/client/schema/SerdeTraitsTest.cpp new file mode 100644 index 000000000000..b625bf8c48fb --- /dev/null +++ b/tests/aws-cpp-sdk-core-tests/smithy/client/schema/SerdeTraitsTest.cpp @@ -0,0 +1,36 @@ +/** + * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. + * SPDX-License-Identifier: Apache-2.0. + */ +#include +#include + +#include +#include +#include + +using namespace smithy::schema; +using namespace Aws::Utils; + +class SerdeTraitsTest : public Aws::Testing::AwsCppSdkGTestSuite {}; + +TEST_F(SerdeTraitsTest, EpochSecondsToDateTimeMatchesLegacyDateTimeDouble) { + // 1.001 is stored as 1.000999..., which DateTime(double) truncates to 1000 ms; schema clients must agree. + EXPECT_EQ(1000, EpochSecondsToDateTime(1.001)->Millis()); + for (const double seconds : {1.001, -1.001, 1.5, 0.0005, 1.9996, 1700000000.123, -4307758798.296}) { + EXPECT_EQ(DateTime(seconds).Millis(), EpochSecondsToDateTime(seconds)->Millis()) << seconds; + } +} + +TEST_F(SerdeTraitsTest, EpochSecondsToDateTimeRejectsUnrepresentableSeconds) { + const double bound = static_cast(std::chrono::duration_cast( + (std::chrono::system_clock::duration::max)()).count()) / 1000.0; + EXPECT_TRUE(EpochSecondsToDateTime(std::nextafter(bound, 0.0)).has_value()); + EXPECT_TRUE(EpochSecondsToDateTime(-std::nextafter(bound, 0.0)).has_value()); + EXPECT_FALSE(EpochSecondsToDateTime(bound).has_value()); + EXPECT_FALSE(EpochSecondsToDateTime(-bound).has_value()); + EXPECT_FALSE(EpochSecondsToDateTime(1e300).has_value()); + EXPECT_FALSE(EpochSecondsToDateTime(std::numeric_limits::infinity()).has_value()); + EXPECT_FALSE(EpochSecondsToDateTime(-std::numeric_limits::infinity()).has_value()); + EXPECT_FALSE(EpochSecondsToDateTime(std::numeric_limits::quiet_NaN()).has_value()); +} diff --git a/tests/aws-cpp-sdk-core-tests/smithy/client/schema/XmlShapeDeserializerTest.cpp b/tests/aws-cpp-sdk-core-tests/smithy/client/schema/XmlShapeDeserializerTest.cpp index 05d80bc3df88..86333f8ba326 100644 --- a/tests/aws-cpp-sdk-core-tests/smithy/client/schema/XmlShapeDeserializerTest.cpp +++ b/tests/aws-cpp-sdk-core-tests/smithy/client/schema/XmlShapeDeserializerTest.cpp @@ -3,7 +3,10 @@ * SPDX-License-Identifier: Apache-2.0. */ #include +#include +#include #include +#include #include #include #include @@ -15,6 +18,7 @@ #include #include +#include #include #include #include @@ -77,19 +81,16 @@ TEST_F(XmlShapeDeserializerTest, Long) { EXPECT_EQ(got.value(), 9876543210LL); } -TEST_F(XmlShapeDeserializerTest, LongClampsPlainDigitOverflow) { +TEST_F(XmlShapeDeserializerTest, LongPlainDigitOverflowRecordsError) { auto root = RootBuilder().PutMember("big", Schema::CreateLong("L")).Build(); - const Aws::Vector> cases = { - {"99999999999999999999", (std::numeric_limits::max)()}, - {"-99999999999999999999", (std::numeric_limits::min)()}, - {" 99999999999999999999", (std::numeric_limits::max)()}, - }; - for (const auto& c : cases) { - XmlShapeDeserializer d(Aws::Crt::ByteCursorFromArray(reinterpret_cast(c.first.data()), c.first.size())); + for (const Aws::String payload : {"99999999999999999999", "-99999999999999999999", + " 99999999999999999999", "9223372036854775808"}) { + XmlShapeDeserializer d(Aws::Crt::ByteCursorFromArray(reinterpret_cast(payload.data()), payload.size())); Aws::Crt::Optional got; d.ReadStruct(*root, [&](const Schema& m, ShapeDeserializer& de) { got = de.ReadLong(m); }); - ASSERT_TRUE(got.has_value()) << c.first; - EXPECT_EQ(got.value(), c.second) << c.first; + EXPECT_FALSE(got.has_value()) << payload; + ASSERT_TRUE(d.GetError().has_value()) << payload; + EXPECT_NE(d.GetError()->GetMessage().find("out of range"), Aws::String::npos) << payload; } } @@ -650,9 +651,653 @@ TEST_F(XmlShapeDeserializerTest, TimestampFormatTraitControlsParsing) { } } +TEST_F(XmlShapeDeserializerTest, EpochTimestampMatchesLegacyDateTimeDouble) { + auto root = + RootBuilder() + .PutMember( + "ts", Schema::CreateTimestamp("T"), + {{TimestampFormatTrait::KEY(), Aws::MakeShared("Test", TimestampFormatTrait::Format::EPOCH_SECONDS)}}) + .Build(); + auto member = root->GetMember("ts").value(); + for (const int64_t millis : {int64_t(-4307758798296), int64_t(4307758798296), int64_t(1296), int64_t(-1296), int64_t(1700000000123)}) { + auto payload = Encode(root, [&](ShapeSerializer& ser) { ser.WriteTimestamp(*member, Aws::Utils::DateTime(millis)); }); + XmlShapeDeserializer d(Aws::Crt::ByteCursorFromArray(reinterpret_cast(payload.data()), payload.size())); + Aws::Crt::Optional got; + d.ReadStruct(*root, [&](const Schema& m, ShapeDeserializer& de) { got = de.ReadTimestamp(m); }); + ASSERT_TRUE(got.has_value()) << payload; + EXPECT_EQ(got.value().Millis(), LegacyEpochMillis(EpochSecondsText(Aws::Utils::DateTime(millis)))) << payload; + } +} + TEST_F(XmlShapeDeserializerTest, ReadDocumentIsUnsupported) { Aws::String xml = ""; XmlShapeDeserializer deser(Aws::Crt::ByteCursorFromArray(reinterpret_cast(xml.c_str()), xml.size())); auto schema = Schema::CreateDocument("smithy.api#Document"); EXPECT_EQ(deser.ReadDocument(*schema), nullptr); } + +TEST_F(XmlShapeDeserializerTest, WhitespaceOnlyInputIsNotAnError) { + const Aws::String xml = " \n\t "; + XmlShapeDeserializer d(Aws::Crt::ByteCursorFromArray(reinterpret_cast(xml.data()), xml.size())); + EXPECT_FALSE(d.GetError().has_value()); +} + +TEST_F(XmlShapeDeserializerTest, UnclosedRootSetsError) { + const Aws::String xml = "1"; + XmlShapeDeserializer d(Aws::Crt::ByteCursorFromArray(reinterpret_cast(xml.data()), xml.size())); + ASSERT_TRUE(d.GetError().has_value()); + EXPECT_NE(d.GetError()->GetMessage().find("unclosed"), Aws::String::npos); +} + +TEST_F(XmlShapeDeserializerTest, TextWithoutRootSetsError) { + const Aws::String xml = "hello"; + XmlShapeDeserializer d(Aws::Crt::ByteCursorFromArray(reinterpret_cast(xml.data()), xml.size())); + EXPECT_TRUE(d.GetError().has_value()); +} + +TEST_F(XmlShapeDeserializerTest, RecursiveStructAtMaxDepthSucceeds) { + RecursiveNodeSchema node; + const Aws::String xml = NestedXmlNodes(MAX_NESTING_DEPTH); + XmlShapeDeserializer d(Aws::Crt::ByteCursorFromArray(reinterpret_cast(xml.data()), xml.size())); + EXPECT_EQ(DescendRecursiveNode(d, node.Get()), MAX_NESTING_DEPTH); + EXPECT_FALSE(d.GetError().has_value()); +} + +TEST_F(XmlShapeDeserializerTest, RecursiveStructPastMaxDepthSetsError) { + RecursiveNodeSchema node; + const Aws::String xml = NestedXmlNodes(MAX_NESTING_DEPTH + 1); + XmlShapeDeserializer d(Aws::Crt::ByteCursorFromArray(reinterpret_cast(xml.data()), xml.size())); + DescendRecursiveNode(d, node.Get()); + ASSERT_TRUE(d.GetError().has_value()); + EXPECT_NE(d.GetError()->GetMessage().find("depth"), Aws::String::npos); +} + +namespace { +template +void WithXmlValue(const Aws::String& text, const std::shared_ptr& member, const Fn& fn, TraitList memberTraits = {}) { + auto root = Schema::StructureBuilder("Root").PutMember("v", member, memberTraits).Build(); + const Aws::String xml = "" + text + ""; + XmlShapeDeserializer d(Aws::Crt::ByteCursorFromArray(reinterpret_cast(xml.data()), xml.size())); + d.ReadStruct(*root, [&](const Schema& m, ShapeDeserializer& de) { fn(m, de); }); + EXPECT_FALSE(d.GetError().has_value()); +} + +template +void ExpectXmlValueOutOfRange(const Aws::String& text, const std::shared_ptr& member, const Fn& read, + TraitList memberTraits = {}) { + auto root = Schema::StructureBuilder("Root").PutMember("v", member, memberTraits).Build(); + const Aws::String xml = "" + text + ""; + XmlShapeDeserializer d(Aws::Crt::ByteCursorFromArray(reinterpret_cast(xml.data()), xml.size())); + bool present = true; + d.ReadStruct(*root, [&](const Schema& m, ShapeDeserializer& de) { present = read(m, de); }); + EXPECT_FALSE(present) << text; + ASSERT_TRUE(d.GetError().has_value()) << text; + EXPECT_NE(d.GetError()->GetMessage().find("out of range"), Aws::String::npos) << text; +} +} // namespace + +TEST_F(XmlShapeDeserializerTest, IntegerBeyondInt32RecordsError) { + ExpectXmlValueOutOfRange("3000000000", Schema::CreateInteger("I"), + [](const Schema& m, ShapeDeserializer& de) { return de.ReadInteger(m).has_value(); }); +} + +TEST_F(XmlShapeDeserializerTest, FloatBeyondFloatRangeRecordsError) { + ExpectXmlValueOutOfRange("1e300", Schema::CreateFloat("F"), + [](const Schema& m, ShapeDeserializer& de) { return de.ReadFloat(m).has_value(); }); +} + +TEST_F(XmlShapeDeserializerTest, UnrepresentableEpochTimestampRecordsError) { + ExpectXmlValueOutOfRange("1e300", Schema::CreateTimestamp("T"), + [](const Schema& m, ShapeDeserializer& de) { return de.ReadTimestamp(m).has_value(); }, + {{TimestampFormatTrait::KEY(), Aws::MakeShared("Test", TimestampFormatTrait::Format::EPOCH_SECONDS)}}); +} + +namespace { +const std::array ROUND_TRIP_DOUBLES = {{2147483647.0, 0.1, 1.0 / 3.0, 9007199254740993.0, (std::numeric_limits::max)(), + (std::numeric_limits::min)(), -123456789.123456789}}; +const std::array ROUND_TRIP_FLOATS = {{(std::numeric_limits::max)(), (std::numeric_limits::min)(), + std::numeric_limits::denorm_min(), 0.1f, 3.14f, std::nextafter(16777216.0f, 1e9f), + std::nextafter(16777216.0f, 0.0f), 2147483647.0f}}; + +std::shared_ptr ElementNumberRoot(ShapeType type) { + return RootBuilder().PutMember("n", type == ShapeType::Float ? Schema::CreateFloat("N") : Schema::CreateDouble("N")).Build(); +} + +std::shared_ptr AttributeNumberRoot(ShapeType type) { + return RootBuilder() + .PutMember("n", type == ShapeType::Float ? Schema::CreateFloat("N") : Schema::CreateDouble("N"), + {{XmlAttributeTrait::KEY(), Aws::MakeShared("Test")}}) + .Build(); +} + +template +Aws::Crt::Optional RoundTripNumber(ShapeType type, const std::shared_ptr& root, T value) { + auto n = root->GetMember("n").value(); + const Aws::String payload = Encode(root, [&](ShapeSerializer& ser) { + if (type == ShapeType::Float) { + ser.WriteFloat(*n, static_cast(value)); + } else { + ser.WriteDouble(*n, static_cast(value)); + } + }); + XmlShapeDeserializer d(Aws::Crt::ByteCursorFromArray(reinterpret_cast(payload.data()), payload.size())); + Aws::Crt::Optional got; + d.ReadStruct(*root, [&](const Schema& m, ShapeDeserializer& de) { + if (type == ShapeType::Float) { + auto v = de.ReadFloat(m); + if (v.has_value()) { + got = static_cast(v.value()); + } + } else { + auto v = de.ReadDouble(m); + if (v.has_value()) { + got = static_cast(v.value()); + } + } + }); + return got; +} +} // namespace + +TEST_F(XmlShapeDeserializerTest, DoubleRoundTripsExactlyAsElementAndAttribute) { + const auto element = ElementNumberRoot(ShapeType::Double); + const auto attribute = AttributeNumberRoot(ShapeType::Double); + for (const double value : ROUND_TRIP_DOUBLES) { + const auto asElement = RoundTripNumber(ShapeType::Double, element, value); + ASSERT_TRUE(asElement.has_value()) << value; + EXPECT_EQ(asElement.value(), value); + const auto asAttribute = RoundTripNumber(ShapeType::Double, attribute, value); + ASSERT_TRUE(asAttribute.has_value()) << value; + EXPECT_EQ(asAttribute.value(), value); + } +} + +TEST_F(XmlShapeDeserializerTest, FloatRoundTripsExactlyAsElementAndAttribute) { + const auto element = ElementNumberRoot(ShapeType::Float); + const auto attribute = AttributeNumberRoot(ShapeType::Float); + for (const float value : ROUND_TRIP_FLOATS) { + const auto asElement = RoundTripNumber(ShapeType::Float, element, value); + ASSERT_TRUE(asElement.has_value()) << value; + EXPECT_EQ(asElement.value(), value); + const auto asAttribute = RoundTripNumber(ShapeType::Float, attribute, value); + ASSERT_TRUE(asAttribute.has_value()) << value; + EXPECT_EQ(asAttribute.value(), value); + } +} + +TEST_F(XmlShapeDeserializerTest, NonFiniteDoubleRoundTripsAsAttribute) { + const auto attribute = AttributeNumberRoot(ShapeType::Double); + const auto nan = RoundTripNumber(ShapeType::Double, attribute, std::numeric_limits::quiet_NaN()); + ASSERT_TRUE(nan.has_value()); + EXPECT_TRUE(std::isnan(nan.value())); + for (const double value : {std::numeric_limits::infinity(), -std::numeric_limits::infinity()}) { + const auto got = RoundTripNumber(ShapeType::Double, attribute, value); + ASSERT_TRUE(got.has_value()) << value; + EXPECT_EQ(got.value(), value); + } +} + +TEST_F(XmlShapeDeserializerTest, AbsentAttributeMemberIsNotDelivered) { + auto root = RootBuilder() + .PutMember("id", Schema::CreateString("S"), {{XmlAttributeTrait::KEY(), Aws::MakeShared("Test")}}) + .Build(); + const Aws::String payload = ""; + XmlShapeDeserializer d(Aws::Crt::ByteCursorFromArray(reinterpret_cast(payload.data()), payload.size())); + int delivered = 0; + d.ReadStruct(*root, [&](const Schema&, ShapeDeserializer&) { ++delivered; }); + EXPECT_EQ(delivered, 0); + EXPECT_FALSE(d.GetError().has_value()); +} + +TEST_F(XmlShapeDeserializerTest, AbsentNumericAttributeMemberIsNotDelivered) { + auto root = RootBuilder() + .PutMember("n", Schema::CreateInteger("I"), {{XmlAttributeTrait::KEY(), Aws::MakeShared("Test")}}) + .Build(); + const Aws::String payload = ""; + XmlShapeDeserializer d(Aws::Crt::ByteCursorFromArray(reinterpret_cast(payload.data()), payload.size())); + int delivered = 0; + d.ReadStruct(*root, [&](const Schema&, ShapeDeserializer&) { ++delivered; }); + EXPECT_EQ(delivered, 0); +} + +TEST_F(XmlShapeDeserializerTest, ExplicitEmptyAttributeIsDeliveredAsEmptyString) { + auto root = RootBuilder() + .PutMember("id", Schema::CreateString("S"), {{XmlAttributeTrait::KEY(), Aws::MakeShared("Test")}}) + .Build(); + const Aws::String payload = ""; + XmlShapeDeserializer d(Aws::Crt::ByteCursorFromArray(reinterpret_cast(payload.data()), payload.size())); + int delivered = 0; + Aws::Crt::Optional got; + d.ReadStruct(*root, [&](const Schema& m, ShapeDeserializer& de) { + ++delivered; + got = de.ReadString(m); + }); + EXPECT_EQ(delivered, 1); + ASSERT_TRUE(got.has_value()); + EXPECT_EQ(got.value(), ""); +} + +namespace { +Aws::Map ReadStringAttributes(const Aws::String& payload, const Aws::Vector& names, + Aws::Crt::Optional* error = nullptr) { + auto builder = RootBuilder(); + for (const auto& name : names) { + builder.PutMember(name.c_str(), Schema::CreateString("S"), {{XmlAttributeTrait::KEY(), Aws::MakeShared("Test")}}); + } + auto root = builder.Build(); + XmlShapeDeserializer d(Aws::Crt::ByteCursorFromArray(reinterpret_cast(payload.data()), payload.size())); + Aws::Map got; + d.ReadStruct(*root, [&](const Schema& m, ShapeDeserializer& de) { + const auto value = de.ReadString(m); + if (value.has_value()) { + got[m.GetMemberName()] = value.value(); + } + }); + if (error) { + *error = d.GetError(); + } + return got; +} +} // namespace + +TEST_F(XmlShapeDeserializerTest, AttributeNameInsideDoubleQuotedValueIsNotMatched) { + const auto got = ReadStringAttributes("", {"a", "b"}); + ASSERT_EQ(got.size(), 2u); + EXPECT_EQ(got.at("a"), "z b='x'"); + EXPECT_EQ(got.at("b"), "y"); +} + +TEST_F(XmlShapeDeserializerTest, AttributeNameInsideSingleQuotedValueIsNotMatched) { + const auto got = ReadStringAttributes("", {"a", "b"}); + ASSERT_EQ(got.size(), 2u); + EXPECT_EQ(got.at("a"), "z b=\"x\""); + EXPECT_EQ(got.at("b"), "y"); +} + +TEST_F(XmlShapeDeserializerTest, AttributeNameInsideValueFollowedByQuoteIsNotMatched) { + const auto got = ReadStringAttributes("", {"attr", "int-attr"}); + ASSERT_EQ(got.size(), 2u); + EXPECT_EQ(got.at("attr"), "x int-attr="); + EXPECT_EQ(got.at("int-attr"), "1"); +} + +TEST_F(XmlShapeDeserializerTest, AttributeValueContainingTagEndIsRead) { + const auto got = ReadStringAttributes("2\" b=\"y\"/>", {"a", "b"}); + ASSERT_EQ(got.size(), 2u); + EXPECT_EQ(got.at("a"), "1>2"); + EXPECT_EQ(got.at("b"), "y"); +} + +TEST_F(XmlShapeDeserializerTest, AttributeNameThatIsSuffixOfAnotherIsNotMatched) { + const auto got = ReadStringAttributes("", {"attr", "int-attr"}); + ASSERT_EQ(got.size(), 2u); + EXPECT_EQ(got.at("attr"), "2"); + EXPECT_EQ(got.at("int-attr"), "1"); +} + +TEST_F(XmlShapeDeserializerTest, AttributeNameThatIsSuffixOfAnotherIsAbsentWhenOnlyLongerPresent) { + const auto got = ReadStringAttributes("", {"attr"}); + EXPECT_TRUE(got.empty()); +} + +TEST_F(XmlShapeDeserializerTest, AttributeWithWhitespaceAroundEqualsIsRead) { + const auto got = ReadStringAttributes("", {"a", "b"}); + ASSERT_EQ(got.size(), 2u); + EXPECT_EQ(got.at("a"), "1"); + EXPECT_EQ(got.at("b"), "2"); +} + +TEST_F(XmlShapeDeserializerTest, MalformedAttributeRecordsError) { + for (const char* payload : {"", "", "", "", "", + "", "", ""}) { + Aws::Crt::Optional error; + EXPECT_TRUE(ReadStringAttributes(payload, {"a"}, &error).empty()) << payload; + EXPECT_TRUE(error.has_value()) << payload; + } +} + +TEST_F(XmlShapeDeserializerTest, AttributesWithoutSeparatingWhitespaceAreRead) { + Aws::Crt::Optional error; + const auto got = ReadStringAttributes("", {"a", "b"}, &error); + EXPECT_FALSE(error.has_value()); + ASSERT_EQ(got.size(), 2u); + EXPECT_EQ(got.at("b"), "2"); +} + +TEST_F(XmlShapeDeserializerTest, AttributeValueIsDecodedAndLineEndsNormalized) { + const auto got = ReadStringAttributes("", {"a"}); + ASSERT_EQ(got.size(), 1u); + EXPECT_EQ(got.at("a"), "xA<\ny\r\tz"); +} + +TEST_F(XmlShapeDeserializerTest, UnterminatedAttributeQuoteBeforeLaterQuoteIsAbsent) { + EXPECT_TRUE(ReadStringAttributes("", {"a"}).empty()); +} + +TEST_F(XmlShapeDeserializerTest, OverflowingFiniteTextRecordsError) { + auto root = RootBuilder().PutMember("d", Schema::CreateDouble("D")).PutMember("f", Schema::CreateFloat("F")).Build(); + for (const char* text : {"1e400", "-1e400"}) { + ExpectXmlValueOutOfRange(text, Schema::CreateDouble("D"), + [](const Schema& m, ShapeDeserializer& de) { return de.ReadDouble(m).has_value(); }); + ExpectXmlValueOutOfRange(text, Schema::CreateFloat("F"), + [](const Schema& m, ShapeDeserializer& de) { return de.ReadFloat(m).has_value(); }); + } + const Aws::String payload = "Infinity"; + XmlShapeDeserializer d(Aws::Crt::ByteCursorFromArray(reinterpret_cast(payload.data()), payload.size())); + Aws::Crt::Optional got; + d.ReadStruct(*root, [&](const Schema& m, ShapeDeserializer& de) { got = de.ReadDouble(m); }); + ASSERT_TRUE(got.has_value()); + EXPECT_TRUE(std::isinf(got.value())); +} + +namespace { +Aws::String ReadText(const Aws::String& payload, Aws::Crt::Optional* error = nullptr) { + auto root = RootBuilder().PutMember("s", Schema::CreateString("S")).Build(); + XmlShapeDeserializer d(Aws::Crt::ByteCursorFromArray(reinterpret_cast(payload.data()), payload.size())); + Aws::Crt::Optional got; + d.ReadStruct(*root, [&](const Schema& m, ShapeDeserializer& de) { got = de.ReadString(m); }); + if (error) { + *error = d.GetError(); + } + return got.has_value() ? got.value() : Aws::String(""); +} +} // namespace + +TEST_F(XmlShapeDeserializerTest, NumericCharacterReferencesDecode) { + EXPECT_EQ(ReadText("ABé😀C"), Aws::String("AB\xC3\xA9\xF0\x9F\x98\x80" "C")); +} + +TEST_F(XmlShapeDeserializerTest, UnrecognisedReferencesAreKeptAsWritten) { + for (const char* text : {"A", "&#;", "&#x;", "�", " a;", "A", "&foo;", "a&b", "&"}) { + EXPECT_EQ(ReadText(Aws::String("") + text + ""), Aws::String(text)) << text; + } +} + +TEST_F(XmlShapeDeserializerTest, CdataIsLiteral) { + EXPECT_EQ(ReadText("&]]>"), Aws::String("&")); + EXPECT_EQ(ReadText("x&z"), Aws::String("xy&z")); +} + +TEST_F(XmlShapeDeserializerTest, CommentsAreNotText) { EXPECT_EQ(ReadText("abc"), Aws::String("abc")); } + +TEST_F(XmlShapeDeserializerTest, LineEndsAreNormalizedAsTinyxml2Does) { + EXPECT_EQ(ReadText("a\r\nb\rc\n\rd\n\ne"), Aws::String("a\nb\nc\nd\n\ne")); + EXPECT_EQ(ReadText(""), Aws::String("a\nb")); + EXPECT_EQ(ReadText("a \nb"), Aws::String("a\r\r\nb")); +} + +TEST_F(XmlShapeDeserializerTest, InvalidElementNameRecordsError) { + for (const char* payload : {"<1v>xa", "<-v/>a", "a", + "a<.x/>"}) { + Aws::Crt::Optional error; + EXPECT_EQ(ReadText(payload, &error), "") << payload; + EXPECT_TRUE(error.has_value()) << payload; + } + Aws::Crt::Optional error; + EXPECT_EQ(ReadText("<_a.b-c:d\xC3\xA9/>a", &error), Aws::String("a")); + EXPECT_FALSE(error.has_value()); +} + +TEST_F(XmlShapeDeserializerTest, MalformedStructureRecordsError) { + auto root = RootBuilder().PutMember("a", Schema::CreateString("S")).Build(); + const Aws::Vector> cases = { + {"y'>", "invalid closing tag"}, + }; + for (const auto& c : cases) { + XmlShapeDeserializer d(Aws::Crt::ByteCursorFromArray(reinterpret_cast(c.first.data()), c.first.size())); + int delivered = 0; + d.ReadStruct(*root, [&](const Schema&, ShapeDeserializer&) { ++delivered; }); + EXPECT_EQ(delivered, 0) << c.first; + ASSERT_TRUE(d.GetError().has_value()) << c.first; + EXPECT_NE(d.GetError()->GetMessage().find(c.second), Aws::String::npos) << c.first << ": " << d.GetError()->GetMessage(); + } +} + +TEST_F(XmlShapeDeserializerTest, MarkupAroundRootIsAccepted) { + auto root = RootBuilder().PutMember("a", Schema::CreateString("S")).Build(); + for (const Aws::String payload : {"\xEF\xBB\xBF\nx\n\n", + "", "x"}) { + XmlShapeDeserializer d(Aws::Crt::ByteCursorFromArray(reinterpret_cast(payload.data()), payload.size())); + Aws::Crt::Optional got; + d.ReadStruct(*root, [&](const Schema& m, ShapeDeserializer& de) { got = de.ReadString(m); }); + EXPECT_FALSE(d.GetError().has_value()) << payload; + EXPECT_TRUE(got.has_value()) << payload; + } +} + +namespace { +Aws::Map ReadStringMembers(const Aws::String& payload, const Aws::Vector& names) { + auto builder = RootBuilder(); + for (const auto& name : names) { + builder.PutMember(name.c_str(), Schema::CreateString("S")); + } + auto root = builder.Build(); + XmlShapeDeserializer d(Aws::Crt::ByteCursorFromArray(reinterpret_cast(payload.data()), payload.size())); + Aws::Map got; + d.ReadStruct(*root, [&](const Schema& m, ShapeDeserializer& de) { + const auto value = de.ReadString(m); + if (value.has_value()) { + got[m.GetMemberName()] = value.value(); + } + }); + EXPECT_FALSE(d.GetError().has_value()) << payload; + return got; +} +} // namespace + +TEST_F(XmlShapeDeserializerTest, DeclarationInsideElementIsSkippedByNavigation) { + const auto got = ReadStringMembers("12", {"a", "b"}); + ASSERT_EQ(got.size(), 2u); + EXPECT_EQ(got.at("a"), "1"); + EXPECT_EQ(got.at("b"), "2"); + EXPECT_EQ(ReadStringMembers("12", {"a", "b"}).at("b"), "2"); +} + +TEST_F(XmlShapeDeserializerTest, ProcessingInstructionEndsAtItsOwnTerminator) { + const auto got = ReadStringMembers("1?>2", {"a", "b"}); + ASSERT_EQ(got.size(), 1u); + EXPECT_EQ(got.at("b"), "2"); +} + +TEST_F(XmlShapeDeserializerTest, RootIsFoundAfterProcessingInstructionContainingTags) { + const auto got = ReadStringMembers("1 ?>2", {"a"}); + ASSERT_EQ(got.size(), 1u); + EXPECT_EQ(got.at("a"), "2"); +} + +TEST_F(XmlShapeDeserializerTest, StructStopsDeliveringMembersAfterFailure) { + auto root = RootBuilder().PutMember("a", Schema::CreateInteger("I")).PutMember("b", Schema::CreateString("S")).Build(); + const Aws::String payload = "99999999999x"; + XmlShapeDeserializer d(Aws::Crt::ByteCursorFromArray(reinterpret_cast(payload.data()), payload.size())); + int delivered = 0; + d.ReadStruct(*root, [&](const Schema& m, ShapeDeserializer& de) { + ++delivered; + de.ReadInteger(m); + }); + EXPECT_EQ(delivered, 1); + EXPECT_TRUE(d.GetError().has_value()); +} + +TEST_F(XmlShapeDeserializerTest, ListStopsDeliveringElementsAfterFailure) { + auto listBuilder = Schema::ListBuilder("Nums"); + auto root = RootBuilder().PutMember("nums", listBuilder).Build(); + auto elem = Schema::CreateMember("member", ShapeType::Integer); + const Aws::String payload = "9999999999912"; + XmlShapeDeserializer d(Aws::Crt::ByteCursorFromArray(reinterpret_cast(payload.data()), payload.size())); + int delivered = 0; + d.ReadStruct(*root, [&](const Schema& m, ShapeDeserializer& de) { + de.ReadList(m, [&](ShapeDeserializer& ede) { + ++delivered; + ede.ReadInteger(*elem); + }); + }); + EXPECT_EQ(delivered, 1); + EXPECT_TRUE(d.GetError().has_value()); +} + +TEST_F(XmlShapeDeserializerTest, MapStopsDeliveringEntriesAfterFailure) { + auto mapBuilder = Schema::MapBuilder("M"); + auto root = RootBuilder().PutMember("m", mapBuilder).Build(); + auto valSchema = Schema::CreateMember("value", ShapeType::Integer); + const Aws::String payload = + "k199999999999k21"; + XmlShapeDeserializer d(Aws::Crt::ByteCursorFromArray(reinterpret_cast(payload.data()), payload.size())); + int delivered = 0; + d.ReadStruct(*root, [&](const Schema& m, ShapeDeserializer& de) { + de.ReadMap(m, [&](const Aws::String&, ShapeDeserializer& vde) { + ++delivered; + vde.ReadInteger(*valSchema); + }); + }); + EXPECT_EQ(delivered, 1); + EXPECT_TRUE(d.GetError().has_value()); +} + +namespace { + +int ReadUnionVariants(XmlShapeDeserializer& d) { + auto variants = Schema::UnionBuilder("U").PutMember("a", Schema::CreateString("S")).PutMember("b", Schema::CreateInteger("I")).Build(); + auto root = RootBuilder().PutMember("u", variants).Build(); + int delivered = 0; + d.ReadStruct(*root, [&](const Schema& m, ShapeDeserializer& de) { + de.ReadStruct(**m.GetMemberTarget(), [&](const Schema& variant, ShapeDeserializer& vde) { + ++delivered; + if (variant.GetMemberName() == Aws::String("a")) { + vde.ReadString(variant); + } else { + vde.ReadInteger(variant); + } + }); + }); + return delivered; +} + +} // namespace + +TEST_F(XmlShapeDeserializerTest, UnionWithTwoMembersSetRecordsError) { + const Aws::String payload = "x1"; + XmlShapeDeserializer d(Aws::Crt::ByteCursorFromArray(reinterpret_cast(payload.data()), payload.size())); + EXPECT_EQ(ReadUnionVariants(d), 1); + ASSERT_TRUE(d.GetError().has_value()); + EXPECT_NE(d.GetError()->GetMessage().find("more than one member"), Aws::String::npos); +} + +TEST_F(XmlShapeDeserializerTest, UnionIgnoresUnmodeledMembers) { + const Aws::String payload = "21"; + XmlShapeDeserializer d(Aws::Crt::ByteCursorFromArray(reinterpret_cast(payload.data()), payload.size())); + EXPECT_EQ(ReadUnionVariants(d), 1); + EXPECT_FALSE(d.GetError().has_value()); +} + +TEST_F(XmlShapeDeserializerTest, ProcessingInstructionOnlyOpensTheDocument) { + auto root = RootBuilder().PutMember("v", Schema::CreateString("S")).Build(); + const auto errorFor = [&](const Aws::String& payload) { + XmlShapeDeserializer d(Aws::Crt::ByteCursorFromArray(reinterpret_cast(payload.data()), payload.size())); + d.ReadStruct(*root, [](const Schema&, ShapeDeserializer&) {}); + return d.GetError(); + }; + EXPECT_FALSE(errorFor("x").has_value()); + EXPECT_FALSE(errorFor(" \n").has_value()); + for (const Aws::String payload : + {"", "", "", ""}) { + const auto error = errorFor(payload); + ASSERT_TRUE(error.has_value()) << payload; + EXPECT_NE(error->GetMessage().find("processing instruction"), Aws::String::npos) << payload; + } +} + +TEST_F(XmlShapeDeserializerTest, NumberBooleanAndTimestampTextIsTrimmedAsLegacyClientsDo) { + const auto epoch = Aws::MakeShared("Test", TimestampFormatTrait::Format::EPOCH_SECONDS); + auto root = RootBuilder() + .PutMember("b", Schema::CreateBoolean("B")) + .PutMember("i", Schema::CreateInteger("I")) + .PutMember("l", Schema::CreateLong("L")) + .PutMember("f", Schema::CreateFloat("F")) + .PutMember("d", Schema::CreateDouble("D")) + .PutMember("n", Schema::CreateDouble("D")) + .PutMember("t", Schema::CreateTimestamp("T")) + .PutMember("e", Schema::CreateTimestamp("T"), {{TimestampFormatTrait::KEY(), epoch}}) + .PutMember("a", Schema::CreateInteger("I"), {{XmlAttributeTrait::KEY(), Aws::MakeShared("Test")}}) + .Build(); + const Aws::String payload = + " true\n\n 42\n\t-9876543210 1.5 2.25 NaN " + " 2020-01-02T03:04:05Z 1.5 "; + XmlShapeDeserializer d(Aws::Crt::ByteCursorFromArray(reinterpret_cast(payload.data()), payload.size())); + Aws::Map got; + d.ReadStruct(*root, [&](const Schema& m, ShapeDeserializer& de) { + const Aws::String name = m.GetMemberName(); + if (name == "b") { + const auto v = de.ReadBoolean(m); + got[name] = v.has_value() ? (v.value() ? "true" : "false") : ""; + } else if (name == "i" || name == "a") { + const auto v = de.ReadInteger(m); + got[name] = v.has_value() ? Aws::Utils::StringUtils::to_string(v.value()) : ""; + } else if (name == "l") { + const auto v = de.ReadLong(m); + got[name] = v.has_value() ? Aws::Utils::StringUtils::to_string(v.value()) : ""; + } else if (name == "f") { + const auto v = de.ReadFloat(m); + got[name] = v.has_value() ? Aws::Utils::StringUtils::to_string(v.value()) : ""; + } else if (name == "d" || name == "n") { + const auto v = de.ReadDouble(m); + got[name] = v.has_value() ? (std::isnan(v.value()) ? "NaN" : Aws::Utils::StringUtils::to_string(v.value())) : ""; + } else { + const auto v = de.ReadTimestamp(m); + got[name] = v.has_value() ? Aws::Utils::StringUtils::to_string(v->Millis()) : ""; + } + }); + EXPECT_FALSE(d.GetError().has_value()); + EXPECT_EQ(got["b"], "true"); + EXPECT_EQ(got["i"], "42"); + EXPECT_EQ(got["l"], "-9876543210"); + EXPECT_EQ(got["f"], "1.5"); + EXPECT_EQ(got["d"], "2.25"); + EXPECT_EQ(got["n"], "NaN"); + EXPECT_EQ(got["t"], "1577934245000"); + EXPECT_EQ(got["e"], "1500"); + EXPECT_EQ(got["a"], "7"); +} + +TEST_F(XmlShapeDeserializerTest, StringAndBlobTextIsNotTrimmed) { + EXPECT_EQ(ReadText(" x\n"), " x\n"); + auto root = RootBuilder().PutMember("b", Schema::CreateBlob("B")).Build(); + const Aws::String payload = " AQI= "; + XmlShapeDeserializer d(Aws::Crt::ByteCursorFromArray(reinterpret_cast(payload.data()), payload.size())); + Aws::Crt::Optional got; + d.ReadStruct(*root, [&](const Schema& m, ShapeDeserializer& de) { got = de.ReadBlob(m); }); + ASSERT_TRUE(got.has_value()); + EXPECT_EQ(got.value(), Aws::Utils::HashingUtils::Base64Decode(" AQI= ")); +} + +TEST_F(XmlShapeDeserializerTest, BooleanTextIsCaseInsensitive) { + auto root = RootBuilder().PutMember("b", Schema::CreateBoolean("B")).Build(); + const auto read = [&](const char* text) { + const Aws::String payload = Aws::String("") + text + ""; + XmlShapeDeserializer d(Aws::Crt::ByteCursorFromArray(reinterpret_cast(payload.data()), payload.size())); + Aws::Crt::Optional got; + d.ReadStruct(*root, [&](const Schema& m, ShapeDeserializer& de) { got = de.ReadBoolean(m); }); + EXPECT_FALSE(d.GetError().has_value()) << text; + return got; + }; + for (const char* text : {"TRUE", "True", "tRuE"}) { + ASSERT_TRUE(read(text).has_value()) << text; + EXPECT_TRUE(read(text).value()) << text; + } + for (const char* text : {"FALSE", "False"}) { + ASSERT_TRUE(read(text).has_value()) << text; + EXPECT_FALSE(read(text).value()) << text; + } + for (const char* text : {"1", "0", "yes", "truee", ""}) { + EXPECT_FALSE(read(text).has_value()) << text; + } +} diff --git a/tests/aws-cpp-sdk-core-tests/smithy/client/schema/XmlShapeSerializerTest.cpp b/tests/aws-cpp-sdk-core-tests/smithy/client/schema/XmlShapeSerializerTest.cpp index 96b03411ec12..1775a378e130 100644 --- a/tests/aws-cpp-sdk-core-tests/smithy/client/schema/XmlShapeSerializerTest.cpp +++ b/tests/aws-cpp-sdk-core-tests/smithy/client/schema/XmlShapeSerializerTest.cpp @@ -14,6 +14,7 @@ #include #include +#include #include "SchemaSerializerTestHelpers.h" @@ -74,6 +75,54 @@ TEST_F(XmlShapeSerializerTest, Double) { EXPECT_NE(s.GetPayload().GetResult().find("3.14"), Aws::String::npos); } +namespace { +Aws::String FloatElementPayload(float value) { + XmlShapeSerializer s; + auto root = Schema::StructureBuilder("Root", {{XmlNameTrait::KEY(), Aws::MakeShared("Schema", "Root")}}).Build(); + auto member = Schema::CreateMember("f", ShapeType::Float); + LambdaStruct rootStruct(*root, [&](ShapeSerializer& ser) { ser.WriteFloat(*member, value); }); + s.WriteStruct(*root, rootStruct); + return s.GetPayload().GetResult(); +} + +Aws::String FloatAttributePayload(float value) { + XmlShapeSerializer s; + auto root = Schema::StructureBuilder("Root", {{XmlNameTrait::KEY(), Aws::MakeShared("Schema", "Root")}}) + .PutMember("f", Schema::CreateFloat("F"), {{XmlAttributeTrait::KEY(), Aws::MakeShared("Schema")}}) + .Build(); + auto member = root->GetMember("f").value(); + LambdaStruct rootStruct(*root, [&](ShapeSerializer& ser) { ser.WriteFloat(*member, value); }); + s.WriteStruct(*root, rootStruct); + return s.GetPayload().GetResult(); +} +} // namespace + +TEST_F(XmlShapeSerializerTest, FloatElementIsFormattedAtFloatPrecision) { + EXPECT_NE(FloatElementPayload(3.14f).find("3.14"), Aws::String::npos); + EXPECT_NE(FloatElementPayload(0.1f).find("0.1"), Aws::String::npos); + EXPECT_NE(FloatElementPayload((std::numeric_limits::max)()).find("3.40282347e+38"), Aws::String::npos); +} + +TEST_F(XmlShapeSerializerTest, FloatAttributeIsFormattedAtFloatPrecision) { + EXPECT_NE(FloatAttributePayload(3.14f).find("f=\"3.14\""), Aws::String::npos); + EXPECT_NE(FloatAttributePayload(0.1f).find("f=\"0.1\""), Aws::String::npos); +} + +TEST_F(XmlShapeSerializerTest, NonFiniteFloatAttributeUsesXmlLiterals) { + EXPECT_NE(FloatAttributePayload(std::numeric_limits::quiet_NaN()).find("f=\"NaN\""), Aws::String::npos); + EXPECT_NE(FloatAttributePayload(std::numeric_limits::infinity()).find("f=\"Infinity\""), Aws::String::npos); + EXPECT_NE(FloatAttributePayload(-std::numeric_limits::infinity()).find("f=\"-Infinity\""), Aws::String::npos); +} + +TEST_F(XmlShapeSerializerTest, LargeIntegralDoubleKeepsAllDigits) { + XmlShapeSerializer s; + auto root = Schema::StructureBuilder("Root", {{XmlNameTrait::KEY(), Aws::MakeShared("Schema", "Root")}}).Build(); + auto member = Schema::CreateMember("big", ShapeType::Double); + LambdaStruct rootStruct(*root, [&](ShapeSerializer& ser) { ser.WriteDouble(*member, 2147483647.0); }); + s.WriteStruct(*root, rootStruct); + EXPECT_NE(s.GetPayload().GetResult().find("2147483647"), Aws::String::npos); +} + TEST_F(XmlShapeSerializerTest, String) { XmlShapeSerializer s; auto root = Schema::StructureBuilder("Root", {{XmlNameTrait::KEY(), Aws::MakeShared("Schema", "Root")}}).Build(); @@ -354,20 +403,13 @@ TEST_F(XmlShapeSerializerTest, EscapesMultipleSpecialChars) { EXPECT_NE(s.GetPayload().GetResult().find("x < 5 & y > 3"), Aws::String::npos); } -TEST_F(XmlShapeSerializerTest, MaxDepthEnforcement) { +TEST_F(XmlShapeSerializerTest, EscapesCarriageReturnSoItSurvivesLineEndNormalization) { XmlShapeSerializer s; auto root = Schema::StructureBuilder("Root", {{XmlNameTrait::KEY(), Aws::MakeShared("Schema", "Root")}}).Build(); - auto nested = Schema::CreateMember("n", ShapeType::Structure); - std::function nest = [&](ShapeSerializer& ser, int remaining) { - if (remaining <= 0) { - return; - } - ser.WriteStruct(*nested, LambdaStruct(*nested, [&, remaining](ShapeSerializer& inner) { nest(inner, remaining - 1); })); - }; - LambdaStruct rootStruct(*root, [&](ShapeSerializer& ser) { nest(ser, 1000); }); + auto member = Schema::CreateMember("msg", ShapeType::String); + LambdaStruct rootStruct(*root, [&](ShapeSerializer& ser) { ser.WriteString(*member, "a\r\nb\n\rc"); }); s.WriteStruct(*root, rootStruct); - bool hitLimit = !s.GetPayload().IsSuccess(); - EXPECT_TRUE(hitLimit); + EXPECT_NE(s.GetPayload().GetResult().find("a \nb\n c"), Aws::String::npos); } TEST_F(XmlShapeSerializerTest, XmlNameOverridesMemberName) { @@ -534,3 +576,19 @@ TEST_F(XmlShapeSerializerTest, DocumentIsUnsupported) { ASSERT_FALSE(outcome.IsSuccess()); EXPECT_EQ(outcome.GetError().GetExceptionName(), "SerializationException"); } + +TEST_F(XmlShapeSerializerTest, NestingAtMaxDepthSucceeds) { + XmlShapeSerializer s; + auto root = Schema::StructureBuilder("Root", {{XmlNameTrait::KEY(), Aws::MakeShared("Schema", "Root")}}).Build(); + WriteNestedStructs(s, *root, MAX_NESTING_DEPTH); + EXPECT_TRUE(s.GetPayload().IsSuccess()); +} + +TEST_F(XmlShapeSerializerTest, NestingPastMaxDepthFails) { + XmlShapeSerializer s; + auto root = Schema::StructureBuilder("Root", {{XmlNameTrait::KEY(), Aws::MakeShared("Schema", "Root")}}).Build(); + WriteNestedStructs(s, *root, MAX_NESTING_DEPTH + 1); + auto outcome = s.GetPayload(); + ASSERT_FALSE(outcome.IsSuccess()); + EXPECT_NE(outcome.GetError().GetMessage().find("depth"), Aws::String::npos); +} diff --git a/tests/aws-cpp-sdk-core-tests/utils/StringUtilsTest.cpp b/tests/aws-cpp-sdk-core-tests/utils/StringUtilsTest.cpp index b1252118b3ad..f1cd68a48270 100644 --- a/tests/aws-cpp-sdk-core-tests/utils/StringUtilsTest.cpp +++ b/tests/aws-cpp-sdk-core-tests/utils/StringUtilsTest.cpp @@ -9,6 +9,12 @@ #include #include #include +#include +#include +#include +#include +#include +#include using namespace Aws::Utils; @@ -601,10 +607,9 @@ TEST_F(StringUtilsTest, TestParseInt64) ASSERT_FALSE(StringUtils::ParseInt64("1-2").has_value()); ASSERT_FALSE(StringUtils::ParseInt64("42 ").has_value()); - // Out-of-range values saturate, matching ConvertToInt64. - ASSERT_EQ(LLONG_MAX, StringUtils::ParseInt64("9223372036854775808").value()); - ASSERT_EQ(LLONG_MIN, StringUtils::ParseInt64("-9223372036854775809").value()); - ASSERT_EQ(LLONG_MAX, StringUtils::ParseInt64("99999999999999999999999999").value()); + ASSERT_FALSE(StringUtils::ParseInt64("9223372036854775808").has_value()); + ASSERT_FALSE(StringUtils::ParseInt64("-9223372036854775809").has_value()); + ASSERT_FALSE(StringUtils::ParseInt64("99999999999999999999999999").has_value()); } TEST_F(StringUtilsTest, TestParseDouble) @@ -624,6 +629,115 @@ TEST_F(StringUtilsTest, TestParseDouble) ASSERT_FALSE(StringUtils::ParseDouble("1.5abc").has_value()); } +namespace +{ +class ScopedNumericLocale +{ +public: + explicit ScopedNumericLocale(const char* name) + : m_previous(std::setlocale(LC_NUMERIC, nullptr)), m_set(std::setlocale(LC_NUMERIC, name) != nullptr) {} + ~ScopedNumericLocale() { std::setlocale(LC_NUMERIC, m_previous.c_str()); } + bool IsSet() const { return m_set; } + +private: + std::string m_previous; + bool m_set; +}; + +// std::locale throws for a missing locale and exceptions are off, so availability is checked through the C API first. +class ScopedGlobalLocale +{ +public: + explicit ScopedGlobalLocale(const char* name) + { + const std::string previous = std::setlocale(LC_ALL, nullptr); + const bool available = std::setlocale(LC_ALL, name) != nullptr; + std::setlocale(LC_ALL, previous.c_str()); + if (available) + { + m_previous = std::locale::global(std::locale(name)); + m_set = true; + } + } + ~ScopedGlobalLocale() + { + if (m_set) + { + std::locale::global(m_previous); + } + } + bool IsSet() const { return m_set; } + +private: + std::locale m_previous; + bool m_set = false; +}; +} + +TEST_F(StringUtilsTest, NumberConversionsIgnoreTheCNumericLocale) +{ + ScopedNumericLocale locale("de_DE.UTF-8"); + if (!locale.IsSet()) + { + GTEST_SKIP() << "de_DE.UTF-8 is not installed"; + } + ASSERT_EQ(',', *std::localeconv()->decimal_point); + ASSERT_TRUE(StringUtils::ParseDouble("3.14").has_value()); + ASSERT_DOUBLE_EQ(3.14, StringUtils::ParseDouble("3.14").value()); + ASSERT_TRUE(StringUtils::ParseDouble("-1.5e-3").has_value()); + ASSERT_DOUBLE_EQ(-1.5e-3, StringUtils::ParseDouble("-1.5e-3").value()); + ASSERT_FALSE(StringUtils::ParseDouble("3,14").has_value()); + ASSERT_EQ("0.1", StringUtils::FormatRoundTripDouble(0.1)); + ASSERT_EQ("0.1", StringUtils::FormatRoundTripFloat(0.1f)); +} + +TEST_F(StringUtilsTest, FormattingIgnoresTheGlobalLocale) +{ + ScopedGlobalLocale locale("de_DE.UTF-8"); + if (!locale.IsSet()) + { + GTEST_SKIP() << "de_DE.UTF-8 is not installed"; + } + ASSERT_EQ("1234567", StringUtils::to_string(1234567)); + ASSERT_EQ("3.14", StringUtils::FormatRoundTripDouble(3.14)); + ASSERT_EQ("3.14", StringUtils::FormatRoundTripFloat(3.14f)); + ASSERT_TRUE(StringUtils::ParseDouble("3.14").has_value()); + ASSERT_DOUBLE_EQ(3.14, StringUtils::ParseDouble("3.14").value()); +} + +TEST_F(StringUtilsTest, TestFormatRoundTripDouble) +{ + ASSERT_EQ("0", StringUtils::FormatRoundTripDouble(0.0)); + ASSERT_EQ("0.1", StringUtils::FormatRoundTripDouble(0.1)); + ASSERT_EQ("-2.5", StringUtils::FormatRoundTripDouble(-2.5)); + ASSERT_EQ("3.14", StringUtils::FormatRoundTripDouble(3.14)); + ASSERT_EQ("2147483647", StringUtils::FormatRoundTripDouble(2147483647.0)); + // 15 digits would print "0.3", which reparses to a different double. + ASSERT_EQ("0.30000000000000004", StringUtils::FormatRoundTripDouble(0.1 + 0.2)); + ASSERT_EQ("1.7976931348623157e+308", StringUtils::FormatRoundTripDouble((std::numeric_limits::max)())); + + for (const double value : {0.1 + 0.2, 1.0 / 3.0, 123456789.123456789, 5e-324, -(std::numeric_limits::max)()}) + { + ASSERT_EQ(value, StringUtils::ParseDouble(StringUtils::FormatRoundTripDouble(value)).value()); + } +} + +TEST_F(StringUtilsTest, TestFormatRoundTripFloat) +{ + ASSERT_EQ("0", StringUtils::FormatRoundTripFloat(0.0f)); + ASSERT_EQ("0.1", StringUtils::FormatRoundTripFloat(0.1f)); + ASSERT_EQ("3.14", StringUtils::FormatRoundTripFloat(3.14f)); + // 6 digits would print "1.67772e+07" and "1", which reparse to different floats. + ASSERT_EQ("16777216", StringUtils::FormatRoundTripFloat(16777216.0f)); + ASSERT_EQ("1.00000012", StringUtils::FormatRoundTripFloat(std::nextafter(1.0f, 2.0f))); + ASSERT_EQ("3.40282347e+38", StringUtils::FormatRoundTripFloat((std::numeric_limits::max)())); + + for (const float value : {1.0f / 3.0f, 123456.789f, (std::numeric_limits::min)(), -(std::numeric_limits::max)()}) + { + ASSERT_EQ(value, std::strtof(StringUtils::FormatRoundTripFloat(value).c_str(), nullptr)); + } +} + TEST_F(StringUtilsTest, TestDoubleURLEncoding) { double doubleValue = 56789432.08; diff --git a/tests/fuzz/CMakeLists.txt b/tests/fuzz/CMakeLists.txt new file mode 100644 index 000000000000..575f5db3a289 --- /dev/null +++ b/tests/fuzz/CMakeLists.txt @@ -0,0 +1,46 @@ +# Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +# SPDX-License-Identifier: Apache-2.0. +# + +file(GLOB SCHEMA_FUZZ_HARNESS_SRC "${CMAKE_CURRENT_SOURCE_DIR}/harness/*.cpp") +file(GLOB SCHEMA_FUZZ_TARGET_SRC "${CMAKE_CURRENT_SOURCE_DIR}/targets/*.cpp") + +# The tinyxml2 version core vendors, fetched so the reference parser outlives core's copy. +include(FetchContent) +set(SCHEMA_FUZZ_TINYXML2_GIT_TAG "9148bdf719e997d1f474be6bcc7943881046dba1" CACHE STRING "tinyxml2 11.0.0 commit used by the fuzz tests") +FetchContent_Declare(schema_fuzz_tinyxml2 + GIT_REPOSITORY https://github.com/leethomason/tinyxml2.git + GIT_TAG ${SCHEMA_FUZZ_TINYXML2_GIT_TAG} + SOURCE_SUBDIR no-cmake) +FetchContent_MakeAvailable(schema_fuzz_tinyxml2) +add_library(schema-fuzz-tinyxml2 STATIC ${schema_fuzz_tinyxml2_SOURCE_DIR}/tinyxml2.cpp) +target_include_directories(schema-fuzz-tinyxml2 SYSTEM PUBLIC ${schema_fuzz_tinyxml2_SOURCE_DIR}) + +add_executable(aws-cpp-sdk-schema-fuzz-tests ${SCHEMA_FUZZ_HARNESS_SRC} ${SCHEMA_FUZZ_TARGET_SRC}) +set_target_properties(aws-cpp-sdk-schema-fuzz-tests PROPERTIES CXX_STANDARD 17 CXX_STANDARD_REQUIRED ON CXX_EXTENSIONS OFF) +# Warnings only: set_compiler_flags would force -fno-exceptions and the SDK's C++ standard, and FuzzTest needs C++17. +set_compiler_warnings(aws-cpp-sdk-schema-fuzz-tests) +target_include_directories(aws-cpp-sdk-schema-fuzz-tests PRIVATE ${CMAKE_CURRENT_SOURCE_DIR}) +target_compile_definitions(aws-cpp-sdk-schema-fuzz-tests PRIVATE + "SCHEMA_FUZZ_CORPUS_DIR=\"${CMAKE_CURRENT_SOURCE_DIR}/corpus\"" + "SCHEMA_FUZZ_DICTIONARY_DIR=\"${CMAKE_CURRENT_SOURCE_DIR}/dictionaries\"" + "SCHEMA_FUZZ_FINDINGS_DIR=\"${CMAKE_CURRENT_SOURCE_DIR}/findings\"") +target_link_libraries(aws-cpp-sdk-schema-fuzz-tests PRIVATE aws-cpp-sdk-core schema-fuzz-tinyxml2) +link_fuzztest(aws-cpp-sdk-schema-fuzz-tests) + +# Fuzzing mode already builds everything with ASan. Sanitizers are otherwise scoped to the serde, not the CRT. +if (FUZZTEST_FUZZING_MODE) + set(SCHEMA_FUZZ_SANITIZERS undefined) +elseif (SCHEMA_FUZZ_UNIT_SANITIZERS) + set(SCHEMA_FUZZ_SANITIZERS address,undefined) +endif () +if (SCHEMA_FUZZ_SANITIZERS) + target_compile_options(aws-cpp-sdk-core PRIVATE -fsanitize=${SCHEMA_FUZZ_SANITIZERS} -fno-sanitize-recover=undefined -fno-omit-frame-pointer) + target_compile_options(aws-cpp-sdk-schema-fuzz-tests PRIVATE -fsanitize=${SCHEMA_FUZZ_SANITIZERS} -fno-sanitize-recover=undefined -fno-omit-frame-pointer) + target_link_options(aws-cpp-sdk-schema-fuzz-tests PRIVATE -fsanitize=${SCHEMA_FUZZ_SANITIZERS}) +endif () + +if (AUTORUN_UNIT_TESTS) + add_custom_command(TARGET aws-cpp-sdk-schema-fuzz-tests POST_BUILD + COMMAND $ "--gtest_brief=1") +endif () diff --git a/tests/fuzz/README.md b/tests/fuzz/README.md new file mode 100644 index 000000000000..acfc76862701 --- /dev/null +++ b/tests/fuzz/README.md @@ -0,0 +1,243 @@ +# Schema serde fuzz tests + +Coverage-guided fuzzing for the schema-based serde in `aws-cpp-sdk-core` +(`src/aws-cpp-sdk-core/{include,source}/smithy/client/schema/`), built on +[google/fuzztest](https://github.com/google/fuzztest). + +FuzzTest requires Linux and Clang, so everything runs in a container. `tools/scripts/run_fuzz.sh` builds the +image, configures a core-only static build with `-DENABLE_SCHEMA_FUZZ_TESTS=ON`, and runs +`aws-cpp-sdk-schema-fuzz-tests`. The build tree lives in the Docker volume `aws-sdk-cpp-schema-fuzz-build`. + +## Commands + +| Command | What it does | +|---|---| +| `tools/scripts/run_fuzz.sh unit [gtest-args]` | Unit-test mode: every target replays its seeds, `corpus/` and `findings/`, plus a short random burst; plain gtests run too. ASan + UBSan (`SCHEMA_FUZZ_UNIT_SANITIZERS`, on by default), no coverage instrumentation. | +| `tools/scripts/run_fuzz.sh fuzz ` | One target, coverage-guided, ASan + UBSan, e.g. `10m`. | +| `tools/scripts/run_fuzz.sh campaign [duration] [jobs]` | Every `FUZZ_TEST` in parallel, `` each (default `30m`); see [Campaign](#campaign). | +| `tools/scripts/run_fuzz.sh repro ` | Runs one raw byte file as the only seed, under sanitizers. Byte targets only; typed targets are rejected with a pointer to `replay`. | +| `tools/scripts/run_fuzz.sh replay ` | Replays a FuzzTest reproducer file with `FUZZTEST_REPLAY=/src/` against the fuzzing-mode binary, under sanitizers. Works for every target. | + +Example: `run_fuzz.sh unit --gtest_filter='JsonCodecFuzz.*'`. + +`fuzz`, `campaign`, `repro` and `replay` pass `--stack_limit_kb=2304`; all but `repro` also pass +`--time_limit_per_input=10s`. Durations are ``; anything else is a usage error. Arguments +reach the container as positional parameters, so spaces, globs and quotes in gtest filters survive. On +macOS these modes run under `caffeinate -i`, because host idle sleep freezes the container clock and +FuzzTest then reports a false per-input timeout (closing the lid on battery still sleeps the host). + +FuzzTest's default stack limit (128 KiB) is too small for the seeds, which nest `MAX_NESTING_DEPTH` deep, so the script +raises it to 2304 KiB. A "stack limit exceeded" failure therefore means the stack used per nesting level has grown. + +`SCHEMA_FUZZ_LOCALE` sets `LC_NUMERIC` before the SDK starts, e.g. `SCHEMA_FUZZ_LOCALE=de_DE.UTF-8 run_fuzz.sh campaign` +runs every target with a ',' decimal point. The image generates `de_DE.UTF-8`. + +On a Linux host the container runs as the calling user, so files it writes under `tests/fuzz/reproducers/` stay yours; +the first such run hands an existing root-owned build volume over to that user. + +### `replay` versus `repro` + +Reproducer files for failures found by `fuzz` and `campaign` are written to `tests/fuzz/reproducers/` +(gitignored) in FuzzTest's serialized format. + +- `replay ` feeds that file back through FuzzTest, so it reproduces any target, byte + or typed. Use it for everything under `reproducers/`. +- `repro ` sets `SCHEMA_FUZZ_REPRO_FILE`, which makes a raw byte file the only seed of + a byte target. Typed targets (`RoundTripsTypedValues`, `SerializeRequestIsWellFormedForm`, + `DocumentRoundTrips`, `NonJsonCodecsRejectDocuments`, `GeneratedValuesAre...`) never read it, so + `repro` refuses them rather than silently testing nothing. The byte targets are listed once, in + `BYTE_TARGET_PATTERN` in `run_fuzz.sh`. + +## Campaign + +`run_fuzz.sh campaign [duration] [jobs]` builds once, lists the targets with `--list_fuzz_tests`, and +runs one process per target in a job pool, so a finding in one target does not stop the others. + +- Each process gets `--rss_limit_mb` (default 2560, override with `FUZZ_CAMPAIGN_RSS_MB`), + `--time_limit_per_input=10s` and `--stack_limit_kb=2304`. FuzzTest itself has no RSS limit by default, + and an ASan build's memory grows with its in-process corpus, so the cap protects the container rather + than looking for leaks. +- Jobs default to the smaller of `min(nproc-2, 8)` and the number of processes whose RSS cap fits in 85% of + the container's memory, computed inside the container. Wall-clock time is roughly + `ceil(targets / jobs) x duration`; give the container more memory to run more jobs. An explicit `jobs` + above what fits is honoured with a warning. +- Logs go to `/build/campaign//.log` in the build volume. Each target writes + its reproducers to `tests/fuzz/reproducers//`; a failed target's log is copied beside them as + `campaign.log`. +- The summary prints one PASS or FAIL line per target with corpus size, edges covered and runs per + second. The exit status is non-zero if any target failed. +- The corpus is not persisted between runs: only Centipede writes `--corpus_database`, and this build + runs FuzzTest in-process. Inputs worth keeping go under `corpus/`. +- `RSS limit exceeded` after many minutes is usually fuzzer-state growth, not a codec bug. Triage it by + replaying the reproducer (`run_fuzz.sh replay ...`) and checking the peak RSS of that single input; only a + single input that needs hundreds of MB is a finding. +- A 20-second smoke run: `run_fuzz.sh campaign 20s 8`. + +## Targets and oracles + +| Target | Input | Passes when | +|---|---|---| +| `{Json,Cbor,Xml}CodecFuzz.DeserializeNeverCrashes` | arbitrary bytes | deserializing against every fuzz schema returns, with no sanitizer fault and no hang | +| `{Json,Cbor,Xml}CodecFuzz.ReserializeIsStable` | arbitrary bytes | anything we can read, we can write and read back unchanged; after that first write the output is a fixed point (writing what we read back gives the same bytes) | +| `{Json,Cbor,Xml}CodecFuzz.RoundTripsTypedValues` | choice tape | a generated value survives serialize, then deserialize | +| `QueryProtocolFuzz.SerializeRequestIsWellFormedForm` | choice tape | output is valid form encoding with one `=` per field, no duplicate keys, and exactly one `Action` and one `Version` with the expected values; its other fields are exactly the ones the value implies, each under the name the awsQuery or ec2Query rules give it (`QueryProtocolOracle.*` pins those names), every string, map key, integer, boolean and blob field holds its exact decoded text, every floating-point field parses back (in the C locale) to the same number, and every timestamp to the same instant (whole seconds for date-time and http-date). `QueryProtocolOracle.*` also runs the seed tapes under `de_DE.UTF-8`. An empty awsQuery list is a bare key; an empty ec2Query list and any empty map are omitted | +| `QueryProtocolFuzz.DeserializeResponseNeverCrashes` | arbitrary bytes | as above, through both query protocols; seeds come bare and inside the `` wrapper | +| `QueryProtocolFuzz.DeserializeResponseIsStable` | arbitrary bytes | two reads of the same body agree: awsQuery with the `Result` wrapper, ec2Query bare | +| `JsonDocumentFuzz.ReadDocumentNeverCrashes` | arbitrary bytes | `ReadDocument` returns, in a struct member and at top level | +| `JsonReferenceFuzz.JsonMatchesReference` | arbitrary bytes | for every fuzz schema, the JSON codec accepts exactly the payloads the CRT's JSON parser parses and reads as acceptable, and reads the same value tree; see [Reference parsers](#reference-parsers) | +| `XmlReferenceFuzz.XmlMatchesReference` | arbitrary bytes | the same for the XML codec against tinyxml2 | +| `QueryProtocolFuzz.DeserializeResponseMatchesReference` | arbitrary bytes | the same for awsQuery responses, entering the `` wrapper when the root has one | +| `JsonReferenceFuzz.JsonSerializedOutputMatchesReference` | choice tape | the CRT's JSON parser accepts what the JSON codec writes for a generated value and reads it back as that value | +| `XmlReferenceFuzz.XmlSerializedOutputMatchesReference` | choice tape | the same for the XML codec against tinyxml2 | +| `JsonDocumentFuzz.DocumentRoundTrips` | choice tape | a generated document survives write, then read | +| `JsonDocumentFuzz.NonJsonCodecsRejectDocuments` | choice tape | CBOR, XML and query refuse to serialize a document | +| `FuzzDomains.GeneratedValuesAreDeterministicBoundedAndJsonSerializable` | choice tape | the generator is deterministic, depth-bounded, and always JSON-serializable | + +Plain gtests that also run in `unit` mode: `FuzzValue.*`, `DynamicStruct.*`, `FuzzSchemas.*`, +`FuzzDomains.*` (non-fuzz), `CodecRegression.*`, `ReferenceParity.*`, `QueryProtocolOracle.*`, `ScalarText.*`. + +Byte targets take a `std::string` of up to `MAX_PAYLOAD_BYTES` (64 KiB) with a per-format AFL dictionary. FuzzTest drops a +larger seed with only a warning, so generated seeds over the limit are left out, and a corpus file, repro file or deep +payload over it aborts. Typed targets +take a `ChoiceTape` (a `vector` of choices plus a `vector` pool), because FuzzTest +domains cannot be built from a runtime `Schema`; the generator consumes the tape to build a value. + +Deserializers never throw. Structural malformation (including trailing data after the root), exceeding +`MAX_NESTING_DEPTH`, and a value the target type cannot represent (an out-of-range or non-integral number, a +non-finite document number, an epoch timestamp beyond the system clock) come back as an error +(`DeserializeShape` returns a non-empty `Optional`). A skipped value (an unmodeled member, or one of +the wrong type) is held to the same grammar, so malformed JSON or XML anywhere in the payload is an error. A well-formed +value of the wrong type leaves the member absent without an error. A union with more than one non-null modeled member set +is an error; null and unmodeled members do not count. Every comparison also fails on a decoded union holding two members. + +## Reference parsers + +The reference targets judge the codecs against the parsers the legacy SDK read with: the CRT's JSON parser (cJSON, called +through `cJSON_ParseWithOpts` so trailing data is rejected) and tinyxml2 11.0.0, the version core vendors, configured as +the legacy `XmlDocument` was (entities processed, whitespace preserved). tinyxml2 is fetched by `tests/fuzz/CMakeLists.txt` +and linked only into the fuzz binary. There is no CBOR reference. + +A reference supplies the tree and the decoded text; `ReferenceParity.cpp` walks that tree along the schema by the +protocol's rules (member names and `jsonName`/`xmlName`, attributes, flattened and wrapped lists and maps, which repeated +member wins, what a value of the wrong type does) and turns scalar text into values as legacy clients did: XML number, +boolean and timestamp text is trimmed, and XML booleans are `true` or `false` in any case. It parses numbers with its own +parsers (`ScalarText.cpp`), not the codecs', so a bug in a codec's number parsing shows up as a mismatch. Where the read +contract deliberately differs from legacy (an out-of-range or non-integral number fails; text a number cannot be read +from is absent rather than 0), the walker follows the contract. +The result, a `ReferenceReading`, is checked three ways: + +- **reads**: on every fuzz schema the codec accepts a payload exactly when the reference parses it and its reading does not + require a failure (out-of-range or non-integral number, a union with two members set, an epoch timestamp beyond the + clock, a second root element, nesting beyond `MAX_NESTING_DEPTH`), and reads the same value tree; +- **writes**: the reference accepts what the codec writes for a generated value, and reads it back as that value under the + codec's round-trip tolerances; +- **awsQuery responses**: reads, through `AwsQueryProtocol`. + +The walker models where the references are laxer than the codecs, which reject: tinyxml2 drops a closing tag that closes +no element (a reading that must fail), and its tokenizer tolerates other malformed markup (whitespace before an element +name, junk in an end tag, `\v` and `\f` between elements), so whether a document holding such markup must be read is not +judged (`TinyXmlLeniency`). Every other rejection and every acceptance is judged. + +The walker follows the codecs where they part from the legacy readers: a repeated JSON member or map key keeps its last +value (cJSON's lookup returned the first), a repeated XML element keeps its first, and a repeated key inside a JSON document +keeps its first. + +What a reference cannot judge is skipped, each for a stated reason in `ReferenceParity.cpp`: + +- input containing NUL or only whitespace: both references read a NUL-terminated copy, and reject blank input that the + codecs read as an empty struct; a string with an escaped NUL (`\u0000`), which cJSON truncates; +- XML nested deeper than 500 elements: tinyxml2's own limit, below `MAX_NESTING_DEPTH`; +- the exact value of a JSON integer beyond 2^53, since cJSON reads numbers as doubles: it is compared after rounding + (`CodecTolerance::ReadByCrtJson`), and whether a long cJSON reads as ±2^63 fits is not judged; +- XML text holding a reference tinyxml2 mangles (an unrecognised entity, or one decoding to U+0000), and text beside an + element, comment or processing instruction; +- the value of whitespace-only XML text: tinyxml2 drops it, the codec deliberately keeps it; +- whether the JSON codec must accept a payload holding a control byte below space other than tab, LF and CR (cJSON skips + them all as whitespace) or a `\u` not followed by four hex digits (cJSON reads it as U+0000); +- the type and sign of a JSON document number: cJSON cannot tell `1` from `1.0` or `-0` from `-0.0`, so document numbers + compare by value. + +## Layout + +- `harness/FuzzValue.*` - the value tree, plus `FindMismatch` and the per-codec `CodecTolerance`. +- `harness/DynamicStruct.*` - one `SerializableStruct` that reads and writes any schema. +- `harness/FuzzSchemas.*` - the fixed schema registry every byte input is replayed against. +- `harness/FuzzDomains.*` - the choice tape, value and document generators, seeds, dictionary loading. +- `harness/CodecFuzzTargets.*` - the three oracles, shared by the codec targets. +- `harness/ReferenceParity.*` - the reference readers and the read, write and awsQuery comparisons. +- `harness/ScalarText.*` - the oracles' own integer, double and epoch-seconds parsers. +- `targets/*.cpp` - one file per codec or protocol: a thin wrapper plus `FUZZ_TEST` registrations. +- `dictionaries/*.dict` - AFL dictionaries from smithy-java (Apache-2.0). +- `corpus//` - raw byte regression inputs, replayed on every `unit` run. +- `findings//` - `.tape` records of failing typed inputs (`choices: {...}` and `strings: {...}` lines, + strings with C escapes). Each typed target adds its records to its seeds, and `CodecRegression.*` replays the codec + ones, so a recorded finding is re-run on every `unit` run. A malformed record aborts. Typed targets never load files + from `corpus/`. + +Corpus files must sit in a directory named for a real `Suite.Test` and stay under 1 KiB. Binary inputs (CBOR) are stored +base64-encoded in a file ending `.b64` and decoded when the seeds load; text inputs (JSON, XML) are stored as written. +`repro` accepts either form. Deep inputs are +generated instead: every byte target is also seeded with its codec's encoding of `Node` nested +`DEEP_SEED_DEPTH` (= `MAX_NESTING_DEPTH`) structures deep (`DeepNodePayload` in `FuzzDomains.cpp`), which exercises the +recursive read and write paths at the limit. JSON targets also get a `DocumentValue` whose document is nested lists to +the same depth (`DeepDocumentPayload`). + +The generator cannot build a timestamp outside what `system_clock` holds, because `DateTime` cannot represent one; on +Linux that ends in 2262. The `ReserializeIsStable` corpus therefore carries years 0001 and 9999 in every timestamp format, +so the out-of-range read paths still run. + +The XML reader does not check the root element's name, so every XML corpus file is read against every schema; a file +named for a float bug can hold an `` body that reaches `FloatValue`. + +## Adding a schema or a target + +Add a schema to `FuzzSchemas.cpp` and every byte target picks it up; keep ids free of `#`, because +the XML serializer uses the shape id as the root element name. For a new codec, add a +`targets/Fuzz.cpp` that calls the shared oracles with that codec and its tolerance. New files +are globbed, so re-run the script (it re-configures CMake). + +Seeds passed to `.WithSeeds` must be lambdas (`.WithSeeds([] { ... })`). `FUZZ_TEST` registration +runs before `Aws::InitAPI`, so an eager seed list would build schema objects too early. + +Fuzzing mode compiles with `-fno-jump-tables`. On AArch64, a late SimplifyCFG pass builds switch lookup +tables of `__sancov_cntrs` pointers outside COMDAT groups, which fails to link across translation units +with "defined in discarded section". + +`ENABLE_SCHEMA_FUZZ_TESTS` cannot be combined with `ENABLE_TESTING`: the vendored gtest and FuzzTest's +gtest collide, and CMake stops with an error. + +## When a target fails + +1. Save the input. FuzzTest prints it, and byte targets also print it base64-encoded. For a byte + target write the input to `corpus//`, as `.b64` if it is binary; for a typed target record + the printed `choices`/`strings` in `findings//.tape`, strings included in full. +2. Reproduce: `run_fuzz.sh replay tests/fuzz/reproducers/` for any target, or, for a + byte target, `run_fuzz.sh repro tests/fuzz/corpus//`. +3. If the codec is wrong, add a failing test in + `tests/aws-cpp-sdk-core-tests/smithy/client/schema/` first, then fix it, then keep the corpus file. +4. If the behaviour is deliberate and an existing core unit test already asserts it, add a named + flag to `CodecTolerance` citing that test, plus a `FuzzValueTest` case. Do not widen an existing flag. +5. If the harness is wrong, fix it and add a harness test. +6. Re-run the target for at least 10 minutes. + +## Byte/Short narrowing + +`ShapeDeserializer` has no `ReadByte`/`ReadShort`: `ReadInteger` covers them, and rejecting +out-of-range values is the caller's job. `DynamicStruct::ReadValue` is the reference for what +generated code must do. + +## Tolerances + +One row per `CodecTolerance` flag, plus the null-document generator rule. Each row names the test or rule +that pins the behaviour. + +| Codec | Tolerance | Pinned by (test or rule) | +|---|---|---| +| CBOR, and XML/JSON date-time and http-date members | timestamps read back as the expected value truncated toward zero to whole seconds; sub-second digits must not leak and rounding up fails (`DateTime::Seconds()` and `system_clock::to_time_t` both truncate toward zero) | `CborShapeSerializerTest.TimestampFractionalSeconds`; `FuzzValue.SecondsPrecisionRequiresTruncationTowardZero` | +| JSON, and XML epoch-seconds members | an epoch-seconds timestamp may read back as exactly what `DateTime(double)` decodes its text to, which can be 1 ms toward zero; no other difference is accepted | `SerdeTraitsTest.EpochSecondsToDateTimeMatchesLegacyDateTimeDouble`; `FuzzValue.EpochSecondsAcceptsOnlyTheLegacyTruncatedDecode` | +| CBOR | pre-epoch timestamps (<= -1000 ms) read back absent, in members, list elements and map values | `CborShapeDeserializerTest.TimestampNegative*ReturnsEmpty`; `FuzzValue.CborPreEpochDropBoundaryIsMinusOneSecond` | +| CBOR | expected `-0.0` may read back as `+0.0`, never the reverse: the CRT's `aws_cbor_encoder_write_float` encodes integral doubles as integers | `CborShapeSerializerTest.NegativeZeroFloatIsEncodedAsIntegerZero`; `FuzzValue.CborMayDropNegativeZeroSign` | +| XML | an empty list or map may read back absent; XML cannot distinguish the two | `FuzzValue.XmlMayDropEmptyContainers` | +| JSON | documents compare an integer to an integer and a double to a double (NaN equals NaN, zero keeps its sign); `JsonShapeSerializer::FormatDouble` keeps a `.0` marker so a double never reads back as an integer | `FuzzValue.DocumentIntegersRequireIntegersOfTheSameValue`, `FuzzValue.DocumentDoublesRequireDoublesWithNanAndSignedZeroPreserved` | +| JSON | a non-finite document double may read back as the string `"NaN"`, `"Infinity"` or `"-Infinity"`, which is how JSON writes it | `JsonShapeSerializerTest.NonFiniteDocumentDoubleIsWrittenAsItsString`; `FuzzValue.NonFiniteDocumentDoublesMayReadBackAsTheirStrings` | +| all | a null document directly in a member, list or map slot is never generated: a null member reads back absent and nothing is `@sparse`. Nulls nested inside a document are generated and round-trip | generator rule in `FuzzDomains` | diff --git a/tests/fuzz/corpus/CborCodecFuzz.DeserializeNeverCrashes/indefinite-bytes-value.b64 b/tests/fuzz/corpus/CborCodecFuzz.DeserializeNeverCrashes/indefinite-bytes-value.b64 new file mode 100644 index 000000000000..2719c51f635c --- /dev/null +++ b/tests/fuzz/corpus/CborCodecFuzz.DeserializeNeverCrashes/indefinite-bytes-value.b64 @@ -0,0 +1 @@ +oWV2YWx1ZV9CAQJBA/8= diff --git a/tests/fuzz/corpus/CborCodecFuzz.DeserializeNeverCrashes/indefinite-text-key.b64 b/tests/fuzz/corpus/CborCodecFuzz.DeserializeNeverCrashes/indefinite-text-key.b64 new file mode 100644 index 000000000000..8200f944b4e5 --- /dev/null +++ b/tests/fuzz/corpus/CborCodecFuzz.DeserializeNeverCrashes/indefinite-text-key.b64 @@ -0,0 +1 @@ +oX9jdmFsYnVl/2F4 diff --git a/tests/fuzz/corpus/CborCodecFuzz.DeserializeNeverCrashes/indefinite-text-value.b64 b/tests/fuzz/corpus/CborCodecFuzz.DeserializeNeverCrashes/indefinite-text-value.b64 new file mode 100644 index 000000000000..c9180e50cfbf --- /dev/null +++ b/tests/fuzz/corpus/CborCodecFuzz.DeserializeNeverCrashes/indefinite-text-value.b64 @@ -0,0 +1 @@ +oWV2YWx1ZX9iYWJgYWP/ diff --git a/tests/fuzz/corpus/CborCodecFuzz.ReserializeIsStable/epoch-tag-year-0001.b64 b/tests/fuzz/corpus/CborCodecFuzz.ReserializeIsStable/epoch-tag-year-0001.b64 new file mode 100644 index 000000000000..213707adbcc2 --- /dev/null +++ b/tests/fuzz/corpus/CborCodecFuzz.ReserializeIsStable/epoch-tag-year-0001.b64 @@ -0,0 +1 @@ +oWV2YWx1ZcE7AAAADneR9v8= diff --git a/tests/fuzz/corpus/CborCodecFuzz.ReserializeIsStable/epoch-tag-year-9999.b64 b/tests/fuzz/corpus/CborCodecFuzz.ReserializeIsStable/epoch-tag-year-9999.b64 new file mode 100644 index 000000000000..6e1b8edbfba6 --- /dev/null +++ b/tests/fuzz/corpus/CborCodecFuzz.ReserializeIsStable/epoch-tag-year-9999.b64 @@ -0,0 +1 @@ +oWV2YWx1ZcEbAAAAOv/0QX8= diff --git a/tests/fuzz/corpus/CborCodecFuzz.ReserializeIsStable/indefinite-bytes-value.b64 b/tests/fuzz/corpus/CborCodecFuzz.ReserializeIsStable/indefinite-bytes-value.b64 new file mode 100644 index 000000000000..2719c51f635c --- /dev/null +++ b/tests/fuzz/corpus/CborCodecFuzz.ReserializeIsStable/indefinite-bytes-value.b64 @@ -0,0 +1 @@ +oWV2YWx1ZV9CAQJBA/8= diff --git a/tests/fuzz/corpus/CborCodecFuzz.ReserializeIsStable/indefinite-text-key.b64 b/tests/fuzz/corpus/CborCodecFuzz.ReserializeIsStable/indefinite-text-key.b64 new file mode 100644 index 000000000000..8200f944b4e5 --- /dev/null +++ b/tests/fuzz/corpus/CborCodecFuzz.ReserializeIsStable/indefinite-text-key.b64 @@ -0,0 +1 @@ +oX9jdmFsYnVl/2F4 diff --git a/tests/fuzz/corpus/CborCodecFuzz.ReserializeIsStable/indefinite-text-value.b64 b/tests/fuzz/corpus/CborCodecFuzz.ReserializeIsStable/indefinite-text-value.b64 new file mode 100644 index 000000000000..c9180e50cfbf --- /dev/null +++ b/tests/fuzz/corpus/CborCodecFuzz.ReserializeIsStable/indefinite-text-value.b64 @@ -0,0 +1 @@ +oWV2YWx1ZX9iYWJgYWP/ diff --git a/tests/fuzz/corpus/CborCodecFuzz.ReserializeIsStable/union-two-members.b64 b/tests/fuzz/corpus/CborCodecFuzz.ReserializeIsStable/union-two-members.b64 new file mode 100644 index 000000000000..d1dbc66cfcb9 --- /dev/null +++ b/tests/fuzz/corpus/CborCodecFuzz.ReserializeIsStable/union-two-members.b64 @@ -0,0 +1 @@ +oWV2YWx1ZaJtc3RyaW5nVmFyaWFudGFhamludFZhcmlhbnQB diff --git a/tests/fuzz/corpus/JsonCodecFuzz.ReserializeIsStable/date-time-and-http-date-year-0001 b/tests/fuzz/corpus/JsonCodecFuzz.ReserializeIsStable/date-time-and-http-date-year-0001 new file mode 100644 index 000000000000..adc8615ed17d --- /dev/null +++ b/tests/fuzz/corpus/JsonCodecFuzz.ReserializeIsStable/date-time-and-http-date-year-0001 @@ -0,0 +1 @@ +{"dateTime":"0001-01-01T00:00:00Z","httpDate":"Mon, 01 Jan 0001 00:00:00 GMT"} \ No newline at end of file diff --git a/tests/fuzz/corpus/JsonCodecFuzz.ReserializeIsStable/date-time-and-http-date-year-9999 b/tests/fuzz/corpus/JsonCodecFuzz.ReserializeIsStable/date-time-and-http-date-year-9999 new file mode 100644 index 000000000000..7500dc6a245d --- /dev/null +++ b/tests/fuzz/corpus/JsonCodecFuzz.ReserializeIsStable/date-time-and-http-date-year-9999 @@ -0,0 +1 @@ +{"dateTime":"9999-12-31T23:59:59Z","httpDate":"Fri, 31 Dec 9999 23:59:59 GMT"} \ No newline at end of file diff --git a/tests/fuzz/corpus/JsonCodecFuzz.ReserializeIsStable/document-number-overflow b/tests/fuzz/corpus/JsonCodecFuzz.ReserializeIsStable/document-number-overflow new file mode 100644 index 000000000000..0426c00cd23a --- /dev/null +++ b/tests/fuzz/corpus/JsonCodecFuzz.ReserializeIsStable/document-number-overflow @@ -0,0 +1 @@ +{"value":1E310} \ No newline at end of file diff --git a/tests/fuzz/corpus/JsonCodecFuzz.ReserializeIsStable/document-number-overflow-no-colon b/tests/fuzz/corpus/JsonCodecFuzz.ReserializeIsStable/document-number-overflow-no-colon new file mode 100644 index 000000000000..5c40d4b8206d --- /dev/null +++ b/tests/fuzz/corpus/JsonCodecFuzz.ReserializeIsStable/document-number-overflow-no-colon @@ -0,0 +1 @@ +{"value"1E310} \ No newline at end of file diff --git a/tests/fuzz/corpus/JsonCodecFuzz.ReserializeIsStable/epoch-seconds-year-0001 b/tests/fuzz/corpus/JsonCodecFuzz.ReserializeIsStable/epoch-seconds-year-0001 new file mode 100644 index 000000000000..7f1ad8b2cb7f --- /dev/null +++ b/tests/fuzz/corpus/JsonCodecFuzz.ReserializeIsStable/epoch-seconds-year-0001 @@ -0,0 +1 @@ +{"value":-62135596800} \ No newline at end of file diff --git a/tests/fuzz/corpus/JsonCodecFuzz.ReserializeIsStable/epoch-seconds-year-9999 b/tests/fuzz/corpus/JsonCodecFuzz.ReserializeIsStable/epoch-seconds-year-9999 new file mode 100644 index 000000000000..2592843e6e84 --- /dev/null +++ b/tests/fuzz/corpus/JsonCodecFuzz.ReserializeIsStable/epoch-seconds-year-9999 @@ -0,0 +1 @@ +{"value":253402300799} \ No newline at end of file diff --git a/tests/fuzz/corpus/JsonCodecFuzz.ReserializeIsStable/union-two-members b/tests/fuzz/corpus/JsonCodecFuzz.ReserializeIsStable/union-two-members new file mode 100644 index 000000000000..d806e3474020 --- /dev/null +++ b/tests/fuzz/corpus/JsonCodecFuzz.ReserializeIsStable/union-two-members @@ -0,0 +1 @@ +{"value":{"stringVariant":"a","intVariant":1}} \ No newline at end of file diff --git a/tests/fuzz/corpus/JsonReferenceFuzz.JsonMatchesReference/document-negative-zero-integer b/tests/fuzz/corpus/JsonReferenceFuzz.JsonMatchesReference/document-negative-zero-integer new file mode 100644 index 000000000000..e17a31214818 --- /dev/null +++ b/tests/fuzz/corpus/JsonReferenceFuzz.JsonMatchesReference/document-negative-zero-integer @@ -0,0 +1 @@ +{"value":{"":-0}} \ No newline at end of file diff --git a/tests/fuzz/corpus/JsonReferenceFuzz.JsonMatchesReference/leading-byte-order-mark.b64 b/tests/fuzz/corpus/JsonReferenceFuzz.JsonMatchesReference/leading-byte-order-mark.b64 new file mode 100644 index 000000000000..cc2337aad847 --- /dev/null +++ b/tests/fuzz/corpus/JsonReferenceFuzz.JsonMatchesReference/leading-byte-order-mark.b64 @@ -0,0 +1 @@ +77u/e30= diff --git a/tests/fuzz/corpus/JsonReferenceFuzz.JsonMatchesReference/lone-high-surrogate b/tests/fuzz/corpus/JsonReferenceFuzz.JsonMatchesReference/lone-high-surrogate new file mode 100644 index 000000000000..2da843341044 --- /dev/null +++ b/tests/fuzz/corpus/JsonReferenceFuzz.JsonMatchesReference/lone-high-surrogate @@ -0,0 +1 @@ +{"value":"\ud800"} \ No newline at end of file diff --git a/tests/fuzz/corpus/JsonReferenceFuzz.JsonMatchesReference/lone-low-surrogate b/tests/fuzz/corpus/JsonReferenceFuzz.JsonMatchesReference/lone-low-surrogate new file mode 100644 index 000000000000..9e97c2f5639c --- /dev/null +++ b/tests/fuzz/corpus/JsonReferenceFuzz.JsonMatchesReference/lone-low-surrogate @@ -0,0 +1 @@ +{"value":"\ude00"} \ No newline at end of file diff --git a/tests/fuzz/corpus/JsonReferenceFuzz.JsonMatchesReference/skipped-array-missing-comma b/tests/fuzz/corpus/JsonReferenceFuzz.JsonMatchesReference/skipped-array-missing-comma new file mode 100644 index 000000000000..c9a76c113899 --- /dev/null +++ b/tests/fuzz/corpus/JsonReferenceFuzz.JsonMatchesReference/skipped-array-missing-comma @@ -0,0 +1 @@ +{"unknown":[1 2],"value":"a"} \ No newline at end of file diff --git a/tests/fuzz/corpus/JsonReferenceFuzz.JsonMatchesReference/skipped-mismatched-bracket b/tests/fuzz/corpus/JsonReferenceFuzz.JsonMatchesReference/skipped-mismatched-bracket new file mode 100644 index 000000000000..06ee48258470 --- /dev/null +++ b/tests/fuzz/corpus/JsonReferenceFuzz.JsonMatchesReference/skipped-mismatched-bracket @@ -0,0 +1 @@ +{"unknown":[1},"value":"a"} \ No newline at end of file diff --git a/tests/fuzz/corpus/JsonReferenceFuzz.JsonMatchesReference/skipped-object-missing-colon b/tests/fuzz/corpus/JsonReferenceFuzz.JsonMatchesReference/skipped-object-missing-colon new file mode 100644 index 000000000000..ba26f0c4fffc --- /dev/null +++ b/tests/fuzz/corpus/JsonReferenceFuzz.JsonMatchesReference/skipped-object-missing-colon @@ -0,0 +1 @@ +{"unknown":{"a" 1},"value":"a"} \ No newline at end of file diff --git a/tests/fuzz/corpus/JsonReferenceFuzz.JsonMatchesReference/skipped-trailing-comma b/tests/fuzz/corpus/JsonReferenceFuzz.JsonMatchesReference/skipped-trailing-comma new file mode 100644 index 000000000000..d62dd2736e4e --- /dev/null +++ b/tests/fuzz/corpus/JsonReferenceFuzz.JsonMatchesReference/skipped-trailing-comma @@ -0,0 +1 @@ +{"value":[1,]} \ No newline at end of file diff --git a/tests/fuzz/corpus/JsonReferenceFuzz.JsonMatchesReference/trailing-control-byte.b64 b/tests/fuzz/corpus/JsonReferenceFuzz.JsonMatchesReference/trailing-control-byte.b64 new file mode 100644 index 000000000000..e366dca2e75d --- /dev/null +++ b/tests/fuzz/corpus/JsonReferenceFuzz.JsonMatchesReference/trailing-control-byte.b64 @@ -0,0 +1 @@ +e30B diff --git a/tests/fuzz/corpus/JsonReferenceFuzz.JsonMatchesReference/unicode-escape-without-hex-digits.b64 b/tests/fuzz/corpus/JsonReferenceFuzz.JsonMatchesReference/unicode-escape-without-hex-digits.b64 new file mode 100644 index 000000000000..4a5713ea5066 --- /dev/null +++ b/tests/fuzz/corpus/JsonReferenceFuzz.JsonMatchesReference/unicode-escape-without-hex-digits.b64 @@ -0,0 +1 @@ +eyIiOiJcdQkJCQkifQ== diff --git a/tests/fuzz/corpus/QueryProtocolFuzz.DeserializeResponseMatchesReference/byte-order-mark-after-whitespace.b64 b/tests/fuzz/corpus/QueryProtocolFuzz.DeserializeResponseMatchesReference/byte-order-mark-after-whitespace.b64 new file mode 100644 index 000000000000..b434b83992b5 --- /dev/null +++ b/tests/fuzz/corpus/QueryProtocolFuzz.DeserializeResponseMatchesReference/byte-order-mark-after-whitespace.b64 @@ -0,0 +1 @@ +Ce+7vzw6Lz4= diff --git a/tests/fuzz/corpus/QueryProtocolFuzz.DeserializeResponseMatchesReference/vertical-tab-after-root.b64 b/tests/fuzz/corpus/QueryProtocolFuzz.DeserializeResponseMatchesReference/vertical-tab-after-root.b64 new file mode 100644 index 000000000000..ab43a6cc061b --- /dev/null +++ b/tests/fuzz/corpus/QueryProtocolFuzz.DeserializeResponseMatchesReference/vertical-tab-after-root.b64 @@ -0,0 +1 @@ +PDovPgs= diff --git a/tests/fuzz/corpus/QueryProtocolFuzz.DeserializeResponseMatchesReference/whitespace-before-element-name.b64 b/tests/fuzz/corpus/QueryProtocolFuzz.DeserializeResponseMatchesReference/whitespace-before-element-name.b64 new file mode 100644 index 000000000000..2fb91fdd6fda --- /dev/null +++ b/tests/fuzz/corpus/QueryProtocolFuzz.DeserializeResponseMatchesReference/whitespace-before-element-name.b64 @@ -0,0 +1 @@ +PAk6Lz4= diff --git a/tests/fuzz/corpus/XmlCodecFuzz.ReserializeIsStable/attribute-name-inside-quoted-value b/tests/fuzz/corpus/XmlCodecFuzz.ReserializeIsStable/attribute-name-inside-quoted-value new file mode 100644 index 000000000000..ec6044b939e7 --- /dev/null +++ b/tests/fuzz/corpus/XmlCodecFuzz.ReserializeIsStable/attribute-name-inside-quoted-value @@ -0,0 +1 @@ +" \ No newline at end of file diff --git a/tests/fuzz/corpus/XmlCodecFuzz.ReserializeIsStable/date-time-year-0001 b/tests/fuzz/corpus/XmlCodecFuzz.ReserializeIsStable/date-time-year-0001 new file mode 100644 index 000000000000..3724f8ba0cfd --- /dev/null +++ b/tests/fuzz/corpus/XmlCodecFuzz.ReserializeIsStable/date-time-year-0001 @@ -0,0 +1 @@ +0001-01-01T00:00:00Z \ No newline at end of file diff --git a/tests/fuzz/corpus/XmlCodecFuzz.ReserializeIsStable/date-time-year-9999 b/tests/fuzz/corpus/XmlCodecFuzz.ReserializeIsStable/date-time-year-9999 new file mode 100644 index 000000000000..2267c087f7df --- /dev/null +++ b/tests/fuzz/corpus/XmlCodecFuzz.ReserializeIsStable/date-time-year-9999 @@ -0,0 +1 @@ +9999-12-31T23:59:59Z \ No newline at end of file diff --git a/tests/fuzz/corpus/XmlCodecFuzz.ReserializeIsStable/epoch-and-http-date-year-9999 b/tests/fuzz/corpus/XmlCodecFuzz.ReserializeIsStable/epoch-and-http-date-year-9999 new file mode 100644 index 000000000000..20535ef6c333 --- /dev/null +++ b/tests/fuzz/corpus/XmlCodecFuzz.ReserializeIsStable/epoch-and-http-date-year-9999 @@ -0,0 +1 @@ +253402300799Fri, 31 Dec 9999 23:59:59 GMT \ No newline at end of file diff --git a/tests/fuzz/corpus/XmlCodecFuzz.ReserializeIsStable/int-max-map-value-read-as-float-six-digit-precision-loss b/tests/fuzz/corpus/XmlCodecFuzz.ReserializeIsStable/int-max-map-value-read-as-float-six-digit-precision-loss new file mode 100644 index 000000000000..34f09804f76f --- /dev/null +++ b/tests/fuzz/corpus/XmlCodecFuzz.ReserializeIsStable/int-max-map-value-read-as-float-six-digit-precision-loss @@ -0,0 +1 @@ +trail 2147483647 \ No newline at end of file diff --git a/tests/fuzz/corpus/XmlCodecFuzz.ReserializeIsStable/int-max-read-as-float-six-digit-precision-loss b/tests/fuzz/corpus/XmlCodecFuzz.ReserializeIsStable/int-max-read-as-float-six-digit-precision-loss new file mode 100644 index 000000000000..27ab77750d0c --- /dev/null +++ b/tests/fuzz/corpus/XmlCodecFuzz.ReserializeIsStable/int-max-read-as-float-six-digit-precision-loss @@ -0,0 +1 @@ +2147483647 \ No newline at end of file diff --git a/tests/fuzz/corpus/XmlCodecFuzz.ReserializeIsStable/union-two-members b/tests/fuzz/corpus/XmlCodecFuzz.ReserializeIsStable/union-two-members new file mode 100644 index 000000000000..a92edde8f466 --- /dev/null +++ b/tests/fuzz/corpus/XmlCodecFuzz.ReserializeIsStable/union-two-members @@ -0,0 +1 @@ +a1 \ No newline at end of file diff --git a/tests/fuzz/corpus/XmlReferenceFuzz.XmlMatchesReference/attribute-reference-to-nul b/tests/fuzz/corpus/XmlReferenceFuzz.XmlMatchesReference/attribute-reference-to-nul new file mode 100644 index 000000000000..34e67197be3e --- /dev/null +++ b/tests/fuzz/corpus/XmlReferenceFuzz.XmlMatchesReference/attribute-reference-to-nul @@ -0,0 +1 @@ +<: attr="�"/> \ No newline at end of file diff --git a/tests/fuzz/corpus/XmlReferenceFuzz.XmlMatchesReference/attribute-references b/tests/fuzz/corpus/XmlReferenceFuzz.XmlMatchesReference/attribute-references new file mode 100644 index 000000000000..fbf9a7362305 --- /dev/null +++ b/tests/fuzz/corpus/XmlReferenceFuzz.XmlMatchesReference/attribute-references @@ -0,0 +1 @@ + \ No newline at end of file diff --git a/tests/fuzz/corpus/XmlReferenceFuzz.XmlMatchesReference/cdata-value b/tests/fuzz/corpus/XmlReferenceFuzz.XmlMatchesReference/cdata-value new file mode 100644 index 000000000000..37f841e0621a --- /dev/null +++ b/tests/fuzz/corpus/XmlReferenceFuzz.XmlMatchesReference/cdata-value @@ -0,0 +1 @@ + \ No newline at end of file diff --git a/tests/fuzz/corpus/XmlReferenceFuzz.XmlMatchesReference/cr-line-ends b/tests/fuzz/corpus/XmlReferenceFuzz.XmlMatchesReference/cr-line-ends new file mode 100644 index 000000000000..23c27766792c --- /dev/null +++ b/tests/fuzz/corpus/XmlReferenceFuzz.XmlMatchesReference/cr-line-ends @@ -0,0 +1,2 @@ +a +b c \ No newline at end of file diff --git a/tests/fuzz/corpus/XmlReferenceFuzz.XmlMatchesReference/duplicate-attribute b/tests/fuzz/corpus/XmlReferenceFuzz.XmlMatchesReference/duplicate-attribute new file mode 100644 index 000000000000..ca93991607f8 --- /dev/null +++ b/tests/fuzz/corpus/XmlReferenceFuzz.XmlMatchesReference/duplicate-attribute @@ -0,0 +1 @@ + \ No newline at end of file diff --git a/tests/fuzz/corpus/XmlReferenceFuzz.XmlMatchesReference/invalid-element-name b/tests/fuzz/corpus/XmlReferenceFuzz.XmlMatchesReference/invalid-element-name new file mode 100644 index 000000000000..d47f3bc9bd13 --- /dev/null +++ b/tests/fuzz/corpus/XmlReferenceFuzz.XmlMatchesReference/invalid-element-name @@ -0,0 +1 @@ +<1v>xa \ No newline at end of file diff --git a/tests/fuzz/corpus/XmlReferenceFuzz.XmlMatchesReference/junk-in-closing-tag.b64 b/tests/fuzz/corpus/XmlReferenceFuzz.XmlMatchesReference/junk-in-closing-tag.b64 new file mode 100644 index 000000000000..e89975b54dab --- /dev/null +++ b/tests/fuzz/corpus/XmlReferenceFuzz.XmlMatchesReference/junk-in-closing-tag.b64 @@ -0,0 +1 @@ +PHQ+PC90CTo9IiI+ diff --git a/tests/fuzz/corpus/XmlReferenceFuzz.XmlMatchesReference/numeric-character-references b/tests/fuzz/corpus/XmlReferenceFuzz.XmlMatchesReference/numeric-character-references new file mode 100644 index 000000000000..65c69434148a --- /dev/null +++ b/tests/fuzz/corpus/XmlReferenceFuzz.XmlMatchesReference/numeric-character-references @@ -0,0 +1 @@ +ABé \ No newline at end of file diff --git a/tests/fuzz/corpus/XmlReferenceFuzz.XmlMatchesReference/processing-instruction-inside-root b/tests/fuzz/corpus/XmlReferenceFuzz.XmlMatchesReference/processing-instruction-inside-root new file mode 100644 index 000000000000..8a2a1185ca6c --- /dev/null +++ b/tests/fuzz/corpus/XmlReferenceFuzz.XmlMatchesReference/processing-instruction-inside-root @@ -0,0 +1 @@ + \ No newline at end of file diff --git a/tests/fuzz/corpus/XmlReferenceFuzz.XmlMatchesReference/stray-closing-tag-after-root b/tests/fuzz/corpus/XmlReferenceFuzz.XmlMatchesReference/stray-closing-tag-after-root new file mode 100644 index 000000000000..7b4472d62f13 --- /dev/null +++ b/tests/fuzz/corpus/XmlReferenceFuzz.XmlMatchesReference/stray-closing-tag-after-root @@ -0,0 +1 @@ + \ No newline at end of file diff --git a/tests/fuzz/corpus/XmlReferenceFuzz.XmlMatchesReference/unquoted-attribute b/tests/fuzz/corpus/XmlReferenceFuzz.XmlMatchesReference/unquoted-attribute new file mode 100644 index 000000000000..42b339f6f227 --- /dev/null +++ b/tests/fuzz/corpus/XmlReferenceFuzz.XmlMatchesReference/unquoted-attribute @@ -0,0 +1 @@ +a \ No newline at end of file diff --git a/tests/fuzz/dictionaries/cbor.dict b/tests/fuzz/dictionaries/cbor.dict new file mode 100644 index 000000000000..0814e6f7f8ef --- /dev/null +++ b/tests/fuzz/dictionaries/cbor.dict @@ -0,0 +1,196 @@ +# From smithy-lang/smithy-java codecs/cbor-codec/src/fuzz/resources/dictionary/codec-fuzz.dict (Apache-2.0) +# ---- Integers (major type 0) ---- +# uint(0) +"\x00" + +# uint(1) +"\x01" + +# uint(10) +"\x0a" + +# uint(23) — boundary of “small” ints +"\x17" + +# uint(24) encoded using 1 follow byte +"\x18\x18" + +# uint(255) +"\x18\xff" + +# uint(256) +"\x19\x01\x00" + +# ---- Negative integers (major type 1) ---- +# -1 +"\x20" + +# -10 +"\x29" + +# -24 (boundary small negative int) +"\x37" + +# -25 (encoded with u8 following) +"\x38\x18" + + +# ---- Byte strings (major type 2) ---- +# byte string length 0 +"\x40" + +# byte string length 1 +"\x41" + +# byte string length 4 +"\x44" + +# byte string length 8 +"\x48" + +# byte string length 16 (u8) +"\x58\x10" + +# byte string length 256 (u16) +"\x59\x01\x00" + +# indefinite-length byte string +"\x5f" + + +# ---- Text strings (major type 3) ---- +# text string length 0 +"\x60" + +# text string length 1 +"\x61" + +# text string length 4 +"\x64" + +# text string length 8 +"\x68" + +# text string length 16 (u8) +"\x78\x10" + +# text string length 256 (u16) +"\x79\x01\x00" + +# indefinite-length text string +"\x7f" + + +# ---- Arrays (major type 4) ---- +# array(0) +"\x80" + +# array(1) +"\x81" + +# array(2) +"\x82" + +# array(3) +"\x83" + +# array(4) +"\x84" + +# array(8) +"\x88" + +# array(16) (u8) +"\x98\x10" + +# array(256) (u16) +"\x99\x01\x00" + +# indefinite-length array +"\x9f" + + +# ---- Maps (major type 5) ---- +# map(0) +"\xa0" + +# map(1) +"\xa1" + +# map(2) +"\xa2" + +# map(3) +"\xa3" + +# map(4) +"\xa4" + +# map(8) +"\xa8" + +# map(16) (u8) +"\xb8\x10" + +# map(256) (u16) +"\xb9\x01\x00" + +# indefinite-length map +"\xbf" + + +# ---- Tags (major type 6) ---- +# tag(0): datetime string +"\xc0" + +# tag(1): epoch datetime +"\xc1" + +# tag(2): positive bignum +"\xc2" + +# tag(3): negative bignum +"\xc3" + +# tag(32): URI +"\xd8\x20" + +# tag(55799) — common CBOR self-describe magic +"\xd9\xd9\xf7" + + +# ---- Simple values, floats, break (major type 7) ---- +# false +"\xf4" + +# true +"\xf5" + +# null +"\xf6" + +# undefined +"\xf7" + +# half float header +"\xf9" + +# float32 header +"\xfa" + +# float64 header +"\xfb" + +# BREAK code for indefinite lengths +"\xff" + + +# ---- Useful prefabricated patterns ---- +# { "a": false } +"\xa1\x61\x61\xf4" + +# [1,2,3] +"\x83\x01\x02\x03" + +# [false, true, null] +"\x83\xf4\xf5\xf6" diff --git a/tests/fuzz/dictionaries/json.dict b/tests/fuzz/dictionaries/json.dict new file mode 100644 index 000000000000..eeae576a5779 --- /dev/null +++ b/tests/fuzz/dictionaries/json.dict @@ -0,0 +1,74 @@ +# From smithy-lang/smithy-java codecs/json-codec/src/fuzz/resources/dictionary/codec-fuzz.dict (Apache-2.0) +"0" +"7" +"," +":" +"2.1e24" + +"true" +"false" +"null" + +"\"\"" +"\"\":" + +"{}" +",{}" +":{}" +"{\"\":0}" +"{{}}" + +"[]" +",[]" +":[]" +"[0]" +"[[]]" + +"''" +"\\" +"\\b" +"\\f" +"\\n" +"\\r" +"\\t" +"\\u0000" +"\\x00" +"\\0" +"\\uD800\\uDC00" +"\\uDBFF\\uDFFF" + +"\"\":0" +"//" +"/**/" + + +# Things like geojson, json-ld, ... +"$ref" +"type" +"coordinates" +"@context" +"@id" +"@type" + +# Strings with truncated special values +"{\"foo\":fa" +"{\"foo\":t" +"{\"foo\":nul" + +"{" +"}" +"\"qty\": 1, \"qty\": -1" +"\"qty\": 1, \"qty\\ud800\": -1" +"\"qty\": 1, \"qt\\y\": -1" +"/*" +"*/" +"\"" +"1.7976931348623157e+308" +"5e-324" +"9007199254740991" +"-9007199254740991" + +"}=" + +",," +"{\"\":" \ No newline at end of file diff --git a/tests/fuzz/dictionaries/xml.dict b/tests/fuzz/dictionaries/xml.dict new file mode 100644 index 000000000000..2642f1d5f769 --- /dev/null +++ b/tests/fuzz/dictionaries/xml.dict @@ -0,0 +1,83 @@ +# From smithy-lang/smithy-java codecs/xml-codec/src/fuzz/resources/dictionary/codec-fuzz.dict (Apache-2.0) +# +# AFL dictionary for XML +# ---------------------- +# +# Several basic syntax elements and attributes, modeled on libxml2. +# +# Created by Michal Zalewski +# + +attr_encoding=" encoding=\"1\"" +attr_generic=" a=\"1\"" +attr_href=" href=\"1\"" +attr_standalone=" standalone=\"no\"" +attr_version=" version=\"1\"" +attr_xml_base=" xml:base=\"1\"" +attr_xml_id=" xml:id=\"1\"" +attr_xml_lang=" xml:lang=\"1\"" +attr_xml_space=" xml:space=\"1\"" +attr_xmlns=" xmlns=\"1\"" + +entity_builtin="<" +entity_decimal="" +entity_external="&a;" +entity_hex="" + +string_any="ANY" +string_brackets="[]" +string_cdata="CDATA" +string_col_fallback=":fallback" +string_col_generic=":a" +string_col_include=":include" +string_dashes="--" +string_empty="EMPTY" +string_empty_dblquotes="\"\"" +string_empty_quotes="''" +string_entities="ENTITIES" +string_entity="ENTITY" +string_fixed="#FIXED" +string_id="ID" +string_idref="IDREF" +string_idrefs="IDREFS" +string_implied="#IMPLIED" +string_nmtoken="NMTOKEN" +string_nmtokens="NMTOKENS" +string_notation="NOTATION" +string_parentheses="()" +string_pcdata="#PCDATA" +string_percent="%a" +string_public="PUBLIC" +string_required="#REQUIRED" +string_schema=":schema" +string_system="SYSTEM" +string_ucs4="UCS-4" +string_utf16="UTF-16" +string_utf8="UTF-8" +string_xmlns="xmlns:" + +tag_attlist="" +tag_doctype="" +tag_open_close="" +tag_open_exclamation="" +tag_xml_q="" + +encoding_utf="UTF-" +encoding_iso1="ISO-8859" +encoding_iso3="ISO-10646-UCS" +encoding_iso5="ISO-LATIN-1" +encoding_jis="SHIFT_JIS" +encoding_utf7="UTF-7" +encoding_utf16le="UTF-16BE" +encoding_utf16le="UTF-16LE" +encoding_ascii="US-ASCII" \ No newline at end of file diff --git a/tests/fuzz/findings/CborCodecFuzz.RoundTripsTypedValues/odd-negint-double-off-by-two.tape b/tests/fuzz/findings/CborCodecFuzz.RoundTripsTypedValues/odd-negint-double-off-by-two.tape new file mode 100644 index 000000000000..7a03ffc65dd3 --- /dev/null +++ b/tests/fuzz/findings/CborCodecFuzz.RoundTripsTypedValues/odd-negint-double-off-by-two.tape @@ -0,0 +1,3 @@ +choices: {18446744073709551615, 18446744073709551615, 18446744073709551615, 18446744073709551615, 18446744073709551615, 18446744073709551615, 18446744073709551615, 18446744073709551615, 18446744073709551615, 18446744073709551615, 14070713522838610127, 9223372036854775807, 18446462598732840959, 15394373617364385893, 18446744073709551598, 9223372036854775807, 9223372036854775807, 9223372036854775807, 18446744073709551613, 18446744073709551613, 18446744073709551615, 18446744073709551615, 9223372036854775807, 9223372036854775802, 10039637858738959069} +strings: {"(", "", "ooonoooooo", "$$)", "N\003\003\003\003\003\003\003\003\003\003\003\003\003\003\003\003\003\003\003\003\003\003\003\003\003\003\003\003\003NNNN", "$", "\021\021\021", "", "{"} +schema: StringDoubleMapValue mismatch: values["trail "] -11943773121102238 read back as -11943773121102236 (CborShapeDeserializer::ReadDouble computes -1.0 - static_cast(n) on a NegInt n > 2^53, rounding twice) diff --git a/tests/fuzz/findings/JsonCodecFuzz.RoundTripsTypedValues/float-max-map-value.tape b/tests/fuzz/findings/JsonCodecFuzz.RoundTripsTypedValues/float-max-map-value.tape new file mode 100644 index 000000000000..3e8cb7b807e9 --- /dev/null +++ b/tests/fuzz/findings/JsonCodecFuzz.RoundTripsTypedValues/float-max-map-value.tape @@ -0,0 +1,3 @@ +choices: {18446744073709551607, 18446744073709551615, 18446744073709551615, 0, 0, 0, 16423293201637202761, 16423293201637202761, 11993971958116216098, 16423293201637202761, 0, 0, 18446744073709551103, 18446744073709551615} +strings: {} +schema: StringFloatMapValue payload: {"values":{"":3.40282347e+38}} diff --git a/tests/fuzz/findings/JsonCodecFuzz.RoundTripsTypedValues/timestamp-map-negative-ms-truncation.tape b/tests/fuzz/findings/JsonCodecFuzz.RoundTripsTypedValues/timestamp-map-negative-ms-truncation.tape new file mode 100644 index 000000000000..03a4379ed9f8 --- /dev/null +++ b/tests/fuzz/findings/JsonCodecFuzz.RoundTripsTypedValues/timestamp-map-negative-ms-truncation.tape @@ -0,0 +1,4 @@ +choices: {1, 0, 1, 0, 3, 4915613238557} +strings: {} +schema: StringTimestampMapValue entry: "" -> -4307758798.296 (decoded -4307758798295ms; DateTime(double) truncates toward zero) +note: timestamps are drawn from the system_clock range, so this tape assumes the container's nanosecond libstdc++ clock diff --git a/tests/fuzz/findings/XmlCodecFuzz.RoundTripsTypedValues/absent-attribute-decodes-empty.tape b/tests/fuzz/findings/XmlCodecFuzz.RoundTripsTypedValues/absent-attribute-decodes-empty.tape new file mode 100644 index 000000000000..f7961ac05420 --- /dev/null +++ b/tests/fuzz/findings/XmlCodecFuzz.RoundTripsTypedValues/absent-attribute-decodes-empty.tape @@ -0,0 +1,3 @@ +choices: {18446744073709289466} +strings: {} +schema: XmlTraits payload: diff --git a/tests/fuzz/harness/CodecFuzzTargets.cpp b/tests/fuzz/harness/CodecFuzzTargets.cpp new file mode 100644 index 000000000000..d4c917bf4f6b --- /dev/null +++ b/tests/fuzz/harness/CodecFuzzTargets.cpp @@ -0,0 +1,101 @@ +/** + * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. + * SPDX-License-Identifier: Apache-2.0. + */ +#include +#include +#include +#include +#include +#include +#include + +#include + +namespace smithy { +namespace schema { +namespace fuzz { + +namespace { + +Aws::Crt::ByteCursor CursorOf(const std::string& bytes) { + return Aws::Crt::ByteCursorFromArray(reinterpret_cast(bytes.data()), bytes.size()); +} + +Aws::String Base64(const std::string& bytes) { + return Aws::Utils::HashingUtils::Base64Encode(Aws::Utils::ByteBuffer(reinterpret_cast(bytes.data()), bytes.size())); +} + +void ExpectReadBackUnchanged(const Codec& codec, const CodecTolerance& tolerance, const std::shared_ptr& schema, + const DynamicStruct& first, const std::string& bytes) { + const auto payload = codec.Serialize(*schema, first); + ASSERT_TRUE(payload.IsSuccess()) << schema->GetId() << ": read but could not write: " << payload.GetError().GetMessage() + << " input(base64)=" << Base64(bytes); + const std::string reserialized(payload.GetResult().data(), payload.GetResult().size()); + DynamicStruct second(schema, FuzzValue::Struct()); + const auto error = codec.DeserializeShape(CursorOf(reserialized), second); + ASSERT_FALSE(error.has_value()) << schema->GetId() << ": own output rejected: " << error->GetMessage() + << " output=" << payload.GetResult(); + EXPECT_EQ(FindMismatch(first.GetValue(), second.GetValue(), *schema, tolerance), "") + << schema->GetId() << " output=" << payload.GetResult() << " input(base64)=" << Base64(bytes); + + // Tolerated losses happen on the first write only. + const auto secondPayload = codec.Serialize(*schema, second); + ASSERT_TRUE(secondPayload.IsSuccess()) << schema->GetId() << ": " << secondPayload.GetError().GetMessage(); + const std::string secondBytes(secondPayload.GetResult().data(), secondPayload.GetResult().size()); + DynamicStruct third(schema, FuzzValue::Struct()); + ASSERT_FALSE(codec.DeserializeShape(CursorOf(secondBytes), third).has_value()) << schema->GetId() << " output=" << secondBytes; + const auto thirdPayload = codec.Serialize(*schema, third); + ASSERT_TRUE(thirdPayload.IsSuccess()) << schema->GetId() << ": " << thirdPayload.GetError().GetMessage(); + EXPECT_EQ(thirdPayload.GetResult(), secondPayload.GetResult()) << schema->GetId() << " input(base64)=" << Base64(bytes); +} + +void ExpectRoundTrip(const Codec& codec, const CodecTolerance& tolerance, const std::shared_ptr& schema, + const FuzzValue& value) { + DynamicStruct original(schema, value); + const auto payload = codec.Serialize(*schema, original); + ASSERT_TRUE(payload.IsSuccess()) << schema->GetId() << ": " << payload.GetError().GetMessage() << " value=" << value.Describe(); + const std::string bytes(payload.GetResult().data(), payload.GetResult().size()); + DynamicStruct decoded(schema, FuzzValue::Struct()); + const auto error = codec.DeserializeShape(CursorOf(bytes), decoded); + ASSERT_FALSE(error.has_value()) << schema->GetId() << ": " << error->GetMessage() << " payload=" << payload.GetResult(); + EXPECT_EQ(FindMismatch(value, decoded.GetValue(), *schema, tolerance), "") + << schema->GetId() << " payload=" << payload.GetResult() << " value=" << value.Describe(); +} + +} // namespace + +void DeserializeNeverCrashes(const Codec& codec, const std::string& bytes) { + for (const auto& schema : FuzzSchemas()) { + DynamicStruct shape(schema, FuzzValue::Struct()); + codec.DeserializeShape(CursorOf(bytes), shape); + } +} + +void ReserializeIsStable(const Codec& codec, const CodecTolerance& tolerance, const std::string& bytes) { + for (const auto& schema : FuzzSchemas()) { + DynamicStruct first(schema, FuzzValue::Struct()); + if (!codec.DeserializeShape(CursorOf(bytes), first).has_value()) { + ExpectReadBackUnchanged(codec, tolerance, schema, first, bytes); + } + } +} + +void RoundTripsTypedValues(const Codec& codec, const CodecTolerance& tolerance, const std::vector& choices, + const std::vector& strings) { + const bool documents = CodecSupportsDocuments(codec); + for (const auto& schema : FuzzSchemas()) { + ChoiceTape tape(choices, strings); + const FuzzValue value = BuildStruct(*schema, tape); + // Only JSON supports documents; NonJsonCodecsRejectDocuments covers the rest. + if (documents || !ContainsDocument(value)) { + ExpectRoundTrip(codec, tolerance, schema, value); + } + } +} + +bool CodecSupportsDocuments(const Codec& codec) { return dynamic_cast(&codec) != nullptr; } + +} // namespace fuzz +} // namespace schema +} // namespace smithy diff --git a/tests/fuzz/harness/CodecFuzzTargets.h b/tests/fuzz/harness/CodecFuzzTargets.h new file mode 100644 index 000000000000..2b9ad2e28896 --- /dev/null +++ b/tests/fuzz/harness/CodecFuzzTargets.h @@ -0,0 +1,29 @@ +/** + * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. + * SPDX-License-Identifier: Apache-2.0. + */ +#pragma once + +#include +#include + +#include +#include +#include + +namespace smithy { +namespace schema { +namespace fuzz { + +void DeserializeNeverCrashes(const Codec& codec, const std::string& bytes); + +void ReserializeIsStable(const Codec& codec, const CodecTolerance& tolerance, const std::string& bytes); + +void RoundTripsTypedValues(const Codec& codec, const CodecTolerance& tolerance, const std::vector& choices, + const std::vector& strings); + +bool CodecSupportsDocuments(const Codec& codec); + +} // namespace fuzz +} // namespace schema +} // namespace smithy diff --git a/tests/fuzz/harness/CodecRegressionTest.cpp b/tests/fuzz/harness/CodecRegressionTest.cpp new file mode 100644 index 000000000000..2443d0f84062 --- /dev/null +++ b/tests/fuzz/harness/CodecRegressionTest.cpp @@ -0,0 +1,53 @@ +/** + * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. + * SPDX-License-Identifier: Apache-2.0. + */ +#include +#include +#include +#include +#include + +#include +#include +#include + +using namespace smithy::schema; +using namespace smithy::schema::fuzz; + +namespace { + +void ReplayFindings(const char* suiteTest, const Codec& codec, const CodecTolerance& tolerance) { + const auto tapes = LoadTapes(suiteTest); + EXPECT_FALSE(tapes.empty()) << suiteTest; + for (const auto& tape : tapes) { + RoundTripsTypedValues(codec, tolerance, std::get<0>(tape), std::get<1>(tape)); + } +} + +void RoundTripSeedsUnderNumericLocale(const char* name) { + const std::string previous = std::setlocale(LC_NUMERIC, nullptr); + if (std::setlocale(LC_NUMERIC, name) == nullptr) { + GTEST_SKIP() << name << " is not installed"; + } + for (const auto& tape : TapeSeeds()) { + RoundTripsTypedValues(JsonCodec(), CodecTolerance::Json(), std::get<0>(tape), std::get<1>(tape)); + RoundTripsTypedValues(CborCodec(), CodecTolerance::Cbor(), std::get<0>(tape), std::get<1>(tape)); + RoundTripsTypedValues(XmlCodec(), CodecTolerance::Xml(), std::get<0>(tape), std::get<1>(tape)); + } + std::setlocale(LC_NUMERIC, previous.c_str()); +} + +} // namespace + +TEST(CodecRegression, RoundTripsUnderCommaDecimalLocale) { RoundTripSeedsUnderNumericLocale("de_DE.UTF-8"); } + +TEST(CodecRegression, JsonFindingsRoundTrip) { + ReplayFindings("JsonCodecFuzz.RoundTripsTypedValues", JsonCodec(), CodecTolerance::Json()); +} + +TEST(CodecRegression, CborFindingsRoundTrip) { + ReplayFindings("CborCodecFuzz.RoundTripsTypedValues", CborCodec(), CodecTolerance::Cbor()); +} + +TEST(CodecRegression, XmlFindingsRoundTrip) { ReplayFindings("XmlCodecFuzz.RoundTripsTypedValues", XmlCodec(), CodecTolerance::Xml()); } diff --git a/tests/fuzz/harness/DynamicStruct.cpp b/tests/fuzz/harness/DynamicStruct.cpp new file mode 100644 index 000000000000..f8d20764678b --- /dev/null +++ b/tests/fuzz/harness/DynamicStruct.cpp @@ -0,0 +1,249 @@ +/** + * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. + * SPDX-License-Identifier: Apache-2.0. + */ +#include +#include + +#include +#include +#include +#include + +namespace smithy { +namespace schema { +namespace fuzz { + +namespace { + +Aws::Crt::Optional ReadIntegerInRange(const Schema& member, ShapeDeserializer& de, int64_t min, int64_t max) { + const auto value = de.ReadInteger(member); + if (!value.has_value() || value.value() < min || value.value() > max) { + return {}; + } + return FuzzValue::Integer(value.value()); +} + +// Inlining the helpers back into ReadValue/WriteValue would give each nesting level one multi-KB frame under ASan. +__attribute__((noinline)) Aws::Crt::Optional ReadScalarValue(const Schema& member, ShapeDeserializer& de) { + switch (member.GetType()) { + case ShapeType::Boolean: { + const auto v = de.ReadBoolean(member); + return v.has_value() ? Aws::Crt::Optional(FuzzValue::Boolean(v.value())) : Aws::Crt::Optional(); + } + case ShapeType::Byte: + return ReadIntegerInRange(member, de, INT8_MIN, INT8_MAX); + case ShapeType::Short: + return ReadIntegerInRange(member, de, INT16_MIN, INT16_MAX); + case ShapeType::Integer: + case ShapeType::IntEnum: + return ReadIntegerInRange(member, de, INT32_MIN, INT32_MAX); + case ShapeType::Long: { + const auto v = de.ReadLong(member); + return v.has_value() ? Aws::Crt::Optional(FuzzValue::Integer(v.value())) : Aws::Crt::Optional(); + } + case ShapeType::Float: { + const auto v = de.ReadFloat(member); + return v.has_value() ? Aws::Crt::Optional(FuzzValue::Double(static_cast(v.value()))) + : Aws::Crt::Optional(); + } + case ShapeType::Double: { + const auto v = de.ReadDouble(member); + return v.has_value() ? Aws::Crt::Optional(FuzzValue::Double(v.value())) : Aws::Crt::Optional(); + } + case ShapeType::String: + case ShapeType::Enum: { + auto v = de.ReadString(member); + return v.has_value() ? Aws::Crt::Optional(FuzzValue::String(std::move(v.value()))) : Aws::Crt::Optional(); + } + case ShapeType::Blob: { + auto v = de.ReadBlob(member); + return v.has_value() ? Aws::Crt::Optional(FuzzValue::Blob(std::move(v.value()))) : Aws::Crt::Optional(); + } + case ShapeType::Timestamp: { + const auto v = de.ReadTimestamp(member); + return v.has_value() ? Aws::Crt::Optional(FuzzValue::Timestamp(v.value())) : Aws::Crt::Optional(); + } + case ShapeType::Document: { + auto v = de.ReadDocument(member); + return v ? Aws::Crt::Optional(FuzzValue::FromDocument(std::move(v))) : Aws::Crt::Optional(); + } + default: + std::fprintf(stderr, "ReadScalarValue: unhandled shape type %d\n", static_cast(member.GetType())); + std::abort(); + } +} + +__attribute__((noinline)) Aws::Crt::Optional ReadListValue(const Schema& member, ShapeDeserializer& de) { + const auto element = member.GetMemberTarget().value()->GetMember("member").value(); + Aws::Vector items; + de.ReadList(member, [&](ShapeDeserializer& d) { + auto v = ReadValue(*element, d); + if (v.has_value()) { + items.push_back(std::move(v.value())); + } + }); + return FuzzValue::List(std::move(items)); +} + +__attribute__((noinline)) Aws::Crt::Optional ReadMapValue(const Schema& member, ShapeDeserializer& de) { + const auto valueMember = member.GetMemberTarget().value()->GetMember("value").value(); + Aws::Map entries; + de.ReadMap(member, [&](const Aws::String& key, ShapeDeserializer& d) { + auto v = ReadValue(*valueMember, d); + if (v.has_value()) { + entries.erase(key); + entries.emplace(key, std::move(v.value())); + } + }); + return FuzzValue::Map(std::move(entries)); +} + +__attribute__((noinline)) Aws::Crt::Optional ReadStructValue(const std::shared_ptr& target, + ShapeDeserializer& de) { + FuzzValue result = FuzzValue::Struct(); + de.ReadStruct(*target, [&](const Schema& m, ShapeDeserializer& d) { + auto v = ReadValue(m, d); + if (v.has_value()) { + result.SetMember(m.GetMemberIndex(), std::move(v.value())); + } + }); + return result; +} + +void SerializeStructMembers(const Schema& schema, const FuzzValue& value, ShapeSerializer& serializer) { + for (const auto& entry : value.GetMembers()) { + const auto member = schema.GetMember(entry.first); + if (member.has_value() && *member) { + WriteValue(**member, entry.second, serializer); + } + } +} + +// Avoids copying the subtree at every nesting level. +class StructWriteView final : public SerializableStruct { + public: + StructWriteView(const Schema& schema, const FuzzValue& value) : m_schema(schema), m_value(value) {} + + const Schema& GetSchema() const override { return m_schema; } + void SerializeMembers(ShapeSerializer& serializer) const override { SerializeStructMembers(m_schema, m_value, serializer); } + void From(const Schema&, ShapeDeserializer&) override {} + + private: + const Schema& m_schema; + const FuzzValue& m_value; +}; + +__attribute__((noinline)) void WriteScalarValue(const Schema& member, const FuzzValue& value, ShapeSerializer& ser) { + switch (member.GetType()) { + case ShapeType::Boolean: + ser.WriteBoolean(member, value.GetBoolean()); + return; + case ShapeType::Byte: + case ShapeType::Short: + case ShapeType::Integer: + case ShapeType::IntEnum: + ser.WriteInteger(member, static_cast(value.GetInteger())); + return; + case ShapeType::Long: + ser.WriteLong(member, value.GetInteger()); + return; + case ShapeType::Float: + ser.WriteFloat(member, static_cast(value.GetDouble())); + return; + case ShapeType::Double: + ser.WriteDouble(member, value.GetDouble()); + return; + case ShapeType::String: + case ShapeType::Enum: + ser.WriteString(member, value.GetString()); + return; + case ShapeType::Blob: + ser.WriteBlob(member, value.GetBlob()); + return; + case ShapeType::Timestamp: + ser.WriteTimestamp(member, value.GetTimestamp()); + return; + case ShapeType::Document: + ser.WriteDocument(member, *value.GetDocument()); + return; + default: + std::fprintf(stderr, "WriteScalarValue: unhandled shape type %d\n", static_cast(member.GetType())); + std::abort(); + } +} + +__attribute__((noinline)) void WriteListValue(const Schema& member, const FuzzValue& value, ShapeSerializer& ser) { + const auto element = member.GetMemberTarget().value()->GetMember("member").value(); + ser.WriteList(member, value.GetList().size(), [&](ShapeSerializer& s) { + for (const auto& item : value.GetList()) { + WriteValue(*element, item, s); + } + }); +} + +__attribute__((noinline)) void WriteMapValue(const Schema& member, const FuzzValue& value, ShapeSerializer& ser) { + const auto valueMember = member.GetMemberTarget().value()->GetMember("value").value(); + ser.WriteMap(member, value.GetMap().size(), [&](MapSerializer& m) { + for (const auto& entry : value.GetMap()) { + m.WriteEntry(entry.first, [&](ShapeSerializer& s) { WriteValue(*valueMember, entry.second, s); }); + } + }); +} + +__attribute__((noinline)) void WriteStructValue(const Schema& member, const FuzzValue& value, ShapeSerializer& ser) { + StructWriteView nested(**member.GetMemberTarget(), value); + ser.WriteStruct(member, nested); +} + +} // namespace + +DynamicStruct::DynamicStruct(std::shared_ptr schema) : m_schema(std::move(schema)), m_value(FuzzValue::Struct()) {} + +DynamicStruct::DynamicStruct(std::shared_ptr schema, FuzzValue value) + : m_schema(std::move(schema)), m_value(std::move(value)) {} + +void DynamicStruct::SerializeMembers(ShapeSerializer& serializer) const { SerializeStructMembers(*m_schema, m_value, serializer); } + +void DynamicStruct::From(const Schema& memberSchema, ShapeDeserializer& deserializer) { + auto value = ReadValue(memberSchema, deserializer); + if (value.has_value()) { + m_value.SetMember(memberSchema.GetMemberIndex(), std::move(value.value())); + } +} + +Aws::Crt::Optional ReadValue(const Schema& member, ShapeDeserializer& de) { + switch (member.GetType()) { + case ShapeType::List: + return ReadListValue(member, de); + case ShapeType::Map: + return ReadMapValue(member, de); + case ShapeType::Structure: + case ShapeType::Union: + return ReadStructValue(member.GetMemberTarget().value(), de); + default: + return ReadScalarValue(member, de); + } +} + +void WriteValue(const Schema& member, const FuzzValue& value, ShapeSerializer& ser) { + switch (member.GetType()) { + case ShapeType::List: + WriteListValue(member, value, ser); + return; + case ShapeType::Map: + WriteMapValue(member, value, ser); + return; + case ShapeType::Structure: + case ShapeType::Union: + WriteStructValue(member, value, ser); + return; + default: + WriteScalarValue(member, value, ser); + return; + } +} + +} // namespace fuzz +} // namespace schema +} // namespace smithy diff --git a/tests/fuzz/harness/DynamicStruct.h b/tests/fuzz/harness/DynamicStruct.h new file mode 100644 index 000000000000..726f50b7d13c --- /dev/null +++ b/tests/fuzz/harness/DynamicStruct.h @@ -0,0 +1,43 @@ +/** + * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. + * SPDX-License-Identifier: Apache-2.0. + */ +#pragma once + +#include +#include +#include +#include +#include +#include + +#include + +namespace smithy { +namespace schema { +namespace fuzz { + +class DynamicStruct final : public SerializableStruct { + public: + explicit DynamicStruct(std::shared_ptr schema); + DynamicStruct(std::shared_ptr schema, FuzzValue value); + + const Schema& GetSchema() const override { return *m_schema; } + void SerializeMembers(ShapeSerializer& serializer) const override; + void From(const Schema& memberSchema, ShapeDeserializer& deserializer) override; + + const FuzzValue& GetValue() const { return m_value; } + + private: + std::shared_ptr m_schema; + FuzzValue m_value; +}; + +// Narrows Byte and Short itself, since ShapeDeserializer has no ReadByte/ReadShort. +Aws::Crt::Optional ReadValue(const Schema& member, ShapeDeserializer& deserializer); + +void WriteValue(const Schema& member, const FuzzValue& value, ShapeSerializer& serializer); + +} // namespace fuzz +} // namespace schema +} // namespace smithy diff --git a/tests/fuzz/harness/DynamicStructTest.cpp b/tests/fuzz/harness/DynamicStructTest.cpp new file mode 100644 index 000000000000..7326f50c625a --- /dev/null +++ b/tests/fuzz/harness/DynamicStructTest.cpp @@ -0,0 +1,161 @@ +/** + * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. + * SPDX-License-Identifier: Apache-2.0. + */ +#include +#include +#include +#include +#include +#include +#include +#include + +using namespace smithy::schema; +using namespace smithy::schema::fuzz; + +namespace { + +std::shared_ptr Inner() { return Schema::StructureBuilder("Inner").PutMember("s", Schema::CreateString("String")).Build(); } + +std::shared_ptr Everything() { + auto strings = Schema::ListBuilder("StringList").PutMember("member", Schema::CreateString("String")).Build(); + auto ints = Schema::MapBuilder("IntMap") + .PutMember("key", Schema::CreateString("String")) + .PutMember("value", Schema::CreateInteger("Integer")) + .Build(); + auto choice = Schema::UnionBuilder("Choice").PutMember("text", Schema::CreateString("String")).PutMember("inner", Inner()).Build(); + return Schema::StructureBuilder("Everything") + .PutMember("bool", Schema::CreateBoolean("Boolean")) + .PutMember("byte", Schema::CreateByte("Byte")) + .PutMember("short", Schema::CreateShort("Short")) + .PutMember("int", Schema::CreateInteger("Integer")) + .PutMember("long", Schema::CreateLong("Long")) + .PutMember("float", Schema::CreateFloat("Float")) + .PutMember("double", Schema::CreateDouble("Double")) + .PutMember("string", Schema::CreateString("String")) + .PutMember("blob", Schema::CreateBlob("Blob")) + .PutMember("ts", Schema::CreateTimestamp("Timestamp")) + .PutMember("list", strings) + .PutMember("map", ints) + .PutMember("inner", Inner()) + .PutMember("choice", choice) + .Build(); +} + +FuzzValue SampleValue() { + FuzzValue inner = FuzzValue::Struct(); + inner.SetMember(0, FuzzValue::String("nested")); + FuzzValue choice = FuzzValue::Struct(); + choice.SetMember(1, inner); + const unsigned char bytes[] = {0x00, 0xFF, 0x10}; + FuzzValue v = FuzzValue::Struct(); + v.SetMember(0, FuzzValue::Boolean(true)); + v.SetMember(1, FuzzValue::Integer(-128)); + v.SetMember(2, FuzzValue::Integer(32767)); + v.SetMember(3, FuzzValue::Integer(-2147483647 - 1)); + v.SetMember(4, FuzzValue::Integer(9007199254740993LL)); + v.SetMember(5, FuzzValue::Double(static_cast(1.5f))); + v.SetMember(6, FuzzValue::Double(0.1)); + v.SetMember(7, FuzzValue::String("a\"b&d")); + v.SetMember(8, FuzzValue::Blob(Aws::Utils::ByteBuffer(bytes, sizeof(bytes)))); + v.SetMember(9, FuzzValue::Timestamp(Aws::Utils::DateTime(static_cast(1700000000000LL)))); + v.SetMember(10, FuzzValue::List({FuzzValue::String("x"), FuzzValue::String("")})); + Aws::Map entries; + entries.emplace("k1", FuzzValue::Integer(1)); + entries.emplace("k2", FuzzValue::Integer(-1)); + v.SetMember(11, FuzzValue::Map(std::move(entries))); + v.SetMember(12, inner); + v.SetMember(13, choice); + return v; +} + +Aws::Crt::ByteCursor CursorOf(const Aws::String& s) { + return Aws::Crt::ByteCursorFromArray(reinterpret_cast(s.data()), s.size()); +} + +void ExpectRoundTrip(const Codec& codec, const CodecTolerance& tolerance) { + auto schema = Everything(); + DynamicStruct original(schema, SampleValue()); + auto payload = codec.Serialize(*schema, original); + ASSERT_TRUE(payload.IsSuccess()) << payload.GetError().GetMessage(); + DynamicStruct decoded(schema); + const auto error = codec.DeserializeShape(CursorOf(payload.GetResult()), decoded); + ASSERT_FALSE(error.has_value()) << error->GetMessage(); + EXPECT_EQ(FindMismatch(original.GetValue(), decoded.GetValue(), *schema, tolerance), "") << payload.GetResult(); +} + +} // namespace + +TEST(DynamicStruct, JsonRoundTrip) { ExpectRoundTrip(JsonCodec(), CodecTolerance::Json()); } +TEST(DynamicStruct, CborRoundTrip) { ExpectRoundTrip(CborCodec(), CodecTolerance::Cbor()); } +TEST(DynamicStruct, XmlRoundTrip) { ExpectRoundTrip(XmlCodec(), CodecTolerance::Xml()); } + +TEST(DynamicStruct, JsonDocumentMemberRoundTrip) { + auto schema = Schema::StructureBuilder("Holder").PutMember("doc", Schema::CreateDocument("Document")).Build(); + Aws::Map> fields{{"n", Document::FromInteger(3)}, {"s", Document::FromString("x")}}; + FuzzValue v = FuzzValue::Struct(); + v.SetMember(0, FuzzValue::FromDocument(Document::FromMap(fields))); + DynamicStruct original(schema, v); + JsonCodec codec; + auto payload = codec.Serialize(*schema, original); + ASSERT_TRUE(payload.IsSuccess()); + DynamicStruct decoded(schema); + ASSERT_FALSE(codec.DeserializeShape(CursorOf(payload.GetResult()), decoded).has_value()); + EXPECT_EQ(FindMismatch(original.GetValue(), decoded.GetValue(), *schema, CodecTolerance::Json()), ""); +} + +TEST(DynamicStruct, OutOfRangeByteIsAbsent) { + auto schema = Schema::StructureBuilder("B").PutMember("b", Schema::CreateByte("Byte")).Build(); + DynamicStruct decoded(schema); + ASSERT_FALSE(JsonCodec().DeserializeShape(CursorOf("{\"b\":300}"), decoded).has_value()); + EXPECT_TRUE(decoded.GetValue().GetMembers().empty()); +} + +TEST(DynamicStruct, XmlRoundTripHonoursMemberTraits) { + auto strings = Schema::ListBuilder("StringList").PutMember("member", Schema::CreateString("String")).Build(); + auto ints = Schema::ListBuilder("IntList").PutMember("member", Schema::CreateInteger("Integer")).Build(); + auto pairs = Schema::MapBuilder("PairMap") + .PutMember("key", Schema::CreateString("String")) + .PutMember("value", Schema::CreateInteger("Integer")) + .Build(); + auto schema = + Schema::StructureBuilder("Traited") + .PutMember("attr", Schema::CreateString("String"), {{XmlAttributeTrait::KEY(), Aws::MakeShared("Tag")}}) + .PutMember("plain", Schema::CreateString("String"), {{XmlNameTrait::KEY(), Aws::MakeShared("Tag", "Renamed")}}) + .PutMember("flat", strings, {{XmlFlattenedTrait::KEY(), Aws::MakeShared("Tag")}}) + .PutMember("items", ints, {{XmlListItemNameTrait::KEY(), Aws::MakeShared("Tag", "item")}}) + .PutMember("pairs", pairs, + {{XmlMapEntryNameTrait::KEY(), Aws::MakeShared("Tag", "pair")}, + {XmlMapKeyNameTrait::KEY(), Aws::MakeShared("Tag", "k")}, + {XmlMapValueNameTrait::KEY(), Aws::MakeShared("Tag", "v")}}) + .Build(); + + Aws::Map entries; + entries.emplace("a", FuzzValue::Integer(1)); + entries.emplace("b", FuzzValue::Integer(2)); + FuzzValue v = FuzzValue::Struct(); + v.SetMember(0, FuzzValue::String("attrValue")); + v.SetMember(1, FuzzValue::String("renamedValue")); + v.SetMember(2, FuzzValue::List({FuzzValue::String("x"), FuzzValue::String("y")})); + v.SetMember(3, FuzzValue::List({FuzzValue::Integer(7), FuzzValue::Integer(8)})); + v.SetMember(4, FuzzValue::Map(std::move(entries))); + + DynamicStruct original(schema, v); + XmlCodec codec; + auto payload = codec.Serialize(*schema, original); + ASSERT_TRUE(payload.IsSuccess()) << payload.GetError().GetMessage(); + DynamicStruct decoded(schema); + const auto error = codec.DeserializeShape(CursorOf(payload.GetResult()), decoded); + ASSERT_FALSE(error.has_value()) << error->GetMessage(); + EXPECT_EQ(FindMismatch(original.GetValue(), decoded.GetValue(), *schema, CodecTolerance::Xml()), "") << payload.GetResult(); + + const Aws::String& xml = payload.GetResult(); + EXPECT_NE(xml.find(""), Aws::String::npos) << xml; + EXPECT_NE(xml.find(""), Aws::String::npos) << xml; + EXPECT_NE(xml.find(""), Aws::String::npos) << xml; + EXPECT_NE(xml.find(""), Aws::String::npos) << xml; + EXPECT_NE(xml.find(""), Aws::String::npos) << xml; + EXPECT_NE(xml.find("attr=\""), Aws::String::npos) << xml; + EXPECT_EQ(xml.find(""), Aws::String::npos) << xml; +} diff --git a/tests/fuzz/harness/FuzzDomains.cpp b/tests/fuzz/harness/FuzzDomains.cpp new file mode 100644 index 000000000000..2449a688a434 --- /dev/null +++ b/tests/fuzz/harness/FuzzDomains.cpp @@ -0,0 +1,629 @@ +/** + * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. + * SPDX-License-Identifier: Apache-2.0. + */ +#include +#include +#include +#include +#include +#include +#include +#include + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +namespace smithy { +namespace schema { +namespace fuzz { + +namespace { + +const Aws::Vector& EdgeStrings() { + static const Aws::Vector strings = { + "", + "a", + "\"", + "\\", + "<", + ">", + "&", + "]]>", + "&", + "\x01\x1f", + Aws::String("\0", 1), + "\xC3\xA9", + "\xF0\x9F\x98\x80", + "\xFF\xFE", + " lead", + "trail ", + "line\nbreak", + "\r\n", + "\t", + "member", + "entry", + "NaN", + "-Infinity", + "1e400", + }; + return strings; +} + +bool IsContainer(ShapeType type) { + return type == ShapeType::List || type == ShapeType::Map || type == ShapeType::Structure || type == ShapeType::Union; +} + +int64_t PickInteger(ChoiceTape& tape, int64_t min, int64_t max) { + const int64_t edges[] = {0, 1, -1, min, max, min + 1, max - 1}; + if (tape.Below(4) < 3) { + return edges[tape.Below(sizeof(edges) / sizeof(edges[0]))]; + } + const uint64_t raw = tape.Next(); + if (min == (std::numeric_limits::min)() && max == (std::numeric_limits::max)()) { + int64_t value = 0; + std::memcpy(&value, &raw, sizeof(value)); + return value; + } + const uint64_t span = static_cast(max) - static_cast(min) + 1; + return min + static_cast(raw % span); +} + +double PickDouble(ChoiceTape& tape) { + const double edges[] = {0.0, + -0.0, + 1.5, + -1.5, + 0.1, + 1e-7, + 1e21, + 9007199254740993.0, + std::numeric_limits::quiet_NaN(), + std::numeric_limits::infinity(), + -std::numeric_limits::infinity(), + (std::numeric_limits::max)(), + -(std::numeric_limits::max)(), + (std::numeric_limits::min)(), + std::numeric_limits::denorm_min(), + static_cast((std::numeric_limits::max)())}; + if (tape.Below(4) < 3) { + return edges[tape.Below(sizeof(edges) / sizeof(edges[0]))]; + } + const uint64_t raw = tape.Next(); + double value = 0.0; + std::memcpy(&value, &raw, sizeof(value)); + return value; +} + +float PickFloat(ChoiceTape& tape) { + const float edges[] = {0.0f, + -0.0f, + 1.5f, + 0.1f, + std::numeric_limits::quiet_NaN(), + std::numeric_limits::infinity(), + -std::numeric_limits::infinity(), + (std::numeric_limits::max)(), + -(std::numeric_limits::max)(), + (std::numeric_limits::min)(), + std::numeric_limits::denorm_min()}; + if (tape.Below(4) < 3) { + return edges[tape.Below(sizeof(edges) / sizeof(edges[0]))]; + } + const uint32_t raw = static_cast(tape.Next()); + float value = 0.0f; + std::memcpy(&value, &raw, sizeof(value)); + return value; +} + +// Years 0001-9999, clipped to what system_clock can hold. +int64_t PickEpochMillis(ChoiceTape& tape) { + const int64_t clockMax = std::chrono::duration_cast((std::chrono::system_clock::duration::max)()).count() - 1; + const int64_t lo = (std::max)(static_cast(-62135596800000LL), -clockMax); + const int64_t hi = (std::min)(static_cast(253402300799999LL), clockMax); + const int64_t edges[] = {0, 1, -1, 999, -999, 1000, 1700000000123LL, lo, hi}; + if (tape.Below(4) < 3) { + return edges[tape.Below(sizeof(edges) / sizeof(edges[0]))]; + } + return lo + static_cast(tape.Next() % static_cast(hi - lo + 1)); +} + +FuzzValue BuildValue(const Schema& member, ChoiceTape& tape, int depthBudget) { + switch (member.GetType()) { + case ShapeType::Boolean: + return FuzzValue::Boolean(tape.Flip()); + case ShapeType::Byte: + return FuzzValue::Integer(PickInteger(tape, INT8_MIN, INT8_MAX)); + case ShapeType::Short: + return FuzzValue::Integer(PickInteger(tape, INT16_MIN, INT16_MAX)); + case ShapeType::Integer: + case ShapeType::IntEnum: + return FuzzValue::Integer(PickInteger(tape, INT32_MIN, INT32_MAX)); + case ShapeType::Long: + return FuzzValue::Integer(PickInteger(tape, (std::numeric_limits::min)(), (std::numeric_limits::max)())); + case ShapeType::Float: + return FuzzValue::Double(static_cast(PickFloat(tape))); + case ShapeType::Double: + return FuzzValue::Double(PickDouble(tape)); + case ShapeType::String: + case ShapeType::Enum: + return FuzzValue::String(tape.NextSizedString()); + case ShapeType::Blob: { + const Aws::String bytes = tape.NextSizedString(); + return FuzzValue::Blob(Aws::Utils::ByteBuffer(reinterpret_cast(bytes.data()), bytes.size())); + } + case ShapeType::Timestamp: + return FuzzValue::Timestamp(Aws::Utils::DateTime(PickEpochMillis(tape))); + case ShapeType::Document: { + auto doc = BuildDocument(tape, depthBudget); + // A null slot reads back absent and nothing is @sparse; nulls nested inside a document are kept. + return FuzzValue::FromDocument(doc->IsNull() ? smithy::schema::Document::FromString("") : std::move(doc)); + } + case ShapeType::List: { + const auto element = member.GetMemberTarget().value()->GetMember("member").value(); + Aws::Vector items; + const uint64_t count = depthBudget > 0 ? tape.NextCount(5) : 0; + for (uint64_t i = 0; i < count; ++i) { + items.push_back(BuildValue(*element, tape, depthBudget - 1)); + } + return FuzzValue::List(std::move(items)); + } + case ShapeType::Map: { + const auto valueMember = member.GetMemberTarget().value()->GetMember("value").value(); + Aws::Map entries; + const uint64_t count = depthBudget > 0 ? tape.NextCount(4) : 0; + for (uint64_t i = 0; i < count; ++i) { + Aws::String key = tape.NextString(); + FuzzValue value = BuildValue(*valueMember, tape, depthBudget - 1); + entries.erase(key); + entries.emplace(std::move(key), std::move(value)); + } + return FuzzValue::Map(std::move(entries)); + } + case ShapeType::Structure: + case ShapeType::Union: + return BuildStruct(*member.GetMemberTarget().value(), tape, depthBudget - 1); + default: + std::fprintf(stderr, "BuildValue: unhandled shape type %d\n", static_cast(member.GetType())); + std::abort(); + } +} + +[[noreturn]] void MalformedTape(const std::string& path, const std::string& reason) { + std::fprintf(stderr, "malformed tape %s: %s\n", path.c_str(), reason.c_str()); + std::abort(); +} + +std::string BracedBody(const std::string& path, const std::string& text, const std::string& label) { + const size_t at = text.find(label); + const size_t open = at == std::string::npos ? std::string::npos : text.find('{', at); + if (open == std::string::npos) { + MalformedTape(path, "missing " + label + " {...}"); + } + bool quoted = false; + for (size_t i = open + 1; i < text.size(); ++i) { + const char c = text[i]; + if (quoted && c == '\\') { + ++i; + } else if (c == '"') { + quoted = !quoted; + } else if (!quoted && c == '}') { + return text.substr(open + 1, i - open - 1); + } + } + MalformedTape(path, "unterminated " + label); +} + +std::vector ParseChoices(const std::string& path, const std::string& body) { + std::vector choices; + size_t at = body.find_first_not_of(" \t\r\n"); + while (at != std::string::npos) { + char* end = nullptr; + errno = 0; + const unsigned long long value = std::strtoull(body.c_str() + at, &end, 10); + if (end == body.c_str() + at || errno == ERANGE || body[at] == '-') { + MalformedTape(path, "bad choice at offset " + std::to_string(at)); + } + choices.push_back(static_cast(value)); + at = body.find_first_not_of(" \t\r\n", static_cast(end - body.c_str())); + if (at != std::string::npos && body[at] != ',') { + MalformedTape(path, "expected ',' at offset " + std::to_string(at)); + } + at = at == std::string::npos ? at : body.find_first_not_of(" \t\r\n", at + 1); + } + return choices; +} + +int HexDigit(char c) { + if (c >= '0' && c <= '9') { + return c - '0'; + } + if (c >= 'a' && c <= 'f') { + return c - 'a' + 10; + } + return c >= 'A' && c <= 'F' ? c - 'A' + 10 : -1; +} + +size_t DecodeEscape(const std::string& path, const std::string& body, size_t at, std::string& out) { + if (at >= body.size()) { + MalformedTape(path, "dangling escape"); + } + const char c = body[at]; + switch (c) { + case 'n': + out += '\n'; + return at + 1; + case 't': + out += '\t'; + return at + 1; + case 'r': + out += '\r'; + return at + 1; + case '"': + case '\'': + case '\\': + case '?': + out += c; + return at + 1; + default: + break; + } + int value = 0; + size_t next = at; + if (c == 'x') { + for (next = at + 1; next < body.size() && next < at + 3 && HexDigit(body[next]) >= 0; ++next) { + value = value * 16 + HexDigit(body[next]); + } + if (next == at + 1) { + MalformedTape(path, "\\x without hex digits"); + } + } else { + for (; next < body.size() && next < at + 3 && body[next] >= '0' && body[next] <= '7'; ++next) { + value = value * 8 + (body[next] - '0'); + } + if (next == at) { + MalformedTape(path, std::string("unknown escape \\") + c); + } + } + out += static_cast(value); + return next; +} + +std::vector ParseStrings(const std::string& path, const std::string& body) { + std::vector strings; + size_t at = body.find_first_not_of(" \t\r\n,"); + while (at != std::string::npos) { + if (body[at] != '"') { + MalformedTape(path, "expected '\"' at offset " + std::to_string(at)); + } + std::string value; + ++at; + while (at < body.size() && body[at] != '"') { + if (body[at] == '\\') { + at = DecodeEscape(path, body, at + 1, value); + } else { + value += body[at++]; + } + } + if (at >= body.size()) { + MalformedTape(path, "unterminated string"); + } + strings.push_back(std::move(value)); + at = body.find_first_not_of(" \t\r\n,", at + 1); + } + return strings; +} + +std::string ReadSeedFile(const std::filesystem::path& path) { + Aws::IFStream in(path.string().c_str(), std::ios::binary); + if (!in) { + std::fprintf(stderr, "cannot read seed file %s\n", path.string().c_str()); + std::abort(); + } + const std::string contents((std::istreambuf_iterator(in)), std::istreambuf_iterator()); + if (path.extension() != ".b64") { + return contents; + } + Aws::String encoded; + for (const char c : contents) { + if (c != ' ' && c != '\t' && c != '\r' && c != '\n') { + encoded += c; + } + } + const auto decoded = Aws::Utils::HashingUtils::Base64Decode(encoded); + if (Aws::Utils::HashingUtils::Base64Encode(decoded) != encoded) { + std::fprintf(stderr, "seed file %s is not valid base64\n", path.string().c_str()); + std::abort(); + } + return std::string(reinterpret_cast(decoded.GetUnderlyingData()), decoded.GetLength()); +} + +// FuzzTest drops an oversized seed with only a warning. +std::string RequireFitsPayload(std::string bytes, const std::string& origin) { + if (bytes.size() > MAX_PAYLOAD_BYTES) { + std::fprintf(stderr, "seed %s is %zu bytes, over the %zu-byte payload limit\n", origin.c_str(), bytes.size(), MAX_PAYLOAD_BYTES); + std::abort(); + } + return bytes; +} + +} // namespace + +Aws::String ChoiceTape::NextString() { + if (m_strings.empty() || Below(3) == 0) { + return EdgeStrings()[Below(EdgeStrings().size())]; + } + const std::string& pooled = m_strings[m_nextString++ % m_strings.size()]; + return Aws::String(pooled.data(), pooled.size()); +} + +Aws::String ChoiceTape::NextSizedString() { + const uint64_t selector = Below(16); + Aws::String base = NextString(); + size_t target = 0; + switch (selector) { + case 1: + target = 24; + break; + case 2: + target = 256; + break; + case 3: + target = 65536; + break; + default: + return base; + } + if (!TakeBytes(target)) { + return base; + } + if (base.empty()) { + base = "a"; + } + Aws::String sized; + sized.reserve(target + base.size()); + while (sized.size() < target) { + sized += base; + } + sized.resize(target); + return sized; +} + +uint64_t ChoiceTape::NextCount(uint64_t smallBound) { + const uint64_t selector = Below(16); + const uint64_t large = selector == 1 ? 24 : (selector == 2 ? 300 : 0); + if (large != 0 && TakeElements(large)) { + return large; + } + return Below(smallBound); +} + +bool ChoiceTape::TakeBytes(size_t count) { + const bool fits = count <= m_bytesLeft; + m_bytesLeft -= fits ? count : 0; + return fits; +} + +bool ChoiceTape::TakeElements(size_t count) { + const bool fits = count <= m_elementsLeft; + m_elementsLeft -= fits ? count : 0; + return fits; +} + +void ChoiceTape::EnterBuild() { + if (m_buildDepth == 0) { + m_bytesLeft = VALUE_BYTE_BUDGET; + m_elementsLeft = VALUE_ELEMENT_BUDGET; + } + ++m_buildDepth; +} + +void ChoiceTape::LeaveBuild() { --m_buildDepth; } + +FuzzValue BuildStruct(const Schema& structSchema, ChoiceTape& tape, int depthBudget) { + tape.EnterBuild(); + FuzzValue result = FuzzValue::Struct(); + Aws::Vector eligible; + for (uint16_t i = 0; i < structSchema.GetMemberCount(); ++i) { + const auto member = structSchema.GetMember(static_cast(i)).value(); + if (depthBudget > 0 || !IsContainer(member->GetType())) { + eligible.push_back(static_cast(i)); + } + } + if (structSchema.GetType() == ShapeType::Union) { + if (eligible.empty()) { + std::fprintf(stderr, "BuildStruct: union %s has no eligible member\n", structSchema.GetId()); + std::abort(); + } + const int index = eligible[tape.Below(eligible.size())]; + result.SetMember(index, BuildValue(*structSchema.GetMember(index).value(), tape, depthBudget)); + } else { + for (int index : eligible) { + if (tape.Flip()) { + result.SetMember(index, BuildValue(*structSchema.GetMember(index).value(), tape, depthBudget)); + } + } + } + tape.LeaveBuild(); + return result; +} + +std::shared_ptr BuildDocument(ChoiceTape& tape, int depthBudget) { + switch (tape.Below(depthBudget > 0 ? 7 : 5)) { + case 0: + return smithy::schema::Document::Null(); + case 1: + return smithy::schema::Document::FromBoolean(tape.Flip()); + case 2: + return smithy::schema::Document::FromInteger( + PickInteger(tape, (std::numeric_limits::min)(), (std::numeric_limits::max)())); + case 3: + return smithy::schema::Document::FromDouble(PickDouble(tape)); + case 4: + return smithy::schema::Document::FromString(tape.NextSizedString()); + case 5: { + Aws::Vector> items; + const uint64_t count = tape.NextCount(4); + for (uint64_t i = 0; i < count; ++i) { + items.push_back(BuildDocument(tape, depthBudget - 1)); + } + return smithy::schema::Document::FromList(std::move(items)); + } + default: { + Aws::Map> fields; + const uint64_t count = tape.NextCount(4); + for (uint64_t i = 0; i < count; ++i) { + Aws::String key = tape.NextString(); + fields[key] = BuildDocument(tape, depthBudget - 1); + } + return smithy::schema::Document::FromMap(std::move(fields)); + } + } +} + +std::vector LoadDictionary(const char* fileName) { + return fuzztest::ReadDictionaryFromFile(std::string(SCHEMA_FUZZ_DICTIONARY_DIR) + "/" + fileName); +} + +std::vector TapeSeeds() { + std::vector ascending; + for (uint64_t i = 0; i < 128; ++i) { + ascending.push_back(i); + } + std::vector alternating; + for (uint64_t i = 0; i < 64; ++i) { + alternating.push_back(i % 2 == 0 ? 0 : 3); + } + const std::vector pool = {"seed", "\xCE\xBA\xCE\xBB\xCE\xB5\xCE\xB9\xCE\xB4\xCE\xAF", "&"}; + return { + TapeSeed{{}, {}}, + TapeSeed{alternating, pool}, + TapeSeed{std::vector(64, 1), pool}, + TapeSeed{ascending, pool}, + TapeSeed{std::vector(64, ~uint64_t{0}), pool}, + TapeSeed{std::vector(64, uint64_t{1} << 63), {}}, + }; +} + +std::vector LoadTapes(const char* suiteTest) { + const std::filesystem::path dir = std::filesystem::path(SCHEMA_FUZZ_FINDINGS_DIR) / suiteTest; + std::vector files; + if (std::filesystem::is_directory(dir)) { + for (const auto& entry : std::filesystem::directory_iterator(dir)) { + if (entry.path().extension() == ".tape") { + files.push_back(entry.path()); + } + } + } + std::sort(files.begin(), files.end()); + std::vector tapes; + for (const auto& file : files) { + Aws::IFStream in(file.string().c_str(), std::ios::binary); + const std::string text((std::istreambuf_iterator(in)), std::istreambuf_iterator()); + const std::string path = file.string(); + tapes.emplace_back(ParseChoices(path, BracedBody(path, text, "choices:")), ParseStrings(path, BracedBody(path, text, "strings:"))); + } + return tapes; +} + +std::vector TapeSeeds(const char* suiteTest) { + auto seeds = TapeSeeds(); + for (auto& tape : LoadTapes(suiteTest)) { + seeds.push_back(std::move(tape)); + } + return seeds; +} + +Aws::String DeepNodePayload(const Codec& codec, int depth) { + const auto node = FindFuzzSchema("Node"); + const int child = node->GetMember("child").value()->GetMemberIndex(); + FuzzValue value = FuzzValue::Struct(); + for (int level = 1; level < depth; ++level) { + FuzzValue parent = FuzzValue::Struct(); + parent.SetMember(child, std::move(value)); + value = std::move(parent); + } + const auto payload = codec.Serialize(*node, DynamicStruct(node, std::move(value))); + if (!payload.IsSuccess()) { + std::fprintf(stderr, "DeepNodePayload: %s\n", payload.GetError().GetMessage().c_str()); + std::abort(); + } + return payload.GetResult(); +} + +Aws::String DeepDocumentPayload(int depth) { + const size_t lists = static_cast(depth > 1 ? depth - 1 : 0); + return "{\"value\":" + Aws::String(lists, '[') + Aws::String(lists, ']') + "}"; +} + +bool ReproFileSet() { + const char* repro = std::getenv("SCHEMA_FUZZ_REPRO_FILE"); + return repro != nullptr && *repro != '\0'; +} + +std::vector> ByteSeeds(const Codec& codec, const char* suiteTest) { + std::vector> seeds; + if (ReproFileSet()) { + const char* repro = std::getenv("SCHEMA_FUZZ_REPRO_FILE"); + if (!std::filesystem::is_regular_file(repro)) { + std::fprintf(stderr, "SCHEMA_FUZZ_REPRO_FILE: cannot read %s\n", repro); + std::abort(); + } + std::string contents = ReadSeedFile(repro); + if (contents.empty()) { + std::fprintf(stderr, "SCHEMA_FUZZ_REPRO_FILE: %s is empty\n", repro); + std::abort(); + } + seeds.emplace_back(RequireFitsPayload(std::move(contents), repro)); + return seeds; + } + for (const auto& tapeSeed : TapeSeeds()) { + for (const auto& schema : FuzzSchemas()) { + ChoiceTape tape(std::get<0>(tapeSeed), std::get<1>(tapeSeed)); + DynamicStruct shape(schema, BuildStruct(*schema, tape)); + const auto payload = codec.Serialize(*schema, shape); + if (payload.IsSuccess() && payload.GetResult().size() <= MAX_PAYLOAD_BYTES) { + seeds.emplace_back(std::string(payload.GetResult().data(), payload.GetResult().size())); + } + } + } + // Mutation rarely builds deep, balanced nesting. + const Aws::String deep = DeepNodePayload(codec, DEEP_SEED_DEPTH); + seeds.emplace_back(RequireFitsPayload(std::string(deep.data(), deep.size()), "DeepNodePayload")); + const std::string corpus = std::string(SCHEMA_FUZZ_CORPUS_DIR) + "/" + suiteTest; + std::vector files; + if (std::filesystem::is_directory(corpus)) { + for (const auto& entry : std::filesystem::directory_iterator(corpus)) { + files.push_back(entry.path()); + } + } + std::sort(files.begin(), files.end()); + for (const auto& file : files) { + seeds.emplace_back(RequireFitsPayload(ReadSeedFile(file), file.string())); + } + return seeds; +} + +std::vector> JsonByteSeeds(const char* suiteTest) { + auto seeds = ByteSeeds(JsonCodec(), suiteTest); + if (!ReproFileSet()) { + const Aws::String deep = DeepDocumentPayload(DEEP_SEED_DEPTH); + seeds.emplace_back(RequireFitsPayload(std::string(deep.data(), deep.size()), "DeepDocumentPayload")); + } + return seeds; +} + +} // namespace fuzz +} // namespace schema +} // namespace smithy diff --git a/tests/fuzz/harness/FuzzDomains.h b/tests/fuzz/harness/FuzzDomains.h new file mode 100644 index 000000000000..01e7a8d54599 --- /dev/null +++ b/tests/fuzz/harness/FuzzDomains.h @@ -0,0 +1,95 @@ +/** + * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. + * SPDX-License-Identifier: Apache-2.0. + */ +#pragma once + +#include +#include +#include +#include +#include + +#include +#include +#include +#include +#include +#include + +namespace smithy { +namespace schema { +namespace fuzz { + +constexpr int VALUE_DEPTH_BUDGET = 8; +constexpr size_t VALUE_BYTE_BUDGET = 256 * 1024; +constexpr size_t VALUE_ELEMENT_BUDGET = 2048; +constexpr int DEEP_SEED_DEPTH = MAX_NESTING_DEPTH; +constexpr size_t MAX_PAYLOAD_BYTES = 64 * 1024; + +// An exhausted tape answers 0, so every tape yields a valid value. +class ChoiceTape final { + public: + ChoiceTape(const std::vector& choices, const std::vector& strings) + : m_choices(choices), m_strings(strings), m_bytesLeft(VALUE_BYTE_BUDGET), m_elementsLeft(VALUE_ELEMENT_BUDGET) {} + + uint64_t Next() { return m_next < m_choices.size() ? m_choices[m_next++] : 0; } + uint64_t Below(uint64_t bound) { return bound == 0 ? 0 : Next() % bound; } + bool Flip() { return Below(2) == 1; } + Aws::String NextString(); + Aws::String NextSizedString(); + uint64_t NextCount(uint64_t smallBound); + void EnterBuild(); + void LeaveBuild(); + + private: + bool TakeBytes(size_t count); + bool TakeElements(size_t count); + + const std::vector& m_choices; + const std::vector& m_strings; + size_t m_next = 0; + size_t m_nextString = 0; + size_t m_buildDepth = 0; + size_t m_bytesLeft; + size_t m_elementsLeft; +}; + +FuzzValue BuildStruct(const Schema& structSchema, ChoiceTape& tape, int depthBudget = VALUE_DEPTH_BUDGET); + +std::shared_ptr BuildDocument(ChoiceTape& tape, int depthBudget = VALUE_DEPTH_BUDGET); + +std::vector LoadDictionary(const char* fileName); + +using TapeSeed = std::tuple, std::vector>; +std::vector TapeSeeds(); + +std::vector LoadTapes(const char* suiteTest); + +std::vector TapeSeeds(const char* suiteTest); + +Aws::String DeepNodePayload(const Codec& codec, int depth); + +Aws::String DeepDocumentPayload(int depth); + +bool ReproFileSet(); + +std::vector> ByteSeeds(const Codec& codec, const char* suiteTest); + +std::vector> JsonByteSeeds(const char* suiteTest); + +inline auto ChoicesDomain() { return fuzztest::VectorOf(fuzztest::Arbitrary()).WithMaxSize(1024); } + +inline auto StringPoolDomain(const char* dictionary) { + return fuzztest::VectorOf(fuzztest::String().WithMaxSize(64).WithDictionary(LoadDictionary(dictionary))).WithMaxSize(32); +} + +inline auto PayloadDomain(std::vector dictionary) { + return fuzztest::String().WithMaxSize(MAX_PAYLOAD_BYTES).WithDictionary(std::move(dictionary)); +} + +inline auto PayloadDomain(const char* dictionary) { return PayloadDomain(LoadDictionary(dictionary)); } + +} // namespace fuzz +} // namespace schema +} // namespace smithy diff --git a/tests/fuzz/harness/FuzzDomainsTest.cpp b/tests/fuzz/harness/FuzzDomainsTest.cpp new file mode 100644 index 000000000000..a357941c5def --- /dev/null +++ b/tests/fuzz/harness/FuzzDomainsTest.cpp @@ -0,0 +1,295 @@ +/** + * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. + * SPDX-License-Identifier: Apache-2.0. + */ +#include +#include +#include +#include +#include +#include +#include + +#include +#include +#include +#include +#include +#include +#include +#include +#include + +using namespace smithy::schema; +using namespace smithy::schema::fuzz; + +namespace { + +int Depth(const FuzzValue& v) { + int deepest = 0; + for (const auto& item : v.GetList()) { + deepest = (std::max)(deepest, Depth(item)); + } + for (const auto& entry : v.GetMap()) { + deepest = (std::max)(deepest, Depth(entry.second)); + } + for (const auto& entry : v.GetMembers()) { + deepest = (std::max)(deepest, Depth(entry.second)); + } + const bool container = + v.GetKind() == FuzzValue::Kind::List || v.GetKind() == FuzzValue::Kind::Map || v.GetKind() == FuzzValue::Kind::Struct; + return deepest + (container ? 1 : 0); +} + +} // namespace + +TEST(FuzzDomains, EmptyTapeBuildsAValueForEverySchema) { + const std::vector choices; + const std::vector strings; + for (const auto& schema : FuzzSchemas()) { + ChoiceTape tape(choices, strings); + const FuzzValue v = BuildStruct(*schema, tape); + EXPECT_EQ(v.GetKind(), FuzzValue::Kind::Struct) << schema->GetId(); + } +} + +TEST(FuzzDomains, ByteValuesStayInRange) { + const auto schema = FindFuzzSchema("ByteValue"); + ASSERT_NE(schema, nullptr); + const std::vector strings; + for (uint64_t raw : {uint64_t{0}, uint64_t{3}, uint64_t{255}, uint64_t{1} << 63, ~uint64_t{0}}) { + for (uint64_t pick = 0; pick < 4; ++pick) { + const std::vector choices = {1, pick, raw, raw}; + ChoiceTape tape(choices, strings); + const FuzzValue v = BuildStruct(*schema, tape); + ASSERT_EQ(v.GetMembers().size(), 1u); + for (const auto& entry : v.GetMembers()) { + EXPECT_GE(entry.second.GetInteger(), INT8_MIN); + EXPECT_LE(entry.second.GetInteger(), INT8_MAX); + } + } + } +} + +TEST(FuzzDomains, ByteSeedsIncludeGeneratedPayloads) { + EXPECT_GT(ByteSeeds(JsonCodec(), "JsonCodecFuzz.DeserializeNeverCrashes").size(), FuzzSchemas().size()); +} + +TEST(FuzzDomains, ByteSeedsIncludeDeepNodePayload) { + const Aws::String deep = DeepNodePayload(CborCodec(), DEEP_SEED_DEPTH); + const auto seeds = ByteSeeds(CborCodec(), "CborCodecFuzz.ReserializeIsStable"); + EXPECT_NE(std::find(seeds.begin(), seeds.end(), std::make_tuple(std::string(deep.data(), deep.size()))), seeds.end()); +} + +// FuzzTest drops an oversized seed with only a warning. +TEST(FuzzDomains, ByteSeedsFitThePayloadLimit) { + const auto fits = [](const std::vector>& seeds) { + return std::all_of(seeds.begin(), seeds.end(), + [](const std::tuple& seed) { return std::get<0>(seed).size() <= MAX_PAYLOAD_BYTES; }); + }; + EXPECT_TRUE(fits(JsonByteSeeds("JsonCodecFuzz.ReserializeIsStable"))); + EXPECT_TRUE(fits(ByteSeeds(CborCodec(), "CborCodecFuzz.ReserializeIsStable"))); + EXPECT_TRUE(fits(ByteSeeds(XmlCodec(), "XmlCodecFuzz.ReserializeIsStable"))); +} + +TEST(FuzzDomains, SomeGeneratedPayloadsExceedThePayloadLimit) { + bool exceeded = false; + for (const auto& tapeSeed : TapeSeeds()) { + for (const auto& schema : FuzzSchemas()) { + ChoiceTape tape(std::get<0>(tapeSeed), std::get<1>(tapeSeed)); + const auto payload = XmlCodec().Serialize(*schema, DynamicStruct(schema, BuildStruct(*schema, tape))); + exceeded = exceeded || (payload.IsSuccess() && payload.GetResult().size() > MAX_PAYLOAD_BYTES); + } + } + EXPECT_TRUE(exceeded); +} + +TEST(FuzzDomainsDeathTest, OversizedReproFileAborts) { + const std::string path = (std::filesystem::temp_directory_path() / "schema-fuzz-oversized.bin").string(); + { + Aws::OFStream out(path.c_str(), std::ios::binary); + out << std::string(MAX_PAYLOAD_BYTES + 1, ' '); + } + setenv("SCHEMA_FUZZ_REPRO_FILE", path.c_str(), 1); + EXPECT_DEATH(ByteSeeds(JsonCodec(), "x"), "over the 65536-byte payload limit"); + unsetenv("SCHEMA_FUZZ_REPRO_FILE"); + std::remove(path.c_str()); +} + +TEST(FuzzDomains, DeepNodePayloadNestsToTheRequestedDepth) { + const Aws::String json = DeepNodePayload(JsonCodec(), 3); + EXPECT_EQ(json, "{\"child\":{\"child\":{}}}"); + const Aws::String cbor = DeepNodePayload(CborCodec(), DEEP_SEED_DEPTH); + size_t children = 0; + for (size_t at = cbor.find("child"); at != Aws::String::npos; at = cbor.find("child", at + 1)) { + ++children; + } + EXPECT_EQ(children, static_cast(DEEP_SEED_DEPTH - 1)); +} + +TEST(FuzzDomains, DeepDocumentPayloadNestsToTheRequestedDepth) { + EXPECT_EQ(DeepDocumentPayload(3), "{\"value\":[[]]}"); + const auto seeds = JsonByteSeeds("JsonDocumentFuzz.ReadDocumentNeverCrashes"); + const Aws::String deep = DeepDocumentPayload(DEEP_SEED_DEPTH); + EXPECT_NE(std::find(seeds.begin(), seeds.end(), std::make_tuple(std::string(deep.data(), deep.size()))), seeds.end()); +} + +TEST(FuzzDomains, FindingTapesParse) { + for (const char* suiteTest : + {"CborCodecFuzz.RoundTripsTypedValues", "JsonCodecFuzz.RoundTripsTypedValues", "XmlCodecFuzz.RoundTripsTypedValues"}) { + const auto tapes = LoadTapes(suiteTest); + ASSERT_FALSE(tapes.empty()) << suiteTest; + for (const auto& tape : tapes) { + EXPECT_FALSE(std::get<0>(tape).empty()) << suiteTest; + } + } + const auto cbor = LoadTapes("CborCodecFuzz.RoundTripsTypedValues"); + ASSERT_EQ(std::get<1>(cbor[0]).size(), 9u); + EXPECT_EQ(std::get<1>(cbor[0])[6], "\021\021\021"); +} + +TEST(FuzzDomains, TimestampFindingTapeBuildsTheRecordedEntry) { + const auto schema = FindFuzzSchema("StringTimestampMapValue"); + bool found = false; + Aws::String built; + for (const auto& tape : LoadTapes("JsonCodecFuzz.RoundTripsTypedValues")) { + ChoiceTape choices(std::get<0>(tape), std::get<1>(tape)); + const FuzzValue value = BuildStruct(*schema, choices); + built += value.Describe() + " "; + const auto member = value.GetMembers().find(0); + if (member != value.GetMembers().end() && member->second.GetMap().size() == 1) { + const auto& entry = *member->second.GetMap().begin(); + found = found || (entry.first.empty() && entry.second.GetTimestamp().Millis() == -4307758798296LL); + } + } + EXPECT_TRUE(found) << built; +} + +TEST(FuzzDomains, TapeSeedsForATargetIncludeItsFindings) { + EXPECT_EQ(TapeSeeds("XmlCodecFuzz.RoundTripsTypedValues").size(), TapeSeeds().size() + 1); + EXPECT_EQ(TapeSeeds("NoSuch.Target").size(), TapeSeeds().size()); +} + +TEST(FuzzDomains, Base64CorpusFilesAreDecoded) { + const auto seeds = ByteSeeds(CborCodec(), "CborCodecFuzz.ReserializeIsStable"); + const std::string indefiniteText("\xA1\x65value\x7F\x62" "ab\x60\x61" "c\xFF", 15); + EXPECT_NE(std::find(seeds.begin(), seeds.end(), std::make_tuple(indefiniteText)), seeds.end()); +} + +TEST(FuzzDomains, Base64ReproFileIsDecoded) { + const std::string path = (std::filesystem::temp_directory_path() / "schema-fuzz-repro-test.b64").string(); + { + Aws::OFStream out(path.c_str(), std::ios::binary); + out << "oAA=\n"; + } + setenv("SCHEMA_FUZZ_REPRO_FILE", path.c_str(), 1); + const auto seeds = ByteSeeds(CborCodec(), "CborCodecFuzz.DeserializeNeverCrashes"); + unsetenv("SCHEMA_FUZZ_REPRO_FILE"); + std::remove(path.c_str()); + ASSERT_EQ(seeds.size(), 1u); + EXPECT_EQ(std::get<0>(seeds[0]), std::string("\xA0\x00", 2)); +} + +TEST(FuzzDomainsDeathTest, InvalidBase64SeedAborts) { + const std::string path = (std::filesystem::temp_directory_path() / "schema-fuzz-invalid.b64").string(); + { + Aws::OFStream out(path.c_str(), std::ios::binary); + out << "not*base64"; + } + setenv("SCHEMA_FUZZ_REPRO_FILE", path.c_str(), 1); + EXPECT_DEATH(ByteSeeds(CborCodec(), "x"), "not valid base64"); + unsetenv("SCHEMA_FUZZ_REPRO_FILE"); + std::remove(path.c_str()); +} + +TEST(FuzzDomains, ReproFileIsTheOnlySeed) { + const std::string path = (std::filesystem::temp_directory_path() / "schema-fuzz-repro-test.bin").string(); + { + Aws::OFStream out(path.c_str(), std::ios::binary); + out << "{\"x\":1}"; + } + const char* previous = std::getenv("SCHEMA_FUZZ_REPRO_FILE"); + const std::string saved = previous != nullptr ? previous : ""; + setenv("SCHEMA_FUZZ_REPRO_FILE", path.c_str(), 1); + const auto seeds = ByteSeeds(JsonCodec(), "JsonCodecFuzz.DeserializeNeverCrashes"); + const auto jsonSeeds = JsonByteSeeds("JsonCodecFuzz.DeserializeNeverCrashes"); + if (previous != nullptr) { + setenv("SCHEMA_FUZZ_REPRO_FILE", saved.c_str(), 1); + } else { + unsetenv("SCHEMA_FUZZ_REPRO_FILE"); + } + std::remove(path.c_str()); + ASSERT_EQ(seeds.size(), 1u); + EXPECT_EQ(std::get<0>(seeds[0]), "{\"x\":1}"); + EXPECT_EQ(jsonSeeds, seeds); +} + +TEST(FuzzDomainsDeathTest, MissingReproFileAborts) { + const char* previous = std::getenv("SCHEMA_FUZZ_REPRO_FILE"); + const std::string saved = previous != nullptr ? previous : ""; + setenv("SCHEMA_FUZZ_REPRO_FILE", "/nonexistent/schema-fuzz-repro.bin", 1); + EXPECT_DEATH(ByteSeeds(JsonCodec(), "x"), "SCHEMA_FUZZ_REPRO_FILE"); + if (previous != nullptr) { + setenv("SCHEMA_FUZZ_REPRO_FILE", saved.c_str(), 1); + } else { + unsetenv("SCHEMA_FUZZ_REPRO_FILE"); + } +} + +TEST(FuzzDomains, TapeCanSelectLargeStringsAndLists) { + const std::vector strings; + { + const std::vector choices = {1, 3, 1}; + ChoiceTape tape(choices, strings); + const FuzzValue v = BuildStruct(*FindFuzzSchema("StringValue"), tape); + ASSERT_EQ(v.GetMembers().size(), 1u); + EXPECT_GT(v.GetMembers().begin()->second.GetString().size(), 65535u); + } + { + const std::vector choices = {1, 1}; + ChoiceTape tape(choices, strings); + const FuzzValue v = BuildStruct(*FindFuzzSchema("StringListValue"), tape); + ASSERT_EQ(v.GetMembers().size(), 1u); + EXPECT_GE(v.GetMembers().begin()->second.GetList().size(), 24u); + } +} + +TEST(FuzzDomains, SizeBudgetFallsBackToSmallSizes) { + const std::vector strings; + std::vector choices; + for (int i = 0; i < 16; ++i) { + choices.push_back(3); + choices.push_back(1); + } + ChoiceTape tape(choices, strings); + size_t total = 0; + for (int i = 0; i < 16; ++i) { + total += tape.NextSizedString().size(); + } + EXPECT_LE(total, VALUE_BYTE_BUDGET + 16); +} + +TEST(FuzzDomains, DictionariesLoad) { + EXPECT_FALSE(LoadDictionary("json.dict").empty()); + EXPECT_FALSE(LoadDictionary("cbor.dict").empty()); + EXPECT_FALSE(LoadDictionary("xml.dict").empty()); +} + +void GeneratedValuesAreDeterministicBoundedAndJsonSerializable(const std::vector& choices, + const std::vector& strings) { + JsonCodec codec; + for (const auto& schema : FuzzSchemas()) { + ChoiceTape first(choices, strings); + ChoiceTape second(choices, strings); + const FuzzValue a = BuildStruct(*schema, first); + const FuzzValue b = BuildStruct(*schema, second); + ASSERT_EQ(FindMismatch(a, b, *schema, CodecTolerance::Exact()), "") << schema->GetId(); + ASSERT_LE(Depth(a), 2 * VALUE_DEPTH_BUDGET + 2) << schema->GetId(); + DynamicStruct shape(schema, a); + const auto payload = codec.Serialize(*schema, shape); + ASSERT_TRUE(payload.IsSuccess()) << schema->GetId() << ": " << payload.GetError().GetMessage(); + } +} +FUZZ_TEST(FuzzDomains, GeneratedValuesAreDeterministicBoundedAndJsonSerializable) + .WithDomains(ChoicesDomain(), StringPoolDomain("json.dict")) + .WithSeeds(TapeSeeds()); diff --git a/tests/fuzz/harness/FuzzEnvironment.cpp b/tests/fuzz/harness/FuzzEnvironment.cpp new file mode 100644 index 000000000000..f47d75d0f6a4 --- /dev/null +++ b/tests/fuzz/harness/FuzzEnvironment.cpp @@ -0,0 +1,33 @@ +/** + * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. + * SPDX-License-Identifier: Apache-2.0. + */ +#include +#include + +#include +#include +#include + +namespace { + +class SdkEnvironment final : public ::testing::Environment { + public: + void SetUp() override { + const char* locale = std::getenv("SCHEMA_FUZZ_LOCALE"); + if (locale != nullptr && *locale != '\0' && std::setlocale(LC_NUMERIC, locale) == nullptr) { + std::fprintf(stderr, "SCHEMA_FUZZ_LOCALE: cannot set LC_NUMERIC to %s\n", locale); + std::abort(); + } + Aws::InitAPI(m_options); + } + void TearDown() override { Aws::ShutdownAPI(m_options); } + + private: + Aws::SDKOptions m_options; +}; + +// gtest takes ownership of the raw pointer. +::testing::Environment* const SDK_ENVIRONMENT = ::testing::AddGlobalTestEnvironment(new SdkEnvironment); + +} // namespace diff --git a/tests/fuzz/harness/FuzzSchemas.cpp b/tests/fuzz/harness/FuzzSchemas.cpp new file mode 100644 index 000000000000..c3b41f185bce --- /dev/null +++ b/tests/fuzz/harness/FuzzSchemas.cpp @@ -0,0 +1,259 @@ +/** + * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. + * SPDX-License-Identifier: Apache-2.0. + */ +#include +#include +#include +#include +#include +#include +#include + +#include + +namespace smithy { +namespace schema { +namespace fuzz { + +namespace { + +const char ALLOC_TAG[] = "FuzzSchemas"; + +template +TraitEntry MakeTrait(Args&&... args) { + return TraitEntry{T::KEY(), Aws::MakeShared(ALLOC_TAG, std::forward(args)...)}; +} + +class ScalarKind { + public: + ScalarKind(ShapeType type, const char* name) : m_type(type), m_name(name) {} + ShapeType Type() const { return m_type; } + const Aws::String& Name() const { return m_name; } + + private: + ShapeType m_type; + Aws::String m_name; +}; + +const Aws::Vector& Scalars() { + static const Aws::Vector scalars = { + {ShapeType::Boolean, "Boolean"}, {ShapeType::Byte, "Byte"}, {ShapeType::Short, "Short"}, + {ShapeType::Integer, "Integer"}, {ShapeType::Long, "Long"}, {ShapeType::Float, "Float"}, + {ShapeType::Double, "Double"}, {ShapeType::String, "String"}, {ShapeType::Enum, "Enum"}, + {ShapeType::IntEnum, "IntEnum"}, {ShapeType::Blob, "Blob"}, {ShapeType::Timestamp, "Timestamp"}, + {ShapeType::Document, "Document"}, + }; + return scalars; +} + +std::shared_ptr CreateScalar(ShapeType type, const char* id, TraitList traits = {}) { + switch (type) { + case ShapeType::Boolean: + return Schema::CreateBoolean(id, traits); + case ShapeType::Byte: + return Schema::CreateByte(id, traits); + case ShapeType::Short: + return Schema::CreateShort(id, traits); + case ShapeType::Integer: + return Schema::CreateInteger(id, traits); + case ShapeType::Long: + return Schema::CreateLong(id, traits); + case ShapeType::Float: + return Schema::CreateFloat(id, traits); + case ShapeType::Double: + return Schema::CreateDouble(id, traits); + case ShapeType::String: + return Schema::CreateString(id, traits); + case ShapeType::Enum: + return Schema::CreateEnum(id, traits); + case ShapeType::IntEnum: + return Schema::CreateIntEnum(id, traits); + case ShapeType::Blob: + return Schema::CreateBlob(id, traits); + case ShapeType::Timestamp: + return Schema::CreateTimestamp(id, traits); + default: + return Schema::CreateDocument(id, traits); + } +} + +class Registry final { + public: + Registry() { + AddScalarWrappers(); + AddNested(); + AddRecursive(); + AddJsonTraits(); + AddXmlTraits(); + AddContainerTraits(); + AddQueryTraits(); + m_documentHolder = Schema::StructureBuilder("DocumentHolder").PutMember("payload", m_scalars.at(ShapeType::Document)).Build(); + } + + const Aws::Vector>& Roots() const { return m_roots; } + const std::shared_ptr& DocumentHolder() const { return m_documentHolder; } + + private: + void AddScalarWrappers() { + for (const auto& scalar : Scalars()) { + const auto shape = CreateScalar(scalar.Type(), scalar.Name().c_str()); + m_scalars[scalar.Type()] = shape; + const auto list = Schema::ListBuilder((scalar.Name() + "List").c_str()).PutMember("member", shape).Build(); + const auto map = + Schema::MapBuilder(("String" + scalar.Name() + "Map").c_str()).PutMember("key", m_mapKey).PutMember("value", shape).Build(); + m_lists[scalar.Type()] = list; + m_maps[scalar.Type()] = map; + m_roots.push_back(Schema::StructureBuilder((scalar.Name() + "Value").c_str()).PutMember("value", shape).Build()); + m_roots.push_back(Schema::StructureBuilder((scalar.Name() + "ListValue").c_str()).PutMember("values", list).Build()); + m_roots.push_back(Schema::StructureBuilder(("String" + scalar.Name() + "MapValue").c_str()).PutMember("values", map).Build()); + } + } + + void AddNested() { + m_simpleNested = Schema::StructureBuilder("SimpleNested") + .PutMember("str", m_scalars.at(ShapeType::String)) + .PutMember("int", m_scalars.at(ShapeType::Integer)) + .Build(); + const auto testUnion = Schema::UnionBuilder("TestUnion") + .PutMember("stringVariant", m_scalars.at(ShapeType::String)) + .PutMember("intVariant", m_scalars.at(ShapeType::Integer)) + .PutMember("structVariant", m_simpleNested) + .Build(); + m_roots.push_back(m_simpleNested); + m_roots.push_back(testUnion); + m_roots.push_back(Schema::StructureBuilder("UnionValue").PutMember("value", testUnion).Build()); + m_roots.push_back(Schema::StructureBuilder("NestedStructureValue") + .PutMember("str", m_scalars.at(ShapeType::String)) + .PutMember("int", m_scalars.at(ShapeType::Integer)) + .PutMember("long", m_scalars.at(ShapeType::Long)) + .PutMember("bool", m_scalars.at(ShapeType::Boolean)) + .PutMember("flt", m_scalars.at(ShapeType::Float)) + .PutMember("dbl", m_scalars.at(ShapeType::Double)) + .PutMember("blob", m_scalars.at(ShapeType::Blob)) + .PutMember("ts", m_scalars.at(ShapeType::Timestamp)) + .PutMember("list", m_lists.at(ShapeType::String)) + .PutMember("map", m_maps.at(ShapeType::Integer)) + .PutMember("nested", m_simpleNested) + .PutMember("union", testUnion) + .Build()); + } + + void AddRecursive() { + auto node = Schema::StructureBuilder("Node"); + auto nodeList = Schema::ListBuilder("NodeList"); + auto nodeMap = Schema::MapBuilder("StringNodeMap"); + nodeList.PutMember("member", node); + nodeMap.PutMember("key", m_scalars.at(ShapeType::String)).PutMember("value", node); + node.PutMember("value", m_scalars.at(ShapeType::Integer)) + .PutMember("child", node) + .PutMember("children", nodeList) + .PutMember("byName", nodeMap); + m_roots.push_back(node.Build()); + // Deferred members hold weak references; keep the list and map shapes alive after the builders go. + m_keepAlive.push_back(nodeList.Build()); + m_keepAlive.push_back(nodeMap.Build()); + } + + void AddJsonTraits() { + m_roots.push_back(Schema::StructureBuilder("JsonNamed") + .PutMember("renamed", m_scalars.at(ShapeType::String), {MakeTrait("Renamed Key")}) + .PutMember("quoted", m_scalars.at(ShapeType::Integer), {MakeTrait("q\"u\\o")}) + .PutMember("plain", m_scalars.at(ShapeType::String)) + .Build()); + const auto ts = m_scalars.at(ShapeType::Timestamp); + m_roots.push_back(Schema::StructureBuilder("Timestamps") + .PutMember("epoch", ts, {MakeTrait(TimestampFormatTrait::Format::EPOCH_SECONDS)}) + .PutMember("dateTime", ts, {MakeTrait(TimestampFormatTrait::Format::DATE_TIME)}) + .PutMember("httpDate", ts, {MakeTrait(TimestampFormatTrait::Format::HTTP_DATE)}) + .PutMember("plain", ts) + .Build()); + } + + void AddXmlTraits() { + m_roots.push_back( + Schema::StructureBuilder("XmlTraits", + {MakeTrait("XmlRoot"), MakeTrait("https://example.com/fuzz")}) + .PutMember("attr", m_scalars.at(ShapeType::String), {MakeTrait()}) + .PutMember("intAttr", m_scalars.at(ShapeType::Integer), {MakeTrait(), MakeTrait("int-attr")}) + .PutMember("fltAttr", m_scalars.at(ShapeType::Float), {MakeTrait()}) + .PutMember("dblAttr", m_scalars.at(ShapeType::Double), {MakeTrait()}) + .PutMember("boolAttr", m_scalars.at(ShapeType::Boolean), {MakeTrait()}) + .PutMember("tsAttr", m_scalars.at(ShapeType::Timestamp), {MakeTrait()}) + .PutMember("nsMember", m_scalars.at(ShapeType::String), {MakeTrait("https://example.com/ns", "p")}) + .PutMember("renamed", m_scalars.at(ShapeType::String), {MakeTrait("Renamed")}) + .PutMember("flatList", m_lists.at(ShapeType::String), {MakeTrait()}) + .PutMember("flatMap", m_maps.at(ShapeType::String), {MakeTrait()}) + .PutMember("namedList", m_lists.at(ShapeType::Integer), {MakeTrait("item")}) + .PutMember("namedMap", m_maps.at(ShapeType::Integer), + {MakeTrait("pair"), MakeTrait("k"), MakeTrait("v")}) + .PutMember("nested", m_simpleNested, {MakeTrait("Inner")}) + .Build()); + } + + void AddContainerTraits() { + const auto ts = m_scalars.at(ShapeType::Timestamp); + const auto dateTimeList = Schema::ListBuilder("DateTimeList") + .PutMember("member", ts, {MakeTrait(TimestampFormatTrait::Format::DATE_TIME)}) + .Build(); + const auto httpDateMap = Schema::MapBuilder("StringHttpDateMap") + .PutMember("key", m_mapKey) + .PutMember("value", ts, {MakeTrait(TimestampFormatTrait::Format::HTTP_DATE)}) + .Build(); + const auto epochList = Schema::ListBuilder("EpochSecondsList") + .PutMember("member", ts, {MakeTrait(TimestampFormatTrait::Format::EPOCH_SECONDS)}) + .Build(); + const auto enumKeyMap = Schema::MapBuilder("EnumStringMap") + .PutMember("key", m_scalars.at(ShapeType::Enum)) + .PutMember("value", m_scalars.at(ShapeType::String)) + .Build(); + m_roots.push_back(Schema::StructureBuilder("ContainerTraits") + .PutMember("dateTimes", dateTimeList) + .PutMember("httpDates", httpDateMap) + .PutMember("epochs", epochList) + .PutMember("byEnum", enumKeyMap) + .Build()); + } + + void AddQueryTraits() { + m_roots.push_back(Schema::StructureBuilder("Ec2Named") + .PutMember("renamed", m_scalars.at(ShapeType::String), {MakeTrait("CustomName")}) + .PutMember("both", m_scalars.at(ShapeType::Integer), + {MakeTrait("Ec2Both"), MakeTrait("xmlBoth")}) + .PutMember("plain", m_scalars.at(ShapeType::String)) + .Build()); + } + + std::shared_ptr m_mapKey = Schema::CreateString("String"); + Aws::Map> m_scalars; + Aws::Map> m_lists; + Aws::Map> m_maps; + std::shared_ptr m_simpleNested; + std::shared_ptr m_documentHolder; + Aws::Vector> m_roots; + Aws::Vector> m_keepAlive; +}; + +const Registry& GetRegistry() { + static const Registry registry; + return registry; +} + +} // namespace + +const Aws::Vector>& FuzzSchemas() { return GetRegistry().Roots(); } + +std::shared_ptr FindFuzzSchema(const Aws::String& id) { + for (const auto& root : FuzzSchemas()) { + if (id == root->GetId()) { + return root; + } + } + return nullptr; +} + +std::shared_ptr DocumentHolderSchema() { return GetRegistry().DocumentHolder(); } + +} // namespace fuzz +} // namespace schema +} // namespace smithy diff --git a/tests/fuzz/harness/FuzzSchemas.h b/tests/fuzz/harness/FuzzSchemas.h new file mode 100644 index 000000000000..8b74094949c7 --- /dev/null +++ b/tests/fuzz/harness/FuzzSchemas.h @@ -0,0 +1,25 @@ +/** + * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. + * SPDX-License-Identifier: Apache-2.0. + */ +#pragma once + +#include +#include +#include + +#include + +namespace smithy { +namespace schema { +namespace fuzz { + +const Aws::Vector>& FuzzSchemas(); + +std::shared_ptr FindFuzzSchema(const Aws::String& id); + +std::shared_ptr DocumentHolderSchema(); + +} // namespace fuzz +} // namespace schema +} // namespace smithy diff --git a/tests/fuzz/harness/FuzzSchemasTest.cpp b/tests/fuzz/harness/FuzzSchemasTest.cpp new file mode 100644 index 000000000000..032197d267d5 --- /dev/null +++ b/tests/fuzz/harness/FuzzSchemasTest.cpp @@ -0,0 +1,76 @@ +/** + * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. + * SPDX-License-Identifier: Apache-2.0. + */ +#include +#include +#include +#include +#include +#include + +#include + +using namespace smithy::schema; +using namespace smithy::schema::fuzz; + +TEST(FuzzSchemas, RootsAreUniqueStructuresOrUnions) { + Aws::Set ids; + for (const auto& root : FuzzSchemas()) { + EXPECT_TRUE(root->GetType() == ShapeType::Structure || root->GetType() == ShapeType::Union) << root->GetId(); + EXPECT_TRUE(ids.insert(root->GetId()).second) << "duplicate " << root->GetId(); + EXPECT_EQ(Aws::String(root->GetId()).find('#'), Aws::String::npos) << root->GetId(); + } + EXPECT_EQ(ids.size(), 49u); +} + +TEST(FuzzSchemas, EveryMemberTargetResolves) { + Aws::Set visited; + std::function walk = [&](const Schema& shape) { + if (!visited.insert(&shape).second) { + return; + } + for (uint16_t i = 0; i < shape.GetMemberCount(); ++i) { + const auto member = shape.GetMember(static_cast(i)); + ASSERT_TRUE(member.has_value() && *member) << shape.GetId() << " member " << i; + const auto target = (*member)->GetMemberTarget(); + ASSERT_TRUE(target.has_value() && *target) << (*member)->GetId(); + walk(**target); + } + }; + for (const auto& root : FuzzSchemas()) { + walk(*root); + } +} + +TEST(FuzzSchemas, NodeIsSelfRecursive) { + const auto node = FindFuzzSchema("Node"); + ASSERT_NE(node, nullptr); + EXPECT_EQ(node->GetMember("child").value()->GetMemberTarget().value().get(), node.get()); + const auto children = node->GetMember("children").value()->GetMemberTarget().value(); + EXPECT_EQ(children->GetMember("member").value()->GetMemberTarget().value().get(), node.get()); +} + +TEST(FuzzSchemas, TraitVariantsCarryTraits) { + const auto xml = FindFuzzSchema("XmlTraits"); + ASSERT_NE(xml, nullptr); + EXPECT_TRUE(xml->HasTrait(XmlNameTrait::KEY())); + EXPECT_TRUE(xml->GetMember("attr").value()->HasTrait(XmlAttributeTrait::KEY())); + EXPECT_TRUE(xml->GetMember("flatList").value()->HasTrait(XmlFlattenedTrait::KEY())); + for (const char* attr : {"fltAttr", "dblAttr", "boolAttr", "tsAttr"}) { + EXPECT_TRUE(xml->GetMember(attr).value()->HasTrait(XmlAttributeTrait::KEY())) << attr; + } + EXPECT_TRUE(xml->GetMember("nsMember").value()->HasTrait(XmlNamespaceTrait::KEY())); + const auto containers = FindFuzzSchema("ContainerTraits"); + ASSERT_NE(containers, nullptr); + const auto dateTimes = containers->GetMember("dateTimes").value()->GetMemberTarget().value(); + EXPECT_TRUE(dateTimes->GetMember("member").value()->HasTrait(TimestampFormatTrait::KEY())); + const auto byEnum = containers->GetMember("byEnum").value()->GetMemberTarget().value(); + EXPECT_EQ(byEnum->GetMember("key").value()->GetType(), ShapeType::Enum); + const auto ec2 = FindFuzzSchema("Ec2Named"); + ASSERT_NE(ec2, nullptr); + EXPECT_TRUE(ec2->GetMember("renamed").value()->HasTrait(Ec2QueryNameTrait::KEY())); + ASSERT_NE(FindFuzzSchema("JsonNamed"), nullptr); + ASSERT_NE(FindFuzzSchema("Timestamps"), nullptr); + EXPECT_EQ(DocumentHolderSchema()->GetMember("payload").value()->GetType(), ShapeType::Document); +} diff --git a/tests/fuzz/harness/FuzzValue.cpp b/tests/fuzz/harness/FuzzValue.cpp new file mode 100644 index 000000000000..77e829dc503e --- /dev/null +++ b/tests/fuzz/harness/FuzzValue.cpp @@ -0,0 +1,447 @@ +/** + * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. + * SPDX-License-Identifier: Apache-2.0. + */ +#include +#include +#include +#include + +#include +#include +#include + +using Aws::Utils::StringUtils; + +namespace smithy { +namespace schema { +namespace fuzz { + +FuzzValue FuzzValue::Boolean(bool value) { + FuzzValue v(Kind::Boolean); + v.m_boolean = value; + return v; +} +FuzzValue FuzzValue::Integer(int64_t value) { + FuzzValue v(Kind::Integer); + v.m_integer = value; + return v; +} +FuzzValue FuzzValue::Double(double value) { + FuzzValue v(Kind::Double); + v.m_double = value; + return v; +} +FuzzValue FuzzValue::String(Aws::String value) { + FuzzValue v(Kind::String); + v.m_string = std::move(value); + return v; +} +FuzzValue FuzzValue::Blob(Aws::Utils::ByteBuffer value) { + FuzzValue v(Kind::Blob); + v.m_blob = std::move(value); + return v; +} +FuzzValue FuzzValue::Timestamp(Aws::Utils::DateTime value) { + FuzzValue v(Kind::Timestamp); + v.m_timestamp = value; + return v; +} +FuzzValue FuzzValue::FromDocument(std::shared_ptr value) { + FuzzValue v(Kind::Document); + v.m_document = std::move(value); + return v; +} +FuzzValue FuzzValue::List(Aws::Vector items) { + FuzzValue v(Kind::List); + v.m_list = std::move(items); + return v; +} +FuzzValue FuzzValue::Map(Aws::Map entries) { + FuzzValue v(Kind::Map); + v.m_map = std::move(entries); + return v; +} +FuzzValue FuzzValue::Struct() { return FuzzValue(Kind::Struct); } + +void FuzzValue::SetMember(int index, FuzzValue value) { + m_members.erase(index); + m_members.emplace(index, std::move(value)); +} + +Aws::String FuzzValue::Describe() const { + switch (m_kind) { + case Kind::Boolean: + return m_boolean ? "true" : "false"; + case Kind::Integer: + return StringUtils::to_string(m_integer); + case Kind::Double: { + Aws::OStringStream ss; + ss.precision(17); + ss << m_double; + return ss.str(); + } + case Kind::String: + return "\"" + m_string + "\""; + case Kind::Blob: + return "blob[" + StringUtils::to_string(m_blob.GetLength()) + "]"; + case Kind::Timestamp: + return "ts(" + StringUtils::to_string(m_timestamp.Millis()) + "ms)"; + case Kind::Document: + return "document"; + case Kind::List: { + Aws::String out = "["; + for (size_t i = 0; i < m_list.size(); ++i) { + out += (i == 0 ? "" : ", ") + m_list[i].Describe(); + } + return out + "]"; + } + case Kind::Map: { + Aws::String out = "{"; + for (const auto& entry : m_map) { + out += (out.size() == 1 ? "" : ", ") + ("\"" + entry.first + "\": ") + entry.second.Describe(); + } + return out + "}"; + } + case Kind::Struct: { + Aws::String out = "{"; + for (const auto& member : m_members) { + out += (out.size() == 1 ? "#" : ", #") + StringUtils::to_string(member.first) + ": " + member.second.Describe(); + } + return out + "}"; + } + } + return {}; +} + +CodecTolerance CodecTolerance::Json() { + Options options; + options.epochSecondsDecodeTruncates = true; + return CodecTolerance(options); +} + +CodecTolerance CodecTolerance::Cbor() { + Options options; + options.timestampsAlwaysInSeconds = true; + options.dropsPreEpochTimestamps = true; + options.cborDropsNegativeZeroSign = true; + return CodecTolerance(options); +} + +CodecTolerance CodecTolerance::Xml() { + Options options; + options.defaultTimestampFormat = TimestampFormatTrait::Format::DATE_TIME; + options.epochSecondsDecodeTruncates = true; + options.emptyContainerMayBeAbsent = true; + return CodecTolerance(options); +} + +CodecTolerance CodecTolerance::Exact() { + Options options; + options.exactTimestamps = true; + return CodecTolerance(options); +} + +CodecTolerance CodecTolerance::ReadByCrtJson() const { + Options options = m_options; + options.numbersReadAsDoubles = true; + return CodecTolerance(options); +} + +bool CodecTolerance::CompareTimestampsInSeconds(const Schema& member) const { + return !m_options.exactTimestamps && + (m_options.timestampsAlwaysInSeconds || + ResolveTimestampFormat(member, m_options.defaultTimestampFormat) != TimestampFormatTrait::Format::EPOCH_SECONDS); +} + +namespace { + +bool SameFloatingPoint(double expected, double actual) { + if (std::isnan(expected) || std::isnan(actual)) { + return std::isnan(expected) && std::isnan(actual); + } + return expected == actual && std::signbit(expected) == std::signbit(actual); +} + +bool SameFloatingPoint(double expected, double actual, const CodecTolerance& tolerance) { + if (tolerance.CborDropsNegativeZeroSign() && expected == 0.0 && actual == 0.0 && std::signbit(expected) && !std::signbit(actual)) { + return true; + } + return SameFloatingPoint(expected, actual); +} + +bool IsIntegral(ShapeType type) { + return type == ShapeType::Byte || type == ShapeType::Short || type == ShapeType::Integer || type == ShapeType::Long; +} + +bool IsFloatingPoint(ShapeType type) { return type == ShapeType::Float || type == ShapeType::Double; } + +// DateTime::Seconds() and system_clock::to_time_t truncate toward zero. +bool SameTimestampTruncatedToSeconds(int64_t expectedMillis, int64_t actualMillis) { + return actualMillis % 1000 == 0 && actualMillis == expectedMillis - expectedMillis % 1000; +} + +int64_t LegacyEpochDecode(int64_t millis) { + const auto seconds = ParseDoubleText(EpochSecondsText(Aws::Utils::DateTime(millis))); + return seconds.has_value() ? Aws::Utils::DateTime(*seconds).Millis() : millis; +} + +bool IsEmptyContainer(const FuzzValue& value) { + return (value.GetKind() == FuzzValue::Kind::List && value.GetList().empty()) || + (value.GetKind() == FuzzValue::Kind::Map && value.GetMap().empty()); +} + +bool DroppedAsPreEpoch(const FuzzValue& value, const CodecTolerance& tolerance) { + return tolerance.DropsPreEpochTimestamps() && value.GetKind() == FuzzValue::Kind::Timestamp && value.GetTimestamp().Millis() <= -1000; +} + +Aws::String CompareValue(const FuzzValue& expected, const FuzzValue& actual, const Schema& member, const CodecTolerance& tolerance, + const Aws::String& path); + +Aws::String CompareStruct(const FuzzValue& expected, const FuzzValue& actual, const Schema& structSchema, const CodecTolerance& tolerance, + const Aws::String& path) { + if (structSchema.GetType() == ShapeType::Union && actual.GetMembers().size() > 1) { + return path + ": union holds " + StringUtils::to_string(actual.GetMembers().size()) + " members"; + } + for (const auto& entry : expected.GetMembers()) { + const auto member = structSchema.GetMember(entry.first); + if (!member.has_value() || !*member) { + return path + ": member #" + StringUtils::to_string(entry.first) + " is not in schema " + structSchema.GetId(); + } + const Aws::String memberPath = path + "." + (*member)->GetMemberName(); + const auto found = actual.GetMembers().find(entry.first); + if (found != actual.GetMembers().end()) { + const Aws::String mismatch = CompareValue(entry.second, found->second, **member, tolerance, memberPath); + if (!mismatch.empty()) { + return mismatch; + } + } else { + const bool droppedEmpty = tolerance.EmptyContainerMayBeAbsent() && IsEmptyContainer(entry.second); + if (!droppedEmpty && !DroppedAsPreEpoch(entry.second, tolerance)) { + return memberPath + ": missing, expected " + entry.second.Describe(); + } + } + } + for (const auto& entry : actual.GetMembers()) { + if (expected.GetMembers().find(entry.first) == expected.GetMembers().end()) { + const auto member = structSchema.GetMember(entry.first); + const Aws::String name = member.has_value() && *member ? (*member)->GetMemberName() : "#" + StringUtils::to_string(entry.first); + return path + "." + name + ": unexpected " + entry.second.Describe(); + } + } + return {}; +} + +__attribute__((noinline)) Aws::String CompareScalar(const FuzzValue& expected, const FuzzValue& actual, const Schema& member, + const CodecTolerance& tolerance, const Aws::String& path) { + if (tolerance.NumbersReadAsDoubles() && IsIntegral(member.GetType()) && expected.GetKind() != actual.GetKind()) { + const auto asDouble = [](const FuzzValue& v) { + return v.GetKind() == FuzzValue::Kind::Integer ? static_cast(v.GetInteger()) : v.GetDouble(); + }; + return asDouble(expected) == asDouble(actual) ? Aws::String() : path + ": " + expected.Describe() + " != " + actual.Describe(); + } + switch (expected.GetKind()) { + case FuzzValue::Kind::Boolean: + return expected.GetBoolean() == actual.GetBoolean() ? Aws::String() : path + ": " + expected.Describe() + " != " + actual.Describe(); + case FuzzValue::Kind::Integer: + return expected.GetInteger() == actual.GetInteger() ? Aws::String() : path + ": " + expected.Describe() + " != " + actual.Describe(); + case FuzzValue::Kind::Double: { + const bool same = member.GetType() == ShapeType::Float + ? SameFloatingPoint(static_cast(expected.GetDouble()), static_cast(actual.GetDouble()), tolerance) + : SameFloatingPoint(expected.GetDouble(), actual.GetDouble(), tolerance); + return same ? Aws::String() : path + ": " + expected.Describe() + " != " + actual.Describe(); + } + case FuzzValue::Kind::String: + return expected.GetString() == actual.GetString() ? Aws::String() : path + ": " + expected.Describe() + " != " + actual.Describe(); + case FuzzValue::Kind::Blob: + return expected.GetBlob() == actual.GetBlob() ? Aws::String() : path + ": blob bytes differ"; + case FuzzValue::Kind::Timestamp: { + const int64_t expectedMillis = expected.GetTimestamp().Millis(); + const int64_t actualMillis = actual.GetTimestamp().Millis(); + const bool same = tolerance.CompareTimestampsInSeconds(member) + ? SameTimestampTruncatedToSeconds(expectedMillis, actualMillis) + : expectedMillis == actualMillis || + (tolerance.EpochSecondsDecodeTruncates() && actualMillis == LegacyEpochDecode(expectedMillis)); + return same ? Aws::String() : path + ": " + expected.Describe() + " != " + actual.Describe(); + } + case FuzzValue::Kind::Document: + return DocumentsEquivalent(*expected.GetDocument(), *actual.GetDocument(), tolerance.NumbersReadAsDoubles()) + ? Aws::String() + : path + ": documents differ"; + default: + return {}; + } +} + +__attribute__((noinline)) Aws::String CompareList(const FuzzValue& expected, const FuzzValue& actual, const Schema& member, + const CodecTolerance& tolerance, const Aws::String& path) { + const auto target = member.GetMemberTarget(); + Aws::Vector kept; + for (const auto& item : expected.GetList()) { + if (!DroppedAsPreEpoch(item, tolerance)) { + kept.push_back(&item); + } + } + if (kept.size() != actual.GetList().size()) { + return path + ": list size " + StringUtils::to_string(kept.size()) + " != " + StringUtils::to_string(actual.GetList().size()); + } + const auto element = (*target)->GetMember("member").value(); + for (size_t i = 0; i < kept.size(); ++i) { + const Aws::String mismatch = + CompareValue(*kept[i], actual.GetList()[i], *element, tolerance, path + "[" + StringUtils::to_string(i) + "]"); + if (!mismatch.empty()) { + return mismatch; + } + } + return {}; +} + +__attribute__((noinline)) Aws::String CompareMap(const FuzzValue& expected, const FuzzValue& actual, const Schema& member, + const CodecTolerance& tolerance, const Aws::String& path) { + const auto target = member.GetMemberTarget(); + size_t keptCount = 0; + for (const auto& entry : expected.GetMap()) { + if (!DroppedAsPreEpoch(entry.second, tolerance)) { + ++keptCount; + } + } + if (keptCount != actual.GetMap().size()) { + return path + ": map size " + StringUtils::to_string(keptCount) + " != " + StringUtils::to_string(actual.GetMap().size()); + } + const auto valueMember = (*target)->GetMember("value").value(); + for (const auto& entry : expected.GetMap()) { + if (!DroppedAsPreEpoch(entry.second, tolerance)) { + const auto found = actual.GetMap().find(entry.first); + if (found == actual.GetMap().end()) { + return path + "[\"" + entry.first + "\"]: missing key"; + } + const Aws::String mismatch = CompareValue(entry.second, found->second, *valueMember, tolerance, path + "[\"" + entry.first + "\"]"); + if (!mismatch.empty()) { + return mismatch; + } + } + } + return {}; +} + +Aws::String CompareValue(const FuzzValue& expected, const FuzzValue& actual, const Schema& member, const CodecTolerance& tolerance, + const Aws::String& path) { + const bool inexactInteger = tolerance.NumbersReadAsDoubles() && IsIntegral(member.GetType()); + if (expected.GetKind() != actual.GetKind() && !inexactInteger) { + return path + ": kind differs, expected " + expected.Describe() + " got " + actual.Describe(); + } + switch (expected.GetKind()) { + case FuzzValue::Kind::List: + return CompareList(expected, actual, member, tolerance, path); + case FuzzValue::Kind::Map: + return CompareMap(expected, actual, member, tolerance, path); + case FuzzValue::Kind::Struct: + return CompareStruct(expected, actual, **member.GetMemberTarget(), tolerance, path); + default: + return CompareScalar(expected, actual, member, tolerance, path); + } +} + +Aws::Crt::Optional NumberAsDouble(const smithy::schema::Document& number) { + if (IsIntegral(number.GetType())) { + const auto value = number.AsLong(); + return value.has_value() ? Aws::Crt::Optional(static_cast(value.value())) : Aws::Crt::Optional(); + } + return IsFloatingPoint(number.GetType()) ? number.AsDouble() : Aws::Crt::Optional(); +} + +} // namespace + +Aws::String FindMismatch(const FuzzValue& expected, const FuzzValue& actual, const Schema& structSchema, const CodecTolerance& tolerance) { + return CompareStruct(expected, actual, structSchema, tolerance, structSchema.GetId()); +} + +bool ContainsDocument(const FuzzValue& value) { + switch (value.GetKind()) { + case FuzzValue::Kind::Document: + return true; + case FuzzValue::Kind::List: + for (const auto& item : value.GetList()) { + if (ContainsDocument(item)) { + return true; + } + } + return false; + case FuzzValue::Kind::Map: + for (const auto& entry : value.GetMap()) { + if (ContainsDocument(entry.second)) { + return true; + } + } + return false; + case FuzzValue::Kind::Struct: + for (const auto& entry : value.GetMembers()) { + if (ContainsDocument(entry.second)) { + return true; + } + } + return false; + default: + return false; + } +} + +bool DocumentsEquivalent(const smithy::schema::Document& expected, const smithy::schema::Document& actual, bool numbersByValue) { + const bool numbers = (IsIntegral(expected.GetType()) || IsFloatingPoint(expected.GetType())) && + (IsIntegral(actual.GetType()) || IsFloatingPoint(actual.GetType())); + if (numbersByValue && numbers) { + const auto e = NumberAsDouble(expected); + const auto a = NumberAsDouble(actual); + return e.has_value() && a.has_value() && (e.value() == a.value() || (std::isnan(e.value()) && std::isnan(a.value()))); + } + if (IsIntegral(expected.GetType())) { + const auto el = expected.AsLong(); + const auto al = IsIntegral(actual.GetType()) ? actual.AsLong() : Aws::Crt::Optional(); + return el.has_value() && al.has_value() && el.value() == al.value(); + } + if (IsFloatingPoint(expected.GetType())) { + const auto e = expected.AsDouble(); + const auto a = IsFloatingPoint(actual.GetType()) ? actual.AsDouble() : Aws::Crt::Optional(); + if (e.has_value() && a.has_value()) { + return SameFloatingPoint(e.value(), a.value()); + } + const auto text = actual.GetType() == ShapeType::String ? actual.AsString() : Aws::Crt::Optional(); + return e.has_value() && !std::isfinite(e.value()) && text.has_value() && + text.value() == (std::isnan(e.value()) ? "NaN" : (e.value() > 0 ? "Infinity" : "-Infinity")); + } + if (expected.GetType() != actual.GetType()) { + return false; + } + if (expected.GetType() == ShapeType::List) { + const auto el = expected.AsList(); + const auto al = actual.AsList(); + if (!el || !al || el->size() != al->size()) { + return false; + } + for (size_t i = 0; i < el->size(); ++i) { + if (!DocumentsEquivalent(*(*el)[i], *(*al)[i], numbersByValue)) { + return false; + } + } + return true; + } + if (expected.GetType() == ShapeType::Map) { + const auto em = expected.AsMap(); + const auto am = actual.AsMap(); + if (!em || !am || em->size() != am->size()) { + return false; + } + for (const auto& entry : *em) { + const auto found = am->find(entry.first); + if (found == am->end() || !DocumentsEquivalent(*entry.second, *found->second, numbersByValue)) { + return false; + } + } + return true; + } + return expected == actual; +} + +} // namespace fuzz +} // namespace schema +} // namespace smithy diff --git a/tests/fuzz/harness/FuzzValue.h b/tests/fuzz/harness/FuzzValue.h new file mode 100644 index 000000000000..379989838388 --- /dev/null +++ b/tests/fuzz/harness/FuzzValue.h @@ -0,0 +1,115 @@ +/** + * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. + * SPDX-License-Identifier: Apache-2.0. + */ +#pragma once + +#include +#include +#include +#include +#include +#include +#include +#include + +#include +#include + +namespace smithy { +namespace schema { +namespace fuzz { + +class FuzzValue final { + public: + enum class Kind { Boolean, Integer, Double, String, Blob, Timestamp, Document, List, Map, Struct }; + + static FuzzValue Boolean(bool value); + static FuzzValue Integer(int64_t value); + static FuzzValue Double(double value); + static FuzzValue String(Aws::String value); + static FuzzValue Blob(Aws::Utils::ByteBuffer value); + static FuzzValue Timestamp(Aws::Utils::DateTime value); + static FuzzValue FromDocument(std::shared_ptr value); + static FuzzValue List(Aws::Vector items); + static FuzzValue Map(Aws::Map entries); + static FuzzValue Struct(); + + Kind GetKind() const { return m_kind; } + bool GetBoolean() const { return m_boolean; } + int64_t GetInteger() const { return m_integer; } + double GetDouble() const { return m_double; } + const Aws::String& GetString() const { return m_string; } + const Aws::Utils::ByteBuffer& GetBlob() const { return m_blob; } + const Aws::Utils::DateTime& GetTimestamp() const { return m_timestamp; } + const std::shared_ptr& GetDocument() const { return m_document; } + const Aws::Vector& GetList() const { return m_list; } + const Aws::Map& GetMap() const { return m_map; } + const Aws::Map& GetMembers() const { return m_members; } + + void SetMember(int index, FuzzValue value); + + Aws::String Describe() const; + + private: + explicit FuzzValue(Kind kind) : m_kind(kind) {} + + Kind m_kind; + bool m_boolean = false; + int64_t m_integer = 0; + double m_double = 0.0; + Aws::String m_string; + Aws::Utils::ByteBuffer m_blob; + Aws::Utils::DateTime m_timestamp; + std::shared_ptr m_document; + Aws::Vector m_list; + Aws::Map m_map; + Aws::Map m_members; +}; + +class CodecTolerance final { + public: + struct Options { + TimestampFormatTrait::Format defaultTimestampFormat = TimestampFormatTrait::Format::EPOCH_SECONDS; + // For comparing two generated values, not a codec round trip. + bool exactTimestamps = false; + bool timestampsAlwaysInSeconds = false; + bool dropsPreEpochTimestamps = false; + bool epochSecondsDecodeTruncates = false; + bool emptyContainerMayBeAbsent = false; + bool cborDropsNegativeZeroSign = false; + // cJSON reads every number as a double. + bool numbersReadAsDoubles = false; + }; + + static CodecTolerance Json(); + static CodecTolerance Cbor(); + static CodecTolerance Xml(); + static CodecTolerance Exact(); + + CodecTolerance ReadByCrtJson() const; + + bool CompareTimestampsInSeconds(const Schema& member) const; + bool DropsPreEpochTimestamps() const { return m_options.dropsPreEpochTimestamps; } + bool EpochSecondsDecodeTruncates() const { return !m_options.exactTimestamps && m_options.epochSecondsDecodeTruncates; } + bool EmptyContainerMayBeAbsent() const { return m_options.emptyContainerMayBeAbsent; } + bool CborDropsNegativeZeroSign() const { return m_options.cborDropsNegativeZeroSign; } + bool NumbersReadAsDoubles() const { return m_options.numbersReadAsDoubles; } + + private: + explicit CodecTolerance(const Options& options) : m_options(options) {} + + Options m_options; +}; + +// Empty when `actual` is an acceptable round trip of `expected`; otherwise the first difference. +Aws::String FindMismatch(const FuzzValue& expected, const FuzzValue& actual, const Schema& structSchema, const CodecTolerance& tolerance); + +bool ContainsDocument(const FuzzValue& value); + +// Numbers keep their type unless `numbersByValue`; a non-finite double may come back as its JSON string. +bool DocumentsEquivalent(const smithy::schema::Document& expected, const smithy::schema::Document& actual, bool numbersByValue = false); + +} // namespace fuzz +} // namespace schema +} // namespace smithy diff --git a/tests/fuzz/harness/FuzzValueTest.cpp b/tests/fuzz/harness/FuzzValueTest.cpp new file mode 100644 index 000000000000..3cb408bd1f1b --- /dev/null +++ b/tests/fuzz/harness/FuzzValueTest.cpp @@ -0,0 +1,258 @@ +/** + * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. + * SPDX-License-Identifier: Apache-2.0. + */ +#include +#include +#include +#include +#include +#include + +#include + +using namespace smithy::schema; +using namespace smithy::schema::fuzz; + +namespace { + +std::shared_ptr Root() { + auto list = Schema::ListBuilder("t#IntList").PutMember("member", Schema::CreateInteger("t#Int")).Build(); + auto timestamps = Schema::ListBuilder("t#TsList").PutMember("member", Schema::CreateTimestamp("t#Ts")).Build(); + return Schema::StructureBuilder("t#Root") + .PutMember("i", Schema::CreateInteger("t#Int")) + .PutMember("f", Schema::CreateFloat("t#Float")) + .PutMember("d", Schema::CreateDouble("t#Double")) + .PutMember("ts", Schema::CreateTimestamp("t#Ts")) + .PutMember("list", list) + .PutMember("doc", Schema::CreateDocument("t#Doc")) + .PutMember("tslist", timestamps) + .PutMember("tsmap", Schema::MapBuilder("t#TsMap") + .PutMember("key", Schema::CreateString("t#Key")) + .PutMember("value", Schema::CreateTimestamp("t#Ts")) + .Build()) + .Build(); +} + +FuzzValue Ts(int64_t millis) { return FuzzValue::Timestamp(Aws::Utils::DateTime(millis)); } + +FuzzValue With(int index, FuzzValue v) { + FuzzValue s = FuzzValue::Struct(); + s.SetMember(index, std::move(v)); + return s; +} + +} // namespace + +TEST(FuzzValue, IdenticalTreesMatch) { + auto root = Root(); + auto v = With(0, FuzzValue::Integer(7)); + EXPECT_EQ(FindMismatch(v, v, *root, CodecTolerance::Json()), ""); +} + +TEST(FuzzValue, DifferentIntegerNamesTheMember) { + auto root = Root(); + const auto mismatch = FindMismatch(With(0, FuzzValue::Integer(7)), With(0, FuzzValue::Integer(8)), *root, CodecTolerance::Json()); + EXPECT_NE(mismatch.find("i"), Aws::String::npos); +} + +TEST(FuzzValue, MissingAndExtraMembersMismatch) { + auto root = Root(); + EXPECT_NE(FindMismatch(With(0, FuzzValue::Integer(1)), FuzzValue::Struct(), *root, CodecTolerance::Json()), ""); + EXPECT_NE(FindMismatch(FuzzValue::Struct(), With(0, FuzzValue::Integer(1)), *root, CodecTolerance::Json()), ""); +} + +TEST(FuzzValue, UnionWithTwoMembersMismatchesEvenAgainstItself) { + auto variants = + Schema::UnionBuilder("t#U").PutMember("a", Schema::CreateInteger("t#Int")).PutMember("b", Schema::CreateInteger("t#Int")).Build(); + FuzzValue both = With(0, FuzzValue::Integer(1)); + both.SetMember(1, FuzzValue::Integer(2)); + EXPECT_NE(FindMismatch(both, both, *variants, CodecTolerance::Exact()).find("union holds 2 members"), Aws::String::npos); + EXPECT_EQ(FindMismatch(With(1, FuzzValue::Integer(2)), With(1, FuzzValue::Integer(2)), *variants, CodecTolerance::Exact()), ""); +} + +TEST(FuzzValue, NanMatchesNanButSignedZeroDoesNotMatchZero) { + auto root = Root(); + const double nan = std::numeric_limits::quiet_NaN(); + EXPECT_EQ(FindMismatch(With(2, FuzzValue::Double(nan)), With(2, FuzzValue::Double(nan)), *root, CodecTolerance::Json()), ""); + EXPECT_NE(FindMismatch(With(2, FuzzValue::Double(-0.0)), With(2, FuzzValue::Double(0.0)), *root, CodecTolerance::Json()), ""); +} + +TEST(FuzzValue, CborMayDropNegativeZeroSign) { + auto root = Root(); + for (const int member : {1, 2}) { + EXPECT_EQ(FindMismatch(With(member, FuzzValue::Double(-0.0)), With(member, FuzzValue::Double(0.0)), *root, CodecTolerance::Cbor()), ""); + EXPECT_NE(FindMismatch(With(member, FuzzValue::Double(0.0)), With(member, FuzzValue::Double(-0.0)), *root, CodecTolerance::Cbor()), ""); + EXPECT_NE(FindMismatch(With(member, FuzzValue::Double(-0.0)), With(member, FuzzValue::Double(0.0)), *root, CodecTolerance::Json()), ""); + EXPECT_NE(FindMismatch(With(member, FuzzValue::Double(-0.0)), With(member, FuzzValue::Double(0.0)), *root, CodecTolerance::Xml()), ""); + EXPECT_NE(FindMismatch(With(member, FuzzValue::Double(1.0)), With(member, FuzzValue::Double(0.0)), *root, CodecTolerance::Cbor()), ""); + EXPECT_NE(FindMismatch(With(member, FuzzValue::Double(0.0)), With(member, FuzzValue::Double(1.0)), *root, CodecTolerance::Cbor()), ""); + } +} + +TEST(FuzzValue, FloatMembersCompareAtFloatPrecision) { + auto root = Root(); + const double widened = static_cast(0.1f); + EXPECT_EQ(FindMismatch(With(1, FuzzValue::Double(widened)), With(1, FuzzValue::Double(widened)), *root, CodecTolerance::Json()), ""); + EXPECT_NE(FindMismatch(With(1, FuzzValue::Double(widened)), With(1, FuzzValue::Double(0.2)), *root, CodecTolerance::Json()), ""); +} + +TEST(FuzzValue, TimestampPrecisionDependsOnCodecAndFormat) { + auto root = Root(); + auto expected = With(3, Ts(1500)); + auto actual = With(3, Ts(1000)); + EXPECT_NE(FindMismatch(expected, actual, *root, CodecTolerance::Json()), ""); // JSON default is epoch-seconds with millis + EXPECT_EQ(FindMismatch(expected, actual, *root, CodecTolerance::Cbor()), ""); // CBOR truncates to seconds by design + EXPECT_EQ(FindMismatch(expected, actual, *root, CodecTolerance::Xml()), ""); // XML default is date-time +} + +TEST(FuzzValue, SecondsPrecisionRequiresTruncationTowardZero) { + auto root = Root(); + for (const auto& tolerance : {CodecTolerance::Cbor(), CodecTolerance::Xml()}) { + EXPECT_EQ(FindMismatch(With(3, Ts(1500)), With(3, Ts(1000)), *root, tolerance), ""); + EXPECT_NE(FindMismatch(With(3, Ts(1500)), With(3, Ts(2000)), *root, tolerance), ""); + EXPECT_NE(FindMismatch(With(3, Ts(1500)), With(3, Ts(1500)), *root, tolerance), ""); + EXPECT_NE(FindMismatch(With(3, Ts(1500)), With(3, Ts(0)), *root, tolerance), ""); + EXPECT_EQ(FindMismatch(With(3, Ts(1000)), With(3, Ts(1000)), *root, tolerance), ""); + EXPECT_EQ(FindMismatch(With(3, Ts(-500)), With(3, Ts(0)), *root, tolerance), ""); + EXPECT_NE(FindMismatch(With(3, Ts(-500)), With(3, Ts(-1000)), *root, tolerance), ""); + EXPECT_NE(FindMismatch(With(3, Ts(-500)), With(3, Ts(-500)), *root, tolerance), ""); + } +} + +TEST(FuzzValue, EpochSecondsAcceptsOnlyTheLegacyTruncatedDecode) { + auto root = Root(); + // "4307758798.296" is stored just below .296, so DateTime(double) gives ...295. + EXPECT_EQ(FindMismatch(With(3, Ts(4307758798296)), With(3, Ts(4307758798295)), *root, CodecTolerance::Json()), ""); + EXPECT_NE(FindMismatch(With(3, Ts(4307758798296)), With(3, Ts(4307758798294)), *root, CodecTolerance::Json()), ""); + EXPECT_NE(FindMismatch(With(3, Ts(4307758798296)), With(3, Ts(4307758798297)), *root, CodecTolerance::Json()), ""); + EXPECT_NE(FindMismatch(With(3, Ts(1500)), With(3, Ts(1499)), *root, CodecTolerance::Json()), ""); + EXPECT_NE(FindMismatch(With(3, Ts(4307758798296)), With(3, Ts(4307758798295)), *root, CodecTolerance::Exact()), ""); +} + +TEST(FuzzValue, ExactToleranceComparesSubSecondTimestampsOnSecondsFormattedMembers) { + auto root = Root(); + EXPECT_EQ(FindMismatch(With(3, Ts(999)), With(3, Ts(999)), *root, CodecTolerance::Exact()), ""); + EXPECT_NE(FindMismatch(With(3, Ts(999)), With(3, Ts(999)), *root, CodecTolerance::Xml()), ""); + EXPECT_NE(FindMismatch(With(3, Ts(999)), With(3, Ts(0)), *root, CodecTolerance::Exact()), ""); +} + +TEST(FuzzValue, XmlSecondsPrecisionTruncatesPreEpochTowardZero) { + auto root = Root(); + EXPECT_EQ(FindMismatch(With(3, Ts(-1500)), With(3, Ts(-1000)), *root, CodecTolerance::Xml()), ""); + EXPECT_NE(FindMismatch(With(3, Ts(-1500)), With(3, Ts(-2000)), *root, CodecTolerance::Xml()), ""); + EXPECT_NE(FindMismatch(With(3, Ts(-1500)), With(3, Ts(-1500)), *root, CodecTolerance::Xml()), ""); +} + +TEST(FuzzValue, CborMayDropPreEpochTimestamps) { + auto root = Root(); + auto expected = With(3, FuzzValue::Timestamp(Aws::Utils::DateTime(static_cast(-5000)))); + EXPECT_EQ(FindMismatch(expected, FuzzValue::Struct(), *root, CodecTolerance::Cbor()), ""); + EXPECT_NE(FindMismatch(expected, FuzzValue::Struct(), *root, CodecTolerance::Json()), ""); +} + +TEST(FuzzValue, CborMayDropPreEpochTimestampsInsideCollections) { + auto root = Root(); + const auto preEpoch = FuzzValue::Timestamp(Aws::Utils::DateTime(static_cast(-5000))); + const auto atEpoch = FuzzValue::Timestamp(Aws::Utils::DateTime(static_cast(1000))); + auto expected = With(6, FuzzValue::List({preEpoch, atEpoch})); + auto actual = With(6, FuzzValue::List({atEpoch})); + EXPECT_EQ(FindMismatch(expected, actual, *root, CodecTolerance::Cbor()), ""); + EXPECT_NE(FindMismatch(expected, actual, *root, CodecTolerance::Json()), ""); + EXPECT_NE(FindMismatch(expected, With(6, FuzzValue::List({preEpoch, atEpoch})), *root, CodecTolerance::Cbor()), ""); + EXPECT_NE(FindMismatch(With(6, FuzzValue::List({atEpoch})), With(6, FuzzValue::List({})), *root, CodecTolerance::Cbor()), ""); + const auto subSecond = FuzzValue::Timestamp(Aws::Utils::DateTime(static_cast(-999))); + EXPECT_NE(FindMismatch(With(6, FuzzValue::List({subSecond})), With(6, FuzzValue::List({})), *root, CodecTolerance::Cbor()), ""); +} + +TEST(FuzzValue, CborMayDropPreEpochTimestampsInMapValues) { + auto root = Root(); + const auto atEpoch = Ts(1000); + auto expected = With(7, FuzzValue::Map({{"a", Ts(-5000)}, {"b", atEpoch}})); + auto actual = With(7, FuzzValue::Map({{"b", atEpoch}})); + EXPECT_EQ(FindMismatch(expected, actual, *root, CodecTolerance::Cbor()), ""); + EXPECT_NE(FindMismatch(expected, actual, *root, CodecTolerance::Json()), ""); + EXPECT_NE(FindMismatch(expected, With(7, FuzzValue::Map({{"a", Ts(-5000)}, {"b", atEpoch}})), *root, CodecTolerance::Cbor()), ""); + EXPECT_NE(FindMismatch(With(7, FuzzValue::Map({{"b", atEpoch}})), With(7, FuzzValue::Map({})), *root, CodecTolerance::Cbor()), ""); + EXPECT_NE(FindMismatch(With(7, FuzzValue::Map({{"b", Ts(-999)}})), With(7, FuzzValue::Map({})), *root, CodecTolerance::Cbor()), ""); +} + +TEST(FuzzValue, CborPreEpochDropBoundaryIsMinusOneSecond) { + auto root = Root(); + EXPECT_EQ(FindMismatch(With(3, Ts(-1000)), FuzzValue::Struct(), *root, CodecTolerance::Cbor()), ""); + EXPECT_NE(FindMismatch(With(3, Ts(-999)), FuzzValue::Struct(), *root, CodecTolerance::Cbor()), ""); + EXPECT_EQ(FindMismatch(With(3, Ts(-999)), With(3, Ts(0)), *root, CodecTolerance::Cbor()), ""); +} + +TEST(FuzzValue, XmlMayDropEmptyContainers) { + auto root = Root(); + auto expected = With(4, FuzzValue::List({})); + EXPECT_EQ(FindMismatch(expected, FuzzValue::Struct(), *root, CodecTolerance::Xml()), ""); + EXPECT_NE(FindMismatch(expected, FuzzValue::Struct(), *root, CodecTolerance::Json()), ""); +} + +TEST(FuzzValue, ListElementDifferenceIsReported) { + auto root = Root(); + auto a = With(4, FuzzValue::List({FuzzValue::Integer(1), FuzzValue::Integer(2)})); + auto b = With(4, FuzzValue::List({FuzzValue::Integer(1), FuzzValue::Integer(3)})); + EXPECT_NE(FindMismatch(a, b, *root, CodecTolerance::Json()).find("[1]"), Aws::String::npos); +} + +namespace { + +std::shared_ptr Obj(Aws::Map> entries) { + return Document::FromMap(std::move(entries)); +} + +} // namespace + +TEST(FuzzValue, DocumentIntegersRequireIntegersOfTheSameValue) { + EXPECT_TRUE(DocumentsEquivalent(*Document::FromInteger(5), *Document::FromInteger(5))); + EXPECT_FALSE(DocumentsEquivalent(*Document::FromInteger(5), *Document::FromInteger(6))); + EXPECT_FALSE(DocumentsEquivalent(*Document::FromInteger(9007199254740993LL), *Document::FromDouble(9007199254740992.0))); + EXPECT_FALSE(DocumentsEquivalent(*Document::FromInteger(5), *Document::FromDouble(5.0))); +} + +TEST(FuzzValue, DocumentDoublesRequireDoublesWithNanAndSignedZeroPreserved) { + const double nan = std::numeric_limits::quiet_NaN(); + EXPECT_TRUE(DocumentsEquivalent(*Document::FromDouble(nan), *Document::FromDouble(nan))); + EXPECT_TRUE(DocumentsEquivalent(*Document::FromDouble(5.5), *Document::FromDouble(5.5))); + EXPECT_FALSE(DocumentsEquivalent(*Document::FromDouble(-0.0), *Document::FromDouble(0.0))); + EXPECT_FALSE(DocumentsEquivalent(*Document::FromDouble(5.0), *Document::FromInteger(5))); + EXPECT_FALSE(DocumentsEquivalent(*Document::FromDouble(5.5), *Document::FromString("5.5"))); +} + +TEST(FuzzValue, DocumentNumbersByValueIgnoreTypeAndTheSignOfZero) { + EXPECT_TRUE(DocumentsEquivalent(*Document::FromDouble(5.0), *Document::FromInteger(5), true)); + EXPECT_TRUE(DocumentsEquivalent(*Document::FromDouble(-0.0), *Document::FromInteger(0), true)); + EXPECT_FALSE(DocumentsEquivalent(*Document::FromDouble(5.5), *Document::FromInteger(5), true)); + EXPECT_FALSE(DocumentsEquivalent(*Document::FromDouble(-0.0), *Document::FromInteger(0))); +} + +TEST(FuzzValue, NonFiniteDocumentDoublesMayReadBackAsTheirStrings) { + const double inf = std::numeric_limits::infinity(); + EXPECT_TRUE(DocumentsEquivalent(*Document::FromDouble(std::numeric_limits::quiet_NaN()), *Document::FromString("NaN"))); + EXPECT_TRUE(DocumentsEquivalent(*Document::FromDouble(inf), *Document::FromString("Infinity"))); + EXPECT_TRUE(DocumentsEquivalent(*Document::FromDouble(-inf), *Document::FromString("-Infinity"))); + EXPECT_FALSE(DocumentsEquivalent(*Document::FromDouble(inf), *Document::FromString("-Infinity"))); + EXPECT_FALSE(DocumentsEquivalent(*Document::FromDouble(std::numeric_limits::quiet_NaN()), *Document::FromString("nan"))); + EXPECT_FALSE(DocumentsEquivalent(*Document::FromDouble(inf), *Document::FromDouble(-inf))); +} + +TEST(FuzzValue, DocumentComparisonRecursesIntoContainers) { + const auto list = [](int64_t last) { return Document::FromList({Document::FromInteger(1), Document::FromInteger(last)}); }; + EXPECT_TRUE(DocumentsEquivalent(*Obj({{"a", list(2)}}), *Obj({{"a", list(2)}}))); + EXPECT_FALSE(DocumentsEquivalent(*Obj({{"a", list(2)}}), *Obj({{"a", list(3)}}))); + EXPECT_FALSE( + DocumentsEquivalent(*Obj({{"a", Obj({{"n", Document::FromInteger(1)}})}}), *Obj({{"a", Obj({{"n", Document::FromDouble(1.0)}})}}))); + EXPECT_FALSE(DocumentsEquivalent(*Obj({{"a", list(2)}}), *Obj({{"b", list(2)}}))); + EXPECT_FALSE(DocumentsEquivalent(*Obj({{"a", list(2)}, {"b", list(2)}}), *Obj({{"a", list(2)}}))); +} + +TEST(FuzzValue, DocumentsCompareScalarsAndContainers) { + Aws::Map> m1{{"a", Document::FromString("x")}}; + Aws::Map> m2{{"a", Document::FromString("y")}}; + EXPECT_FALSE(DocumentsEquivalent(*Document::FromMap(m1), *Document::FromMap(m2))); + EXPECT_TRUE(ContainsDocument(With(5, FuzzValue::FromDocument(Document::Null())))); + EXPECT_FALSE(ContainsDocument(With(0, FuzzValue::Integer(1)))); +} diff --git a/tests/fuzz/harness/ReferenceParity.cpp b/tests/fuzz/harness/ReferenceParity.cpp new file mode 100644 index 000000000000..553c4c6f84c9 --- /dev/null +++ b/tests/fuzz/harness/ReferenceParity.cpp @@ -0,0 +1,918 @@ +/** + * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. + * SPDX-License-Identifier: Apache-2.0. + */ +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include +#include +#include +#include +#include +#include +#include +#include + +// Called directly for require_null_terminated: aws_json_value_new_from_string accepts trailing data. +extern "C" { +struct cJSON; +cJSON* cJSON_ParseWithOpts(const char* value, const char** return_parse_end, int require_null_terminated); +} + +using Aws::Utils::StringUtils; + +namespace smithy { +namespace schema { +namespace fuzz { + +namespace { + +constexpr double EXACT_INTEGER_LIMIT = 9007199254740992.0; +constexpr double INT64_LIMIT = 9223372036854775808.0; +const char CONTROL_BYTES[] = + "\x01\x02\x03\x04\x05\x06\x07\x08\x0B\x0C\x0E\x0F\x10\x11\x12\x13\x14\x15\x16\x17\x18\x19\x1A\x1B\x1C\x1D\x1E\x1F"; + +Aws::Crt::ByteCursor CursorOf(const std::string& bytes) { + return Aws::Crt::ByteCursorFromArray(reinterpret_cast(bytes.data()), bytes.size()); +} + +// Both references read a NUL-terminated copy and reject blank input, which the codecs read as an empty struct. +bool ReferenceCanJudge(const std::string& bytes) { + const size_t start = bytes.compare(0, 3, "\xEF\xBB\xBF") == 0 ? 3 : 0; + return bytes.find('\0') == std::string::npos && bytes.find_first_not_of(" \t\r\n", start) != std::string::npos; +} + +void Note(Aws::String& slot, const Aws::String& why) { + if (slot.empty()) { + slot = why; + } +} + +using TextTest = bool (*)(const Aws::String&); + +bool AnyText(const Document& document, TextTest test) { + if (document.GetType() == ShapeType::String) { + const auto text = document.AsString(); + return text.has_value() && test(text.value()); + } + if (document.GetType() == ShapeType::List) { + const auto list = document.AsList(); + return list && + std::any_of(list->begin(), list->end(), [test](const std::shared_ptr& item) { return AnyText(*item, test); }); + } + if (document.GetType() == ShapeType::Map) { + const auto map = document.AsMap(); + return map && std::any_of(map->begin(), map->end(), [test](const std::pair>& entry) { + return test(entry.first) || AnyText(*entry.second, test); + }); + } + return false; +} + +bool AnyText(const FuzzValue& value, TextTest test) { + switch (value.GetKind()) { + case FuzzValue::Kind::String: + return test(value.GetString()); + case FuzzValue::Kind::Document: + return value.GetDocument() && AnyText(*value.GetDocument(), test); + case FuzzValue::Kind::List: + return std::any_of(value.GetList().begin(), value.GetList().end(), [test](const FuzzValue& item) { return AnyText(item, test); }); + case FuzzValue::Kind::Map: + return std::any_of(value.GetMap().begin(), value.GetMap().end(), [test](const std::pair& entry) { + return test(entry.first) || AnyText(entry.second, test); + }); + case FuzzValue::Kind::Struct: + return std::any_of(value.GetMembers().begin(), value.GetMembers().end(), + [test](const std::pair& entry) { return AnyText(entry.second, test); }); + default: + return false; + } +} + +// Both references truncate a string at a NUL. +bool HasNul(const Aws::String& text) { return text.find('\0') != Aws::String::npos; } + +// tinyxml2 drops whitespace-only text; the codec keeps it. +bool IsWhitespaceOnly(const Aws::String& text) { return !text.empty() && text.find_first_not_of(" \t\n\v\f\r") == Aws::String::npos; } + +Aws::Crt::Optional IntegerFor(const Schema& member, int64_t value, ReferenceReading& reading) { + if (member.GetType() == ShapeType::Long) { + return FuzzValue::Integer(value); + } + if (value < INT32_MIN || value > INT32_MAX) { + Note(reading.mustReject, "integer out of range"); + return {}; + } + const int64_t limit = member.GetType() == ShapeType::Byte ? INT8_MAX : (member.GetType() == ShapeType::Short ? INT16_MAX : INT32_MAX); + if (value < -limit - 1 || value > limit) { + return {}; + } + return FuzzValue::Integer(value); +} + +Aws::Crt::Optional IntegerFromText(const Schema& member, const Aws::String& text, ReferenceReading& reading) { + const IntegerText parsed = ParseIntegerText(text); + if (parsed.kind == IntegerTextKind::OUT_OF_RANGE) { + Note(reading.mustReject, "integer text out of range"); + } + return parsed.kind == IntegerTextKind::VALUE ? IntegerFor(member, parsed.value, reading) : Aws::Crt::Optional(); +} + +Aws::Crt::Optional FloatingPointFor(const Schema& member, double value, ReferenceReading& reading) { + if (member.GetType() == ShapeType::Double) { + return FuzzValue::Double(value); + } + const double floatMax = static_cast((std::numeric_limits::max)()); + if (std::isfinite(value) && std::fabs(value) > floatMax) { + if (std::fabs(value) >= floatMax + std::ldexp(1.0, 103)) { + Note(reading.mustReject, "float out of range"); + return {}; + } + return FuzzValue::Double(value < 0 ? -floatMax : floatMax); + } + return FuzzValue::Double(static_cast(static_cast(value))); +} + +Aws::Crt::Optional NonFiniteFromText(const Aws::String& text) { + if (text == "NaN") { + return FuzzValue::Double(std::numeric_limits::quiet_NaN()); + } + if (text == "Infinity" || text == "-Infinity") { + return FuzzValue::Double(text[0] == '-' ? -std::numeric_limits::infinity() : std::numeric_limits::infinity()); + } + return {}; +} + +Aws::Crt::Optional EpochTimestamp(double seconds, ReferenceReading& reading) { + const auto timestamp = EpochSecondsInClockRange(seconds); + if (!timestamp.has_value()) { + Note(reading.mustReject, "epoch timestamp beyond the system clock"); + return {}; + } + return FuzzValue::Timestamp(timestamp.value()); +} + +Aws::Crt::Optional TextTimestamp(const Aws::String& text, TimestampFormatTrait::Format format) { + const Aws::Utils::DateTime parsed( + text, format == TimestampFormatTrait::Format::HTTP_DATE ? Aws::Utils::DateFormat::RFC822 : Aws::Utils::DateFormat::ISO_8601); + return parsed.WasParseSuccessful() ? Aws::Crt::Optional(FuzzValue::Timestamp(parsed)) : Aws::Crt::Optional(); +} + +FuzzValue BlobFromText(const Aws::String& text) { return FuzzValue::Blob(Aws::Utils::HashingUtils::Base64Decode(text)); } + +std::shared_ptr ListElement(const Schema& member) { return (*member.GetMemberTarget())->GetMember("member").value(); } + +std::shared_ptr MapValue(const Schema& member) { return (*member.GetMemberTarget())->GetMember("value").value(); } + +template +Aws::String TraitValue(const Schema& member, const TraitKey& key, const Aws::String& fallback) { + const auto trait = member.GetTrait(key); + return trait ? trait->GetValue() : fallback; +} + +class CrtJson final { + public: + explicit CrtJson(const std::string& text) { + const Aws::String copy(text.data(), text.size()); + m_value = reinterpret_cast(cJSON_ParseWithOpts(copy.c_str(), nullptr, 1)); + } + ~CrtJson() { + if (m_value != nullptr) { + aws_json_value_destroy(m_value); + } + } + CrtJson(const CrtJson&) = delete; + CrtJson& operator=(const CrtJson&) = delete; + + const aws_json_value* Get() const { return m_value; } + + private: + aws_json_value* m_value = nullptr; +}; + +using JsonMembers = Aws::Vector>; + +int CollectMember(const aws_byte_cursor* key, const aws_json_value* value, bool*, void* members) { + static_cast(members)->emplace_back(Aws::String(reinterpret_cast(key->ptr), key->len), value); + return AWS_OP_SUCCESS; +} + +int CollectItem(size_t, const aws_json_value* value, bool*, void* items) { + static_cast*>(items)->push_back(value); + return AWS_OP_SUCCESS; +} + +JsonMembers ObjectMembers(const aws_json_value* object) { + JsonMembers members; + aws_json_const_iterate_object(object, &CollectMember, &members); + return members; +} + +Aws::Vector ArrayItems(const aws_json_value* array) { + Aws::Vector items; + aws_json_const_iterate_array(array, &CollectItem, &items); + return items; +} + +Aws::String JsonText(const aws_json_value* node) { + aws_byte_cursor text; + aws_json_value_get_string(node, &text); + return Aws::String(reinterpret_cast(text.ptr), text.len); +} + +double JsonNumber(const aws_json_value* node) { + double number = 0.0; + aws_json_value_get_number(node, &number); + return number; +} + +// The codecs count skipped values against MAX_NESTING_DEPTH too. +size_t JsonDepth(const aws_json_value* node) { + size_t deepest = 0; + if (aws_json_value_is_object(node)) { + for (const auto& member : ObjectMembers(node)) { + deepest = (std::max)(deepest, JsonDepth(member.second)); + } + } else if (aws_json_value_is_array(node)) { + for (const auto* item : ArrayItems(node)) { + deepest = (std::max)(deepest, JsonDepth(item)); + } + } else { + return 0; + } + return deepest + 1; +} + +class JsonReference final { + public: + explicit JsonReference(ReferenceReading& reading) : m_reading(reading) {} + + __attribute__((noinline)) FuzzValue Struct(const Schema& structSchema, const aws_json_value* node) { + FuzzValue result = FuzzValue::Struct(); + if (!aws_json_value_is_object(node)) { + return result; + } + bool delivered = false; + for (const auto& entry : ObjectMembers(node)) { + const auto member = Member(structSchema, entry.first); + const bool modeled = member && !aws_json_value_is_null(entry.second); + if (modeled && delivered && structSchema.GetType() == ShapeType::Union) { + Note(m_reading.mustReject, "union has more than one member set"); + } + delivered = delivered || modeled; + auto value = modeled ? Value(*member, entry.second) : Aws::Crt::Optional(); + if (value.has_value()) { + result.SetMember(member->GetMemberIndex(), std::move(value.value())); + } + } + return result; + } + + private: + static std::shared_ptr Member(const Schema& structSchema, const Aws::String& name) { + for (uint16_t i = 0; i < structSchema.GetMemberCount(); ++i) { + const auto member = structSchema.GetMember(static_cast(i)).value(); + if (TraitValue(*member, JsonNameTrait::KEY(), member->GetMemberName()) == name) { + return member; + } + } + return nullptr; + } + + Aws::Crt::Optional Value(const Schema& member, const aws_json_value* node) { + switch (member.GetType()) { + case ShapeType::List: + return List(member, node); + case ShapeType::Map: + return Map(member, node); + case ShapeType::Structure: + case ShapeType::Union: + return Struct(**member.GetMemberTarget(), node); + default: + return Scalar(member, node); + } + } + + __attribute__((noinline)) FuzzValue List(const Schema& member, const aws_json_value* node) { + Aws::Vector items; + if (aws_json_value_is_array(node)) { + const auto element = ListElement(member); + for (const auto* item : ArrayItems(node)) { + auto value = aws_json_value_is_null(item) ? Aws::Crt::Optional() : Value(*element, item); + if (value.has_value()) { + items.push_back(std::move(value.value())); + } + } + } + return FuzzValue::List(std::move(items)); + } + + __attribute__((noinline)) FuzzValue Map(const Schema& member, const aws_json_value* node) { + Aws::Map entries; + if (aws_json_value_is_object(node)) { + const auto valueMember = MapValue(member); + for (const auto& entry : ObjectMembers(node)) { + auto value = aws_json_value_is_null(entry.second) ? Aws::Crt::Optional() : Value(*valueMember, entry.second); + if (value.has_value()) { + entries.erase(entry.first); + entries.emplace(entry.first, std::move(value.value())); + } + } + } + return FuzzValue::Map(std::move(entries)); + } + + __attribute__((noinline)) Aws::Crt::Optional Scalar(const Schema& member, const aws_json_value* node) { + const bool number = aws_json_value_is_number(node); + const bool text = aws_json_value_is_string(node); + switch (member.GetType()) { + case ShapeType::Boolean: { + bool flag = false; + return aws_json_value_is_boolean(node) && aws_json_value_get_boolean(node, &flag) == AWS_OP_SUCCESS + ? Aws::Crt::Optional(FuzzValue::Boolean(flag)) + : Aws::Crt::Optional(); + } + case ShapeType::Byte: + case ShapeType::Short: + case ShapeType::Integer: + case ShapeType::IntEnum: + case ShapeType::Long: + return number ? Integral(member, JsonNumber(node)) : Aws::Crt::Optional(); + case ShapeType::Float: + case ShapeType::Double: + if (number && !std::isfinite(JsonNumber(node))) { + Note(m_reading.mustReject, "non-finite number"); + return {}; + } + return number ? FloatingPointFor(member, JsonNumber(node), m_reading) + : (text ? NonFiniteFromText(JsonText(node)) : Aws::Crt::Optional()); + case ShapeType::String: + case ShapeType::Enum: + return text ? Aws::Crt::Optional(FuzzValue::String(JsonText(node))) : Aws::Crt::Optional(); + case ShapeType::Blob: + return text ? Aws::Crt::Optional(BlobFromText(JsonText(node))) : Aws::Crt::Optional(); + case ShapeType::Timestamp: + if (number) { + return EpochTimestamp(JsonNumber(node), m_reading); + } + return text ? TextTimestamp(JsonText(node), ResolveTimestampFormat(member, TimestampFormatTrait::Format::EPOCH_SECONDS)) + : Aws::Crt::Optional(); + case ShapeType::Document: + return FuzzValue::FromDocument(ToDocument(node)); + default: + return {}; + } + } + + // cJSON reads numbers as doubles, exact only below 2^53. + Aws::Crt::Optional Integral(const Schema& member, double number) { + if (std::fabs(number) < EXACT_INTEGER_LIMIT) { + if (std::trunc(number) != number) { + Note(m_reading.mustReject, "non-integral number for an integer member"); + return {}; + } + return IntegerFor(member, static_cast(number), m_reading); + } + if (member.GetType() != ShapeType::Long || !(std::fabs(number) <= INT64_LIMIT)) { + Note(m_reading.mustReject, "integer out of range"); + return {}; + } + if (std::fabs(number) == INT64_LIMIT) { + Note(m_reading.acceptanceUnjudged, "a long the reference reads as 2^63 may or may not fit"); + } + return FuzzValue::Double(number); + } + + // A repeated object key keeps its first value, as the codec's document reader does. + __attribute__((noinline)) std::shared_ptr ToDocument(const aws_json_value* node) { + if (aws_json_value_is_object(node)) { + Aws::Map> fields; + for (const auto& entry : ObjectMembers(node)) { + auto value = ToDocument(entry.second); + fields.emplace(entry.first, std::move(value)); + } + return Document::FromMap(std::move(fields)); + } + if (aws_json_value_is_array(node)) { + Aws::Vector> items; + for (const auto* item : ArrayItems(node)) { + items.push_back(ToDocument(item)); + } + return Document::FromList(std::move(items)); + } + bool flag = false; + if (aws_json_value_is_boolean(node) && aws_json_value_get_boolean(node, &flag) == AWS_OP_SUCCESS) { + return Document::FromBoolean(flag); + } + if (aws_json_value_is_string(node)) { + return Document::FromString(JsonText(node)); + } + if (aws_json_value_is_number(node)) { + if (!std::isfinite(JsonNumber(node))) { + Note(m_reading.mustReject, "non-finite document number"); + } + return Document::FromDouble(JsonNumber(node)); + } + return Document::Null(); + } + + ReferenceReading& m_reading; +}; + +// cJSON reads \u without four hex digits as U+0000; the codec rejects it. +bool HasBadUnicodeEscape(const std::string& bytes) { + for (size_t at = bytes.find("\\u"); at != std::string::npos; at = bytes.find("\\u", at + 2)) { + const bool hex = at + 6 <= bytes.size() && std::all_of(bytes.begin() + at + 2, bytes.begin() + at + 6, + [](char c) { return std::isxdigit(static_cast(c)) != 0; }); + if (!hex) { + return true; + } + } + return false; +} + +ReferenceReading ReadJson(const Schema& root, const aws_json_value* document) { + ReferenceReading reading; + if (JsonDepth(document) > static_cast(MAX_NESTING_DEPTH)) { + Note(reading.mustReject, "nested deeper than MAX_NESTING_DEPTH"); + } + if (!aws_json_value_is_object(document) && !aws_json_value_is_null(document)) { + Note(reading.mustReject, "root is not an object"); + } + JsonReference reader(reading); + reading.value = reader.Struct(root, document); + return reading; +} + +// tinyxml2 truncates at a reference to U+0000 and leaves a stale byte after an unrecognised entity. Its decoded text is a C +// string, so the NUL itself never shows; the reference is found in the raw text. +bool ReferenceDecodesFaithfully(const Aws::String& raw) { + for (size_t amp = raw.find('&'); amp != Aws::String::npos; amp = raw.find('&', amp + 1)) { + const size_t semicolon = raw.find(';', amp); + const Aws::String body = semicolon == Aws::String::npos ? Aws::String() : raw.substr(amp + 1, semicolon - amp - 1); + const bool named = body == "lt" || body == "gt" || body == "amp" || body == "quot" || body == "apos"; + const bool hex = body.size() > 2 && body.size() <= 8 && body.compare(0, 2, "#x") == 0 && + body.find_first_not_of("0123456789abcdefABCDEF", 2) == Aws::String::npos; + const bool decimal = body.size() > 1 && body.size() <= 8 && body[0] == '#' && body.find_first_not_of("0123456789", 1) == Aws::String::npos; + if (!named && !hex && !decimal) { + return false; + } + if ((hex || decimal) && body.find_first_not_of(hex ? "#x0" : "#0") == Aws::String::npos) { + return false; + } + } + return true; +} + +Aws::String FindTinyXmlLeniency(const std::string& bytes) { + const char* whitespace = " \t\r\n"; + if (bytes.find_first_of("\v\f") != std::string::npos) { + return "a \\v or \\f, which tinyxml2 skips as whitespace"; + } + const size_t start = bytes.find_first_not_of(whitespace); + if (start != 0 && start != std::string::npos && bytes.compare(start, 3, "\xEF\xBB\xBF") == 0) { + return "a byte-order mark after leading whitespace"; + } + for (size_t lt = bytes.find('<'); lt != std::string::npos; lt = bytes.find('<', lt + 1)) { + if (lt + 1 < bytes.size() && Aws::String(whitespace).find(bytes[lt + 1]) != Aws::String::npos) { + return "whitespace before an element name"; + } + if (lt + 1 < bytes.size() && bytes[lt + 1] == '/') { + const size_t nameEnd = bytes.find_first_of(" \t\r\n<>", lt + 2); + const size_t afterName = nameEnd == std::string::npos ? nameEnd : bytes.find_first_not_of(whitespace, nameEnd); + if (afterName != std::string::npos && bytes[afterName] != '>') { + return "junk in a closing tag"; + } + } + } + return {}; +} + +struct XmlNode { + const tinyxml2::XMLElement* decoded; + const tinyxml2::XMLElement* raw; +}; + +Aws::Vector Children(const XmlNode& parent, const Aws::String& name) { + Aws::Vector children; + const tinyxml2::XMLElement* raw = parent.raw->FirstChildElement(name.c_str()); + for (const tinyxml2::XMLElement* child = parent.decoded->FirstChildElement(name.c_str()); child != nullptr && raw != nullptr; + child = child->NextSiblingElement(name.c_str()), raw = raw->NextSiblingElement(name.c_str())) { + children.push_back(XmlNode{child, raw}); + } + return children; +} + +// Empty beside an element, comment or processing instruction, where tinyxml2 drops whitespace. +Aws::Crt::Optional ElementText(const tinyxml2::XMLElement& element) { + Aws::String text; + for (const tinyxml2::XMLNode* child = element.FirstChild(); child != nullptr; child = child->NextSibling()) { + if (child->ToText() == nullptr) { + return {}; + } + text += child->Value(); + } + return text; +} + +class XmlReference final { + public: + explicit XmlReference(ReferenceReading& reading) : m_reading(reading) {} + + __attribute__((noinline)) FuzzValue Struct(const Schema& structSchema, const XmlNode& node) { + FuzzValue result = FuzzValue::Struct(); + bool delivered = false; + for (uint16_t i = 0; i < structSchema.GetMemberCount(); ++i) { + const auto member = structSchema.GetMember(static_cast(i)).value(); + const Aws::String name = TraitValue(*member, XmlNameTrait::KEY(), member->GetMemberName()); + const bool container = member->GetType() == ShapeType::List || member->GetType() == ShapeType::Map; + bool present = false; + Aws::Crt::Optional value; + if (member->HasTrait(XmlAttributeTrait::KEY())) { + const char* decoded = node.decoded->Attribute(name.c_str()); + const char* raw = node.raw->Attribute(name.c_str()); + present = decoded != nullptr; + value = present ? Scalar(*member, AttributeText(decoded, raw)) : value; + } else if (container && member->HasTrait(XmlFlattenedTrait::KEY())) { + const auto siblings = Children(node, name); + present = !siblings.empty(); + value = present ? Container(*member, siblings) : value; + } else { + const auto children = Children(node, name); + present = !children.empty(); + value = present ? Value(*member, children.front()) : value; + } + if (present && delivered && structSchema.GetType() == ShapeType::Union) { + Note(m_reading.mustReject, "union has more than one member set"); + } + delivered = delivered || present; + if (value.has_value()) { + result.SetMember(member->GetMemberIndex(), std::move(value.value())); + } + } + return result; + } + + private: + Aws::Crt::Optional Value(const Schema& member, const XmlNode& node) { + switch (member.GetType()) { + case ShapeType::List: + return Container(member, Children(node, TraitValue(member, XmlListItemNameTrait::KEY(), "member"))); + case ShapeType::Map: + return Container(member, Children(node, TraitValue(member, XmlMapEntryNameTrait::KEY(), "entry"))); + case ShapeType::Structure: + case ShapeType::Union: + return Struct(**member.GetMemberTarget(), node); + case ShapeType::Document: + return {}; + default: + return Scalar(member, Text(node)); + } + } + + __attribute__((noinline)) FuzzValue Container(const Schema& member, const Aws::Vector& nodes) { + if (member.GetType() == ShapeType::List) { + const auto element = ListElement(member); + Aws::Vector items; + for (const auto& item : nodes) { + auto value = Value(*element, item); + if (value.has_value()) { + items.push_back(std::move(value.value())); + } + } + return FuzzValue::List(std::move(items)); + } + const auto valueMember = MapValue(member); + const Aws::String keyName = TraitValue(member, XmlMapKeyNameTrait::KEY(), "key"); + const Aws::String valueName = TraitValue(member, XmlMapValueNameTrait::KEY(), "value"); + Aws::Map entries; + for (const auto& entry : nodes) { + const auto values = Children(entry, valueName); + const auto keys = Children(entry, keyName); + const auto key = keys.empty() ? Aws::Crt::Optional(Aws::String()) : Text(keys.front()); + auto value = values.empty() ? Aws::Crt::Optional() : Value(*valueMember, values.front()); + if (value.has_value() && key.has_value()) { + entries.erase(key.value()); + entries.emplace(key.value(), std::move(value.value())); + } + } + return FuzzValue::Map(std::move(entries)); + } + + Aws::Crt::Optional Text(const XmlNode& node) { + const auto decoded = ElementText(*node.decoded); + const auto raw = ElementText(*node.raw); + if (!decoded.has_value() || !raw.has_value()) { + Note(m_reading.valueUnjudged, "text beside other markup"); + return {}; + } + if (!ReferenceDecodesFaithfully(raw.value())) { + Note(m_reading.valueUnjudged, "a reference tinyxml2 decodes unfaithfully"); + return {}; + } + return decoded; + } + + Aws::Crt::Optional AttributeText(const char* decoded, const char* raw) { + if (raw == nullptr || !ReferenceDecodesFaithfully(raw)) { + Note(m_reading.valueUnjudged, "a reference tinyxml2 decodes unfaithfully"); + return {}; + } + return Aws::String(decoded); + } + + __attribute__((noinline)) Aws::Crt::Optional Scalar(const Schema& member, const Aws::Crt::Optional& text) { + if (!text.has_value()) { + return {}; + } + // Legacy clients trim number, boolean and timestamp text; strings and blobs are read as written. + const Aws::String trimmed = StringUtils::Trim(text.value().c_str()); + switch (member.GetType()) { + case ShapeType::Boolean: { + const Aws::String lower = StringUtils::ToLower(trimmed.c_str()); + return lower == "true" || lower == "false" ? Aws::Crt::Optional(FuzzValue::Boolean(lower == "true")) + : Aws::Crt::Optional(); + } + case ShapeType::Byte: + case ShapeType::Short: + case ShapeType::Integer: + case ShapeType::IntEnum: + case ShapeType::Long: + return IntegerFromText(member, trimmed, m_reading); + case ShapeType::Float: + case ShapeType::Double: { + const auto nonFinite = NonFiniteFromText(trimmed); + if (nonFinite.has_value()) { + return FloatingPointFor(member, nonFinite->GetDouble(), m_reading); + } + const auto number = ParseDoubleText(trimmed); + if (number.has_value() && !std::isfinite(number.value())) { + Note(m_reading.mustReject, "non-finite number"); + return {}; + } + return number.has_value() ? FloatingPointFor(member, number.value(), m_reading) : Aws::Crt::Optional(); + } + case ShapeType::String: + case ShapeType::Enum: + return FuzzValue::String(text.value()); + case ShapeType::Blob: + return BlobFromText(text.value()); + case ShapeType::Timestamp: { + const auto format = ResolveTimestampFormat(member, TimestampFormatTrait::Format::DATE_TIME); + if (format != TimestampFormatTrait::Format::EPOCH_SECONDS) { + return TextTimestamp(trimmed, format); + } + const auto seconds = ParseDoubleText(trimmed); + return seconds.has_value() ? EpochTimestamp(seconds.value(), m_reading) : Aws::Crt::Optional(); + } + default: + return {}; + } + } + + ReferenceReading& m_reading; +}; + +size_t ClosingMarks(const char* text) { + size_t marks = 0; + for (const char* at = text != nullptr ? std::strstr(text, "NextSibling()) { + const tinyxml2::XMLElement* element = child->ToElement(); + if (element != nullptr) { + marks += element->ClosingType() == tinyxml2::XMLElement::OPEN ? 1 : 0; + for (const tinyxml2::XMLAttribute* attribute = element->FirstAttribute(); attribute != nullptr; attribute = attribute->Next()) { + marks += ClosingMarks(attribute->Value()); + } + marks += ClosingMarksInTree(*child); + } else { + marks += ClosingMarks(child->Value()); + } + } + return marks; +} + +class XmlReferenceDocuments final { + public: + explicit XmlReferenceDocuments(const std::string& bytes) + : m_decoded(true, tinyxml2::PRESERVE_WHITESPACE), m_raw(false, tinyxml2::PRESERVE_WHITESPACE) { + m_status = m_decoded.Parse(bytes.data(), bytes.size()); + m_parsed = m_status == tinyxml2::XML_SUCCESS && m_decoded.RootElement() != nullptr && + m_raw.Parse(bytes.data(), bytes.size()) == tinyxml2::XML_SUCCESS && m_raw.RootElement() != nullptr; + const Aws::String copy(bytes.data(), bytes.size()); + // tinyxml2 drops a closing tag outside the root without an error. + m_strayClosingTag = m_parsed && ClosingMarks(copy.c_str()) > ClosingMarksInTree(m_raw); + } + + // tinyxml2 stops at 500 nested elements, below MAX_NESTING_DEPTH. + bool TooDeep() const { return m_status == tinyxml2::XML_ELEMENT_DEPTH_EXCEEDED; } + bool Parsed() const { return m_parsed; } + const char* Error() const { return m_decoded.ErrorStr(); } + + ReferenceReading Read(const Schema& root, const Aws::String& resultWrapper) const { + ReferenceReading reading; + int elements = 0; + for (const tinyxml2::XMLNode* node = m_decoded.FirstChild(); node != nullptr; node = node->NextSibling()) { + elements += node->ToElement() != nullptr ? 1 : 0; + const Aws::String text = node->ToText() != nullptr ? node->Value() : ""; + if (text.find_first_not_of(" \t\r\n") != Aws::String::npos) { + Note(reading.mustReject, "text outside the root element"); + } + } + if (elements > 1) { + Note(reading.mustReject, "more than one root element"); + } + if (m_strayClosingTag) { + Note(reading.mustReject, "a closing tag that closes no element"); + } + XmlNode node{m_decoded.RootElement(), m_raw.RootElement()}; + const tinyxml2::XMLElement* wrapper = resultWrapper.empty() ? nullptr : node.decoded->FirstChildElement(resultWrapper.c_str()); + if (wrapper != nullptr) { + node = XmlNode{wrapper, node.raw->FirstChildElement(resultWrapper.c_str())}; + } + XmlReference reader(reading); + reading.value = reader.Struct(root, node); + return reading; + } + + private: + tinyxml2::XMLDocument m_decoded; + tinyxml2::XMLDocument m_raw; + tinyxml2::XMLError m_status = tinyxml2::XML_SUCCESS; + bool m_parsed = false; + bool m_strayClosingTag = false; +}; + +template +Aws::String ReadMismatch(bool parsed, const char* referenceError, ReadCodec readCodec, ReadReference readReference, + const CodecTolerance& tolerance, TextTest unjudged) { + for (const auto& schema : FuzzSchemas()) { + DynamicStruct shape(schema); + const auto error = readCodec(*schema, shape); + const bool accepted = !error.has_value(); + Aws::String mismatch; + if (!parsed) { + mismatch = accepted ? Aws::String("accepted, the reference rejects: ") + referenceError : Aws::String(); + } else { + const ReferenceReading reading = readReference(*schema); + if (reading.acceptanceUnjudged.empty() && accepted != reading.mustReject.empty()) { + mismatch = accepted ? "accepted, but the reference reading must fail: " + reading.mustReject + : "rejected what the reference reads: " + error->GetMessage(); + } else if (accepted && reading.mustReject.empty() && reading.valueUnjudged.empty() && + (unjudged == nullptr || !AnyText(shape.GetValue(), unjudged))) { + mismatch = FindMismatch(reading.value, shape.GetValue(), *schema, tolerance); + } + } + if (!mismatch.empty()) { + return Aws::String(schema->GetId()) + ": " + mismatch; + } + } + return {}; +} + +Aws::String XmlReadMismatch(const std::string& bytes, const Aws::String& resultWrapper, + const std::function(const Schema&, DynamicStruct&)>& readCodec) { + if (!ReferenceCanJudge(bytes)) { + return {}; + } + const XmlReferenceDocuments reference(bytes); + if (reference.TooDeep()) { + return {}; + } + const Aws::String leniency = FindTinyXmlLeniency(bytes); + return ReadMismatch( + reference.Parsed(), reference.Error(), readCodec, + [&](const Schema& schema) { + ReferenceReading reading = reference.Read(schema, resultWrapper); + if (!leniency.empty()) { + Note(reading.acceptanceUnjudged, leniency); + } + return reading; + }, + CodecTolerance::Exact(), &IsWhitespaceOnly); +} + +} // namespace + +Aws::Crt::Optional ReadJsonReference(const Schema& root, const std::string& bytes) { + const CrtJson reference(bytes); + return reference.Get() == nullptr ? Aws::Crt::Optional() : ReadJson(root, reference.Get()); +} + +Aws::Crt::Optional ReadXmlReference(const Schema& root, const std::string& bytes, const Aws::String& resultWrapper) { + const XmlReferenceDocuments reference(bytes); + return reference.Parsed() ? reference.Read(root, resultWrapper) : Aws::Crt::Optional(); +} + +Aws::String JsonReferenceMismatch(const std::string& bytes) { + if (!ReferenceCanJudge(bytes)) { + return {}; + } + const CrtJson reference(bytes); + // The reference stops a string at an escaped NUL, which can also turn an unmodeled key into a modeled one. + const bool escapedNul = bytes.find("\\u0000") != std::string::npos; + // cJSON skips every control byte as whitespace; JSON allows only tab, LF and CR. + const bool controlBytes = bytes.find_first_of(CONTROL_BYTES) != std::string::npos; + const bool badUnicodeEscape = HasBadUnicodeEscape(bytes); + return ReadMismatch( + reference.Get() != nullptr, "invalid JSON", + [&](const Schema&, DynamicStruct& shape) { return JsonCodec().DeserializeShape(CursorOf(bytes), shape); }, + [&](const Schema& schema) { + ReferenceReading reading = ReadJson(schema, reference.Get()); + if (escapedNul) { + Note(reading.acceptanceUnjudged, "escaped NUL"); + Note(reading.valueUnjudged, "escaped NUL"); + } + if (controlBytes) { + Note(reading.acceptanceUnjudged, "a control byte cJSON skips as whitespace"); + } + if (badUnicodeEscape) { + Note(reading.acceptanceUnjudged, "a \\u escape cJSON reads as U+0000"); + } + return reading; + }, + CodecTolerance::Exact().ReadByCrtJson(), nullptr); +} + +Aws::String XmlReferenceMismatch(const std::string& bytes) { + return XmlReadMismatch(bytes, "", + [&](const Schema&, DynamicStruct& shape) { return XmlCodec().DeserializeShape(CursorOf(bytes), shape); }); +} + +Aws::String AwsQueryResponseReferenceMismatch(const std::string& bytes) { + const OperationRequestContext context{"FuzzOperation", "2026-10-06"}; + return XmlReadMismatch(bytes, "FuzzOperationResult", [&](const Schema&, DynamicStruct& shape) { + return AwsQueryProtocol().DeserializeResponse(context, CursorOf(bytes), shape); + }); +} + +Aws::String JsonOutputReferenceMismatch(const std::shared_ptr& schema, const FuzzValue& value) { + const auto payload = JsonCodec().Serialize(*schema, DynamicStruct(schema, value)); + if (!payload.IsSuccess()) { + return "could not serialize: " + payload.GetError().GetMessage(); + } + const std::string output(payload.GetResult().data(), payload.GetResult().size()); + const CrtJson reference(output); + if (reference.Get() == nullptr) { + return "the reference rejects the output " + payload.GetResult(); + } + const ReferenceReading reading = ReadJson(*schema, reference.Get()); + if (!reading.mustReject.empty()) { + return "the output must fail to read (" + reading.mustReject + "): " + payload.GetResult(); + } + const Aws::String mismatch = + AnyText(value, &HasNul) ? Aws::String() : FindMismatch(value, reading.value, *schema, CodecTolerance::Json().ReadByCrtJson()); + return mismatch.empty() ? mismatch : mismatch + " output=" + payload.GetResult(); +} + +Aws::String XmlOutputReferenceMismatch(const std::shared_ptr& schema, const FuzzValue& value) { + // XML 1.0 cannot carry U+0000, and the XML codec writes no documents. + if (AnyText(value, &HasNul) || ContainsDocument(value)) { + return {}; + } + const auto payload = XmlCodec().Serialize(*schema, DynamicStruct(schema, value)); + if (!payload.IsSuccess()) { + return "could not serialize: " + payload.GetError().GetMessage(); + } + const std::string output(payload.GetResult().data(), payload.GetResult().size()); + const XmlReferenceDocuments reference(output); + if (!reference.Parsed()) { + return Aws::String("the reference rejects the output (") + reference.Error() + "): " + payload.GetResult(); + } + const ReferenceReading reading = reference.Read(*schema, ""); + if (!reading.mustReject.empty()) { + return "the output must fail to read (" + reading.mustReject + "): " + payload.GetResult(); + } + // tinyxml2 drops whitespace-only character data, so such values are checked for acceptance only. + const bool judged = reading.valueUnjudged.empty() && !AnyText(value, &IsWhitespaceOnly); + const Aws::String mismatch = judged ? FindMismatch(value, reading.value, *schema, CodecTolerance::Xml()) : Aws::String(); + return mismatch.empty() ? mismatch : mismatch + " output=" + payload.GetResult(); +} + +Aws::String TinyXmlLeniency(const std::string& bytes) { return FindTinyXmlLeniency(bytes); } + +} // namespace fuzz +} // namespace schema +} // namespace smithy diff --git a/tests/fuzz/harness/ReferenceParity.h b/tests/fuzz/harness/ReferenceParity.h new file mode 100644 index 000000000000..8a4696c50248 --- /dev/null +++ b/tests/fuzz/harness/ReferenceParity.h @@ -0,0 +1,44 @@ +/** + * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. + * SPDX-License-Identifier: Apache-2.0. + */ +#pragma once + +#include +#include +#include +#include + +#include +#include + +namespace smithy { +namespace schema { +namespace fuzz { + +// Walked by the protocol's rules, not the codec's code. Each reason is empty when it does not apply. +struct ReferenceReading { + FuzzValue value = FuzzValue::Struct(); + Aws::String mustReject; + Aws::String acceptanceUnjudged; + Aws::String valueUnjudged; +}; + +// Why tinyxml2 tolerates markup in `bytes` that the codec rejects; empty when it does not. +Aws::String TinyXmlLeniency(const std::string& bytes); + +Aws::Crt::Optional ReadJsonReference(const Schema& root, const std::string& bytes); + +// `resultWrapper` is entered first when the root has that child, as awsQuery responses are read. +Aws::Crt::Optional ReadXmlReference(const Schema& root, const std::string& bytes, const Aws::String& resultWrapper = ""); + +// Each returns empty when the codec and its reference agree, otherwise the first difference. +Aws::String JsonReferenceMismatch(const std::string& bytes); +Aws::String XmlReferenceMismatch(const std::string& bytes); +Aws::String AwsQueryResponseReferenceMismatch(const std::string& bytes); +Aws::String JsonOutputReferenceMismatch(const std::shared_ptr& schema, const FuzzValue& value); +Aws::String XmlOutputReferenceMismatch(const std::shared_ptr& schema, const FuzzValue& value); + +} // namespace fuzz +} // namespace schema +} // namespace smithy diff --git a/tests/fuzz/harness/ReferenceParityTest.cpp b/tests/fuzz/harness/ReferenceParityTest.cpp new file mode 100644 index 000000000000..487b47010249 --- /dev/null +++ b/tests/fuzz/harness/ReferenceParityTest.cpp @@ -0,0 +1,210 @@ +/** + * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. + * SPDX-License-Identifier: Apache-2.0. + */ +#include +#include +#include +#include +#include + +#include +#include + +using namespace smithy::schema; +using namespace smithy::schema::fuzz; + +namespace { + +ReferenceReading Json(const char* schemaId, const std::string& bytes) { + const auto reading = ReadJsonReference(*FindFuzzSchema(schemaId), bytes); + EXPECT_TRUE(reading.has_value()) << bytes; + return reading.has_value() ? reading.value() : ReferenceReading(); +} + +ReferenceReading Xml(const char* schemaId, const std::string& bytes) { + const auto reading = ReadXmlReference(*FindFuzzSchema(schemaId), bytes); + EXPECT_TRUE(reading.has_value()) << bytes; + return reading.has_value() ? reading.value() : ReferenceReading(); +} + +int MemberIndex(const char* schemaId, const char* member) { return FindFuzzSchema(schemaId)->GetMember(member).value()->GetMemberIndex(); } + +FuzzValue With(int index, FuzzValue value) { + FuzzValue s = FuzzValue::Struct(); + s.SetMember(index, std::move(value)); + return s; +} + +void ExpectReads(const ReferenceReading& reading, const char* schemaId, const FuzzValue& expected) { + EXPECT_EQ(reading.mustReject, ""); + EXPECT_EQ(reading.acceptanceUnjudged, ""); + EXPECT_EQ(reading.valueUnjudged, ""); + EXPECT_EQ(FindMismatch(expected, reading.value, *FindFuzzSchema(schemaId), CodecTolerance::Exact().ReadByCrtJson()), ""); +} + +} // namespace + +TEST(ReferenceParity, CodecOutputMatchesTheReferences) { + for (const auto& seed : JsonByteSeeds("JsonReferenceFuzz.JsonMatchesReference")) { + EXPECT_EQ(JsonReferenceMismatch(std::get<0>(seed)), "") << std::get<0>(seed); + } + for (const auto& seed : ByteSeeds(XmlCodec(), "XmlReferenceFuzz.XmlMatchesReference")) { + EXPECT_EQ(XmlReferenceMismatch(std::get<0>(seed)), "") << std::get<0>(seed); + } +} + +TEST(ReferenceParity, SeedValuesSerializeToWhatTheReferencesRead) { + for (const auto& tapeSeed : TapeSeeds()) { + for (const auto& schema : FuzzSchemas()) { + ChoiceTape jsonTape(std::get<0>(tapeSeed), std::get<1>(tapeSeed)); + const FuzzValue jsonValue = BuildStruct(*schema, jsonTape); + EXPECT_EQ(JsonOutputReferenceMismatch(schema, jsonValue), "") << schema->GetId(); + ChoiceTape xmlTape(std::get<0>(tapeSeed), std::get<1>(tapeSeed)); + const FuzzValue xmlValue = BuildStruct(*schema, xmlTape); + EXPECT_EQ(XmlOutputReferenceMismatch(schema, xmlValue), "") << schema->GetId(); + } + } +} + +TEST(ReferenceParity, SimplePayloadsMatch) { + EXPECT_EQ(JsonReferenceMismatch("{\"value\":\"a\"}"), ""); + EXPECT_EQ(XmlReferenceMismatch("a & b"), ""); + EXPECT_EQ(AwsQueryResponseReferenceMismatch("7"), ""); +} + +TEST(ReferenceParity, JsonReadsTheWholeTree) { + const auto reading = Json("NestedStructureValue", + "{\"str\":\"s\",\"list\":[\"a\",null,5,\"b\"],\"map\":{\"k\":1,\"k\":2,\"x\":\"no\"},\"nested\":{\"int\":3}," + "\"union\":{\"intVariant\":4},\"unmodeled\":[1]}"); + const auto schema = FindFuzzSchema("NestedStructureValue"); + FuzzValue expected = FuzzValue::Struct(); + expected.SetMember(MemberIndex("NestedStructureValue", "str"), FuzzValue::String("s")); + expected.SetMember(MemberIndex("NestedStructureValue", "list"), FuzzValue::List({FuzzValue::String("a"), FuzzValue::String("b")})); + expected.SetMember(MemberIndex("NestedStructureValue", "map"), FuzzValue::Map({{"k", FuzzValue::Integer(2)}})); + expected.SetMember( + MemberIndex("NestedStructureValue", "nested"), + With((*schema->GetMember("nested").value()->GetMemberTarget())->GetMember("int").value()->GetMemberIndex(), FuzzValue::Integer(3))); + expected.SetMember(MemberIndex("NestedStructureValue", "union"), + With((*schema->GetMember("union").value()->GetMemberTarget())->GetMember("intVariant").value()->GetMemberIndex(), + FuzzValue::Integer(4))); + ExpectReads(reading, "NestedStructureValue", expected); +} + +TEST(ReferenceParity, JsonContainerOfTheWrongTypeIsEmptyAndScalarIsAbsent) { + ExpectReads(Json("StringListValue", "{\"values\":5}"), "StringListValue", With(0, FuzzValue::List({}))); + ExpectReads(Json("IntegerValue", "{\"value\":\"5\"}"), "IntegerValue", FuzzValue::Struct()); + ExpectReads(Json("ByteValue", "{\"value\":200}"), "ByteValue", FuzzValue::Struct()); +} + +TEST(ReferenceParity, JsonReadingsTheCodecMustReject) { + EXPECT_NE(Json("IntegerValue", "{\"value\":1.5}").mustReject, ""); + EXPECT_NE(Json("IntegerValue", "{\"value\":3000000000}").mustReject, ""); + EXPECT_NE(Json("DoubleValue", "{\"value\":1e400}").mustReject, ""); + EXPECT_NE(Json("TimestampValue", "{\"value\":1e300}").mustReject, ""); + EXPECT_NE(Json("UnionValue", "{\"value\":{\"stringVariant\":\"a\",\"intVariant\":1}}").mustReject, ""); + EXPECT_NE(Json("StringValue", "[1]").mustReject, ""); + EXPECT_EQ(Json("UnionValue", "{\"value\":{\"stringVariant\":\"a\",\"intVariant\":null}}").mustReject, ""); +} + +TEST(ReferenceParity, JsonLongsBeyondTwoToThe53AreKnownOnlyToTheNearestDouble) { + const auto reading = Json("LongValue", "{\"value\":9007199254740993}"); + EXPECT_EQ(reading.mustReject, ""); + EXPECT_EQ(FindMismatch(With(0, FuzzValue::Integer(9007199254740993LL)), reading.value, *FindFuzzSchema("LongValue"), + CodecTolerance::Exact().ReadByCrtJson()), + ""); + EXPECT_NE(Json("LongValue", "{\"value\":9223372036854775807}").acceptanceUnjudged, ""); + EXPECT_NE(Json("LongValue", "{\"value\":1e19}").mustReject, ""); +} + +TEST(ReferenceParity, JsonReferenceRejectsTrailingData) { + EXPECT_FALSE(ReadJsonReference(*FindFuzzSchema("StringValue"), "{} {}").has_value()); +} + +TEST(ReferenceParity, XmlReadsAttributesAndNamedAndFlattenedContainers) { + const auto reading = Xml("XmlTraits", + "xy" + "1no2" + "a1b2" + "ssecond"); + const auto schema = FindFuzzSchema("XmlTraits"); + FuzzValue expected = FuzzValue::Struct(); + expected.SetMember(MemberIndex("XmlTraits", "attr"), FuzzValue::String("a")); + expected.SetMember(MemberIndex("XmlTraits", "intAttr"), FuzzValue::Integer(3)); + expected.SetMember(MemberIndex("XmlTraits", "flatList"), FuzzValue::List({FuzzValue::String("x"), FuzzValue::String("y")})); + expected.SetMember(MemberIndex("XmlTraits", "namedList"), FuzzValue::List({FuzzValue::Integer(1), FuzzValue::Integer(2)})); + expected.SetMember(MemberIndex("XmlTraits", "namedMap"), FuzzValue::Map({{"a", FuzzValue::Integer(1)}, {"", FuzzValue::Integer(2)}})); + expected.SetMember( + MemberIndex("XmlTraits", "nested"), + With((*schema->GetMember("nested").value()->GetMemberTarget())->GetMember("str").value()->GetMemberIndex(), FuzzValue::String("s"))); + ExpectReads(reading, "XmlTraits", expected); +} + +TEST(ReferenceParity, XmlReadingsTheCodecMustRejectOrCannotJudge) { + EXPECT_NE(Xml("StringValue", "").mustReject, ""); + EXPECT_NE(Xml("IntegerValue", "99999999999").mustReject, ""); + EXPECT_NE(Xml("UnionValue", "a1").mustReject, ""); + EXPECT_NE(Xml("StringValue", "ac").valueUnjudged, ""); + EXPECT_EQ(Xml("IntegerValue", "abc").mustReject, ""); +} + +TEST(ReferenceParity, AwsQueryResultWrapperIsEnteredWhenPresent) { + const auto wrapped = + ReadXmlReference(*FindFuzzSchema("StringValue"), + "inout", "FuzzOperationResult"); + ASSERT_TRUE(wrapped.has_value()); + ExpectReads(wrapped.value(), "StringValue", With(0, FuzzValue::String("in"))); + const auto bare = ReadXmlReference(*FindFuzzSchema("StringValue"), "out", "FuzzOperationResult"); + ASSERT_TRUE(bare.has_value()); + ExpectReads(bare.value(), "StringValue", With(0, FuzzValue::String("out"))); +} + +TEST(ReferenceParity, LegacyLeniencyOutsideTheRootIsModeled) { + // tinyxml2 drops a closing tag that closes nothing; the codec must reject it. + EXPECT_NE(Xml("StringValue", "").mustReject, ""); + // Before the root, tinyxml2 loses the root as well, so it rejects the document itself. + EXPECT_FALSE(ReadXmlReference(*FindFuzzSchema("StringValue"), "").has_value()); + EXPECT_EQ(Xml("StringValue", "\"/>").mustReject, ""); + EXPECT_EQ(XmlReferenceMismatch(""), ""); + EXPECT_EQ(XmlReferenceMismatch(std::string("<:/>\v")), ""); + // tinyxml2 tolerates whitespace before an element name and junk in an end tag; the codec rejects both as malformed. + EXPECT_EQ(XmlReferenceMismatch(std::string("<\t:/>")), ""); + EXPECT_EQ(XmlReferenceMismatch(""), ""); + // cJSON skips control bytes as whitespace; both skip a leading byte-order mark. + EXPECT_EQ(JsonReferenceMismatch(std::string("{}\x01")), ""); + EXPECT_EQ(JsonReferenceMismatch("\xEF\xBB\xBF{}"), ""); + // cJSON reads a \u escape without four hex digits as U+0000; the codec rejects it. + EXPECT_EQ(JsonReferenceMismatch("{\"\":\"\\u\t\t\t\t\"}"), ""); + EXPECT_EQ(XmlReferenceMismatch(""), ""); +} + +TEST(ReferenceParity, XmlScalarsAreReadAsLegacyClientsReadThem) { + ExpectReads(Xml("IntegerValue", "\n 42\t"), "IntegerValue", With(0, FuzzValue::Integer(42))); + ExpectReads(Xml("BooleanValue", " TRUE "), "BooleanValue", With(0, FuzzValue::Boolean(true))); + ExpectReads(Xml("BooleanValue", "1"), "BooleanValue", FuzzValue::Struct()); + ExpectReads(Xml("StringValue", " x "), "StringValue", With(0, FuzzValue::String(" x "))); + EXPECT_NE(Xml("LongValue", " 9223372036854775808 ").mustReject, ""); + for (const char* payload : + {" 42 ", " 2.5 ", + "False", " 2020-01-02T03:04:05Z "}) { + EXPECT_EQ(XmlReferenceMismatch(payload), "") << payload; + } +} + +TEST(ReferenceParity, OnlyMarkupTinyXmlToleratesIsExcusedFromTheAcceptanceCheck) { + for (const char* lenient : {"< a/>", "", "", "\f", "\v"}) { + EXPECT_NE(TinyXmlLeniency(lenient), "") << lenient; + } + EXPECT_NE(TinyXmlLeniency("\t\xEF\xBB\xBF"), ""); + EXPECT_EQ(TinyXmlLeniency("\xEF\xBB\xBF"), ""); + for (const char* strict : {"1", "", "", "", "1 < 2"}) { + EXPECT_EQ(TinyXmlLeniency(strict), "") << strict; + } +} + +TEST(ReferenceParity, XmlReferencesToNulAreNotJudged) { + EXPECT_NE(Xml("XmlTraits", "").valueUnjudged, ""); + EXPECT_NE(Xml("XmlTraits", "").valueUnjudged, ""); + EXPECT_NE(Xml("StringValue", "a�b").valueUnjudged, ""); + EXPECT_EQ(Xml("StringValue", " 0").valueUnjudged, ""); +} diff --git a/tests/fuzz/harness/ScalarText.cpp b/tests/fuzz/harness/ScalarText.cpp new file mode 100644 index 000000000000..ab8e388114ad --- /dev/null +++ b/tests/fuzz/harness/ScalarText.cpp @@ -0,0 +1,71 @@ +/** + * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. + * SPDX-License-Identifier: Apache-2.0. + */ +#include +#include + +#include +#include +#include +#include + +namespace smithy { +namespace schema { +namespace fuzz { + +IntegerText ParseIntegerText(const Aws::String& text) { + size_t at = 0; + const bool negative = !text.empty() && text[0] == '-'; + if (!text.empty() && (text[0] == '-' || text[0] == '+')) { + ++at; + } + if (at == text.size()) { + return {}; + } + // Accumulate the magnitude as unsigned so INT64_MIN fits. + const uint64_t limit = negative ? uint64_t{1} << 63 : static_cast((std::numeric_limits::max)()); + uint64_t magnitude = 0; + bool overflow = false; + for (; at < text.size(); ++at) { + if (text[at] < '0' || text[at] > '9') { + return {}; + } + const uint64_t digit = static_cast(text[at] - '0'); + overflow = overflow || magnitude > (limit - digit) / 10; + magnitude = overflow ? magnitude : magnitude * 10 + digit; + } + if (overflow) { + return {IntegerTextKind::OUT_OF_RANGE, 0}; + } + const int64_t value = negative ? static_cast(0 - magnitude) : static_cast(magnitude); + return {IntegerTextKind::VALUE, value}; +} + +Aws::Crt::Optional ParseDoubleText(const Aws::String& text) { + static const locale_t cLocale = newlocale(LC_NUMERIC_MASK, "C", static_cast(0)); + // strtod would skip leading whitespace. + if (text.empty() || text.find('\0') != Aws::String::npos || Aws::String(" \t\n\v\f\r").find(text[0]) != Aws::String::npos) { + return {}; + } + char* end = nullptr; + const double value = strtod_l(text.c_str(), &end, cLocale); + if (end != text.c_str() + text.size()) { + return {}; + } + return value; +} + +Aws::Crt::Optional EpochSecondsInClockRange(double seconds) { + const double clockSeconds = + static_cast(std::chrono::duration_cast((std::chrono::system_clock::duration::max)()).count()) / + 1000.0; + if (!std::isfinite(seconds) || !(std::fabs(seconds) < clockSeconds)) { + return {}; + } + return Aws::Utils::DateTime(seconds); +} + +} // namespace fuzz +} // namespace schema +} // namespace smithy diff --git a/tests/fuzz/harness/ScalarText.h b/tests/fuzz/harness/ScalarText.h new file mode 100644 index 000000000000..d3efdb0a6b92 --- /dev/null +++ b/tests/fuzz/harness/ScalarText.h @@ -0,0 +1,37 @@ +/** + * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. + * SPDX-License-Identifier: Apache-2.0. + */ +#pragma once + +#include +#include +#include + +#include + +namespace smithy { +namespace schema { +namespace fuzz { + +// Scalar text parsers for the reference oracles, written apart from the codecs' so a bug in theirs shows up as a mismatch. + +enum class IntegerTextKind { VALUE, OUT_OF_RANGE, NOT_AN_INTEGER }; + +struct IntegerText { + IntegerTextKind kind = IntegerTextKind::NOT_AN_INTEGER; + int64_t value = 0; +}; + +// An optional sign, then decimal digits only. +IntegerText ParseIntegerText(const Aws::String& text); + +// The whole text as strtod reads it in the C locale; empty if any of it is left over. +Aws::Crt::Optional ParseDoubleText(const Aws::String& text); + +// DateTime(seconds), empty where system_clock cannot hold the value. +Aws::Crt::Optional EpochSecondsInClockRange(double seconds); + +} // namespace fuzz +} // namespace schema +} // namespace smithy diff --git a/tests/fuzz/harness/ScalarTextTest.cpp b/tests/fuzz/harness/ScalarTextTest.cpp new file mode 100644 index 000000000000..b82a25865ee1 --- /dev/null +++ b/tests/fuzz/harness/ScalarTextTest.cpp @@ -0,0 +1,56 @@ +/** + * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. + * SPDX-License-Identifier: Apache-2.0. + */ +#include +#include + +#include +#include +#include + +using namespace smithy::schema::fuzz; + +namespace { + +IntegerText Parse(const char* text) { return ParseIntegerText(text); } + +} // namespace + +TEST(ScalarText, IntegersAreASignAndDigitsOnly) { + EXPECT_EQ(Parse("0").value, 0); + EXPECT_EQ(Parse("+7").value, 7); + EXPECT_EQ(Parse("-007").value, -7); + EXPECT_EQ(Parse("9223372036854775807").value, INT64_MAX); + EXPECT_EQ(Parse("-9223372036854775808").value, INT64_MIN); + EXPECT_EQ(Parse("-9223372036854775808").kind, IntegerTextKind::VALUE); + for (const char* outOfRange : {"9223372036854775808", "-9223372036854775809", "99999999999999999999999"}) { + EXPECT_EQ(Parse(outOfRange).kind, IntegerTextKind::OUT_OF_RANGE) << outOfRange; + } + for (const char* notInteger : {"", "-", "+", " 1", "1 ", "1.0", "1e3", "0x10", "--1", "1-"}) { + EXPECT_EQ(Parse(notInteger).kind, IntegerTextKind::NOT_AN_INTEGER) << notInteger; + } +} + +TEST(ScalarText, DoublesMustBeWholeAndIgnoreTheNumericLocale) { + EXPECT_EQ(ParseDoubleText("2.5").value(), 2.5); + EXPECT_EQ(ParseDoubleText("-1e-3").value(), -1e-3); + EXPECT_TRUE(std::isinf(ParseDoubleText("1e400").value())); + for (const char* partial : {"", "2.5x", " 2.5", "2,5"}) { + EXPECT_FALSE(ParseDoubleText(partial).has_value()) << partial; + } + const std::string previous = std::setlocale(LC_NUMERIC, nullptr); + if (std::setlocale(LC_NUMERIC, "de_DE.UTF-8") != nullptr) { + EXPECT_EQ(ParseDoubleText("2.5").value(), 2.5); + EXPECT_FALSE(ParseDoubleText("2,5").has_value()); + } + std::setlocale(LC_NUMERIC, previous.c_str()); +} + +TEST(ScalarText, EpochSecondsMustFitTheSystemClock) { + EXPECT_EQ(EpochSecondsInClockRange(1.5)->Millis(), 1500); + EXPECT_EQ(EpochSecondsInClockRange(-1.5)->Millis(), -1500); + EXPECT_FALSE(EpochSecondsInClockRange(1e300).has_value()); + EXPECT_FALSE(EpochSecondsInClockRange(std::nan("")).has_value()); + EXPECT_FALSE(EpochSecondsInClockRange(INFINITY).has_value()); +} diff --git a/tests/fuzz/targets/CborCodecFuzz.cpp b/tests/fuzz/targets/CborCodecFuzz.cpp new file mode 100644 index 000000000000..837d0bec14ab --- /dev/null +++ b/tests/fuzz/targets/CborCodecFuzz.cpp @@ -0,0 +1,39 @@ +/** + * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. + * SPDX-License-Identifier: Apache-2.0. + */ +#include +#include +#include +#include +#include + +#include +#include +#include + +using namespace smithy::schema; + +namespace { + +void DeserializeNeverCrashes(const std::string& bytes) { fuzz::DeserializeNeverCrashes(CborCodec(), bytes); } + +void ReserializeIsStable(const std::string& bytes) { fuzz::ReserializeIsStable(CborCodec(), fuzz::CodecTolerance::Cbor(), bytes); } + +void RoundTripsTypedValues(const std::vector& choices, const std::vector& strings) { + fuzz::RoundTripsTypedValues(CborCodec(), fuzz::CodecTolerance::Cbor(), choices, strings); +} + +} // namespace + +FUZZ_TEST(CborCodecFuzz, DeserializeNeverCrashes).WithDomains(fuzz::PayloadDomain("cbor.dict")).WithSeeds([] { + return fuzz::ByteSeeds(CborCodec(), "CborCodecFuzz.DeserializeNeverCrashes"); +}); + +FUZZ_TEST(CborCodecFuzz, ReserializeIsStable).WithDomains(fuzz::PayloadDomain("cbor.dict")).WithSeeds([] { + return fuzz::ByteSeeds(CborCodec(), "CborCodecFuzz.ReserializeIsStable"); +}); + +FUZZ_TEST(CborCodecFuzz, RoundTripsTypedValues).WithDomains(fuzz::ChoicesDomain(), fuzz::StringPoolDomain("cbor.dict")).WithSeeds([] { + return fuzz::TapeSeeds("CborCodecFuzz.RoundTripsTypedValues"); +}); diff --git a/tests/fuzz/targets/JsonCodecFuzz.cpp b/tests/fuzz/targets/JsonCodecFuzz.cpp new file mode 100644 index 000000000000..a025c61bde45 --- /dev/null +++ b/tests/fuzz/targets/JsonCodecFuzz.cpp @@ -0,0 +1,39 @@ +/** + * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. + * SPDX-License-Identifier: Apache-2.0. + */ +#include +#include +#include +#include +#include + +#include +#include +#include + +using namespace smithy::schema; + +namespace { + +void DeserializeNeverCrashes(const std::string& bytes) { fuzz::DeserializeNeverCrashes(JsonCodec(), bytes); } + +void ReserializeIsStable(const std::string& bytes) { fuzz::ReserializeIsStable(JsonCodec(), fuzz::CodecTolerance::Json(), bytes); } + +void RoundTripsTypedValues(const std::vector& choices, const std::vector& strings) { + fuzz::RoundTripsTypedValues(JsonCodec(), fuzz::CodecTolerance::Json(), choices, strings); +} + +} // namespace + +FUZZ_TEST(JsonCodecFuzz, DeserializeNeverCrashes).WithDomains(fuzz::PayloadDomain("json.dict")).WithSeeds([] { + return fuzz::JsonByteSeeds("JsonCodecFuzz.DeserializeNeverCrashes"); +}); + +FUZZ_TEST(JsonCodecFuzz, ReserializeIsStable).WithDomains(fuzz::PayloadDomain("json.dict")).WithSeeds([] { + return fuzz::JsonByteSeeds("JsonCodecFuzz.ReserializeIsStable"); +}); + +FUZZ_TEST(JsonCodecFuzz, RoundTripsTypedValues).WithDomains(fuzz::ChoicesDomain(), fuzz::StringPoolDomain("json.dict")).WithSeeds([] { + return fuzz::TapeSeeds("JsonCodecFuzz.RoundTripsTypedValues"); +}); diff --git a/tests/fuzz/targets/JsonDocumentFuzz.cpp b/tests/fuzz/targets/JsonDocumentFuzz.cpp new file mode 100644 index 000000000000..7a52a8159da4 --- /dev/null +++ b/tests/fuzz/targets/JsonDocumentFuzz.cpp @@ -0,0 +1,93 @@ +/** + * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. + * SPDX-License-Identifier: Apache-2.0. + */ +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include +#include +#include + +using namespace smithy::schema; + +namespace { + +Aws::Crt::ByteCursor CursorOf(const std::string& bytes) { + return Aws::Crt::ByteCursorFromArray(reinterpret_cast(bytes.data()), bytes.size()); +} + +fuzz::FuzzValue HolderWith(const std::shared_ptr& document) { + fuzz::FuzzValue value = fuzz::FuzzValue::Struct(); + value.SetMember(0, fuzz::FuzzValue::FromDocument(document)); + return value; +} + +void ReadDocumentNeverCrashes(const std::string& bytes) { + fuzz::DynamicStruct shape(fuzz::DocumentHolderSchema(), fuzz::FuzzValue::Struct()); + JsonCodec().DeserializeShape(CursorOf(bytes), shape); + + const auto document = Schema::CreateDocument("FuzzDocument"); + JsonShapeDeserializer bare(CursorOf(bytes)); + bare.ReadDocument(*document); +} + +void DocumentRoundTrips(const std::vector& choices, const std::vector& strings) { + fuzz::ChoiceTape tape(choices, strings); + const auto document = fuzz::BuildDocument(tape); + ASSERT_NE(document, nullptr); + const auto holder = fuzz::DocumentHolderSchema(); + fuzz::DynamicStruct original(holder, HolderWith(document)); + + const JsonCodec codec; + const auto payload = codec.Serialize(*holder, original); + ASSERT_TRUE(payload.IsSuccess()) << payload.GetError().GetMessage(); + const std::string bytes(payload.GetResult().data(), payload.GetResult().size()); + fuzz::DynamicStruct decoded(holder, fuzz::FuzzValue::Struct()); + const auto error = codec.DeserializeShape(CursorOf(bytes), decoded); + ASSERT_FALSE(error.has_value()) << error->GetMessage() << " payload=" << payload.GetResult(); + + const auto& members = decoded.GetValue().GetMembers(); + const auto member = members.find(0); + // A null member reads back as absent: readers skip null members. + if (document->IsNull()) { + EXPECT_EQ(member, members.end()) << payload.GetResult(); + } else { + ASSERT_NE(member, members.end()) << "document member lost: " << payload.GetResult(); + EXPECT_TRUE(fuzz::DocumentsEquivalent(*document, *member->second.GetDocument())) << payload.GetResult(); + } +} + +void NonJsonCodecsRejectDocuments(const std::vector& choices, const std::vector& strings) { + fuzz::ChoiceTape tape(choices, strings); + const auto holder = fuzz::DocumentHolderSchema(); + fuzz::DynamicStruct shape(holder, HolderWith(fuzz::BuildDocument(tape))); + + EXPECT_FALSE(CborCodec().Serialize(*holder, shape).IsSuccess()); + EXPECT_FALSE(XmlCodec().Serialize(*holder, shape).IsSuccess()); + const OperationRequestContext context{"FuzzOperation", "2026-10-06"}; + EXPECT_FALSE(AwsQueryProtocol().SerializeRequest(context, *holder, shape).IsSuccess()); +} + +} // namespace + +FUZZ_TEST(JsonDocumentFuzz, ReadDocumentNeverCrashes).WithDomains(fuzz::PayloadDomain("json.dict")).WithSeeds([] { + return fuzz::JsonByteSeeds("JsonDocumentFuzz.ReadDocumentNeverCrashes"); +}); + +FUZZ_TEST(JsonDocumentFuzz, DocumentRoundTrips).WithDomains(fuzz::ChoicesDomain(), fuzz::StringPoolDomain("json.dict")).WithSeeds([] { + return fuzz::TapeSeeds("JsonDocumentFuzz.DocumentRoundTrips"); +}); + +FUZZ_TEST(JsonDocumentFuzz, NonJsonCodecsRejectDocuments) + .WithDomains(fuzz::ChoicesDomain(), fuzz::StringPoolDomain("json.dict")) + .WithSeeds([] { return fuzz::TapeSeeds("JsonDocumentFuzz.NonJsonCodecsRejectDocuments"); }); diff --git a/tests/fuzz/targets/QueryProtocolFuzz.cpp b/tests/fuzz/targets/QueryProtocolFuzz.cpp new file mode 100644 index 000000000000..66368797b86c --- /dev/null +++ b/tests/fuzz/targets/QueryProtocolFuzz.cpp @@ -0,0 +1,519 @@ +/** + * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. + * SPDX-License-Identifier: Apache-2.0. + */ +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include +#include +#include +#include +#include +#include +#include +#include + +using namespace smithy::schema; + +namespace { + +const OperationRequestContext& FuzzContext() { + static const OperationRequestContext ctx{"FuzzOperation", "2026-10-06"}; + return ctx; +} + +std::string WrapResult(const std::string& inner) { + return "" + inner + ""; +} + +std::vector> ResponseSeeds(const char* suiteTest) { + auto seeds = fuzz::ByteSeeds(XmlCodec(), suiteTest); + if (!fuzz::ReproFileSet()) { + const size_t bare = seeds.size(); + for (size_t i = 0; i < bare; ++i) { + std::string wrapped = WrapResult(std::get<0>(seeds[i])); + if (wrapped.size() <= fuzz::MAX_PAYLOAD_BYTES) { + seeds.emplace_back(std::move(wrapped)); + } + } + } + return seeds; +} + +std::vector ResponseDictionary() { + auto entries = fuzz::LoadDictionary("xml.dict"); + for (const char* token : {"", "", "", "", + "", ""}) { + entries.emplace_back(token); + } + return entries; +} + +Aws::Crt::ByteCursor ResponseCursor(const std::string& bytes) { + return Aws::Crt::ByteCursorFromArray(reinterpret_cast(bytes.data()), bytes.size()); +} + +// The serializer encodes '=' inside values, so a field has exactly one separator. +Aws::String ExpectWellFormedField(const Aws::String& field, const Aws::String& body) { + const size_t eq = field.find('='); + EXPECT_NE(eq, Aws::String::npos) << "field without '=': " << field << " body=" << body; + if (eq == Aws::String::npos) { + return {}; + } + EXPECT_NE(eq, 0u) << "empty key: " << body; + EXPECT_EQ(field.find('=', eq + 1), Aws::String::npos) << "more than one '=' in field: " << field << " body=" << body; + const Aws::String hex("0123456789ABCDEFabcdef"); + size_t i = 0; + while (i < field.size()) { + const char c = field[i]; + if (c == '%') { + EXPECT_LT(i + 2, field.size()) << "truncated escape: " << body; + if (i + 2 >= field.size()) { + break; + } + EXPECT_NE(hex.find(field[i + 1]), Aws::String::npos) << "bad escape: " << body; + EXPECT_NE(hex.find(field[i + 2]), Aws::String::npos) << "bad escape: " << body; + i += 3; + } else { + const bool allowed = + (c >= 'A' && c <= 'Z') || (c >= 'a' && c <= 'z') || (c >= '0' && c <= '9') || Aws::String("-_.~=").find(c) != Aws::String::npos; + EXPECT_TRUE(allowed) << "unencoded byte '" << c << "' in " << body; + ++i; + } + } + return field.substr(0, eq); +} + +// Empty when a field's decoded text is right; otherwise what is wrong with it. +using TextCheck = std::function; + +TextCheck ExactText(const Aws::String& expected) { + return [expected](const Aws::String& text) { return text == expected ? Aws::String() : "expected \"" + expected + "\""; }; +} + +TextCheck PresentOnly() { + return [](const Aws::String&) { return Aws::String(); }; +} + +bool SameNumber(double expected, double actual) { return expected == actual && std::signbit(expected) == std::signbit(actual); } + +TextCheck FloatingPointText(double expected, bool singlePrecision) { + return [expected, singlePrecision](const Aws::String& text) { + if (!std::isfinite(expected)) { + const char* name = std::isnan(expected) ? "NaN" : (expected > 0 ? "Infinity" : "-Infinity"); + return text == name ? Aws::String() : Aws::String("expected ") + name; + } + const auto parsed = fuzz::ParseDoubleText(text); + const bool same = + parsed.has_value() && std::isfinite(*parsed) && + (singlePrecision ? SameNumber(static_cast(expected), static_cast(*parsed)) : SameNumber(expected, *parsed)); + return same ? Aws::String() : "expected the number " + Aws::Utils::StringUtils::to_string(expected); + }; +} + +// Date-time and http-date text carries whole seconds, truncated toward zero. +TextCheck TimestampText(int64_t expectedMillis, TimestampFormatTrait::Format format) { + return [expectedMillis, format](const Aws::String& text) { + Aws::Crt::Optional millis; + int64_t expected = expectedMillis; + if (format == TimestampFormatTrait::Format::EPOCH_SECONDS) { + const auto seconds = fuzz::ParseDoubleText(text); + millis = seconds.has_value() && std::isfinite(*seconds) ? Aws::Crt::Optional(std::llround(*seconds * 1000.0)) + : Aws::Crt::Optional(); + } else { + const Aws::Utils::DateTime parsed( + text, format == TimestampFormatTrait::Format::HTTP_DATE ? Aws::Utils::DateFormat::RFC822 : Aws::Utils::DateFormat::ISO_8601); + millis = parsed.WasParseSuccessful() ? Aws::Crt::Optional(parsed.Millis()) : Aws::Crt::Optional(); + expected -= expectedMillis % 1000; + } + return millis.has_value() && *millis == expected ? Aws::String() : "expected " + Aws::Utils::StringUtils::to_string(expected) + " ms"; + }; +} + +// Names follow the awsQuery and ec2Query rules, not the serializer. +class ExpectedFields final { + public: + using Fields = Aws::Map; + + explicit ExpectedFields(bool ec2) : m_ec2(ec2) {} + + void AddStruct(const Aws::String& prefix, const Schema& structSchema, const fuzz::FuzzValue& value) { + for (const auto& entry : value.GetMembers()) { + const Schema& member = *structSchema.GetMember(entry.first).value(); + AddValue(prefix.empty() ? MemberName(member) : prefix + "." + MemberName(member), member, entry.second); + } + } + + const Fields& Get() const { return m_fields; } + + private: + template + static Aws::String TraitValue(const Schema& member, const TraitKey& key, const Aws::String& fallback) { + const auto trait = member.GetTrait(key); + return trait ? trait->GetValue() : fallback; + } + + Aws::String MemberName(const Schema& member) const { + const auto ec2Name = member.GetTrait(Ec2QueryNameTrait::KEY()); + if (m_ec2 && ec2Name) { + return ec2Name->GetValue(); + } + Aws::String name = TraitValue(member, XmlNameTrait::KEY(), member.GetMemberName()); + if (m_ec2 && !name.empty()) { + name[0] = static_cast(std::toupper(static_cast(name[0]))); + } + return name; + } + + void AddValue(const Aws::String& key, const Schema& member, const fuzz::FuzzValue& value) { + switch (member.GetType()) { + case ShapeType::List: + AddList(key, member, value); + return; + case ShapeType::Map: + AddMap(key, member, value); + return; + case ShapeType::Structure: + case ShapeType::Union: + AddStruct(key, **member.GetMemberTarget(), value); + return; + default: + AddScalar(key, member, value); + return; + } + } + + // awsQuery sends an empty list as a bare key; ec2Query omits it. + void AddList(const Aws::String& key, const Schema& member, const fuzz::FuzzValue& value) { + if (value.GetList().empty() && !m_ec2) { + Add(key, Aws::String()); + } + const bool flattened = m_ec2 || member.HasTrait(XmlFlattenedTrait::KEY()); + const Aws::String itemBase = flattened ? key : key + "." + TraitValue(member, XmlListItemNameTrait::KEY(), "member"); + const auto element = (*member.GetMemberTarget())->GetMember("member").value(); + for (size_t i = 0; i < value.GetList().size(); ++i) { + AddValue(itemBase + "." + Aws::Utils::StringUtils::to_string(i + 1), *element, value.GetList()[i]); + } + } + + // Both protocols omit an empty map. + void AddMap(const Aws::String& key, const Schema& member, const fuzz::FuzzValue& value) { + const bool flattened = member.HasTrait(XmlFlattenedTrait::KEY()); + const Aws::String entryBase = flattened ? key : key + "." + TraitValue(member, XmlMapEntryNameTrait::KEY(), "entry"); + const Aws::String keyName = TraitValue(member, XmlMapKeyNameTrait::KEY(), "key"); + const Aws::String valueName = TraitValue(member, XmlMapValueNameTrait::KEY(), "value"); + const auto valueMember = (*member.GetMemberTarget())->GetMember("value").value(); + size_t index = 0; + for (const auto& entry : value.GetMap()) { + const Aws::String entryKey = entryBase + "." + Aws::Utils::StringUtils::to_string(++index); + Add(entryKey + "." + keyName, entry.first); + AddValue(entryKey + "." + valueName, *valueMember, entry.second); + } + } + + void AddScalar(const Aws::String& key, const Schema& member, const fuzz::FuzzValue& value) { + switch (member.GetType()) { + case ShapeType::Boolean: + Add(key, value.GetBoolean() ? "true" : "false"); + return; + case ShapeType::Byte: + case ShapeType::Short: + case ShapeType::Integer: + case ShapeType::IntEnum: + case ShapeType::Long: + Add(key, Aws::Utils::StringUtils::to_string(value.GetInteger())); + return; + case ShapeType::String: + case ShapeType::Enum: + Add(key, value.GetString()); + return; + case ShapeType::Blob: + Add(key, Aws::Utils::HashingUtils::Base64Encode(value.GetBlob())); + return; + case ShapeType::Float: + case ShapeType::Double: + Insert(key, FloatingPointText(value.GetDouble(), member.GetType() == ShapeType::Float)); + return; + default: + Insert(key, TimestampText(value.GetTimestamp().Millis(), ResolveTimestampFormat(member, TimestampFormatTrait::Format::DATE_TIME))); + return; + } + } + + // URLEncode stops at a NUL, so text holding one must be present but is not compared. + void Add(const Aws::String& key, const Aws::String& text) { + Insert(key, text.find('\0') != Aws::String::npos ? PresentOnly() : ExactText(text)); + } + + void Insert(const Aws::String& key, TextCheck check) { + if (!m_fields.emplace(key, std::move(check)).second) { + ADD_FAILURE() << "the value implies field '" << key << "' twice"; + } + } + + bool m_ec2; + Fields m_fields; +}; + +Aws::String FieldMismatch(const ExpectedFields::Fields& expected, const Aws::Map& actual) { + for (const auto& field : expected) { + const auto found = actual.find(field.first); + if (found == actual.end()) { + return "missing field '" + field.first + "'"; + } + const Aws::String wrong = field.second(found->second); + if (!wrong.empty()) { + return "field '" + field.first + "' is \"" + found->second + "\", " + wrong; + } + } + for (const auto& field : actual) { + if (expected.find(field.first) == expected.end()) { + return "unexpected field '" + field.first + "'"; + } + } + return {}; +} + +void ExpectWellFormedBody(const ClientProtocol& protocol, bool ec2, const std::shared_ptr& schema, + const fuzz::FuzzValue& value) { + fuzz::DynamicStruct input(schema, value); + const auto payload = protocol.SerializeRequest(FuzzContext(), *schema, input); + ASSERT_TRUE(payload.IsSuccess()) << schema->GetId() << ": " << payload.GetError().GetMessage(); + const Aws::String body = payload.GetResult(); + int actions = 0; + int versions = 0; + Aws::Map fields; + for (const auto& field : Aws::Utils::StringUtils::Split(body, '&', Aws::Utils::StringUtils::SplitOptions::INCLUDE_EMPTY_ENTRIES)) { + const Aws::String key = ExpectWellFormedField(field, body); + if (key == "Action") { + ++actions; + EXPECT_EQ(field, "Action=FuzzOperation") << body; + } else if (key == "Version") { + ++versions; + EXPECT_EQ(field, "Version=2026-10-06") << body; + } else if (!key.empty()) { + const bool unique = fields.emplace(key, Aws::Utils::StringUtils::URLDecode(field.substr(key.size() + 1).c_str())).second; + EXPECT_TRUE(unique) << "duplicate key '" << key << "' in " << body; + } + } + EXPECT_EQ(actions, 1) << body; + EXPECT_EQ(versions, 1) << body; + ExpectedFields expected(ec2); + expected.AddStruct("", *schema, value); + EXPECT_EQ(FieldMismatch(expected.Get(), fields), "") << schema->GetId() << " body=" << body; +} + +void ExpectWellFormedRequest(const ClientProtocol& protocol, bool ec2, const std::vector& choices, + const std::vector& strings) { + for (const auto& schema : fuzz::FuzzSchemas()) { + fuzz::ChoiceTape tape(choices, strings); + const fuzz::FuzzValue value = fuzz::BuildStruct(*schema, tape); + // Query cannot serialize documents; WriteDocument fails by design. + if (!fuzz::ContainsDocument(value)) { + ExpectWellFormedBody(protocol, ec2, schema, value); + } + } +} + +void SerializeRequestIsWellFormedForm(const std::vector& choices, const std::vector& strings) { + ExpectWellFormedRequest(AwsQueryProtocol(), false, choices, strings); + ExpectWellFormedRequest(Ec2QueryProtocol(), true, choices, strings); +} + +void DeserializeResponseNeverCrashes(const std::string& bytes) { + for (const auto& schema : fuzz::FuzzSchemas()) { + fuzz::DynamicStruct awsQuery(schema, fuzz::FuzzValue::Struct()); + AwsQueryProtocol().DeserializeResponse(FuzzContext(), ResponseCursor(bytes), awsQuery); + fuzz::DynamicStruct ec2(schema, fuzz::FuzzValue::Struct()); + Ec2QueryProtocol().DeserializeResponse(FuzzContext(), ResponseCursor(bytes), ec2); + } +} + +void ExpectStableReads(const ClientProtocol& protocol, const std::string& body) { + for (const auto& schema : fuzz::FuzzSchemas()) { + fuzz::DynamicStruct first(schema, fuzz::FuzzValue::Struct()); + if (!protocol.DeserializeResponse(FuzzContext(), ResponseCursor(body), first).has_value()) { + fuzz::DynamicStruct second(schema, fuzz::FuzzValue::Struct()); + EXPECT_FALSE(protocol.DeserializeResponse(FuzzContext(), ResponseCursor(body), second).has_value()) << body; + EXPECT_EQ(fuzz::FindMismatch(first.GetValue(), second.GetValue(), *schema, fuzz::CodecTolerance::Xml()), "") << body; + } + } +} + +// The wrapper is peeled on read but not written back, so compare two reads of the same body. +void DeserializeResponseIsStable(const std::string& inner) { + ExpectStableReads(AwsQueryProtocol(), WrapResult(inner)); + ExpectStableReads(Ec2QueryProtocol(), inner); +} + +// ec2Query responses are bare XML, which XmlReferenceFuzz already judges. +void DeserializeResponseMatchesReference(const std::string& bytes) { + EXPECT_EQ(fuzz::AwsQueryResponseReferenceMismatch(bytes), "") << bytes; +} + +} // namespace + +namespace { + +int MemberIndex(const Schema& schema, const char* name) { return schema.GetMember(name).value()->GetMemberIndex(); } + +fuzz::FuzzValue XmlTraitsValue(const Schema& schema) { + fuzz::FuzzValue nested = fuzz::FuzzValue::Struct(); + nested.SetMember(MemberIndex(**schema.GetMember("nested").value()->GetMemberTarget(), "str"), fuzz::FuzzValue::String("x")); + fuzz::FuzzValue value = fuzz::FuzzValue::Struct(); + value.SetMember(MemberIndex(schema, "intAttr"), fuzz::FuzzValue::Integer(3)); + value.SetMember(MemberIndex(schema, "flatList"), fuzz::FuzzValue::List({fuzz::FuzzValue::String("s")})); + value.SetMember(MemberIndex(schema, "namedList"), fuzz::FuzzValue::List({fuzz::FuzzValue::Integer(7)})); + value.SetMember(MemberIndex(schema, "flatMap"), fuzz::FuzzValue::Map({{"fk", fuzz::FuzzValue::String("fv")}})); + value.SetMember(MemberIndex(schema, "namedMap"), fuzz::FuzzValue::Map({{"a", fuzz::FuzzValue::Integer(2)}})); + value.SetMember(MemberIndex(schema, "nested"), std::move(nested)); + return value; +} + +Aws::Map Texts(std::initializer_list> fields) { + Aws::Map result; + for (const auto& field : fields) { + result.emplace(field.first, field.second); + } + return result; +} + +ExpectedFields::Fields Expect(std::initializer_list> fields) { + ExpectedFields::Fields result; + for (const auto& field : fields) { + result.emplace(field.first, ExactText(field.second)); + } + return result; +} + +} // namespace + +// The expected names are written out from the protocol rules, so the model is checked independently of the serializer. +TEST(QueryProtocolOracle, AwsQueryNamesFollowXmlTraits) { + const auto schema = fuzz::FindFuzzSchema("XmlTraits"); + ExpectedFields expected(false); + expected.AddStruct("", *schema, XmlTraitsValue(*schema)); + EXPECT_EQ(FieldMismatch(expected.Get(), Texts({{"int-attr", "3"}, + {"flatList.1", "s"}, + {"namedList.item.1", "7"}, + {"flatMap.1.key", "fk"}, + {"flatMap.1.value", "fv"}, + {"namedMap.pair.1.k", "a"}, + {"namedMap.pair.1.v", "2"}, + {"Inner.str", "x"}})), + ""); + ExpectWellFormedBody(AwsQueryProtocol(), false, schema, XmlTraitsValue(*schema)); +} + +TEST(QueryProtocolOracle, Ec2QueryFlattensListsAndCapitalizesNames) { + const auto schema = fuzz::FindFuzzSchema("XmlTraits"); + ExpectedFields expected(true); + expected.AddStruct("", *schema, XmlTraitsValue(*schema)); + EXPECT_EQ(FieldMismatch(expected.Get(), Texts({{"Int-attr", "3"}, + {"FlatList.1", "s"}, + {"NamedList.1", "7"}, + {"FlatMap.1.key", "fk"}, + {"FlatMap.1.value", "fv"}, + {"NamedMap.pair.1.k", "a"}, + {"NamedMap.pair.1.v", "2"}, + {"Inner.Str", "x"}})), + ""); + ExpectWellFormedBody(Ec2QueryProtocol(), true, schema, XmlTraitsValue(*schema)); +} + +TEST(QueryProtocolOracle, Ec2QueryNamePrecedesXmlName) { + const auto schema = fuzz::FindFuzzSchema("Ec2Named"); + fuzz::FuzzValue value = fuzz::FuzzValue::Struct(); + value.SetMember(MemberIndex(*schema, "renamed"), fuzz::FuzzValue::String("r")); + value.SetMember(MemberIndex(*schema, "both"), fuzz::FuzzValue::Integer(1)); + value.SetMember(MemberIndex(*schema, "plain"), fuzz::FuzzValue::String("p")); + ExpectedFields awsQuery(false); + awsQuery.AddStruct("", *schema, value); + EXPECT_EQ(FieldMismatch(awsQuery.Get(), Texts({{"renamed", "r"}, {"xmlBoth", "1"}, {"plain", "p"}})), ""); + ExpectedFields ec2(true); + ec2.AddStruct("", *schema, value); + EXPECT_EQ(FieldMismatch(ec2.Get(), Texts({{"CustomName", "r"}, {"Ec2Both", "1"}, {"Plain", "p"}})), ""); +} + +TEST(QueryProtocolOracle, EmptyListIsABareKeyOnlyInAwsQuery) { + const auto schema = fuzz::FindFuzzSchema("StringListValue"); + const fuzz::FuzzValue value = [&] { + fuzz::FuzzValue v = fuzz::FuzzValue::Struct(); + v.SetMember(MemberIndex(*schema, "values"), fuzz::FuzzValue::List({})); + return v; + }(); + ExpectedFields awsQuery(false); + awsQuery.AddStruct("", *schema, value); + EXPECT_EQ(FieldMismatch(awsQuery.Get(), Texts({{"values", ""}})), ""); + ExpectedFields ec2(true); + ec2.AddStruct("", *schema, value); + EXPECT_TRUE(ec2.Get().empty()); +} + +TEST(QueryProtocolOracle, SwappedKeyAndValueOrWrongIndexIsReported) { + const auto expected = Expect({{"m.entry.1.key", "k"}, {"m.entry.1.value", "v"}}); + EXPECT_EQ(FieldMismatch(expected, Texts({{"m.entry.1.key", "k"}, {"m.entry.1.value", "v"}})), ""); + EXPECT_NE(FieldMismatch(expected, Texts({{"m.entry.1.key", "v"}, {"m.entry.1.value", "k"}})), ""); + EXPECT_NE(FieldMismatch(expected, Texts({{"m.entry.0.key", "k"}, {"m.entry.0.value", "v"}})), ""); + EXPECT_NE(FieldMismatch(expected, Texts({{"m.entry.1.key", "k"}})), ""); +} + +TEST(QueryProtocolOracle, FloatingPointAndTimestampTextIsChecked) { + EXPECT_EQ(FloatingPointText(1.5, false)("1.5"), ""); + EXPECT_NE(FloatingPointText(1.5, false)("1,5"), ""); + EXPECT_NE(FloatingPointText(1.5, false)("1.25"), ""); + EXPECT_NE(FloatingPointText(-0.0, false)("0"), ""); + EXPECT_EQ(FloatingPointText(static_cast(3.14f), true)("3.14"), ""); + EXPECT_EQ(FloatingPointText(INFINITY, false)("Infinity"), ""); + EXPECT_NE(FloatingPointText(INFINITY, false)("inf"), ""); + EXPECT_EQ(FloatingPointText(std::nan(""), true)("NaN"), ""); + EXPECT_EQ(TimestampText(1001, TimestampFormatTrait::Format::EPOCH_SECONDS)("1.001"), ""); + EXPECT_NE(TimestampText(1001, TimestampFormatTrait::Format::EPOCH_SECONDS)("1"), ""); + EXPECT_EQ(TimestampText(1577934245678, TimestampFormatTrait::Format::DATE_TIME)("2020-01-02T03:04:05Z"), ""); + EXPECT_NE(TimestampText(1577934245678, TimestampFormatTrait::Format::DATE_TIME)("2020-01-02T03:04:06Z"), ""); + EXPECT_EQ(TimestampText(-1500, TimestampFormatTrait::Format::HTTP_DATE)("Wed, 31 Dec 1969 23:59:59 GMT"), ""); +} + +TEST(QueryProtocolOracle, RequestsAreWellFormedUnderCommaDecimalLocale) { + const std::string previous = std::setlocale(LC_NUMERIC, nullptr); + if (std::setlocale(LC_NUMERIC, "de_DE.UTF-8") == nullptr) { + GTEST_SKIP() << "de_DE.UTF-8 is not installed"; + } + for (const auto& tape : fuzz::TapeSeeds("QueryProtocolFuzz.SerializeRequestIsWellFormedForm")) { + SerializeRequestIsWellFormedForm(std::get<0>(tape), std::get<1>(tape)); + } + std::setlocale(LC_NUMERIC, previous.c_str()); +} + +FUZZ_TEST(QueryProtocolFuzz, SerializeRequestIsWellFormedForm) + .WithDomains(fuzz::ChoicesDomain(), fuzz::StringPoolDomain("xml.dict")) + .WithSeeds([] { return fuzz::TapeSeeds("QueryProtocolFuzz.SerializeRequestIsWellFormedForm"); }); + +FUZZ_TEST(QueryProtocolFuzz, DeserializeResponseNeverCrashes).WithDomains(fuzz::PayloadDomain(ResponseDictionary())).WithSeeds([] { + return ResponseSeeds("QueryProtocolFuzz.DeserializeResponseNeverCrashes"); +}); + +FUZZ_TEST(QueryProtocolFuzz, DeserializeResponseMatchesReference).WithDomains(fuzz::PayloadDomain(ResponseDictionary())).WithSeeds([] { + return ResponseSeeds("QueryProtocolFuzz.DeserializeResponseMatchesReference"); +}); + +FUZZ_TEST(QueryProtocolFuzz, DeserializeResponseIsStable).WithDomains(fuzz::PayloadDomain("xml.dict")).WithSeeds([] { + return fuzz::ByteSeeds(XmlCodec(), "QueryProtocolFuzz.DeserializeResponseIsStable"); +}); diff --git a/tests/fuzz/targets/ReferenceParserFuzz.cpp b/tests/fuzz/targets/ReferenceParserFuzz.cpp new file mode 100644 index 000000000000..0dcd3f76139d --- /dev/null +++ b/tests/fuzz/targets/ReferenceParserFuzz.cpp @@ -0,0 +1,58 @@ +/** + * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. + * SPDX-License-Identifier: Apache-2.0. + */ +#include +#include +#include +#include +#include +#include + +#include +#include +#include + +using namespace smithy::schema; + +namespace { + +void JsonMatchesReference(const std::string& bytes) { EXPECT_EQ(fuzz::JsonReferenceMismatch(bytes), "") << bytes; } + +void XmlMatchesReference(const std::string& bytes) { EXPECT_EQ(fuzz::XmlReferenceMismatch(bytes), "") << bytes; } + +template +void ExpectOutputMatchesReference(OutputMismatch outputMismatch, const std::vector& choices, + const std::vector& strings) { + for (const auto& schema : fuzz::FuzzSchemas()) { + fuzz::ChoiceTape tape(choices, strings); + const fuzz::FuzzValue value = fuzz::BuildStruct(*schema, tape); + EXPECT_EQ(outputMismatch(schema, value), "") << schema->GetId() << " value=" << value.Describe(); + } +} + +void JsonSerializedOutputMatchesReference(const std::vector& choices, const std::vector& strings) { + ExpectOutputMatchesReference(&fuzz::JsonOutputReferenceMismatch, choices, strings); +} + +void XmlSerializedOutputMatchesReference(const std::vector& choices, const std::vector& strings) { + ExpectOutputMatchesReference(&fuzz::XmlOutputReferenceMismatch, choices, strings); +} + +} // namespace + +FUZZ_TEST(JsonReferenceFuzz, JsonMatchesReference).WithDomains(fuzz::PayloadDomain("json.dict")).WithSeeds([] { + return fuzz::JsonByteSeeds("JsonReferenceFuzz.JsonMatchesReference"); +}); + +FUZZ_TEST(XmlReferenceFuzz, XmlMatchesReference).WithDomains(fuzz::PayloadDomain("xml.dict")).WithSeeds([] { + return fuzz::ByteSeeds(XmlCodec(), "XmlReferenceFuzz.XmlMatchesReference"); +}); + +FUZZ_TEST(JsonReferenceFuzz, JsonSerializedOutputMatchesReference) + .WithDomains(fuzz::ChoicesDomain(), fuzz::StringPoolDomain("json.dict")) + .WithSeeds([] { return fuzz::TapeSeeds("JsonReferenceFuzz.JsonSerializedOutputMatchesReference"); }); + +FUZZ_TEST(XmlReferenceFuzz, XmlSerializedOutputMatchesReference) + .WithDomains(fuzz::ChoicesDomain(), fuzz::StringPoolDomain("xml.dict")) + .WithSeeds([] { return fuzz::TapeSeeds("XmlReferenceFuzz.XmlSerializedOutputMatchesReference"); }); diff --git a/tests/fuzz/targets/XmlCodecFuzz.cpp b/tests/fuzz/targets/XmlCodecFuzz.cpp new file mode 100644 index 000000000000..6844f0f5f256 --- /dev/null +++ b/tests/fuzz/targets/XmlCodecFuzz.cpp @@ -0,0 +1,39 @@ +/** + * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. + * SPDX-License-Identifier: Apache-2.0. + */ +#include +#include +#include +#include +#include + +#include +#include +#include + +using namespace smithy::schema; + +namespace { + +void DeserializeNeverCrashes(const std::string& bytes) { fuzz::DeserializeNeverCrashes(XmlCodec(), bytes); } + +void ReserializeIsStable(const std::string& bytes) { fuzz::ReserializeIsStable(XmlCodec(), fuzz::CodecTolerance::Xml(), bytes); } + +void RoundTripsTypedValues(const std::vector& choices, const std::vector& strings) { + fuzz::RoundTripsTypedValues(XmlCodec(), fuzz::CodecTolerance::Xml(), choices, strings); +} + +} // namespace + +FUZZ_TEST(XmlCodecFuzz, DeserializeNeverCrashes).WithDomains(fuzz::PayloadDomain("xml.dict")).WithSeeds([] { + return fuzz::ByteSeeds(XmlCodec(), "XmlCodecFuzz.DeserializeNeverCrashes"); +}); + +FUZZ_TEST(XmlCodecFuzz, ReserializeIsStable).WithDomains(fuzz::PayloadDomain("xml.dict")).WithSeeds([] { + return fuzz::ByteSeeds(XmlCodec(), "XmlCodecFuzz.ReserializeIsStable"); +}); + +FUZZ_TEST(XmlCodecFuzz, RoundTripsTypedValues).WithDomains(fuzz::ChoicesDomain(), fuzz::StringPoolDomain("xml.dict")).WithSeeds([] { + return fuzz::TapeSeeds("XmlCodecFuzz.RoundTripsTypedValues"); +}); diff --git a/tools/CI/docker-file/Ubuntu/24.04-schema-fuzz/Dockerfile b/tools/CI/docker-file/Ubuntu/24.04-schema-fuzz/Dockerfile new file mode 100644 index 000000000000..85e465fbee07 --- /dev/null +++ b/tools/CI/docker-file/Ubuntu/24.04-schema-fuzz/Dockerfile @@ -0,0 +1,10 @@ +# ubuntu:24.04 multi-arch index; bump deliberately. +FROM ubuntu:24.04@sha256:fe81560990fae86dc4743202b2cb4d3df98d0bcf949173cecef15ce1127ef852 + +RUN apt-get update && DEBIAN_FRONTEND=noninteractive apt-get install -y --no-install-recommends \ + ca-certificates clang-18 cmake git libclang-rt-18-dev libcurl4-openssl-dev libssl-dev llvm-18 locales ninja-build python3 zlib1g-dev \ + && rm -rf /var/lib/apt/lists/* \ + && sed -i 's/^# *de_DE.UTF-8/de_DE.UTF-8/' /etc/locale.gen && locale-gen + +ENV CC=clang-18 CXX=clang++-18 ASAN_SYMBOLIZER_PATH=/usr/lib/llvm-18/bin/llvm-symbolizer +WORKDIR /src diff --git a/tools/scripts/run_fuzz.sh b/tools/scripts/run_fuzz.sh new file mode 100755 index 000000000000..f4f06e38b7f5 --- /dev/null +++ b/tools/scripts/run_fuzz.sh @@ -0,0 +1,192 @@ +#!/usr/bin/env bash +# Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +# SPDX-License-Identifier: Apache-2.0. + +# Builds and runs aws-cpp-sdk-schema-fuzz-tests in a Linux container. See tests/fuzz/README.md. +set -euo pipefail + +REPO_ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd)" +IMAGE="aws-sdk-cpp-schema-fuzz" +MODE="${1:-unit}" +shift || true + +STACK_LIMIT_KB=2304 +CAMPAIGN_RSS_MB="${FUZZ_CAMPAIGN_RSS_MB:-2560}" +DURATION_PATTERN='^[0-9]+[smh]$' +# Typed targets ignore a raw file, so repro rejects them. +BYTE_TARGET_PATTERN='\.(DeserializeNeverCrashes|ReserializeIsStable|DeserializeResponse[A-Za-z]*|ReadDocumentNeverCrashes|JsonMatchesReference|XmlMatchesReference)$' + +usage() { + echo "usage: $0 unit [gtest-args...] | fuzz | campaign [duration] [jobs] | repro | replay " >&2 + echo " duration is , e.g. 90s, 10m, 2h" >&2 + exit 2 +} + +require_duration() { + if [[ ! "$1" =~ ${DURATION_PATTERN} ]]; then + echo "invalid duration '$1': expected , e.g. 90s, 10m, 2h" >&2 + usage + fi +} + +require_repo_file() { + if [[ "$1" == /* || "$1" == *..* || ! -f "${REPO_ROOT}/$1" ]]; then + echo "'$1' must be an existing file path relative to the repository root" >&2 + usage + fi +} + +ENV_ARGS=() +RUN_ARGS=() +REPRODUCERS_OUT_DIR="" +case "${MODE}" in + unit) FUZZING=OFF; RUN_ARGS=("$@") ;; + fuzz) + [ $# -eq 2 ] || usage + require_duration "$2" + FUZZING=ON + RUN_ARGS=("--fuzz=$1" "--fuzz_for=$2" "--time_limit_per_input=10s" "--stack_limit_kb=${STACK_LIMIT_KB}") + REPRODUCERS_OUT_DIR="/src/tests/fuzz/reproducers" ;; + campaign) + [ $# -le 2 ] || usage + FUZZING=ON + RUN_ARGS=("${1:-30m}" "${2:-}") + require_duration "${RUN_ARGS[0]}" + if [[ -n "${RUN_ARGS[1]}" && ! "${RUN_ARGS[1]}" =~ ^[1-9][0-9]*$ ]]; then + echo "jobs must be a positive integer" >&2; usage + fi + REPRODUCERS_OUT_DIR="/src/tests/fuzz/reproducers" ;; + repro) + [ $# -eq 2 ] || usage + require_repo_file "$2" + if [[ ! "$1" =~ ${BYTE_TARGET_PATTERN} ]]; then + echo "repro feeds a raw byte file and $1 is a typed target that never reads it; use: $0 replay $1 $2" >&2 + exit 2 + fi + FUZZING=ON + RUN_ARGS=("--gtest_filter=$1" "--stack_limit_kb=${STACK_LIMIT_KB}") + ENV_ARGS=(-e "SCHEMA_FUZZ_REPRO_FILE=/src/$2") ;; + replay) + [ $# -eq 2 ] || usage + require_repo_file "$2" + FUZZING=ON + RUN_ARGS=("--gtest_filter=$1" "--time_limit_per_input=10s" "--stack_limit_kb=${STACK_LIMIT_KB}") + # FuzzTest treats a set-but-empty FUZZTEST_REPLAY as active, so the variable is only passed in this mode. + ENV_ARGS=(-e "FUZZTEST_REPLAY=/src/$2") ;; + *) usage ;; +esac + +# One process per FUZZ_TEST, so a finding in one does not stop the others. +read -r -d '' CAMPAIGN_RUN <<'EOS' || true +BIN="$1"; DURATION="$2"; JOBS="$3"; RSS_MB="$4"; STACK_KB="$5" +# Raise when fuzz targets are added. +MIN_FUZZ_TESTS=21 +MEM_MB=$(( $(awk '/^MemTotal:/ {print $2}' /proc/meminfo) / 1024 )) +# Every process may grow to the RSS cap, so the pool must fit in memory. +FIT_JOBS=$(( MEM_MB * 85 / 100 / RSS_MB )) +if [ -z "${JOBS}" ]; then + JOBS=$(( $(nproc) - 2 )); [ "${JOBS}" -gt 8 ] && JOBS=8 + [ "${JOBS}" -gt "${FIT_JOBS}" ] && JOBS="${FIT_JOBS}" + [ "${JOBS}" -lt 1 ] && JOBS=1 +elif [ "${JOBS}" -gt "${FIT_JOBS}" ]; then + echo "campaign: warning: ${JOBS} jobs x ${RSS_MB} MiB exceeds 85% of ${MEM_MB} MiB; runs may be OOM-killed" >&2 +fi +OUT="/build/campaign/$(date +%Y%m%d-%H%M%S)" +REPRO_ROOT="/src/tests/fuzz/reproducers" +mkdir -p "${OUT}" +if ! LISTING=$("${BIN}" --list_fuzz_tests); then + echo "campaign: ${BIN} --list_fuzz_tests failed" >&2; exit 1 +fi +TESTS=$(printf '%s\n' "${LISTING}" | sed -n 's/^\[\*\] Fuzz test: //p') +NUM_TESTS=$(printf '%s\n' "${TESTS}" | grep -c . || true) +if [ "${NUM_TESTS}" -lt "${MIN_FUZZ_TESTS}" ]; then + echo "campaign: listed ${NUM_TESTS} fuzz tests, expected at least ${MIN_FUZZ_TESTS}" >&2; exit 1 +fi +echo "=== campaign: $(echo "${TESTS}" | wc -w) tests, ${DURATION} each, ${JOBS} parallel, rss limit ${RSS_MB} MiB, memory ${MEM_MB} MiB, logs in ${OUT}" +run_one() { + local t="$1" + mkdir -p "${REPRO_ROOT}/${t}" + FUZZTEST_REPRODUCERS_OUT_DIR="${REPRO_ROOT}/${t}" "${BIN}" --fuzz="${t}" --fuzz_for="${DURATION}" \ + --time_limit_per_input=10s --stack_limit_kb="${STACK_KB}" --rss_limit_mb="${RSS_MB}" > "${OUT}/${t}.log" 2>&1 + echo $? > "${OUT}/${t}.rc" + rmdir "${REPRO_ROOT}/${t}" 2>/dev/null || true + echo "done ${t} rc=$(cat "${OUT}/${t}.rc")" +} +for T in ${TESTS}; do + while [ "$(jobs -rp | wc -l)" -ge "${JOBS}" ]; do wait -n || true; done + echo "start ${T}" + run_one "${T}" & +done +wait +FAILED=0 +echo "=== campaign summary" +for T in ${TESTS}; do + RC=$(cat "${OUT}/${T}.rc" 2>/dev/null || echo 255) + STATS=$(tr -cd '\11\12\15\40-\176' < "${OUT}/${T}.log" | grep 'Corpus size.*Edges covered' | tail -1 \ + | sed -E 's/.*Corpus size: *([0-9]+).*Edges covered: *([0-9]+).*Runs\/secs: *([0-9]+).*/corpus=\1 edges=\2 runs\/s=\3/') + if [ "${RC}" -eq 0 ]; then + printf 'PASS %-70s %s\n' "${T}" "${STATS}" + else + FAILED=$(( FAILED + 1 )) + printf 'FAIL %-70s rc=%s %s\n' "${T}" "${RC}" "${STATS}" + mkdir -p "${REPRO_ROOT}/${T}" + cp "${OUT}/${T}.log" "${REPRO_ROOT}/${T}/campaign.log" + tr -cd '\11\12\15\40-\176' < "${OUT}/${T}.log" | tail -15 | cut -c1-300 | sed 's/^/ /' + fi +done +echo "=== campaign: ${FAILED} failed" +[ "${FAILED}" -eq 0 ] +EOS + +BUILD_DIR="/build/schema-fuzz-$(echo "${FUZZING}" | tr '[:upper:]' '[:lower:]')" + +docker build -q -t "${IMAGE}" "${REPO_ROOT}/tools/CI/docker-file/Ubuntu/24.04-schema-fuzz" >/dev/null + +DOCKER_RUN=(docker run --rm) +if [ "${MODE}" != "unit" ] && command -v caffeinate >/dev/null 2>&1; then + # Idle sleep freezes the container clock and FuzzTest then reports a false per-input timeout. + DOCKER_RUN=(caffeinate -i docker run --rm) +fi +if [ "$(uname -s)" = Linux ]; then + # Run as the caller so files written through the bind mount stay theirs; take over a volume an earlier root run created. + HOST_IDS="$(id -u):$(id -g)" + docker run --rm -v aws-sdk-cpp-schema-fuzz-build:/build "${IMAGE}" \ + sh -c '[ "$(stat -c %u:%g /build)" = "$1" ] || chown -R "$1" /build' _ "${HOST_IDS}" + DOCKER_RUN+=(--user "${HOST_IDS}" -e HOME=/tmp) +fi + +# User-controlled values reach the inner script only as environment variables or positional parameters. +read -r -d '' INNER_RUN <<'EOS' || true +set -euo pipefail +if [ -n "${FUZZTEST_REPRODUCERS_OUT_DIR}" ]; then mkdir -p "${FUZZTEST_REPRODUCERS_OUT_DIR}"; fi +cmake -S /src -B "${BUILD_DIR}" -G Ninja \ + -DCMAKE_BUILD_TYPE=RelWithDebInfo \ + -DBUILD_ONLY=sts \ + -DBUILD_SHARED_LIBS=OFF \ + -DENABLE_TESTING=OFF \ + -DAUTORUN_UNIT_TESTS=OFF \ + -DUSE_CRT_HTTP_CLIENT=ON \ + -DENABLE_SCHEMA_FUZZ_TESTS=ON \ + -DFUZZTEST_FUZZING_MODE="${FUZZING}" >/dev/null +cmake --build "${BUILD_DIR}" --target aws-cpp-sdk-schema-fuzz-tests +BIN="${BUILD_DIR}/tests/fuzz/aws-cpp-sdk-schema-fuzz-tests" +if [ "${FUZZ_MODE}" = campaign ]; then + bash -c "${CAMPAIGN_RUN}" _ "${BIN}" "$1" "$2" "${CAMPAIGN_RSS_MB}" "${STACK_LIMIT_KB}" +else + "${BIN}" "$@" +fi +EOS + +"${DOCKER_RUN[@]}" \ + -v "${REPO_ROOT}:/src" \ + -v aws-sdk-cpp-schema-fuzz-build:/build \ + ${ENV_ARGS[@]+"${ENV_ARGS[@]}"} \ + -e FUZZTEST_REPRODUCERS_OUT_DIR="${REPRODUCERS_OUT_DIR}" \ + -e CAMPAIGN_RUN="${CAMPAIGN_RUN}" \ + -e FUZZ_MODE="${MODE}" \ + -e FUZZING="${FUZZING}" \ + -e BUILD_DIR="${BUILD_DIR}" \ + -e CAMPAIGN_RSS_MB="${CAMPAIGN_RSS_MB}" \ + -e STACK_LIMIT_KB="${STACK_LIMIT_KB}" \ + -e SCHEMA_FUZZ_LOCALE="${SCHEMA_FUZZ_LOCALE:-}" \ + "${IMAGE}" bash -c "${INNER_RUN}" _ ${RUN_ARGS[@]+"${RUN_ARGS[@]}"}