Add FuzzTest-based fuzzing for schema serde and fix the codec bugs it…

.gitignore

@@ -95,3 +95,6 @@ src/aws-cpp-sdk-core/include/aws/core/SDKConfig.h
 
 #temp codegen files
 tools/code-generation/smithy/codegen/cpp-smoke-tests/codegen_output/
+
+# FuzzTest reproducers (run_fuzz.sh)
+tests/fuzz/reproducers/

CMakeLists.txt

@@ -74,6 +74,7 @@ if (LEGACY_BUILD)
     option(DISABLE_DNS_REQUIRED_TESTS "Disable unit tests that require DNS lookup to succeed, useful when using a http client that does not perform DNS lookup" OFF)
     option(AWS_APPSTORE_SAFE "Remove reference to private Apple APIs for AES GCM in Common Crypto. If set to OFF you application will get rejected from the apple app store." OFF)
     option(AWS_ENABLE_CORE_INTEGRATION_TEST "Enables the core integration tests to be built which contains dependencies on other clients for setup and tear down" OFF)
+    option(ENABLE_SCHEMA_FUZZ_TESTS "Build google/fuzztest fuzz tests for the schema serde (Linux + Clang, static libs)" OFF)
 
 
     set(AWS_USER_AGENT_CUSTOMIZATION "" CACHE STRING "User agent extension")
@@ -230,6 +231,11 @@ if (LEGACY_BUILD)
             MinSizeRel              # Like Release, but optimized for memory rather than speed.
             )
 
+    if (ENABLE_SCHEMA_FUZZ_TESTS)
+        include(schema_fuzz_tests)
+        setup_schema_fuzz_tests()
+    endif ()
+
     # build third-party targets
     if (BUILD_DEPS)
         set(CMAKE_INSTALL_RPATH "$ORIGIN")
@@ -346,6 +352,9 @@ if (LEGACY_BUILD)
 
     add_sdks()
     include(tests)
+    if (ENABLE_SCHEMA_FUZZ_TESTS)
+        add_schema_fuzz_tests()
+    endif ()
 
     # Performance tests for services
     if (BUILD_PERFORMANCE_TESTS)

cmake/schema_fuzz_tests.cmake

@@ -0,0 +1,45 @@
+# Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved.
+# SPDX-License-Identifier: Apache-2.0.
+#
+
+set(SCHEMA_FUZZTEST_GIT_TAG "305251260059201f842e5de6bb10e7b73efb1a6e" CACHE STRING "google/fuzztest commit used by ENABLE_SCHEMA_FUZZ_TESTS")
+option(SCHEMA_FUZZ_UNIT_SANITIZERS "Build the non-fuzzing (unit mode) schema fuzz tests with ASan and UBSan" ON)
+
+# Must run before the CRT and core are defined so fuzzing mode instruments them.
+macro(setup_schema_fuzz_tests)
+    if (NOT CMAKE_SYSTEM_NAME STREQUAL "Linux")
+        message(FATAL_ERROR "ENABLE_SCHEMA_FUZZ_TESTS requires Linux (use tools/scripts/run_fuzz.sh)")
+    endif ()
+    if (NOT CMAKE_CXX_COMPILER_ID STREQUAL "Clang")
+        message(FATAL_ERROR "ENABLE_SCHEMA_FUZZ_TESTS requires Clang; got ${CMAKE_CXX_COMPILER_ID}")
+    endif ()
+    if (BUILD_SHARED_LIBS)
+        message(FATAL_ERROR "ENABLE_SCHEMA_FUZZ_TESTS requires -DBUILD_SHARED_LIBS=OFF")
+    endif ()
+    if (ENABLE_TESTING)
+        message(FATAL_ERROR "ENABLE_SCHEMA_FUZZ_TESTS cannot be combined with ENABLE_TESTING: the vendored gtest and FuzzTest's gtest collide. Use -DENABLE_TESTING=OFF")
+    endif ()
+    if (CMAKE_VERSION VERSION_LESS 3.25)
+        message(FATAL_ERROR "ENABLE_SCHEMA_FUZZ_TESTS requires CMake >= 3.25 (FuzzTest, FetchContent SYSTEM)")
+    endif ()
+
+    include(FetchContent)
+    FetchContent_Declare(fuzztest
+            GIT_REPOSITORY https://github.com/google/fuzztest.git
+            GIT_TAG ${SCHEMA_FUZZTEST_GIT_TAG}
+            # Keeps FuzzTest and Abseil headers out of the SDK warning set.
+            SYSTEM)
+    FetchContent_MakeAvailable(fuzztest)
+
+    fuzztest_setup_fuzzing_flags()
+    if (FUZZTEST_FUZZING_MODE)
+        # -fno-jump-tables: on AArch64, switch lookup tables of __sancov_cntrs pointers built after SanitizerCoverage
+        # land outside COMDAT groups and fail to link with "defined in discarded section".
+        set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -fno-omit-frame-pointer -fno-jump-tables")
+        set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -fno-omit-frame-pointer -fno-jump-tables")
+    endif ()
+endmacro()
+
+function(add_schema_fuzz_tests)
+    add_subdirectory(${CMAKE_SOURCE_DIR}/tests/fuzz ${CMAKE_BINARY_DIR}/tests/fuzz)
+endfunction()

src/aws-cpp-sdk-core/include/aws/core/utils/StringUtils.h

@@ -13,6 +13,8 @@
 #include <aws/common/byte_buf.h>
 #include <aws/crt/Optional.h>
 
+#include <locale>
+
 
 namespace Aws
 {
@@ -176,8 +178,7 @@ namespace Aws
 
 
             /**
-             * Parses the entire string as a base-10 int64. Returns empty if the string is empty or has
-             * characters after the number. Out-of-range values saturate, matching ConvertToInt64.
+             * Parses the entire string as a base-10 int64. Returns empty if it is malformed or out of range.
              */
             static Aws::Crt::Optional<int64_t> ParseInt64(const Aws::String& source);
 
@@ -188,6 +189,18 @@ namespace Aws
              */
             static Aws::Crt::Optional<double> ParseDouble(const Aws::String& source);
 
+            /**
+             * Formats a double with 15 significant digits, or 17 when 15 does not round-trip.
+             */
+            static Aws::String FormatRoundTripDouble(double value);
+
+
+            /**
+             * Formats a finite float with 6 significant digits, or 9 when 6 does not round-trip.
+             */
+            static Aws::String FormatRoundTripFloat(float value);
+
+
 
 #ifdef _WIN32
             /**
@@ -202,12 +215,13 @@ namespace Aws
 #endif
 
             /**
-             * not all platforms (Android) have std::to_string
+             * not all platforms (Android) have std::to_string. Formats in the classic locale.
              */
             template< typename T >
             static Aws::String to_string(T value)
             {
                 Aws::OStringStream os;
+                os.imbue(std::locale::classic());
                 os << value;
                 return os.str();
             }

src/aws-cpp-sdk-core/include/smithy/client/schema/CborShapeDeserializer.h

@@ -28,6 +28,7 @@ class SMITHY_API CborShapeDeserializer final : public ShapeDeserializer {
   std::shared_ptr<const Document> ReadDocument(const Schema& schema) override;
 
   bool IsNull() override;
+  Aws::Crt::Optional<DeserializerError> GetError() const override;
 
  private:
   class Impl;

src/aws-cpp-sdk-core/include/smithy/client/schema/ClientProtocol.h

@@ -28,8 +28,8 @@ class SMITHY_API ClientProtocol {
   virtual SerializerOutcome SerializeRequest(const OperationRequestContext& operation, const Schema& inputSchema,
                                              const SerializableStruct& input) const;
 
-  virtual void DeserializeResponse(const OperationRequestContext& operation, Aws::Crt::ByteCursor data,
-                                   SerializableStruct& output) const;
+  virtual Aws::Crt::Optional<DeserializerError> DeserializeResponse(const OperationRequestContext& operation, Aws::Crt::ByteCursor data,
+                                                                    SerializableStruct& output) const;
 };
 
 class SMITHY_API RestJsonProtocol final : public ClientProtocol {
@@ -95,8 +95,8 @@ class SMITHY_API AwsQueryProtocol final : public ClientProtocol {
   Aws::UniquePtr<ShapeDeserializer> CreateOutputDeserializer(Aws::Crt::ByteCursor data) const override;
   SerializerOutcome SerializeRequest(const OperationRequestContext& operation, const Schema& inputSchema,
                                      const SerializableStruct& input) const override;
-  void DeserializeResponse(const OperationRequestContext& operation, Aws::Crt::ByteCursor data,
-                           SerializableStruct& output) const override;
+  Aws::Crt::Optional<DeserializerError> DeserializeResponse(const OperationRequestContext& operation, Aws::Crt::ByteCursor data,
+                                                            SerializableStruct& output) const override;
 };
 
 class SMITHY_API Ec2QueryProtocol final : public ClientProtocol {

src/aws-cpp-sdk-core/include/smithy/client/schema/Codec.h

@@ -43,8 +43,10 @@ class SMITHY_API Codec {
 
   virtual Aws::UniquePtr<ShapeDeserializer> CreateDeserializer(Aws::Crt::ByteCursor data) const = 0;
 
-  void DeserializeShape(Aws::Crt::ByteCursor data, SerializableStruct& shape) const {
-    shape.Deserialize(*CreateDeserializer(data));
+  Aws::Crt::Optional<DeserializerError> DeserializeShape(Aws::Crt::ByteCursor data, SerializableStruct& shape) const {
+    auto deserializer = CreateDeserializer(data);
+    shape.Deserialize(*deserializer);
+    return deserializer->GetError();
   }
 };
 

src/aws-cpp-sdk-core/include/smithy/client/schema/JsonShapeDeserializer.h

@@ -30,6 +30,7 @@ class SMITHY_API JsonShapeDeserializer final : public ShapeDeserializer {
   std::shared_ptr<const Document> ReadDocument(const Schema& schema) override;
 
   bool IsNull() override;
+  Aws::Crt::Optional<DeserializerError> GetError() const override;
 
  private:
   class Impl;

src/aws-cpp-sdk-core/include/smithy/client/schema/Schema.h

@@ -12,6 +12,9 @@
 namespace smithy {
 namespace schema {
 
+// Container nesting limit, root included, shared by every serializer and deserializer.
+constexpr int MAX_NESTING_DEPTH = 1000;
+
 class SchemaBuilder;
 
 enum class ShapeType : uint8_t {

src/aws-cpp-sdk-core/include/smithy/client/schema/SerdeTraits.h

@@ -7,10 +7,13 @@
 #include <aws/core/utils/DateTime.h>
 #include <aws/core/utils/StringUtils.h>
 #include <aws/core/utils/memory/stl/AWSString.h>
+#include <aws/crt/Optional.h>
 #include <smithy/client/schema/Schema.h>
 #include <smithy/client/schema/Trait.h>
 #include <smithy/client/schema/TraitKey.h>
 
+#include <chrono>
+#include <cmath>
 #include <cstdint>
 
 namespace smithy {
@@ -64,6 +67,16 @@ inline Aws::String EpochSecondsText(const Aws::Utils::DateTime& value) {
   return out;
 }
 
+// Decodes as DateTime(double) does, for parity with legacy clients. Empty where DateTime(double) would overflow.
+inline Aws::Crt::Optional<Aws::Utils::DateTime> EpochSecondsToDateTime(double seconds) {
+  static const double maxSeconds = static_cast<double>(std::chrono::duration_cast<std::chrono::milliseconds>(
+                                       (std::chrono::system_clock::duration::max)()).count()) / 1000.0;
+  if (!std::isfinite(seconds) || std::fabs(seconds) >= maxSeconds) {
+    return {};
+  }
+  return Aws::Utils::DateTime(seconds);
+}
+
 inline Aws::String FormatTimestampText(const Aws::Utils::DateTime& value, TimestampFormatTrait::Format format) {
   switch (format) {
     case TimestampFormatTrait::Format::HTTP_DATE:

src/aws-cpp-sdk-core/include/smithy/client/schema/ShapeDeserializer.h

@@ -1,20 +1,28 @@
 #pragma once
 
+#include <aws/core/client/AWSError.h>
+#include <aws/core/client/CoreErrors.h>
 #include <aws/core/utils/Array.h>
 #include <aws/core/utils/DateTime.h>
+#include <aws/core/utils/StringUtils.h>
 #include <aws/core/utils/memory/stl/AWSString.h>
 #include <aws/crt/Optional.h>
 #include <smithy/Smithy_EXPORTS.h>
 #include <smithy/client/schema/Document.h>
 #include <smithy/client/schema/Schema.h>
+#include <smithy/client/schema/SerdeTraits.h>
 
+#include <cmath>
 #include <cstdint>
 #include <functional>
+#include <limits>
 #include <memory>
 
 namespace smithy {
 namespace schema {
 
+using DeserializerError = Aws::Client::AWSError<Aws::Client::CoreErrors>;
+
 class SMITHY_API ShapeDeserializer {
  public:
   virtual ~ShapeDeserializer() = default;
@@ -37,10 +45,117 @@ class SMITHY_API ShapeDeserializer {
   virtual Aws::Crt::Optional<Aws::String> ReadString(const Schema& schema) = 0;
   virtual Aws::Crt::Optional<Aws::Utils::DateTime> ReadTimestamp(const Schema& schema) = 0;
   virtual Aws::Crt::Optional<Aws::Utils::ByteBuffer> ReadBlob(const Schema& schema) = 0;
-  // Null when the member is absent, unparsable, or the protocol does not support documents.
   virtual std::shared_ptr<const Document> ReadDocument(const Schema& schema) = 0;
 
   virtual bool IsNull() = 0;
+
+  // First failure wins; once set, reads return empty.
+  virtual Aws::Crt::Optional<DeserializerError> GetError() const { return m_error; }
+
+ protected:
+  void Fail(const Aws::String& message) {
+    if (!m_error.has_value()) {
+      m_error = DeserializerError(Aws::Client::CoreErrors::INTERNAL_FAILURE, "SerializationException", message, false);
+    }
+  }
+  bool HasFailed() const { return m_error.has_value(); }
+  bool AtRoot() const { return m_depth == 0; }
+
+  void FailUnrepresentable(const char* type) { Fail(Aws::String("Value out of range for ") + type); }
+
+  // Call before delivering each non-null modeled member; a second member of a union fails the read.
+  bool AdmitMember(const Schema& schema, bool& delivered) {
+    if (delivered && schema.GetType() == ShapeType::Union) {
+      Fail("Union has more than one member set");
+      return false;
+    }
+    delivered = true;
+    return true;
+  }
+
+  Aws::Crt::Optional<int> NarrowToInt(int64_t value) {
+    if (value < (std::numeric_limits<int>::min)() || value > (std::numeric_limits<int>::max)()) {
+      FailUnrepresentable("integer");
+      return {};
+    }
+    return static_cast<int>(value);
+  }
+
+  // Float text printed to 9 digits can parse just above FLT_MAX, so values below the rounding point to infinity clamp.
+  Aws::Crt::Optional<float> NarrowToFloat(double value) {
+    const float floatMax = (std::numeric_limits<float>::max)();
+    const double roundsToInfinity = static_cast<double>(floatMax) + std::ldexp(1.0, 103);
+    if (std::isfinite(value) && std::fabs(value) > static_cast<double>(floatMax)) {
+      if (std::fabs(value) >= roundsToInfinity) {
+        FailUnrepresentable("float");
+        return {};
+      }
+      return value < 0 ? -floatMax : floatMax;
+    }
+    return static_cast<float>(value);
+  }
+
+  // Out-of-range integer text fails the read; other non-integer text just returns empty.
+  Aws::Crt::Optional<int64_t> ParseLong(const Aws::String& text) {
+    const auto value = Aws::Utils::StringUtils::ParseInt64(text);
+    if (!value.has_value() && IsPlainInteger(text)) {
+      FailUnrepresentable("long");
+    }
+    return value;
+  }
+
+  // Must accept what strtoll does.
+  static bool IsPlainInteger(const Aws::String& text) {
+    size_t start = text.find_first_not_of(" \t\n\v\f\r");
+    if (start == Aws::String::npos) {
+      return false;
+    }
+    if (text[start] == '-' || text[start] == '+') {
+      ++start;
+    }
+    return text.size() > start && text.find_first_not_of("0123456789", start) == Aws::String::npos;
+  }
+
+  Aws::Crt::Optional<Aws::Utils::DateTime> EpochSecondsToTimestamp(double seconds) {
+    auto parsed = EpochSecondsToDateTime(seconds);
+    if (!parsed.has_value()) {
+      FailUnrepresentable("timestamp");
+    }
+    return parsed;
+  }
+
+  class ContainerScope final {
+   public:
+    explicit ContainerScope(ShapeDeserializer& owner) : m_owner(owner), m_entered(owner.EnterContainer()) {}
+    ~ContainerScope() {
+      if (m_entered) {
+        --m_owner.m_depth;
+      }
+    }
+    ContainerScope(const ContainerScope&) = delete;
+    ContainerScope& operator=(const ContainerScope&) = delete;
+    bool Entered() const { return m_entered; }
+
+   private:
+    ShapeDeserializer& m_owner;
+    bool m_entered;
+  };
+
+ private:
+  bool EnterContainer() {
+    if (HasFailed()) {
+      return false;
+    }
+    if (m_depth >= MAX_NESTING_DEPTH) {
+      Fail("Maximum nesting depth exceeded");
+      return false;
+    }
+    ++m_depth;
+    return true;
+  }
+
+  Aws::Crt::Optional<DeserializerError> m_error;
+  int m_depth = 0;
 };
 
 }  // namespace schema