From 4dd8394a7d3eb4fce79b2fbd6ea0300df2e790d9 Mon Sep 17 00:00:00 2001 From: chaokunyang Date: Fri, 28 Aug 2026 01:23:05 +0800 Subject: [PATCH 001/168] fix(cpp): freeze registries before root access --- .agents/languages/cpp.md | 5 + .../serialization/collection_serializer.h | 48 +++-- cpp/fory/serialization/fory.h | 174 ++++++++++------ cpp/fory/serialization/serialization_test.cc | 197 +++++++++++++++++- cpp/fory/serialization/type_resolver.cc | 38 +++- cpp/fory/serialization/type_resolver.h | 43 ++-- .../cpp/type-registration.md | 6 + 7 files changed, 416 insertions(+), 95 deletions(-) diff --git a/.agents/languages/cpp.md b/.agents/languages/cpp.md index c5a45fa410..8c23dba28d 100644 --- a/.agents/languages/cpp.md +++ b/.agents/languages/cpp.md @@ -18,6 +18,11 @@ Load this file when changing `cpp/`, Cython build plumbing, or C++ xlang behavio resource amplification, publish reference or cache state that survives root cleanup, or return success past the required safepoint. Do not add per-field checks, cursor rollback, or tests that pin the first detection point solely to make an error earlier or more precise. +- Every public `Fory` and `ThreadSafeFory` root overload freezes facade registration as its first + action, before constructing stream wrappers, accessing stream buffers, validating arguments, or + acquiring pooled instances. `BaseFory` and the source `TypeResolver` keep separate owner-local + freeze gates so direct resolver registration cannot bypass the facade gate. Both reject before + mutation; do not collapse these gates or describe permanent registry freeze as finalization. - Put private methods last in class definitions, immediately before private fields. - Do not redesign alias-based or low-level public type shapes to add convenience methods unless the user explicitly asks for that API change. - For cross-language feature ports, match protocol behavior but use idiomatic C++ ownership and layering instead of mirroring Java structure literally. diff --git a/cpp/fory/serialization/collection_serializer.h b/cpp/fory/serialization/collection_serializer.h index bafe03a46e..f6e23ad933 100644 --- a/cpp/fory/serialization/collection_serializer.h +++ b/cpp/fory/serialization/collection_serializer.h @@ -562,6 +562,9 @@ inline bool read_declared_same_type_collection(Container &result, auto elem = Serializer::read(ctx, RefMode::None, false); collection_insert(result, std::move(elem)); } + if (FORY_PREDICT_FALSE(ctx.has_error())) { + return false; + } return true; } @@ -583,9 +586,15 @@ inline bool read_declared_same_type_collection(Container &result, checkpoint_byte = ctx.buffer().logical_reader_index(); } } - return checkpoint_item == length || - detail::settle_unbacked_container_items(ctx, length - checkpoint_item, - checkpoint_byte); + if (FORY_PREDICT_FALSE(ctx.has_error())) { + return false; + } + if (checkpoint_item != length && + FORY_PREDICT_FALSE(!detail::settle_unbacked_container_items( + ctx, length - checkpoint_item, checkpoint_byte))) { + return false; + } + return true; } template @@ -616,10 +625,15 @@ read_declared_same_type_collection(std::forward_list &result, } } } + if (FORY_PREDICT_FALSE(ctx.has_error())) { + return false; + } if constexpr (!read_data_always_advances_v) { - return checkpoint_item == length || - detail::settle_unbacked_container_items( - ctx, length - checkpoint_item, checkpoint_byte); + if (checkpoint_item != length && + FORY_PREDICT_FALSE(!detail::settle_unbacked_container_items( + ctx, length - checkpoint_item, checkpoint_byte))) { + return false; + } } return true; } @@ -665,10 +679,15 @@ read_same_type_info_collection_body(Container &result, ReadContext &ctx, } } } + if (FORY_PREDICT_FALSE(ctx.has_error())) { + return false; + } if constexpr (MeasureProgress) { - return checkpoint_item == length || - detail::settle_unbacked_container_items( - ctx, length - checkpoint_item, checkpoint_byte); + if (checkpoint_item != length && + FORY_PREDICT_FALSE(!detail::settle_unbacked_container_items( + ctx, length - checkpoint_item, checkpoint_byte))) { + return false; + } } return true; } @@ -701,10 +720,15 @@ inline bool read_same_type_info_collection_body( } } } + if (FORY_PREDICT_FALSE(ctx.has_error())) { + return false; + } if constexpr (MeasureProgress) { - return checkpoint_item == length || - detail::settle_unbacked_container_items( - ctx, length - checkpoint_item, checkpoint_byte); + if (checkpoint_item != length && + FORY_PREDICT_FALSE(!detail::settle_unbacked_container_items( + ctx, length - checkpoint_item, checkpoint_byte))) { + return false; + } } return true; } diff --git a/cpp/fory/serialization/fory.h b/cpp/fory/serialization/fory.h index d9087038c1..71226c3d5f 100644 --- a/cpp/fory/serialization/fory.h +++ b/cpp/fory/serialization/fory.h @@ -510,7 +510,7 @@ class BaseFory { template Result register_type(RegisterFn &&fn) { std::lock_guard lock(registration_mutex_); - if (FORY_PREDICT_FALSE(registration_locked_)) { + if (FORY_PREDICT_FALSE(registration_frozen_)) { return Unexpected(Error::invalid( "Cannot register types after first serialize/deserialize call")); } @@ -518,9 +518,10 @@ class BaseFory { } protected: - void lock_registration() const { + Result, Error> finalize_type_resolver() const { std::lock_guard lock(registration_mutex_); - registration_locked_ = true; + registration_frozen_ = true; + return type_resolver_->build_final_type_resolver(); } /// Protected constructor - only derived classes can instantiate. @@ -539,7 +540,7 @@ class BaseFory { Config config_; std::shared_ptr type_resolver_; mutable std::mutex registration_mutex_; - mutable bool registration_locked_{false}; + mutable bool registration_frozen_{false}; }; // ============================================================================ @@ -595,27 +596,7 @@ class Fory : public BaseFory { if (FORY_PREDICT_FALSE(!finalized_)) { ensure_finalized(); } - WriteContextGuard guard(*write_ctx_); - output_stream.reset(); - write_ctx_->set_output_stream(&output_stream); - Buffer &buffer = write_ctx_->buffer(); - buffer.bind_output_stream(&output_stream); - auto serialize_result = serialize_impl(obj, buffer); - if (FORY_PREDICT_FALSE(!serialize_result.ok())) { - buffer.clear_output_stream(); - write_ctx_->set_output_stream(nullptr); - return Unexpected(std::move(serialize_result).error()); - } - output_stream.force_flush(); - buffer.clear_output_stream(); - write_ctx_->set_output_stream(nullptr); - if (FORY_PREDICT_FALSE(output_stream.has_error())) { - return Unexpected(output_stream.error()); - } - if (FORY_PREDICT_FALSE(write_ctx_->has_error())) { - return Unexpected(write_ctx_->take_error()); - } - return output_stream.flushed_bytes(); + return serialize_stream(output_stream, obj); } /// Serialize an object to a std::ostream. @@ -626,8 +607,11 @@ class Fory : public BaseFory { /// @return Number of bytes written, or error. template Result serialize(std::ostream &ostream, const T &obj) { + if (FORY_PREDICT_FALSE(!finalized_)) { + ensure_finalized(); + } StdOutputStream output_stream(ostream); - return serialize(output_stream, obj); + return serialize_stream(output_stream, obj); } /// Serialize an object to an existing Buffer (fastest path). @@ -642,13 +626,7 @@ class Fory : public BaseFory { if (FORY_PREDICT_FALSE(!finalized_)) { ensure_finalized(); } - // Swap in the caller's buffer so all writes go there. - buffer.swap(write_ctx_->buffer()); - auto result = serialize_impl(obj, write_ctx_->buffer()); - buffer.swap(write_ctx_->buffer()); - // reset internal state after use without clobbering caller buffer. - write_ctx_->reset(); - return result; + return serialize_buffer(buffer, obj); } /// Serialize an object to an existing byte vector (zero-copy). @@ -664,12 +642,14 @@ class Fory : public BaseFory { template Result serialize_to(std::vector &output, const T &obj) { + if (FORY_PREDICT_FALSE(!finalized_)) { + ensure_finalized(); + } // Wrap the output vector in a Buffer for zero-copy serialization // writer_index starts at output.size() for appending Buffer buffer(output); - // Forward to Buffer version - auto result = serialize_to(buffer, obj); + auto result = serialize_buffer(buffer, obj); // Resize vector to actual written size output.resize(buffer.writer_index()); @@ -687,16 +667,7 @@ class Fory : public BaseFory { if (FORY_PREDICT_FALSE(!finalized_)) { ensure_finalized(); } - if (data == nullptr) { - return Unexpected(Error::invalid("Data pointer is null")); - } - if (size == 0) { - return Unexpected(Error::invalid("Data size is zero")); - } - - Buffer buffer(const_cast(data), static_cast(size), - false); - return deserialize_buffer(buffer); + return deserialize_bytes(data, size); } /// Deserialize an object from a byte vector. @@ -706,7 +677,10 @@ class Fory : public BaseFory { /// @return Deserialized object, or error. template Result deserialize(const std::vector &data) { - return deserialize(data.data(), data.size()); + if (FORY_PREDICT_FALSE(!finalized_)) { + ensure_finalized(); + } + return deserialize_bytes(data.data(), data.size()); } /// Deserialize an object from a Buffer, updating the buffer's reader_index. @@ -735,20 +709,10 @@ class Fory : public BaseFory { /// @return Deserialized object, or error. template Result deserialize(InputStream &input_stream) { - struct StreamShrinkGuard { - InputStream *input_stream = nullptr; - ~StreamShrinkGuard() { - if (input_stream != nullptr) { - input_stream->shrink_buffer(); - } - } - }; - StreamShrinkGuard shrink_guard{&input_stream}; - Buffer &buffer = input_stream.get_buffer(); if (FORY_PREDICT_FALSE(!finalized_)) { ensure_finalized(); } - return deserialize_buffer(buffer); + return deserialize_stream(input_stream); } /// Deserialize an object from StdInputStream. @@ -757,7 +721,10 @@ class Fory : public BaseFory { /// @param stream Input stream wrapper to read from. /// @return Deserialized object, or error. template Result deserialize(StdInputStream &stream) { - return deserialize(static_cast(stream)); + if (FORY_PREDICT_FALSE(!finalized_)) { + ensure_finalized(); + } + return deserialize_stream(stream); } // ========================================================================== @@ -795,8 +762,7 @@ class Fory : public BaseFory { /// Finalize the type resolver on first use. void ensure_finalized() { if (!finalized_) { - lock_registration(); - auto final_result = type_resolver_->build_final_type_resolver(); + auto final_result = finalize_type_resolver(); FORY_CHECK(final_result.ok()) << "Failed to build finalized TypeResolver: " << final_result.error().to_string(); @@ -837,6 +803,73 @@ class Fory : public BaseFory { ", local xlang=" + std::string(config_.xlang ? "true" : "false")); } + template + Result serialize_stream(OutputStream &output_stream, + const T &obj) { + WriteContextGuard guard(*write_ctx_); + output_stream.reset(); + write_ctx_->set_output_stream(&output_stream); + Buffer &buffer = write_ctx_->buffer(); + buffer.bind_output_stream(&output_stream); + auto serialize_result = serialize_impl(obj, buffer); + if (FORY_PREDICT_FALSE(!serialize_result.ok())) { + buffer.clear_output_stream(); + write_ctx_->set_output_stream(nullptr); + return Unexpected(std::move(serialize_result).error()); + } + output_stream.force_flush(); + buffer.clear_output_stream(); + write_ctx_->set_output_stream(nullptr); + if (FORY_PREDICT_FALSE(output_stream.has_error())) { + return Unexpected(output_stream.error()); + } + if (FORY_PREDICT_FALSE(write_ctx_->has_error())) { + return Unexpected(write_ctx_->take_error()); + } + return output_stream.flushed_bytes(); + } + + template + FORY_ALWAYS_INLINE Result serialize_buffer(Buffer &buffer, + const T &obj) { + // Swap in the caller's buffer so all writes go there. + buffer.swap(write_ctx_->buffer()); + auto result = serialize_impl(obj, write_ctx_->buffer()); + buffer.swap(write_ctx_->buffer()); + // reset internal state after use without clobbering caller buffer. + write_ctx_->reset(); + return result; + } + + template + Result deserialize_bytes(const uint8_t *data, size_t size) { + if (data == nullptr) { + return Unexpected(Error::invalid("Data pointer is null")); + } + if (size == 0) { + return Unexpected(Error::invalid("Data size is zero")); + } + + Buffer buffer(const_cast(data), static_cast(size), + false); + return deserialize_buffer(buffer); + } + + template + Result deserialize_stream(InputStream &input_stream) { + struct StreamShrinkGuard { + InputStream *input_stream = nullptr; + ~StreamShrinkGuard() { + if (input_stream != nullptr) { + input_stream->shrink_buffer(); + } + } + }; + StreamShrinkGuard shrink_guard{&input_stream}; + Buffer &buffer = input_stream.get_buffer(); + return deserialize_buffer(buffer); + } + /// Core serialization implementation. /// TypeMeta is written inline using streaming protocol (no deferred writing). template @@ -962,24 +995,28 @@ class ThreadSafeFory : public BaseFory { public: template Result, Error> serialize(const T &obj) { + ensure_finalized(); auto fory_handle = fory_pool_.acquire(); return fory_handle->serialize(obj); } template Result serialize(OutputStream &output_stream, const T &obj) { + ensure_finalized(); auto fory_handle = fory_pool_.acquire(); return fory_handle->serialize(output_stream, obj); } template Result serialize(std::ostream &ostream, const T &obj) { + ensure_finalized(); auto fory_handle = fory_pool_.acquire(); return fory_handle->serialize(ostream, obj); } template Result serialize_to(Buffer &buffer, const T &obj) { + ensure_finalized(); auto fory_handle = fory_pool_.acquire(); return fory_handle->serialize_to(buffer, obj); } @@ -987,28 +1024,34 @@ class ThreadSafeFory : public BaseFory { template Result serialize_to(std::vector &output, const T &obj) { + ensure_finalized(); auto fory_handle = fory_pool_.acquire(); return fory_handle->serialize_to(output, obj); } template Result deserialize(const uint8_t *data, size_t size) { + ensure_finalized(); auto fory_handle = fory_pool_.acquire(); return fory_handle->template deserialize(data, size); } template Result deserialize(const std::vector &data) { - return deserialize(data.data(), data.size()); + ensure_finalized(); + auto fory_handle = fory_pool_.acquire(); + return fory_handle->template deserialize(data.data(), data.size()); } template Result deserialize(InputStream &input_stream) { + ensure_finalized(); auto fory_handle = fory_pool_.acquire(); return fory_handle->template deserialize(input_stream); } template Result deserialize(StdInputStream &stream) { + ensure_finalized(); auto fory_handle = fory_pool_.acquire(); return fory_handle->template deserialize(stream); } @@ -1022,15 +1065,18 @@ class ThreadSafeFory : public BaseFory { config_, get_finalized_resolver(), Fory::PreFinalized{})); }) {} - std::shared_ptr get_finalized_resolver() const { + void ensure_finalized() const { std::call_once(finalized_once_flag_, [this]() { - lock_registration(); - auto final_result = type_resolver_->build_final_type_resolver(); + auto final_result = finalize_type_resolver(); FORY_CHECK(final_result.ok()) << "Failed to build finalized TypeResolver: " << final_result.error().to_string(); finalized_resolver_ = std::move(final_result).value(); }); + } + + std::shared_ptr get_finalized_resolver() const { + ensure_finalized(); return finalized_resolver_->clone(); } diff --git a/cpp/fory/serialization/serialization_test.cc b/cpp/fory/serialization/serialization_test.cc index e7ea4989a9..7c420f9dd0 100644 --- a/cpp/fory/serialization/serialization_test.cc +++ b/cpp/fory/serialization/serialization_test.cc @@ -181,6 +181,45 @@ inline std::vector buffer_bytes(Buffer &buffer) { buffer.data() + buffer.writer_index()); } +class RegistryProbeInputStream final : public InputStream { +public: + explicit RegistryProbeInputStream(Fory &fory) : fory_(fory) {} + + Result fill_buffer(uint32_t) override { + return Unexpected(Error::io_error("No input available")); + } + + Result read_to(uint8_t *, uint32_t) override { + return Unexpected(Error::io_error("No input available")); + } + + Result skip(uint32_t) override { + return Unexpected(Error::io_error("No input available")); + } + + Result unread(uint32_t) override { + return Unexpected(Error::io_error("No input available")); + } + + void shrink_buffer() override {} + + Buffer &get_buffer() override { + auto result = fory_.register_struct<::SimpleStruct>(1); + registration_rejected_ = !result.ok(); + return active_buffer_ == nullptr ? buffer_ : *active_buffer_; + } + + void bind_buffer(Buffer *buffer) override { active_buffer_ = buffer; } + + bool registration_rejected() const { return registration_rejected_; } + +private: + Fory &fory_; + Buffer buffer_; + Buffer *active_buffer_ = nullptr; + bool registration_rejected_ = false; +}; + template void test_roundtrip(const T &original, bool should_equal = true) { auto fory = @@ -934,6 +973,30 @@ TEST(SerializationTest, SkipNoneListConsumesBudget) { ASSERT_TRUE(ctx.has_error()); } +TEST(SerializationTest, LastElementErrorSafepoints) { + Config config; + + std::vector declared_bytes{2}; + Buffer declared_buffer(declared_bytes); + ReadContext declared_ctx(config, std::make_unique()); + declared_ctx.attach(declared_buffer); + std::vector declared_values; + EXPECT_FALSE(read_declared_same_type_collection(declared_values, + declared_ctx, 2)); + EXPECT_TRUE(declared_ctx.has_error()); + + std::vector type_info_bytes{2}; + Buffer type_info_buffer(type_info_bytes); + ReadContext type_info_ctx(config, std::make_unique()); + type_info_ctx.attach(type_info_buffer); + TypeInfo type_info; + type_info.harness.read_data_always_advances = true; + std::vector type_info_values; + EXPECT_FALSE(read_same_type_info_collection( + type_info_values, type_info_ctx, 2, type_info)); + EXPECT_TRUE(type_info_ctx.has_error()); +} + // ============================================================================ // Character Type Tests (C++ native only) // ============================================================================ @@ -2443,6 +2506,97 @@ TEST(SerializationTest, ConfigurationBuilder) { // Thread Safety Tests // ============================================================================ +static void expect_finalized_source(TypeResolver &resolver) { + auto source_info = resolver.get_type_info<::SimpleStruct>(); + ASSERT_TRUE(source_info.ok()) << source_info.error().to_string(); + ASSERT_NE(source_info.value()->type_meta, nullptr); + ASSERT_FALSE(source_info.value()->type_def.empty()); + + std::vector source_type_def = source_info.value()->type_def; + Buffer source_bytes(source_type_def); + auto parsed_source = TypeMeta::from_bytes(source_bytes, nullptr); + ASSERT_TRUE(parsed_source.ok()) << parsed_source.error().to_string(); + EXPECT_EQ(source_bytes.remaining_size(), 0u); + EXPECT_EQ(parsed_source.value()->field_infos.size(), 2u); + + auto cloned = resolver.clone(); + auto cloned_info = cloned->get_type_info<::SimpleStruct>(); + ASSERT_TRUE(cloned_info.ok()) << cloned_info.error().to_string(); + ASSERT_NE(cloned_info.value()->type_meta, nullptr); + EXPECT_EQ(cloned_info.value()->type_def, source_info.value()->type_def); + EXPECT_EQ(cloned_info.value()->type_meta->field_infos.size(), 2u); + + auto rebuilt = resolver.build_final_type_resolver(); + ASSERT_TRUE(rebuilt.ok()) << rebuilt.error().to_string(); + auto rebuilt_info = rebuilt.value()->get_type_info<::SimpleStruct>(); + ASSERT_TRUE(rebuilt_info.ok()) << rebuilt_info.error().to_string(); + ASSERT_NE(rebuilt_info.value()->type_meta, nullptr); + EXPECT_EQ(rebuilt_info.value()->type_def, source_info.value()->type_def); +} + +TEST(SerializationTest, SourceResolverFinalizes) { + auto source_resolver = std::make_shared(); + auto fory = Fory::builder() + .xlang(true) + .compatible(false) + .track_ref(false) + .type_resolver(source_resolver) + .build(); + ASSERT_TRUE(fory.register_struct<::SimpleStruct>(1).ok()); + auto pending = source_resolver->get_type_info<::SimpleStruct>(); + ASSERT_TRUE(pending.ok()) << pending.error().to_string(); + EXPECT_EQ(pending.value()->type_meta, nullptr); + + auto bytes = fory.serialize(::SimpleStruct{1, 2}); + ASSERT_TRUE(bytes.ok()) << bytes.error().to_string(); + expect_finalized_source(*source_resolver); +} + +TEST(SerializationTest, DirectFailedRootFreezes) { + auto source_resolver = std::make_shared(); + auto fory = Fory::builder() + .xlang(true) + .compatible(false) + .track_ref(false) + .type_resolver(source_resolver) + .build(); + + auto root_result = fory.deserialize(nullptr, 0); + ASSERT_FALSE(root_result.ok()); + + auto facade_registration = fory.register_struct<::SimpleStruct>(1); + ASSERT_FALSE(facade_registration.ok()); + + auto type_info = source_resolver->get_type_info_by_id( + static_cast(TypeId::STRING)); + ASSERT_TRUE(type_info.ok()); + ASSERT_EQ(type_info.value()->harness.any_write_fn, nullptr); + ASSERT_EQ(type_info.value()->harness.any_read_fn, nullptr); + + auto late_registration = register_any_type(*source_resolver); + ASSERT_FALSE(late_registration.ok()); + EXPECT_EQ(type_info.value()->harness.any_write_fn, nullptr); + EXPECT_EQ(type_info.value()->harness.any_read_fn, nullptr); + EXPECT_FALSE( + source_resolver->get_type_info(std::type_index(typeid(std::string))) + .ok()); +} + +TEST(SerializationTest, ThreadSafeSourceFinalizes) { + auto source_resolver = std::make_shared(); + auto fory = Fory::builder() + .xlang(true) + .compatible(false) + .track_ref(false) + .type_resolver(source_resolver) + .build_thread_safe(); + ASSERT_TRUE(fory.register_struct<::SimpleStruct>(1).ok()); + + auto bytes = fory.serialize(::SimpleStruct{1, 2}); + ASSERT_TRUE(bytes.ok()) << bytes.error().to_string(); + expect_finalized_source(*source_resolver); +} + TEST(SerializationTest, ThreadSafeForyMultiThread) { auto fory = Fory::builder() .xlang(true) @@ -2484,7 +2638,7 @@ TEST(SerializationTest, ThreadSafeForyMultiThread) { EXPECT_EQ(success_count.load(), k_num_threads * k_iterations_per_thread); } -TEST(SerializationTest, ThreadSafeForyRejectsRegistrationAfterFirstSerialize) { +TEST(SerializationTest, ThreadSafeRegistrationFreezes) { auto fory = Fory::builder() .xlang(true) .compatible(false) @@ -2499,10 +2653,45 @@ TEST(SerializationTest, ThreadSafeForyRejectsRegistrationAfterFirstSerialize) { auto late_registration = fory.register_struct<::SimpleStruct>(2); EXPECT_FALSE(late_registration.ok()); +} + +TEST(SerializationTest, InputStreamFreezesBeforeAccess) { + auto fory = + Fory::builder().xlang(true).compatible(false).track_ref(false).build(); + RegistryProbeInputStream input_stream(fory); + + EXPECT_FALSE(fory.deserialize(input_stream).ok()); + EXPECT_TRUE(input_stream.registration_rejected()); +} + +TEST(SerializationTest, ThreadSafeFailedRootFreezes) { + auto source_resolver = std::make_shared(); + auto fory = Fory::builder() + .xlang(true) + .compatible(false) + .track_ref(false) + .type_resolver(source_resolver) + .build_thread_safe(); + + auto root_result = fory.deserialize(nullptr, 0); + ASSERT_FALSE(root_result.ok()); + + auto facade_registration = fory.register_struct<::SimpleStruct>(1); + ASSERT_FALSE(facade_registration.ok()); + + auto type_info = source_resolver->get_type_info_by_id( + static_cast(TypeId::STRING)); + ASSERT_TRUE(type_info.ok()); + ASSERT_EQ(type_info.value()->harness.any_write_fn, nullptr); + ASSERT_EQ(type_info.value()->harness.any_read_fn, nullptr); + + auto late_registration = register_any_type(*source_resolver); ASSERT_FALSE(late_registration.ok()); - EXPECT_EQ(late_registration.error().code(), ErrorCode::Invalid); - EXPECT_NE(late_registration.error().to_string().find("Cannot register types"), - std::string::npos); + EXPECT_EQ(type_info.value()->harness.any_write_fn, nullptr); + EXPECT_EQ(type_info.value()->harness.any_read_fn, nullptr); + EXPECT_FALSE( + source_resolver->get_type_info(std::type_index(typeid(std::string))) + .ok()); } TEST(SerializationTest, TemporalCarriersAreHashable) { diff --git a/cpp/fory/serialization/type_resolver.cc b/cpp/fory/serialization/type_resolver.cc index edf8fc60d3..f61d4fbd6f 100644 --- a/cpp/fory/serialization/type_resolver.cc +++ b/cpp/fory/serialization/type_resolver.cc @@ -1762,8 +1762,19 @@ TypeResolver::get_type_info(const std::type_index &type_index) const { return entry->second; } +FORY_NOINLINE Result TypeResolver::registration_frozen_error() { + return Unexpected(Error::invalid( + "TypeResolver registry is frozen, cannot register more types")); +} + Result, Error> TypeResolver::build_final_type_resolver() { + std::lock_guard lock(registration_mutex_); + // Freeze the source before building so even failed finalization permanently + // rejects later registration. Holding the registration mutex makes first use + // linearizable with direct registration helpers. ThreadSafeFory retains this + // source resolver after publishing finalized pool owners. + registry_frozen_ = true; auto final_resolver = std::make_unique(); // copy configuration @@ -1771,7 +1782,7 @@ TypeResolver::build_final_type_resolver() { final_resolver->xlang_ = xlang_; final_resolver->check_struct_version_ = check_struct_version_; final_resolver->track_ref_ = track_ref_; - final_resolver->finalized_ = true; + final_resolver->registry_frozen_ = true; // Build mapping from old pointers to new pointers for rebuilding lookup maps fory::flat_hash_map ptr_map; @@ -1846,6 +1857,29 @@ TypeResolver::build_final_type_resolver() { // Clear partial_type_infos in the final resolver since they're all completed final_resolver->partial_type_infos_.clear(); + // ThreadSafeFory retains the source resolver after publishing the finalized + // clone. Prepare every metadata update before mutating the source so failed + // finalization cannot leave it partially completed. + struct FinalizedPartial { + TypeInfo *source; + std::vector type_def; + std::unique_ptr type_meta; + }; + std::vector finalized_partials; + for (const auto &[key, source_ptr] : partial_type_infos_) { + (void)key; + TypeInfo *completed_ptr = remap_type_info(source_ptr); + FORY_CHECK(completed_ptr->type_meta != nullptr); + finalized_partials.push_back( + {source_ptr, completed_ptr->type_def, + std::make_unique(*completed_ptr->type_meta)}); + } + for (auto &partial : finalized_partials) { + partial.source->type_def = std::move(partial.type_def); + partial.source->type_meta = std::move(partial.type_meta); + } + partial_type_infos_.clear(); + return final_resolver; } @@ -1857,7 +1891,7 @@ std::unique_ptr TypeResolver::clone() const { cloned->xlang_ = xlang_; cloned->check_struct_version_ = check_struct_version_; cloned->track_ref_ = track_ref_; - cloned->finalized_ = finalized_; + cloned->registry_frozen_ = registry_frozen_; // Build mapping from old pointers to new pointers fory::flat_hash_map ptr_map; diff --git a/cpp/fory/serialization/type_resolver.h b/cpp/fory/serialization/type_resolver.h index 07344e68a3..00be162cb2 100644 --- a/cpp/fory/serialization/type_resolver.h +++ b/cpp/fory/serialization/type_resolver.h @@ -28,6 +28,7 @@ #include #include #include +#include #include #include #include @@ -1372,6 +1373,9 @@ class TypeResolver { /// 3. Builds complete TypeMeta and serializes it to bytes /// 4. Returns a new TypeResolver with all type infos fully initialized /// + /// Calling this method permanently freezes registration on the source + /// resolver before construction starts, including when construction fails. + /// /// @return A new TypeResolver with all type infos fully initialized and ready /// for use. Result, Error> build_final_type_resolver(); @@ -1516,7 +1520,10 @@ class TypeResolver { void register_type_internal_runtime(const std::type_index &type_index, TypeInfo *info); - void check_registration_thread(); + /// Validate registration state while registration_mutex_ is held. + Result check_registration(); + + static FORY_NOINLINE Result registration_frozen_error(); void register_builtin_types(); @@ -1526,7 +1533,8 @@ class TypeResolver { bool track_ref_; std::thread::id registration_thread_id_; - bool finalized_; + bool registry_frozen_; + std::mutex registration_mutex_; // Primary storage - owns all TypeInfo objects std::vector> type_infos_; @@ -1554,7 +1562,7 @@ class TypeResolver { inline TypeResolver::TypeResolver() : compatible_(false), xlang_(false), check_struct_version_(true), track_ref_(true), registration_thread_id_(std::this_thread::get_id()), - finalized_(false) { + registry_frozen_(false) { register_builtin_types(); } @@ -1565,12 +1573,14 @@ inline void TypeResolver::apply_config(const Config &config) { track_ref_ = config.track_ref; } -inline void TypeResolver::check_registration_thread() { +inline Result TypeResolver::check_registration() { + if (FORY_PREDICT_FALSE(registry_frozen_)) { + return registration_frozen_error(); + } FORY_CHECK(std::this_thread::get_id() == registration_thread_id_) << "TypeResolver registration methods must be called from the same " "thread that created the TypeResolver"; - FORY_CHECK(!finalized_) - << "TypeResolver has been finalized, cannot register more types"; + return Result(); } template @@ -1752,7 +1762,8 @@ get_type_info_with_resolver(TypeResolver &resolver) { } template Result TypeResolver::register_any_type() { - check_registration_thread(); + std::lock_guard lock(registration_mutex_); + FORY_RETURN_IF_ERROR(check_registration()); using ChronoTimestamp = std::chrono::time_point; if constexpr (std::is_same_v || @@ -1790,7 +1801,8 @@ template Result TypeResolver::register_any_type() { template Result TypeResolver::register_by_id(uint32_t type_id) { - check_registration_thread(); + std::lock_guard lock(registration_mutex_); + FORY_RETURN_IF_ERROR(check_registration()); if (type_id == kInvalidUserTypeId) { return Unexpected(Error::invalid( "type_id must be in range [0, 0xfffffffe] for register_by_id")); @@ -1845,7 +1857,8 @@ template Result TypeResolver::register_by_name(const std::string &ns, const std::string &type_name) { - check_registration_thread(); + std::lock_guard lock(registration_mutex_); + FORY_RETURN_IF_ERROR(check_registration()); if (type_name.empty()) { return Unexpected( Error::invalid("type_name must be non-empty for register_by_name")); @@ -1897,7 +1910,8 @@ TypeResolver::register_by_name(const std::string &ns, template Result TypeResolver::register_ext_type_by_id(uint32_t type_id) { - check_registration_thread(); + std::lock_guard lock(registration_mutex_); + FORY_RETURN_IF_ERROR(check_registration()); if (type_id == kInvalidUserTypeId) { return Unexpected(Error::invalid("type_id must be in range [0, 0xfffffffe] " "for register_ext_type_by_id")); @@ -1921,7 +1935,8 @@ template Result TypeResolver::register_ext_type_by_name(const std::string &ns, const std::string &type_name) { - check_registration_thread(); + std::lock_guard lock(registration_mutex_); + FORY_RETURN_IF_ERROR(check_registration()); if (type_name.empty()) { return Unexpected(Error::invalid( "type_name must be non-empty for register_ext_type_by_name")); @@ -1945,7 +1960,8 @@ TypeResolver::register_ext_type_by_name(const std::string &ns, template Result TypeResolver::register_union_by_id(uint32_t type_id) { - check_registration_thread(); + std::lock_guard lock(registration_mutex_); + FORY_RETURN_IF_ERROR(check_registration()); if (type_id == kInvalidUserTypeId) { return Unexpected(Error::invalid( "type_id must be in range [0, 0xfffffffe] for register_union_by_id")); @@ -1968,7 +1984,8 @@ template Result TypeResolver::register_union_by_name(const std::string &ns, const std::string &type_name) { - check_registration_thread(); + std::lock_guard lock(registration_mutex_); + FORY_RETURN_IF_ERROR(check_registration()); if (type_name.empty()) { return Unexpected(Error::invalid( "type_name must be non-empty for register_union_by_name")); diff --git a/docs/object-serialization/cpp/type-registration.md b/docs/object-serialization/cpp/type-registration.md index 62f525ef94..13169fc4bf 100644 --- a/docs/object-serialization/cpp/type-registration.md +++ b/docs/object-serialization/cpp/type-registration.md @@ -29,6 +29,12 @@ Apache Fory™ requires explicit type registration for struct types. This design - **Type Safety**: Detects type mismatches at deserialization time - **Polymorphic Serialization**: Enables serialization of polymorphic objects via smart pointers +## Registration Lifecycle + +Complete all registrations before the first root serialization or deserialization call. The first +root attempt permanently freezes that Fory instance's registry, even when the operation fails. +Create a new Fory instance when a different registry is required. + ## Registering Structs Use `register_struct(type_id)` to register a struct type: From 0efb35522dfeb77f5ca221147931a576cb30b5be Mon Sep 17 00:00:00 2001 From: chaokunyang Date: Fri, 28 Aug 2026 01:23:13 +0800 Subject: [PATCH 002/168] fix(csharp): freeze registries and clear failed roots --- .agents/languages/csharp.md | 5 + csharp/src/Fory/Fory.cs | 84 ++++- csharp/src/Fory/ReadContext.cs | 11 + csharp/src/Fory/ThreadSafeFory.cs | 91 +++++- .../tests/Fory.Tests/ClassInheritanceTests.cs | 4 +- .../ExternalTypeSerializationTests.cs | 2 +- csharp/tests/Fory.Tests/ForyRuntimeTests.cs | 10 +- .../tests/Fory.Tests/RuntimeEdgeCaseTests.cs | 295 +++++++++++++++++- .../csharp/thread-safety.md | 7 +- .../csharp/type-registration.md | 16 +- 10 files changed, 485 insertions(+), 40 deletions(-) diff --git a/.agents/languages/csharp.md b/.agents/languages/csharp.md index d8c856b59a..8fdbac5105 100644 --- a/.agents/languages/csharp.md +++ b/.agents/languages/csharp.md @@ -9,6 +9,11 @@ Load this file when changing `csharp/` or C# xlang behavior. - C# code must build without compiler or analyzer warnings. Treat warnings as blockers in project, test, and generated code. - Fory C# requires .NET SDK `8.0+` and C# `12+`. - Use `dotnet format` to keep C# code style consistent. +- A direct C# `Fory` owns permanent registration freeze at first root entry, including a failed + root. `ThreadSafeFory` linearizes root entry and registration with its registration lock and + frozen state, validates registration on its staging `Fory`, and appends only successful actions + to the replay log. New per-thread runtimes replay that log; do not mutate existing runtimes or + introduce another freeze owner. - Generated C# gRPC service companions are compiler-owned files that depend on application-provided gRPC packages, not `csharp/src/Fory`. Keep gRPC package references out of the Fory runtime package. - C# generated schema modules are source-file owners. Service companions must use that module's `ThreadSafeFory` and must not introduce namespace-owned aliases or duplicate serializer registration paths. - C# external-type serialization is target-keyed. A local diff --git a/csharp/src/Fory/Fory.cs b/csharp/src/Fory/Fory.cs index cf0684b2e9..e6dd5c5d17 100644 --- a/csharp/src/Fory/Fory.cs +++ b/csharp/src/Fory/Fory.cs @@ -30,6 +30,7 @@ public sealed class Fory private readonly TypeResolver _typeResolver; private WriteContext _writeContext; private ReadContext _readContext; + private bool _registryFrozen; internal Fory(Config config) { @@ -67,8 +68,11 @@ public static ForyBuilder Builder() /// Type to register. /// Numeric type identifier used on the wire. /// The same runtime instance. + /// Registration closes permanently when the first serialization or deserialization attempt begins, including an attempt that fails. + /// Registration has closed because a root operation was attempted. public Fory Register(uint typeId) { + EnsureRegistrationOpen(); _typeResolver.Register(typeof(T), typeId); return this; } @@ -79,8 +83,11 @@ public Fory Register(uint typeId) /// Type to register. /// Name used on the wire. A dotted name is split at the last dot. /// The same runtime instance. + /// Registration closes permanently when the first serialization or deserialization attempt begins, including an attempt that fails. + /// Registration has closed because a root operation was attempted. public Fory Register(string name) { + EnsureRegistrationOpen(); (string namespaceName, string typeName) = TypeResolver.SplitTypeName(name); _typeResolver.Register(typeof(T), namespaceName, typeName); return this; @@ -93,8 +100,11 @@ public Fory Register(string name) /// Namespace used on the wire. /// Type name used on the wire. /// The same runtime instance. + /// Registration closes permanently when the first serialization or deserialization attempt begins, including an attempt that fails. + /// Registration has closed because a root operation was attempted. public Fory Register(string typeNamespace, string typeName) { + EnsureRegistrationOpen(); _typeResolver.Register(typeof(T), typeNamespace, typeName); return this; } @@ -106,9 +116,12 @@ public Fory Register(string typeNamespace, string typeName) /// Serializer implementation used for . /// Numeric type identifier used on the wire. /// The same runtime instance. + /// Registration closes permanently when the first serialization or deserialization attempt begins, including an attempt that fails. + /// Registration has closed because a root operation was attempted. public Fory Register(uint typeId) where TSerializer : Serializer, new() { + EnsureRegistrationOpen(); TypeInfo typeInfo = _typeResolver.RegisterSerializer(); _typeResolver.Register(typeof(T), typeId, typeInfo); return this; @@ -121,9 +134,12 @@ public Fory Register(uint typeId) /// Serializer implementation used for . /// Name used on the wire. A dotted name is split at the last dot. /// The same runtime instance. + /// Registration closes permanently when the first serialization or deserialization attempt begins, including an attempt that fails. + /// Registration has closed because a root operation was attempted. public Fory Register(string name) where TSerializer : Serializer, new() { + EnsureRegistrationOpen(); (string namespaceName, string typeName) = TypeResolver.SplitTypeName(name); TypeInfo typeInfo = _typeResolver.RegisterSerializer(); _typeResolver.Register(typeof(T), namespaceName, typeName, typeInfo); @@ -138,9 +154,12 @@ public Fory Register(string name) /// Namespace used on the wire. /// Type name used on the wire. /// The same runtime instance. + /// Registration closes permanently when the first serialization or deserialization attempt begins, including an attempt that fails. + /// Registration has closed because a root operation was attempted. public Fory Register(string typeNamespace, string typeName) where TSerializer : Serializer, new() { + EnsureRegistrationOpen(); TypeResolver.ValidateSplitTypeName(typeNamespace, typeName); TypeInfo typeInfo = _typeResolver.RegisterSerializer(); _typeResolver.Register(typeof(T), typeNamespace, typeName, typeInfo); @@ -155,14 +174,25 @@ public Fory Register(string typeNamespace, string typeName) /// Serialized bytes. public byte[] Serialize(in T value) { + FreezeRegistry(); ByteWriter writer = _writeContext.Writer; writer.Reset(); Serializer serializer = _typeResolver.GetSerializer(); WriteHead(writer); _writeContext.ResetFor(writer); - RefMode refMode = Config.TrackRef ? RefMode.Tracking : RefMode.NullOnly; - serializer.Write(_writeContext, value, refMode, true, false); - _writeContext.RefWriter.Reset(); + try + { + RefMode refMode = Config.TrackRef ? RefMode.Tracking : RefMode.NullOnly; + serializer.Write(_writeContext, value, refMode, true, false); + _writeContext.RefWriter.Reset(); + } + catch + { + // A custom serializer can fail after reference and metadata publication. The root + // facade must release that operation-local state before the runtime is reused. + _writeContext.Reset(); + throw; + } return writer.ToArray(); } @@ -188,11 +218,13 @@ public void Serialize(IBufferWriter output, in T value) /// Thrown when trailing bytes remain after decoding. public T Deserialize(ReadOnlySpan payload) { + FreezeRegistry(); ByteReader reader = _readContext.Reader; reader.Reset(payload); - T value = DeserializeFromReader(reader); + T value = DeserializeFromReaderCore(reader); if (reader.Remaining != 0) { + _readContext.ResetAfterFailure(); ThrowUnexpectedTrailingBytes(); } @@ -208,11 +240,13 @@ public T Deserialize(ReadOnlySpan payload) /// Thrown when trailing bytes remain after decoding. public T Deserialize(byte[] payload) { + FreezeRegistry(); ByteReader reader = _readContext.Reader; reader.Reset(payload); - T value = DeserializeFromReader(reader); + T value = DeserializeFromReaderCore(reader); if (reader.Remaining != 0) { + _readContext.ResetAfterFailure(); ThrowUnexpectedTrailingBytes(); } @@ -257,6 +291,13 @@ private static void ThrowInvalidRootHeader(byte bitmap) => [MethodImpl(MethodImplOptions.AggressiveInlining)] internal T DeserializeFromReader(ByteReader reader) + { + FreezeRegistry(); + return DeserializeFromReaderCore(reader); + } + + [MethodImpl(MethodImplOptions.AggressiveInlining)] + private T DeserializeFromReaderCore(ByteReader reader) { ReadContext readContext = _readContext; readContext.ResetFor(reader); @@ -283,9 +324,40 @@ internal T DeserializeFromReader(ByteReader reader) { // Failed roots can leave partially published refs, metadata refs, or graph-budget state. // Keep the success path minimal, but fully reset failed reads before this context is reused. - readContext.Reset(); + readContext.ResetAfterFailure(); throw; } } + [MethodImpl(MethodImplOptions.AggressiveInlining)] + private void FreezeRegistry() + { + // Generated descriptors and serializers become operation-visible at the first root, so + // failed roots freeze registration just as successful roots do. + if (!_registryFrozen) + { + FreezeRegistrySlow(); + } + } + + [MethodImpl(MethodImplOptions.NoInlining)] + private void FreezeRegistrySlow() + { + _registryFrozen = true; + } + + [MethodImpl(MethodImplOptions.AggressiveInlining)] + private void EnsureRegistrationOpen() + { + if (_registryFrozen) + { + ThrowRegistryFrozen(); + } + } + + [MethodImpl(MethodImplOptions.NoInlining)] + private static void ThrowRegistryFrozen() => + throw new InvalidOperationException( + "types and serializers must be registered before the first serialization or deserialization operation"); + } diff --git a/csharp/src/Fory/ReadContext.cs b/csharp/src/Fory/ReadContext.cs index 34ea7ff2f4..dd83200539 100644 --- a/csharp/src/Fory/ReadContext.cs +++ b/csharp/src/Fory/ReadContext.cs @@ -564,4 +564,15 @@ internal void Reset() _readMetaStrings.Clear(); _remainingUnbackedContainerItems = 0; } + + internal void ResetAfterFailure() + { + Reset(); + // Remote metadata may be accepted before the owning value body fails. Failed roots must + // not accumulate that decoded state across operations, while successful roots keep using + // this map as the sole checked-cache owner. + _typeMetasByHash.Clear(); + _remoteSchemaVersionsByType.Clear(); + _totalAcceptedSchemaVersions = 0; + } } diff --git a/csharp/src/Fory/ThreadSafeFory.cs b/csharp/src/Fory/ThreadSafeFory.cs index 9639f9161d..6900acff09 100644 --- a/csharp/src/Fory/ThreadSafeFory.cs +++ b/csharp/src/Fory/ThreadSafeFory.cs @@ -16,6 +16,7 @@ // under the License. using System.Buffers; +using System.Runtime.CompilerServices; namespace Apache.Fory; @@ -28,6 +29,8 @@ public sealed class ThreadSafeFory : IDisposable private readonly object _registrationLock = new(); private readonly List> _registrations = []; private readonly ThreadLocal _threadLocalFory; + private Fory? _registrationFory; + private int _registryFrozen; private bool _disposed; internal ThreadSafeFory(Config config) @@ -42,11 +45,13 @@ internal ThreadSafeFory(Config config) public Config Config => _config; /// - /// Registers a user type by numeric type identifier for all current and future thread-local runtimes. + /// Registers a user type by numeric type identifier. /// /// Type to register. /// Numeric type identifier used on the wire. /// The same runtime instance. + /// Registration closes permanently when the first serialization or deserialization attempt begins, including an attempt that fails. + /// Registration has closed because a root operation was attempted. public ThreadSafeFory Register(uint typeId) { ApplyRegistration(fory => fory.Register(typeId)); @@ -54,39 +59,43 @@ public ThreadSafeFory Register(uint typeId) } /// - /// Registers a user type by name for all current and future thread-local runtimes. + /// Registers a user type by name. /// /// Type to register. /// Name used on the wire. A dotted name is split at the last dot. /// The same runtime instance. + /// Registration closes permanently when the first serialization or deserialization attempt begins, including an attempt that fails. + /// Registration has closed because a root operation was attempted. public ThreadSafeFory Register(string name) { - _ = TypeResolver.SplitTypeName(name); ApplyRegistration(fory => fory.Register(name)); return this; } /// - /// Registers a user type by namespace and name for all current and future thread-local runtimes. + /// Registers a user type by namespace and name. /// /// Type to register. /// Namespace used on the wire. /// Type name used on the wire. /// The same runtime instance. + /// Registration closes permanently when the first serialization or deserialization attempt begins, including an attempt that fails. + /// Registration has closed because a root operation was attempted. public ThreadSafeFory Register(string typeNamespace, string typeName) { - TypeResolver.ValidateSplitTypeName(typeNamespace, typeName); ApplyRegistration(fory => fory.Register(typeNamespace, typeName)); return this; } /// - /// Registers a user type by numeric type identifier with a custom serializer for all thread-local runtimes. + /// Registers a user type by numeric type identifier with a custom serializer. /// /// Type to register. /// Serializer implementation used for . /// Numeric type identifier used on the wire. /// The same runtime instance. + /// Registration closes permanently when the first serialization or deserialization attempt begins, including an attempt that fails. + /// Registration has closed because a root operation was attempted. public ThreadSafeFory Register(uint typeId) where TSerializer : Serializer, new() { @@ -95,32 +104,34 @@ public ThreadSafeFory Register(uint typeId) } /// - /// Registers a user type by name with a custom serializer for all thread-local runtimes. + /// Registers a user type by name with a custom serializer. /// /// Type to register. /// Serializer implementation used for . /// Name used on the wire. A dotted name is split at the last dot. /// The same runtime instance. + /// Registration closes permanently when the first serialization or deserialization attempt begins, including an attempt that fails. + /// Registration has closed because a root operation was attempted. public ThreadSafeFory Register(string name) where TSerializer : Serializer, new() { - _ = TypeResolver.SplitTypeName(name); ApplyRegistration(fory => fory.Register(name)); return this; } /// - /// Registers a user type by namespace and name with a custom serializer for all thread-local runtimes. + /// Registers a user type by namespace and name with a custom serializer. /// /// Type to register. /// Serializer implementation used for . /// Namespace used on the wire. /// Type name used on the wire. /// The same runtime instance. + /// Registration closes permanently when the first serialization or deserialization attempt begins, including an attempt that fails. + /// Registration has closed because a root operation was attempted. public ThreadSafeFory Register(string typeNamespace, string typeName) where TSerializer : Serializer, new() { - TypeResolver.ValidateSplitTypeName(typeNamespace, typeName); ApplyRegistration(fory => fory.Register(typeNamespace, typeName)); return this; } @@ -133,6 +144,7 @@ public ThreadSafeFory Register(string typeNamespace, string type /// Serialized bytes. public byte[] Serialize(in T value) { + BeginRoot(); return Current.Serialize(in value); } @@ -144,6 +156,7 @@ public byte[] Serialize(in T value) /// Value to serialize. public void Serialize(IBufferWriter output, in T value) { + BeginRoot(); Current.Serialize(output, in value); } @@ -155,6 +168,7 @@ public void Serialize(IBufferWriter output, in T value) /// Deserialized value. public T Deserialize(ReadOnlySpan payload) { + BeginRoot(); return Current.Deserialize(payload); } @@ -172,6 +186,7 @@ public void Dispose() _threadLocalFory.Dispose(); _registrations.Clear(); + _registrationFory = null; _disposed = true; } } @@ -209,14 +224,66 @@ private void ApplyRegistration(Action registration) lock (_registrationLock) { ThrowIfDisposed(); + if (_registryFrozen != 0) + { + ThrowRegistryFrozen(); + } + + try + { + registration(_registrationFory ??= new Fory(_config)); + } + catch + { + _registrationFory = _registrations.Count == 0 ? null : RebuildRegistrationFory(); + throw; + } + _registrations.Add(registration); - foreach (Fory fory in _threadLocalFory.Values) + } + } + + [MethodImpl(MethodImplOptions.AggressiveInlining)] + private void BeginRoot() + { + // Freeze before Current can create a per-thread runtime so roots and registrations + // linearize against one boundary and every runtime replays the same immutable log. + if (Volatile.Read(ref _registryFrozen) == 0) + { + FreezeRegistry(); + } + } + + [MethodImpl(MethodImplOptions.NoInlining)] + private void FreezeRegistry() + { + lock (_registrationLock) + { + ThrowIfDisposed(); + if (_registryFrozen == 0) { - registration(fory); + _registrationFory = null; + Volatile.Write(ref _registryFrozen, 1); } } } + private Fory RebuildRegistrationFory() + { + Fory fory = new(_config); + foreach (Action registration in _registrations) + { + registration(fory); + } + + return fory; + } + + [MethodImpl(MethodImplOptions.NoInlining)] + private static void ThrowRegistryFrozen() => + throw new InvalidOperationException( + "types and serializers must be registered before the first serialization or deserialization operation"); + private void ThrowIfDisposed() { if (_disposed) diff --git a/csharp/tests/Fory.Tests/ClassInheritanceTests.cs b/csharp/tests/Fory.Tests/ClassInheritanceTests.cs index 32153a21db..47bd44ff51 100644 --- a/csharp/tests/Fory.Tests/ClassInheritanceTests.cs +++ b/csharp/tests/Fory.Tests/ClassInheritanceTests.cs @@ -180,7 +180,8 @@ public void FlattenedHierarchyRoundTrips(bool compatible, bool trackRef) .Compatible(compatible) .TrackRef(trackRef) .Build() - .Register(6401); + .Register(6401) + .Register(6408); InheritedLeaf value = new() { PublicValue = 13, @@ -205,7 +206,6 @@ public void FlattenedHierarchyRoundTrips(bool compatible, bool trackRef) Assert.Equal(23, decoded.HiddenValue); Assert.Equal(trackRef, ReferenceEquals(decoded, decoded.Self)); - fory.Register(6408); InheritedMiddle middle = new() { PublicValue = 29, diff --git a/csharp/tests/Fory.Tests/ExternalTypeSerializationTests.cs b/csharp/tests/Fory.Tests/ExternalTypeSerializationTests.cs index 3df1b9cb4b..2839269a81 100644 --- a/csharp/tests/Fory.Tests/ExternalTypeSerializationTests.cs +++ b/csharp/tests/Fory.Tests/ExternalTypeSerializationTests.cs @@ -411,7 +411,7 @@ public void CustomSerializerReplacesGenerated() Assert.NotEqual(generatedBytes, customBytes); Assert.Equal(value.Count, decoded.Count); Assert.Equal(value.Name, decoded.Name); - Assert.Throws( + Assert.Throws( () => generated.Register(6107)); } diff --git a/csharp/tests/Fory.Tests/ForyRuntimeTests.cs b/csharp/tests/Fory.Tests/ForyRuntimeTests.cs index 36e6ccfbce..630fab26c7 100644 --- a/csharp/tests/Fory.Tests/ForyRuntimeTests.cs +++ b/csharp/tests/Fory.Tests/ForyRuntimeTests.cs @@ -834,18 +834,12 @@ public void ThreadSafeForyPropagatesRegistrationsToThreads() } [Fact] - public void ThreadSafeForyRegistrationAppliesToInitializedThreadLocalInstance() + public void ThreadSafeForyRejectsLateRegister() { using ThreadSafeFory fory = ForyRuntime.Builder().TrackRef(true).BuildThreadSafe(); _ = fory.Serialize(1); - fory.Register(952); - Node source = new() { Value = 7 }; - source.Next = source; - Node decoded = fory.Deserialize(fory.Serialize(source)); - Assert.Equal(7, decoded.Value); - Assert.NotNull(decoded.Next); - Assert.Same(decoded, decoded.Next); + Assert.Throws(() => fory.Register(952)); } [Fact] diff --git a/csharp/tests/Fory.Tests/RuntimeEdgeCaseTests.cs b/csharp/tests/Fory.Tests/RuntimeEdgeCaseTests.cs index b817b99036..e2ba161195 100644 --- a/csharp/tests/Fory.Tests/RuntimeEdgeCaseTests.cs +++ b/csharp/tests/Fory.Tests/RuntimeEdgeCaseTests.cs @@ -15,6 +15,7 @@ // specific language governing permissions and limitations // under the License. +using System.Buffers; using System.Numerics; using Apache.Fory; using ForyRuntime = Apache.Fory.Fory; @@ -77,6 +78,58 @@ public override CustomPayload ReadData(ReadContext context) } } +[ForyStruct] +public sealed class FrozenPayload +{ + public int Value { get; set; } +} + +public sealed class FrozenPayloadSerializer : Serializer +{ + public static int Constructions; + + public FrozenPayloadSerializer() + { + Interlocked.Increment(ref Constructions); + } + + public override FrozenPayload DefaultValue => null!; + + public override void WriteData(WriteContext context, in FrozenPayload value, bool hasGenerics) + { + _ = hasGenerics; + context.Writer.WriteVarInt32(value.Value); + } + + public override FrozenPayload ReadData(ReadContext context) + { + return new FrozenPayload { Value = context.Reader.ReadVarInt32() }; + } +} + +[ForyStruct] +public sealed class FailingWritePayload +{ + public int Value { get; set; } +} + +public sealed class FailingWriteSerializer : Serializer +{ + public override void WriteData(WriteContext context, in FailingWritePayload value, bool hasGenerics) + { + _ = context; + _ = value; + _ = hasGenerics; + throw new InvalidOperationException("write failure"); + } + + public override FailingWritePayload ReadData(ReadContext context) + { + _ = context; + return new(); + } +} + public sealed class RuntimeEdgeCaseTests { [Fact] @@ -739,10 +792,9 @@ public void SplitTypeNameRejectsDots() } [Fact] - public void ThreadSafeDottedSerializerNameRoundTrip() + public void ThreadSafeDottedNameRoundTrip() { using ThreadSafeFory fory = ForyRuntime.Builder().BuildThreadSafe(); - _ = fory.Serialize(1); fory.Register("test.custom_payload"); CustomPayload decoded = fory.Deserialize( @@ -753,14 +805,245 @@ public void ThreadSafeDottedSerializerNameRoundTrip() } [Fact] - public void DeserializeRejectsTrailingBytes() + public void RegistryFreezesAfterSuccessfulRoot() { ForyRuntime fory = ForyRuntime.Builder().Build(); - byte[] payload = fory.Serialize(123); + Assert.Equal(1, fory.Deserialize(fory.Serialize(1))); + + Assert.Throws(() => fory.Register(710)); + } + + [Fact] + public void FrozenRegistryRejectsBeforeMutation() + { + ForyRuntime fory = ForyRuntime.Builder().Build(); + _ = fory.Serialize(1); + FrozenPayloadSerializer.Constructions = 0; + + Action[] registrations = + [ + () => fory.Register(711), + () => fory.Register(string.Empty), + () => fory.Register("test", "bad.name"), + () => fory.Register(712), + () => fory.Register(string.Empty), + () => fory.Register("test", "bad.name"), + ]; + + foreach (Action registration in registrations) + { + Assert.Throws(registration); + } + + Assert.Equal(0, FrozenPayloadSerializer.Constructions); + } + + [Fact] + public void FailedRootFreezesRegistry() + { + ForyRuntime fory = ForyRuntime.Builder().Build(); + + Assert.ThrowsAny(() => fory.Deserialize(Array.Empty())); + Assert.Throws(() => fory.Register(713)); + } + + [Fact] + public void FailedWriteClearsRootState() + { + ForyRuntime fory = ForyRuntime.Builder().TrackRef(true).Build(); + fory.Register(718); + FailingWritePayload value = new() { Value = 1 }; + + Assert.Throws(() => fory.Serialize(value)); + Assert.Throws(() => fory.Register(719)); + + ByteWriter probe = new(); + Assert.False(WriteContextFor(fory).RefWriter.TryWriteRef(probe, value)); + Assert.Equal(7, fory.Deserialize(fory.Serialize(7))); + } + + [Fact] + public void FailedReaderRootFreezesRegistry() + { + ForyRuntime fory = ForyRuntime.Builder().Build(); + + Assert.ThrowsAny( + () => fory.DeserializeFromReader(new ByteReader(Array.Empty()))); + Assert.Throws(() => fory.Register(714)); + } + + [Fact] + public void ThreadSafeFailedRootFreezesRegistry() + { + using ThreadSafeFory fory = ForyRuntime.Builder().BuildThreadSafe(); + + Assert.ThrowsAny(() => fory.Deserialize(Array.Empty())); + Assert.Throws(() => fory.Register(715)); + FrozenPayloadSerializer.Constructions = 0; + Assert.Throws( + () => fory.Register(string.Empty)); + Assert.Equal(0, FrozenPayloadSerializer.Constructions); + } + + [Fact] + public void ThreadSafeOutputFreezesRegistry() + { + using ThreadSafeFory fory = ForyRuntime.Builder().BuildThreadSafe(); + ArrayBufferWriter output = new(); + + fory.Serialize(output, 1); + + Assert.Throws(() => fory.Register(720)); + } + + [Fact] + public async Task ThreadSafeRootAndRegistrationRace() + { + using ThreadSafeFory fory = ForyRuntime.Builder().BuildThreadSafe(); + using Barrier start = new(2); + Exception? registrationError = null; + + Task root = Task.Run(() => + { + start.SignalAndWait(); + _ = fory.Serialize(1); + }); + Task registration = Task.Run(() => + { + start.SignalAndWait(); + try + { + fory.Register(716); + } + catch (Exception error) + { + registrationError = error; + } + }); + + await Task.WhenAll(root, registration); + Assert.True(registrationError is null or InvalidOperationException); + Assert.Throws(() => fory.Register(717)); + + if (registrationError is null) + { + FrozenPayload value = new() { Value = 42 }; + Assert.Equal(value.Value, fory.Deserialize(fory.Serialize(value)).Value); + } + } + + [Theory] + [InlineData(false)] + [InlineData(true)] + public void TrailingBytesResetReadState(bool useSpan) + { + ForyRuntime writer = NewCompatibleTimeFory(); + byte[] payload = writer.Serialize(new TimeEnvelope { Dates = [new DateOnly(2024, 1, 2)] }); + ForyRuntime probe = NewCompatibleTimeFory(); + _ = probe.DeserializeFromReader(new ByteReader(payload)); + Assert.NotNull(ReadContextFor(probe).GetTypeMetaRef(0)); + + ForyRuntime reader = NewCompatibleTimeFory(); byte[] invalidPayload = [.. payload, 0x7F]; - InvalidDataException exception = Assert.Throws(() => fory.Deserialize(invalidPayload)); - Assert.Contains("unexpected trailing bytes", exception.Message, StringComparison.Ordinal); + _ = useSpan + ? Assert.ThrowsAny(() => DeserializeSpan(reader, invalidPayload)) + : Assert.ThrowsAny(() => reader.Deserialize(invalidPayload)); + ReadContext context = ReadContextFor(reader); + Assert.Null(context.GetTypeMetaRef(0)); + Assert.Null(context.GetReadMetaString(0)); + } + + [Fact] + public void RootHeaderFailureClearsMetaCache() + { + ForyRuntime fory = ForyRuntime.Builder() + .Compatible(false) + .MaxSchemaVersionsPerType(1) + .Build(); + ReadContext context = ReadContextFor(fory); + TypeMeta first = ReadAndStoreTypeMeta(context, RemoteStructTypeMeta(901, "first")); + ulong firstHash = EncodedTypeMetaHash(first); + + Assert.ThrowsAny(() => fory.Deserialize([0])); + + Assert.False(context.TryGetTypeMetaByHash(firstHash, out _)); + TypeMeta second = ReadAndStoreTypeMeta(context, RemoteStructTypeMeta(901, "second")); + Assert.True(context.TryGetTypeMetaByHash(EncodedTypeMetaHash(second), out _)); + } + + [Theory] + [InlineData(false)] + [InlineData(true)] + public void TrailingFailureClearsTypeMetaCache(bool useSpan) + { + ForyRuntime fory = ForyRuntime.Builder() + .Compatible(false) + .MaxSchemaVersionsPerType(1) + .Build(); + ReadContext context = ReadContextFor(fory); + TypeMeta first = ReadAndStoreTypeMeta(context, RemoteStructTypeMeta(901, "first")); + ulong firstHash = EncodedTypeMetaHash(first); + byte[] invalidPayload = [.. fory.Serialize(123), 0x7F]; + + if (useSpan) + { + Assert.ThrowsAny(() => DeserializeIntSpan(fory, invalidPayload)); + } + else + { + Assert.ThrowsAny(() => fory.Deserialize(invalidPayload)); + } + + Assert.False(context.TryGetTypeMetaByHash(firstHash, out _)); + TypeMeta second = ReadAndStoreTypeMeta(context, RemoteStructTypeMeta(901, "second")); + Assert.True(context.TryGetTypeMetaByHash(EncodedTypeMetaHash(second), out _)); + } + + private static ForyRuntime NewCompatibleTimeFory() + { + ForyRuntime fory = ForyRuntime.Builder().Compatible(true).Build(); + fory.Register(701); + return fory; + } + + private static void DeserializeSpan(ForyRuntime fory, byte[] payload) + { + _ = fory.Deserialize(payload.AsSpan()); + } + + private static void DeserializeIntSpan(ForyRuntime fory, byte[] payload) + { + _ = fory.Deserialize(payload.AsSpan()); + } + + private static ReadContext ReadContextFor(ForyRuntime fory) + { + System.Reflection.FieldInfo? field = typeof(ForyRuntime).GetField( + "_readContext", + System.Reflection.BindingFlags.Instance | System.Reflection.BindingFlags.NonPublic); + Assert.NotNull(field); + return Assert.IsType(field.GetValue(fory)); + } + + private static WriteContext WriteContextFor(ForyRuntime fory) + { + System.Reflection.FieldInfo? field = typeof(ForyRuntime).GetField( + "_writeContext", + System.Reflection.BindingFlags.Instance | System.Reflection.BindingFlags.NonPublic); + Assert.NotNull(field); + return Assert.IsType(field.GetValue(fory)); + } + + [Fact] + public void DeserializeFromReaderReadsFrames() + { + ForyRuntime fory = ForyRuntime.Builder().Build(); + ByteReader reader = new([.. fory.Serialize(123), .. fory.Serialize(456)]); + + Assert.Equal(123, fory.DeserializeFromReader(reader)); + Assert.Equal(456, fory.DeserializeFromReader(reader)); + Assert.Equal(0, reader.Remaining); } [Fact] diff --git a/docs/object-serialization/csharp/thread-safety.md b/docs/object-serialization/csharp/thread-safety.md index 0681c04219..8a7f105bbd 100644 --- a/docs/object-serialization/csharp/thread-safety.md +++ b/docs/object-serialization/csharp/thread-safety.md @@ -53,9 +53,10 @@ Parallel.For(0, 64, i => ## Registration Behavior -- `ThreadSafeFory.Register(...)` stores registrations centrally. -- Existing per-thread Fory instances are updated. -- New threads receive all previous registrations automatically. +- Register every type before the first serialization or deserialization attempt. +- Starting the first root permanently freezes registration, including when that root fails. +- `ThreadSafeFory` serializes registration against the first root. If the root wins a concurrent + race, registration throws `InvalidOperationException` before changing any runtime. ## Disposal diff --git a/docs/object-serialization/csharp/type-registration.md b/docs/object-serialization/csharp/type-registration.md index 0c6b629310..37118c3fab 100644 --- a/docs/object-serialization/csharp/type-registration.md +++ b/docs/object-serialization/csharp/type-registration.md @@ -67,9 +67,21 @@ Name-based custom serializer registration is also supported: fory.Register("com.example.MyType"); ``` +## Registration Lifecycle + +A `Fory` instance accepts registration only before its first root serialization or deserialization +attempt. Starting that operation permanently freezes the instance's registry, even when the +operation fails. Every later registration throws `InvalidOperationException`. + +To configure additional types, build a new `Fory` instance, complete its registrations, and then +use that new instance for serialization or deserialization. + ## Thread-Safe Registration -`ThreadSafeFory` exposes the same registration APIs. Registrations are propagated to all per-thread Fory instances. +`ThreadSafeFory` exposes the same registration APIs. Register every type before the first +serialization or deserialization attempt. Starting the first root permanently freezes +registration, even when the root fails. A later registration throws `InvalidOperationException` +before changing any per-thread runtime. ```csharp using ThreadSafeFory fory = Fory.Builder().BuildThreadSafe(); @@ -90,7 +102,7 @@ fory.Register(101); Registering a derived class does not make an unannotated base class serializable. - For the split overloads, `typeName` must be non-empty and must not contain dots. -- Register before high-volume serialization workloads to avoid missing type metadata. +- Complete registration before the first root serialization or deserialization attempt. ## Related Topics From cccebac17f7ca815148aba50193e20a27f966243 Mon Sep 17 00:00:00 2001 From: chaokunyang Date: Fri, 28 Aug 2026 01:23:21 +0800 Subject: [PATCH 003/168] fix(go): freeze registries and restore root buffers --- .agents/languages/go.md | 5 + docs/object-serialization/go/configuration.md | 4 +- docs/object-serialization/go/native.md | 17 +- docs/object-serialization/go/security.md | 2 +- docs/object-serialization/go/thread-safety.md | 82 +++---- .../go/type-registration.md | 4 +- go/fory/fory.go | 86 +++++-- go/fory/fory_test.go | 26 +- go/fory/fory_typed_test.go | 11 +- go/fory/registry_freeze_lifecycle_test.go | 230 ++++++++++++++++++ go/fory/stream.go | 10 +- go/fory/threadsafe/fory.go | 117 +++++++-- go/fory/threadsafe/fory_test.go | 5 +- .../registry_freeze_lifecycle_test.go | 140 +++++++++++ go/fory/type_resolver.go | 52 +++- go/fory/writer.go | 1 + 16 files changed, 674 insertions(+), 118 deletions(-) create mode 100644 go/fory/registry_freeze_lifecycle_test.go create mode 100644 go/fory/threadsafe/registry_freeze_lifecycle_test.go diff --git a/.agents/languages/go.md b/.agents/languages/go.md index 5192fc3e7b..30ef97d383 100644 --- a/.agents/languages/go.md +++ b/.agents/languages/go.md @@ -7,6 +7,11 @@ Load this file when changing `go/fory/` or Go xlang behavior. - Run Go commands from within `go/fory/`. - Changes under `go/` must pass formatting and tests. - The Go implementation focuses on fast serializers. +- A Go `Fory` instance owns permanent registry freeze at the start of its first root, including a + failed root. Exported resolver registration entries recheck that facade-owned state before + mutation. `threadsafe.Fory` owns the cross-pool boundary with one frozen state, one prepared + validation instance, and one log of successful named-struct registrations. Failed registrations + are not logged; pool misses after freeze replay the immutable successful log. - Go `ReadContext` intentionally defers codec errors to existing `HasError` or `CheckError` boundaries. After an error, work may continue only while it remains panic- and bounds-safe and cannot cause disproportionate work or allocation, publish state that survives root cleanup, or diff --git a/docs/object-serialization/go/configuration.md b/docs/object-serialization/go/configuration.md index 8f59db06e2..df738a4d8b 100644 --- a/docs/object-serialization/go/configuration.md +++ b/docs/object-serialization/go/configuration.md @@ -390,7 +390,9 @@ f := threadsafe.New( fory.WithXlang(true), fory.WithMaxDepth(30), ) -f.RegisterStruct(Request{}, 1) +if err := f.RegisterStructByName(Request{}, "example.Request"); err != nil { + panic(err) +} // Process requests concurrently for req := range requests { diff --git a/docs/object-serialization/go/native.md b/docs/object-serialization/go/native.md index cf72cb866c..d226807d78 100644 --- a/docs/object-serialization/go/native.md +++ b/docs/object-serialization/go/native.md @@ -80,7 +80,9 @@ import ( ) f := threadsafe.New(fory.WithXlang(false), fory.WithTrackRef(true)) -_ = f.RegisterStruct(Order{}, 100) +if err := f.RegisterStructByName(Order{}, "example.Order"); err != nil { + panic(err) +} ``` ## Schema Evolution @@ -104,14 +106,21 @@ every reader and writer always uses the same Go struct schema. Register structs before serializing them. Prefer explicit numeric IDs for long-lived payloads: ```go -_ = f.RegisterStruct(Order{}, 100) -_ = f.RegisterStruct(LineItem{}, 101) +f := fory.New(fory.WithXlang(false)) +if err := f.RegisterStruct(Order{}, 100); err != nil { + panic(err) +} +if err := f.RegisterStruct(LineItem{}, 101); err != nil { + panic(err) +} ``` Name-based registration is useful when ID coordination is harder: ```go -_ = f.RegisterStructByName(Order{}, "example.Order") +if err := f.RegisterStructByName(Order{}, "example.Order"); err != nil { + panic(err) +} ``` If you register without stable IDs, every writer and reader must make the same registration choices. diff --git a/docs/object-serialization/go/security.md b/docs/object-serialization/go/security.md index 0ad71c5513..3a6af4586e 100644 --- a/docs/object-serialization/go/security.md +++ b/docs/object-serialization/go/security.md @@ -30,7 +30,7 @@ Before deserialization: - Authenticate the sender and protect message integrity at the transport or storage layer. - Enforce request or file size, timeout, and concurrency limits outside Fory. - Register only the application types the endpoint accepts and configure the reader before its - first root operation. + first root operation. The first root attempt freezes registration even if it fails. - Validate the deserialized value against application authorization and domain rules before use. ## Built-in safeguards diff --git a/docs/object-serialization/go/thread-safety.md b/docs/object-serialization/go/thread-safety.md index 89b5367667..3a82823b64 100644 --- a/docs/object-serialization/go/thread-safety.md +++ b/docs/object-serialization/go/thread-safety.md @@ -66,33 +66,6 @@ go func() { }() ``` -### How It Works - -The thread-safe wrapper uses `sync.Pool`: - -1. **Acquire**: Gets a Fory instance from the pool -2. **Use**: Performs serialization/deserialization -3. **Copy**: Copies result data (buffer will be reused) -4. **Release**: Returns instance to pool - -```go -// Simplified implementation -func (f *Fory) Serialize(v any) ([]byte, error) { - fory := f.pool.Get().(*fory.Fory) - defer f.pool.Put(fory) - - data, err := fory.Serialize(v) - if err != nil { - return nil, err - } - - // Copy because underlying buffer will be reused - result := make([]byte, len(data)) - copy(result, data) - return result, nil -} -``` - ### API ```go @@ -114,14 +87,20 @@ err = threadsafe.Unmarshal(data, &target) ## Type Registration -Type registration should be done before concurrent use: +Register every type before the first serialization or deserialization attempt. Starting a root +operation permanently freezes registration on that Fory instance, including when the operation +fails: ```go f := threadsafe.New() // Register types BEFORE concurrent access -f.RegisterStruct(User{}, 1) -f.RegisterStruct(Order{}, 2) +if err := f.RegisterStructByName(User{}, "example.User"); err != nil { + panic(err) +} +if err := f.RegisterStructByName(Order{}, "example.Order"); err != nil { + panic(err) +} // Now safe to use concurrently go func() { @@ -131,15 +110,19 @@ go func() { ### Thread-Safe Registration -The thread-safe wrapper handles registration safely: +The thread-safe wrapper exposes named struct registration and serializes it against the first root +operation: ```go -// Safe: Registration is synchronized f := threadsafe.New() -f.RegisterStruct(User{}, 1) // Thread-safe +if err := f.RegisterStructByName(User{}, "example.User"); err != nil { + panic(err) +} ``` -However, for best performance, register all types at startup before concurrent use. +If registration races with the first root, one operation wins the boundary. When the root wins, +the registration call returns `fory.ErrRegistryFrozen` without changing the registry. Register all +types during startup so the application does not depend on race ordering. ## Zero-Copy Considerations @@ -185,7 +168,9 @@ This is safer but has allocation overhead. ```go func BenchmarkNonThreadSafe(b *testing.B) { f := fory.New(fory.WithXlang(true)) - f.RegisterStruct(User{}, 1) + if err := f.RegisterStruct(User{}, 1); err != nil { + b.Fatal(err) + } user := &User{ID: 1, Name: "Alice"} for i := 0; i < b.N; i++ { @@ -196,7 +181,9 @@ func BenchmarkNonThreadSafe(b *testing.B) { func BenchmarkThreadSafe(b *testing.B) { f := threadsafe.New() - f.RegisterStruct(User{}, 1) + if err := f.RegisterStructByName(User{}, "example.User"); err != nil { + b.Fatal(err) + } user := &User{ID: 1, Name: "Alice"} for i := 0; i < b.N; i++ { @@ -216,7 +203,9 @@ For maximum performance with known goroutine count: func worker(id int) { // Each worker has its own Fory instance f := fory.New(fory.WithXlang(true)) - f.RegisterStruct(User{}, 1) + if err := f.RegisterStruct(User{}, 1); err != nil { + panic(err) + } for task := range tasks { data, _ := f.Serialize(task) @@ -239,7 +228,9 @@ For dynamic goroutine count or simplicity: var f = threadsafe.New() func init() { - f.RegisterStruct(User{}, 1) + if err := f.RegisterStructByName(User{}, "example.User"); err != nil { + panic(err) + } } func handleRequest(user *User) []byte { @@ -255,7 +246,9 @@ func handleRequest(user *User) []byte { var fory = threadsafe.New() func init() { - fory.RegisterStruct(Response{}, 1) + if err := fory.RegisterStructByName(Response{}, "example.Response"); err != nil { + panic(err) + } } func handler(w http.ResponseWriter, r *http.Request) { @@ -322,16 +315,19 @@ data, _ := f.Serialize(value1) // Already copied ### Registering Types Concurrently ```go -// RISKY: Concurrent registration +// The root may freeze the registry first. go func() { - f.RegisterStruct(TypeA{}, 1) + if err := f.RegisterStructByName(TypeA{}, "example.TypeA"); err != nil { + panic(err) + } }() go func() { - f.Serialize(value) // May not see TypeA + _, _ = f.Serialize(value) }() ``` -**Fix**: Register all types before concurrent use. +If serialization wins, registration returns `fory.ErrRegistryFrozen`. Register all types before +starting concurrent roots. ## Best Practices diff --git a/docs/object-serialization/go/type-registration.md b/docs/object-serialization/go/type-registration.md index 9367affdd3..f794f694e9 100644 --- a/docs/object-serialization/go/type-registration.md +++ b/docs/object-serialization/go/type-registration.md @@ -128,7 +128,9 @@ f2.RegisterStruct(User{}, 1) ## Registration Timing -Register types after creating a Fory instance and before any serialize/deserialize calls: +Register types after creating a Fory instance and before the first serialization or deserialization +attempt. Starting that first root permanently freezes the instance registry, even when the root +fails. Later registration returns `fory.ErrRegistryFrozen` without changing the registry: ```go f := fory.New(fory.WithXlang(true)) diff --git a/go/fory/fory.go b/go/fory/fory.go index 82c2b8c0d5..a2be046313 100644 --- a/go/fory/fory.go +++ b/go/fory/fory.go @@ -33,6 +33,9 @@ import ( // ErrNoSerializer indicates no serializer is registered for a type var ErrNoSerializer = errors.New("fory: no serializer registered for type") +// ErrRegistryFrozen indicates registration was attempted after a root operation started. +var ErrRegistryFrozen = errors.New("fory: types and serializers must be registered before the first root serialization or deserialization operation") + // Public named registration accepts one dotted name; resolver primitives receive // the split wire metadata components because named TypeDefs store them separately. func splitRegisteredName(name string) (string, string, error) { @@ -201,10 +204,12 @@ func WithMaxAverageSchemaVersionsPerType(size int) Option { // Fory is the main serialization instance. // Note: Fory is NOT thread-safe. Use ThreadSafeFory for concurrent use. +// Type and serializer registration must finish before its first root operation. type Fory struct { - config Config - metaContext *MetaContext - compatibleSet bool + config Config + metaContext *MetaContext + compatibleSet bool + registryFrozen bool // Reusable contexts - avoid allocation on each SerializeWithCallback/DeserializeWithCallbackBuffers call writeCtx *WriteContext @@ -294,6 +299,20 @@ func validateUserTypeID(typeID uint32) error { return nil } +//go:noinline +func (f *Fory) checkRegistrationOpen() error { + if f.registryFrozen { + return ErrRegistryFrozen + } + return nil +} + +func (f *Fory) beginRoot() { + if !f.registryFrozen { + f.registryFrozen = true + } +} + // RegisterStruct registers a struct type with a numeric ID for cross-language serialization. // This is compatible with Java's fory.register(Class, int) method. // type_ can be either a reflect.Type or an instance of the type @@ -302,6 +321,9 @@ func validateUserTypeID(typeID uint32) error { // //go:noinline func (f *Fory) RegisterStruct(type_ any, typeID uint32) error { + if err := f.checkRegistrationOpen(); err != nil { + return err + } if err := validateUserTypeID(typeID); err != nil { return err } @@ -335,6 +357,9 @@ func (f *Fory) RegisterStruct(type_ any, typeID uint32) error { // //go:noinline func (f *Fory) RegisterUnion(type_ any, typeID uint32, serializer Serializer) error { + if err := f.checkRegistrationOpen(); err != nil { + return err + } if serializer == nil { return fmt.Errorf("RegisterUnion requires a non-nil serializer") } @@ -363,6 +388,9 @@ func (f *Fory) RegisterUnion(type_ any, typeID uint32, serializer Serializer) er // //go:noinline func (f *Fory) RegisterUnionByName(type_ any, name string, serializer Serializer) error { + if err := f.checkRegistrationOpen(); err != nil { + return err + } if serializer == nil { return fmt.Errorf("RegisterUnionByName requires a non-nil serializer") } @@ -392,6 +420,9 @@ func (f *Fory) RegisterUnionByName(type_ any, name string, serializer Serializer // //go:noinline func (f *Fory) RegisterStructByName(type_ any, name string) error { + if err := f.checkRegistrationOpen(); err != nil { + return err + } var t reflect.Type if rt, ok := type_.(reflect.Type); ok { t = rt @@ -419,6 +450,9 @@ func (f *Fory) RegisterStructByName(type_ any, name string) error { // //go:noinline func (f *Fory) RegisterEnum(type_ any, typeID uint32) error { + if err := f.checkRegistrationOpen(); err != nil { + return err + } if err := validateUserTypeID(typeID); err != nil { return err } @@ -451,6 +485,9 @@ func (f *Fory) RegisterEnum(type_ any, typeID uint32) error { // //go:noinline func (f *Fory) RegisterEnumByName(type_ any, name string) error { + if err := f.checkRegistrationOpen(); err != nil { + return err + } var t reflect.Type if rt, ok := type_.(reflect.Type); ok { t = rt @@ -483,6 +520,9 @@ func (f *Fory) RegisterEnumByName(type_ any, name string) error { // //go:noinline func (f *Fory) RegisterExtension(type_ any, typeID uint32, serializer ExtensionSerializer) error { + if err := f.checkRegistrationOpen(); err != nil { + return err + } if err := validateUserTypeID(typeID); err != nil { return err } @@ -522,6 +562,9 @@ func (f *Fory) RegisterExtension(type_ any, typeID uint32, serializer ExtensionS // //go:noinline func (f *Fory) RegisterExtensionByName(type_ any, name string, serializer ExtensionSerializer) error { + if err := f.checkRegistrationOpen(); err != nil { + return err + } var t reflect.Type if rt, ok := type_.(reflect.Type); ok { t = rt @@ -538,7 +581,7 @@ func (f *Fory) RegisterExtensionByName(type_ any, name string, serializer Extens return f.typeResolver.registerExtensionByName(t, namespace, typeName, serializer) } -// Reset clears internal state for reuse +// Reset clears root operation state for reuse. It does not reopen registration. func (f *Fory) Reset() { f.writeCtx.Reset() f.readCtx.Reset() @@ -563,6 +606,7 @@ func (f *Fory) Reset() { // // For thread-safe usage, use threadsafe.Fory which copies the data internally. func (f *Fory) Serialize(value any) ([]byte, error) { + f.beginRoot() defer f.resetWriteState() if !validateRootDecimal(f.writeCtx.Err(), value) { return nil, f.writeCtx.TakeError() @@ -598,6 +642,7 @@ func (f *Fory) rootRefMode() RefMode { // Deserialize deserializes data directly into the provided target value. // The target must be a pointer to the value to deserialize into. func (f *Fory) Deserialize(data []byte, v any) error { + f.beginRoot() defer f.resetReadState() f.readCtx.SetData(data) target := reflect.ValueOf(v).Elem() @@ -637,13 +682,19 @@ func (f *Fory) resetWriteState() { // This is useful when you need to write multiple serialized values to the same buffer. // Returns error if serialization fails. func (f *Fory) SerializeTo(buf *ByteBuffer, value any) error { - defer f.resetWriteState() + f.beginRoot() + origBuffer := f.writeCtx.buffer + defer func() { + // Restore the owned buffer before reset so a serializer panic cannot reset or retain the + // caller-owned buffer. + f.writeCtx.buffer = origBuffer + f.resetWriteState() + }() if !validateRootDecimal(f.writeCtx.Err(), value) { return f.writeCtx.TakeError() } // Temporarily swap buffer - origBuffer := f.writeCtx.buffer f.writeCtx.buffer = buf // Write protocol header @@ -666,10 +717,8 @@ func (f *Fory) SerializeTo(buf *ByteBuffer, value any) error { typeInfo.Serializer.WriteData(f.writeCtx, elemValue) } if f.writeCtx.HasError() { - f.writeCtx.buffer = origBuffer return f.writeCtx.TakeError() } - f.writeCtx.buffer = origBuffer return nil } } @@ -677,12 +726,9 @@ func (f *Fory) SerializeTo(buf *ByteBuffer, value any) error { // Standard path - TypeMeta is written inline using streaming protocol f.writeCtx.WriteValue(rv, f.rootRefMode(), true) if f.writeCtx.HasError() { - f.writeCtx.buffer = origBuffer return f.writeCtx.TakeError() } - // Restore original buffer - f.writeCtx.buffer = origBuffer return nil } @@ -690,32 +736,32 @@ func (f *Fory) SerializeTo(buf *ByteBuffer, value any) error { // The buffer's reader index is advanced as data is read. // This is useful when reading multiple serialized values from the same buffer. func (f *Fory) DeserializeFrom(buf *ByteBuffer, v any) error { + f.beginRoot() // Reset contexts for each independent serialized object - defer f.resetReadState() - // Temporarily swap buffer origBuffer := f.readCtx.buffer f.readCtx.buffer = buf + defer func() { + // Restore the owned buffer before root cleanup so an escaping panic cannot + // leave a caller-owned buffer installed for the next operation. + f.readCtx.buffer = origBuffer + f.resetReadState() + }() target := reflect.ValueOf(v).Elem() f.readCtx.remainingGraphMemoryBytes = f.config.MaxGraphMemoryBytes f.readCtx.remainingUnbackedContainerItems = f.config.MaxUnbackedContainerItems readHeader(f.readCtx) if f.readCtx.HasError() { - f.readCtx.buffer = origBuffer return f.readCtx.TakeError() } // Deserialize the value - TypeMeta is read inline using streaming protocol f.readCtx.ReadValue(target, f.rootRefMode(), true) if f.readCtx.HasError() { - f.readCtx.buffer = origBuffer return f.readCtx.TakeError() } - // Restore original buffer - f.readCtx.buffer = origBuffer - return nil } @@ -743,6 +789,7 @@ func (f *Fory) Unmarshal(data []byte, v any) error { // If callback is provided, it will be called for each BufferObject during serialization. // Return true from callback to write in-band, false for out-of-band. func (f *Fory) SerializeWithCallback(buffer *ByteBuffer, v any, callback func(BufferObject) bool) error { + f.beginRoot() buf := f.writeCtx.buffer defer func() { // Reset internal state but NOT the buffer - caller manages buffer state @@ -786,6 +833,7 @@ func (f *Fory) SerializeWithCallback(buffer *ByteBuffer, v any, callback func(Bu // DeserializeWithCallbackBuffers deserializes from buffer into the provided value (for streaming/cross-language use). // The third parameter is optional external buffers for out-of-band data (can be nil). func (f *Fory) DeserializeWithCallbackBuffers(buffer *ByteBuffer, v any, buffers []*ByteBuffer) error { + f.beginRoot() // Use the caller buffer only for this root; later stream roots reuse the // original internal buffer. origBuffer := f.readCtx.buffer @@ -919,6 +967,7 @@ func readHeaderSlow(ctx *ReadContext, bitmap byte) { // // For thread-safe usage, use threadsafe.Serialize which copies the data internally. func Serialize[T any](f *Fory, value T) ([]byte, error) { + f.beginRoot() defer f.resetWriteState() v := any(value) if !validateRootDecimal(f.writeCtx.Err(), v) { @@ -1075,6 +1124,7 @@ func Serialize[T any](f *Fory, value T) ([]byte, error) { // For structs, it reads directly into the struct fields. // Note: Fory instance is NOT thread-safe. Use ThreadSafeFory for concurrent use. func Deserialize[T any](f *Fory, data []byte, target *T) error { + f.beginRoot() // Generic roots share the same reusable read and metadata owners as the // method API, so both entry and every exit must start from a root-clean state. f.resetReadState() diff --git a/go/fory/fory_test.go b/go/fory/fory_test.go index 950ec5c1a8..bfb9dc8de7 100644 --- a/go/fory/fory_test.go +++ b/go/fory/fory_test.go @@ -274,17 +274,16 @@ func TestSerializeStructSimple(t *testing.T) { type A struct { F1 []string } - require.Nil(t, fory.RegisterStructByName(A{}, "example.A")) - serde(t, fory, A{}) - serde(t, fory, &A{}) - serde(t, fory, A{F1: []string{"str1", "", "str2"}}) - serde(t, fory, &A{F1: []string{"str1", "", "str2"}}) - type SimpleB struct { F1 []string F2 map[string]int32 } + require.Nil(t, fory.RegisterStructByName(A{}, "example.A")) require.Nil(t, fory.RegisterStructByName(SimpleB{}, "example.SimpleB")) + serde(t, fory, A{}) + serde(t, fory, &A{}) + serde(t, fory, A{F1: []string{"str1", "", "str2"}}) + serde(t, fory, &A{F1: []string{"str1", "", "str2"}}) serde(t, fory, SimpleB{}) serde(t, fory, SimpleB{ F1: []string{"str1", "", "str2"}, @@ -410,24 +409,24 @@ func newFoo() Foo { func TestSerializeStruct(t *testing.T) { for _, referenceTracking := range []bool{false, true} { fory := NewFory(WithXlang(true), WithCompatible(false), WithRefTracking(referenceTracking)) + type A struct { + F1 Bar + F2 any + } require.Nil(t, fory.RegisterStructByName(Bar{}, "example.Bar")) + require.Nil(t, fory.RegisterStructByName(A{}, "example.A")) + require.Nil(t, fory.RegisterStructByName(Foo{}, "example.Foo")) serde(t, fory, &Bar{}) bar := Bar{F1: 1, F2: "str"} serde(t, fory, bar) serde(t, fory, &bar) - type A struct { - F1 Bar - F2 any - } - require.Nil(t, fory.RegisterStructByName(A{}, "example.A")) serde(t, fory, A{}) serde(t, fory, &A{}) // Use int64 for any fields since xlang deserializes integers to int64 serde(t, fory, A{F1: Bar{F1: 1, F2: "str"}, F2: int64(-1)}) serde(t, fory, &A{F1: Bar{F1: 1, F2: "str"}, F2: int64(-1)}) - require.Nil(t, fory.RegisterStructByName(Foo{}, "example.Foo")) foo := newFoo() serde(t, fory, foo) serde(t, fory, &foo) @@ -435,8 +434,8 @@ func TestSerializeStruct(t *testing.T) { } func TestSerializeCircularReference(t *testing.T) { - fory := NewFory(WithXlang(true), WithCompatible(false), WithRefTracking(true)) { + fory := NewFory(WithXlang(true), WithCompatible(false), WithRefTracking(true)) type A struct { A1 *A } @@ -455,6 +454,7 @@ func TestSerializeCircularReference(t *testing.T) { require.Same(t, a1, a1.A1) } { + fory := NewFory(WithXlang(true), WithCompatible(false), WithRefTracking(true)) type CircularRefB struct { F1 string F2 *CircularRefB diff --git a/go/fory/fory_typed_test.go b/go/fory/fory_typed_test.go index 3a7da012ad..5e8764f386 100644 --- a/go/fory/fory_typed_test.go +++ b/go/fory/fory_typed_test.go @@ -173,9 +173,8 @@ func TestDeserializeByteSliceAcceptsUint8ArrayRootType(t *testing.T) { // TestSerializeGenericComplex tests Serialize[T]/DeserializeWithCallbackBuffers[T] with complex types. // Struct wrappers must be registered explicitly before fast serializer use. func TestSerializeGenericComplex(t *testing.T) { - f := NewFory(WithXlang(false), WithRefTracking(true), WithCompatible(false)) - t.Run("Struct", func(t *testing.T) { + f := NewFory(WithXlang(false), WithRefTracking(true), WithCompatible(false)) type TestStruct struct { Name string Value int32 @@ -195,6 +194,7 @@ func TestSerializeGenericComplex(t *testing.T) { }) t.Run("Slice", func(t *testing.T) { + f := NewFory(WithXlang(false), WithRefTracking(true), WithCompatible(false)) // Note: *[]T is not supported, use wrapper struct instead type SliceWrapper struct { Items []int32 @@ -211,6 +211,7 @@ func TestSerializeGenericComplex(t *testing.T) { }) t.Run("Map", func(t *testing.T) { + f := NewFory(WithXlang(false), WithRefTracking(true), WithCompatible(false)) // Note: *map[K]V is not supported, use wrapper struct instead type MapWrapper struct { Items map[string]int32 @@ -229,10 +230,9 @@ func TestSerializeGenericComplex(t *testing.T) { // TestSerializeDeserializeRoundTrip tests that serialized data can be correctly deserialized. func TestSerializeDeserializeRoundTrip(t *testing.T) { - f := NewFory(WithXlang(false), WithRefTracking(true), WithCompatible(false)) - // Test that SerializeWithCallback[T] uses pointer-based fast path when available t.Run("TypedSerializerPath", func(t *testing.T) { + f := NewFory(WithXlang(false), WithRefTracking(true), WithCompatible(false)) // Int32 has a registered fast path original := int32(999) data, err := Serialize(f, &original) @@ -246,12 +246,13 @@ func TestSerializeDeserializeRoundTrip(t *testing.T) { }) t.Run("FastSerializerFallbackPath", func(t *testing.T) { + f := NewFory(WithXlang(false), WithRefTracking(true), WithCompatible(false)) // Custom struct uses the fast serializer fallback path. type CustomStruct struct { ID int64 Name string } - f.RegisterStructByName(CustomStruct{}, "test.CustomStruct") + require.NoError(t, f.RegisterStructByName(CustomStruct{}, "test.CustomStruct")) original := CustomStruct{ID: 123, Name: "test"} data, err := Serialize(f, &original) diff --git a/go/fory/registry_freeze_lifecycle_test.go b/go/fory/registry_freeze_lifecycle_test.go new file mode 100644 index 0000000000..5253251859 --- /dev/null +++ b/go/fory/registry_freeze_lifecycle_test.go @@ -0,0 +1,230 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package fory + +import ( + "bytes" + "reflect" + "strings" + "testing" + + "github.com/stretchr/testify/require" +) + +type registryFreezeStruct struct { + Value int32 +} + +type registryFreezeUnion struct{} + +type registryFreezeEnum int32 + +type registryFreezeExtension struct { + Value int32 +} + +type registryPanicSerializer struct{} + +func (registryPanicSerializer) WriteData(ctx *WriteContext, _ reflect.Value) { + ctx.Err().SetError(SerializationError("write failure")) + panic("write failure") +} + +func (registryPanicSerializer) ReadData(*ReadContext, reflect.Value) {} + +type registryFreezeSnapshot struct { + serializers int + typeNames int + typeIDs int + userTypeIDs int + types int + namespacedTypes int + namedTypes int + typeDefs int + definitionIDs int + typePointers int + unionTypes int + typeIDCounter uint32 + dynamicWriteIndex uint32 +} + +func takeRegistryFreezeSnapshot(r *TypeResolver) registryFreezeSnapshot { + return registryFreezeSnapshot{ + serializers: len(r.typeToSerializers), + typeNames: len(r.typeToTypeInfo), + typeIDs: len(r.typeIDToTypeInfo), + userTypeIDs: len(r.userTypeIdToTypeInfo), + types: len(r.typesInfo), + namespacedTypes: len(r.nsTypeToTypeInfo), + namedTypes: len(r.namedTypeToTypeInfo), + typeDefs: len(r.typeToTypeDef), + definitionIDs: len(r.defIdToTypeDef), + typePointers: len(r.typePointerCache), + unionTypes: len(r.unionTypeCache), + typeIDCounter: r.typeIDCounter, + dynamicWriteIndex: r.dynamicWriteStringID, + } +} + +func TestRegistryFreezeRegistrations(t *testing.T) { + f := New(WithXlang(false), WithCompatible(false)) + _, err := f.Serialize(int32(1)) + require.NoError(t, err) + f.Reset() + + before := takeRegistryFreezeSnapshot(f.typeResolver) + attempts := []struct { + name string + call func() error + }{ + {"struct ID", func() error { return f.RegisterStruct(registryFreezeStruct{}, 7101) }}, + {"struct name", func() error { return f.RegisterStructByName(registryFreezeStruct{}, "test.RegistryFreezeStruct") }}, + {"union ID", func() error { return f.RegisterUnion(registryFreezeUnion{}, 7102, nil) }}, + {"union name", func() error { return f.RegisterUnionByName(registryFreezeUnion{}, "test.RegistryFreezeUnion", nil) }}, + {"enum ID", func() error { return f.RegisterEnum(registryFreezeEnum(0), 7103) }}, + {"enum name", func() error { return f.RegisterEnumByName(registryFreezeEnum(0), "test.RegistryFreezeEnum") }}, + {"extension ID", func() error { return f.RegisterExtension(registryFreezeExtension{}, 7104, nil) }}, + {"extension name", func() error { + return f.RegisterExtensionByName(registryFreezeExtension{}, "test.RegistryFreezeExtension", nil) + }}, + } + for _, attempt := range attempts { + t.Run(attempt.name, func(t *testing.T) { + require.ErrorIs(t, attempt.call(), ErrRegistryFrozen) + }) + } + type_ := reflect.TypeOf(registryFreezeStruct{}) + require.ErrorIs(t, + f.typeResolver.RegisterStruct(type_, f.typeResolver.structTypeID(type_, false), 7105), + ErrRegistryFrozen) + require.Equal(t, before, takeRegistryFreezeSnapshot(f.typeResolver)) +} + +func TestNamedEncoderPreflight(t *testing.T) { + f := New(WithXlang(false), WithCompatible(false)) + before := takeRegistryFreezeSnapshot(f.typeResolver) + overlong := strings.Repeat("a", 32_768) + attempts := []struct { + name string + wireName string + }{ + {"namespace", overlong + ".RegistryFreezeStruct"}, + {"type name", overlong}, + } + for _, attempt := range attempts { + t.Run(attempt.name, func(t *testing.T) { + err := f.RegisterStructByName(registryFreezeStruct{}, attempt.wireName) + require.Error(t, err) + require.Equal(t, before, takeRegistryFreezeSnapshot(f.typeResolver)) + }) + } + + require.NoError(t, + f.RegisterStructByName(registryFreezeStruct{}, "test.RegistryFreezeStruct")) +} + +func TestRegistryFreezeRoots(t *testing.T) { + badDecimal := Decimal{Scale: maxDecimalScale + 1} + tests := []struct { + name string + root func(*Fory) error + }{ + {"Serialize", func(f *Fory) error { _, err := f.Serialize(badDecimal); return err }}, + {"Deserialize", func(f *Fory) error { return f.Deserialize(nil, new(int32)) }}, + {"SerializeTo", func(f *Fory) error { return f.SerializeTo(NewByteBuffer(nil), badDecimal) }}, + {"DeserializeFrom", func(f *Fory) error { return f.DeserializeFrom(NewByteBuffer(nil), new(int32)) }}, + {"SerializeWithCallback", func(f *Fory) error { return f.SerializeWithCallback(NewByteBuffer(nil), badDecimal, nil) }}, + {"DeserializeWithCallbackBuffers", func(f *Fory) error { + return f.DeserializeWithCallbackBuffers(NewByteBuffer(nil), nil, nil) + }}, + {"generic Serialize", func(f *Fory) error { _, err := Serialize(f, badDecimal); return err }}, + {"generic Deserialize", func(f *Fory) error { return Deserialize(f, nil, new(int32)) }}, + {"DeserializeFromStream", func(f *Fory) error { + return f.DeserializeFromStream(NewInputStream(bytes.NewReader(nil)), new(int32)) + }}, + {"DeserializeFromReader", func(f *Fory) error { return f.DeserializeFromReader(bytes.NewReader(nil), new(int32)) }}, + } + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + f := New(WithXlang(false), WithCompatible(false)) + require.Error(t, test.root(f)) + require.ErrorIs(t, + f.RegisterStructByName(registryFreezeStruct{}, "test.RegistryFreezeRoot"), + ErrRegistryFrozen) + }) + } +} + +func TestBorrowedBufferPanicRestore(t *testing.T) { + writer := New(WithXlang(false), WithCompatible(false)) + data, err := writer.Serialize(int32(7)) + require.NoError(t, err) + data = bytes.Clone(data) + + f := New(WithXlang(false), WithCompatible(false)) + borrowed := NewByteBuffer(bytes.Clone(data)) + require.Panics(t, func() { + _ = f.DeserializeFrom(borrowed, int32(0)) + }) + var value int32 + require.NoError(t, f.Deserialize(data, &value)) + require.Equal(t, int32(7), value) + require.ErrorIs(t, + f.RegisterStructByName(registryFreezeStruct{}, "test.RegistryFreezePanic"), + ErrRegistryFrozen) +} + +func TestStreamBufferPanicRestore(t *testing.T) { + f := New(WithXlang(false), WithCompatible(false)) + owned := f.readCtx.buffer + stream := NewInputStream(bytes.NewReader(nil)) + + require.Panics(t, func() { + _ = f.DeserializeFromStream(stream, int32(0)) + }) + require.Same(t, owned, f.readCtx.buffer) +} + +func TestSerializeToPanicRestore(t *testing.T) { + f := New(WithXlang(false), WithCompatible(false)) + require.NoError(t, f.RegisterExtension(registryFreezeExtension{}, 7104, registryPanicSerializer{})) + owned := f.writeCtx.buffer + borrowed := NewByteBuffer(nil) + + require.Panics(t, func() { + _ = f.SerializeTo(borrowed, ®istryFreezeExtension{Value: 1}) + }) + require.Same(t, owned, f.writeCtx.buffer) + require.NotZero(t, borrowed.WriterIndex()) + + data, err := f.Serialize(int32(7)) + require.NoError(t, err) + require.NotEmpty(t, data) +} + +func TestCallbackPanicCleanup(t *testing.T) { + f := New(WithXlang(false), WithCompatible(false)) + require.NoError(t, f.RegisterExtension(registryFreezeExtension{}, 7104, registryPanicSerializer{})) + + require.Panics(t, func() { + _ = f.SerializeWithCallback( + NewByteBuffer(nil), ®istryFreezeExtension{Value: 1}, nil) + }) + + require.NoError(t, f.SerializeWithCallback(NewByteBuffer(nil), int32(7), nil)) +} diff --git a/go/fory/stream.go b/go/fory/stream.go index 018f797460..0f304ba818 100644 --- a/go/fory/stream.go +++ b/go/fory/stream.go @@ -96,15 +96,18 @@ func (is *InputStream) Shrink() { // DeserializeFromStream reads the next object from the stream into the provided value. // It preserves the stream buffer while clearing root-scoped read metadata between calls. func (f *Fory) DeserializeFromStream(is *InputStream, v any) error { + f.beginRoot() origBuffer := f.readCtx.buffer f.readCtx.buffer = is.buffer - target := reflect.ValueOf(v).Elem() - f.readCtx.remainingGraphMemoryBytes = f.config.MaxGraphMemoryBytes - f.readCtx.remainingUnbackedContainerItems = f.config.MaxUnbackedContainerItems defer func() { + // Restore the owned buffer before reset so caller validation panics cannot retain the + // stream-owned buffer. f.readCtx.buffer = origBuffer f.resetReadState() }() + target := reflect.ValueOf(v).Elem() + f.readCtx.remainingGraphMemoryBytes = f.config.MaxGraphMemoryBytes + f.readCtx.remainingUnbackedContainerItems = f.config.MaxUnbackedContainerItems readHeader(f.readCtx) if f.readCtx.HasError() { @@ -124,6 +127,7 @@ func (f *Fory) DeserializeFromStream(is *InputStream, v any) error { // each call, discarding any prefetched data and type metadata. // For sequential multi-object reads on the same stream, use NewInputStream instead. func (f *Fory) DeserializeFromReader(r io.Reader, v any) error { + f.beginRoot() defer f.resetReadState() // Always reset to enforce stateless semantics. f.readCtx.buffer.ResetWithReader(r, 0) diff --git a/go/fory/threadsafe/fory.go b/go/fory/threadsafe/fory.go index 4afdfa0de9..1178824554 100644 --- a/go/fory/threadsafe/fory.go +++ b/go/fory/threadsafe/fory.go @@ -19,15 +19,29 @@ package threadsafe import ( + "fmt" + "reflect" "sync" + "sync/atomic" "github.com/apache/fory/go/fory" ) +type structRegistration struct { + typ reflect.Type + name string +} + // Fory is a thread-safe wrapper around fory.Fory using sync.Pool. // It provides the same API as fory.Fory but is safe for concurrent use. +// Registration must finish before its first root operation. type Fory struct { - pool sync.Pool + pool sync.Pool + registrationMu sync.Mutex + registryFrozen atomic.Bool + factory func() *fory.Fory + registrations []structRegistration + prepared *fory.Fory } // New creates a new thread-safe Fory instance. @@ -42,21 +56,44 @@ func NewWithFactory(factory func() *fory.Fory) *Fory { if factory == nil { panic("threadsafe.NewWithFactory requires a non-nil factory") } - f := &Fory{} - f.pool = sync.Pool{ - New: func() any { - inner := factory() - if inner == nil { - panic("threadsafe.NewWithFactory factory returned nil") - } - return inner - }, + return &Fory{factory: factory} +} + +func (f *Fory) newInner() (*fory.Fory, error) { + inner := f.factory() + if inner == nil { + panic("threadsafe.NewWithFactory factory returned nil") + } + // Before the first root, callers hold registrationMu. After registryFrozen + // is published, registrations are immutable, so pool misses can replay them + // without extending the root hot-path lock. + for _, registration := range f.registrations { + if err := inner.RegisterStructByName(registration.typ, registration.name); err != nil { + return nil, fmt.Errorf("apply registration %q to new Fory instance: %w", registration.name, err) + } } - return f + return inner, nil } -func (f *Fory) acquire() *fory.Fory { - return f.pool.Get().(*fory.Fory) +func (f *Fory) acquire() (*fory.Fory, error) { + if !f.registryFrozen.Load() { + f.registrationMu.Lock() + if !f.registryFrozen.Load() { + f.registryFrozen.Store(true) + inner := f.prepared + f.prepared = nil + f.registrationMu.Unlock() + if inner != nil { + return inner, nil + } + return f.newInner() + } + f.registrationMu.Unlock() + } + if pooled := f.pool.Get(); pooled != nil { + return pooled.(*fory.Fory), nil + } + return f.newInner() } func (f *Fory) release(inner *fory.Fory) { @@ -70,7 +107,10 @@ func (f *Fory) release(inner *fory.Fory) { // Serialize serializes a value using a pooled Fory instance func (f *Fory) Serialize(v any) ([]byte, error) { - inner := f.acquire() + inner, err := f.acquire() + if err != nil { + return nil, err + } data, err := inner.Serialize(v) if err != nil { f.release(inner) @@ -85,16 +125,45 @@ func (f *Fory) Serialize(v any) ([]byte, error) { // Deserialize deserializes data into the provided value using a pooled Fory instance func (f *Fory) Deserialize(data []byte, v any) error { - inner := f.acquire() + inner, err := f.acquire() + if err != nil { + return err + } defer f.release(inner) return inner.Deserialize(data, v) } -// RegisterStructByName registers a struct type by name for cross-language serialization. +// RegisterStructByName registers a struct type by name before the first root operation. func (f *Fory) RegisterStructByName(type_ any, name string) error { - inner := f.acquire() - defer f.release(inner) - return inner.RegisterStructByName(type_, name) + f.registrationMu.Lock() + defer f.registrationMu.Unlock() + if f.registryFrozen.Load() { + return fory.ErrRegistryFrozen + } + if f.prepared == nil { + inner, err := f.newInner() + if err != nil { + return err + } + f.prepared = inner + } + registration := structRegistration{name: name} + if err := f.prepared.RegisterStructByName(type_, name); err != nil { + // A failed registration is not part of the facade registry. Rebuild from + // the successful log before the next registration or first root. + f.prepared = nil + return err + } + if registeredType, ok := type_.(reflect.Type); ok { + registration.typ = registeredType + } else { + registration.typ = reflect.TypeOf(type_) + if registration.typ.Kind() == reflect.Ptr { + registration.typ = registration.typ.Elem() + } + } + f.registrations = append(f.registrations, registration) + return nil } // ============================================================================ @@ -104,7 +173,10 @@ func (f *Fory) RegisterStructByName(type_ any, name string) error { // Serialize serializes a value with type T inferred, thread-safe. // Takes pointer to avoid interface heap allocation and struct copy. func Serialize[T any](f *Fory, value *T) ([]byte, error) { - inner := f.acquire() + inner, err := f.acquire() + if err != nil { + return nil, err + } data, err := fory.Serialize(inner, value) if err != nil { f.release(inner) @@ -120,7 +192,10 @@ func Serialize[T any](f *Fory, value *T) ([]byte, error) { // Deserialize deserializes data directly into the provided target, thread-safe. // Takes pointer to avoid interface heap allocation and enable direct writes. func Deserialize[T any](f *Fory, data []byte, target *T) error { - inner := f.acquire() + inner, err := f.acquire() + if err != nil { + return err + } defer f.release(inner) return fory.Deserialize(inner, data, target) } diff --git a/go/fory/threadsafe/fory_test.go b/go/fory/threadsafe/fory_test.go index 37b4a0ecf0..acfd15212f 100644 --- a/go/fory/threadsafe/fory_test.go +++ b/go/fory/threadsafe/fory_test.go @@ -115,9 +115,8 @@ func TestSerializeAny(t *testing.T) { // TestDeserialize tests the Deserialize generic function func TestDeserialize(t *testing.T) { - f := New(fory.WithXlang(false), fory.WithRefTracking(true), fory.WithCompatible(false)) - t.Run("Int32", func(t *testing.T) { + f := New(fory.WithXlang(false), fory.WithRefTracking(true), fory.WithCompatible(false)) val := int32(42) data, err := Serialize(f, &val) require.NoError(t, err) @@ -129,6 +128,7 @@ func TestDeserialize(t *testing.T) { }) t.Run("String", func(t *testing.T) { + f := New(fory.WithXlang(false), fory.WithRefTracking(true), fory.WithCompatible(false)) val := "hello" data, err := Serialize(f, &val) require.NoError(t, err) @@ -140,6 +140,7 @@ func TestDeserialize(t *testing.T) { }) t.Run("Slice", func(t *testing.T) { + f := New(fory.WithXlang(false), fory.WithRefTracking(true), fory.WithCompatible(false)) // Serialize a struct containing the slice since *[]T is not supported type SliceWrapper struct { Items []int32 diff --git a/go/fory/threadsafe/registry_freeze_lifecycle_test.go b/go/fory/threadsafe/registry_freeze_lifecycle_test.go new file mode 100644 index 0000000000..7c0e619baf --- /dev/null +++ b/go/fory/threadsafe/registry_freeze_lifecycle_test.go @@ -0,0 +1,140 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package threadsafe + +import ( + "sync" + "sync/atomic" + "testing" + + "github.com/apache/fory/go/fory" + "github.com/stretchr/testify/require" +) + +type registryFreezePooled struct { + Value int32 +} + +type registryFreezeRace struct { + Value int32 +} + +func TestRegistryFreezePropagation(t *testing.T) { + var factoryCalls atomic.Int32 + f := NewWithFactory(func() *fory.Fory { + factoryCalls.Add(1) + return fory.New(fory.WithXlang(false), fory.WithCompatible(false)) + }) + require.NoError(t, f.RegisterStructByName(registryFreezePooled{}, "test.RegistryFreezePooled")) + + const innerCount = 8 + inners := make([]*fory.Fory, 0, innerCount) + for i := 0; i < innerCount; i++ { + inner, err := f.acquire() + require.NoError(t, err) + inners = append(inners, inner) + value := registryFreezePooled{Value: int32(i)} + data, err := inner.Serialize(&value) + require.NoError(t, err) + var result registryFreezePooled + require.NoError(t, inner.Deserialize(data, &result)) + require.Equal(t, value, result) + } + require.Equal(t, int32(innerCount), factoryCalls.Load()) + require.ErrorIs(t, + f.RegisterStructByName(registryFreezePooled{}, "test.RegistryFreezeLate"), + fory.ErrRegistryFrozen) + for _, inner := range inners { + f.release(inner) + } +} + +func TestRegistryFreezeOnFailure(t *testing.T) { + f := New(fory.WithXlang(false), fory.WithCompatible(false)) + _, err := f.Serialize(fory.Decimal{Scale: 10_001}) + require.Error(t, err) + require.ErrorIs(t, + f.RegisterStructByName(registryFreezePooled{}, "test.RegistryFreezeFailure"), + fory.ErrRegistryFrozen) +} + +func TestRegistryFreezeReplayFailure(t *testing.T) { + frozenInner := fory.New(fory.WithXlang(false), fory.WithCompatible(false)) + _, err := frozenInner.Serialize(int32(1)) + require.NoError(t, err) + + var factoryCalls atomic.Int32 + f := NewWithFactory(func() *fory.Fory { + if factoryCalls.Add(1) == 1 { + return fory.New(fory.WithXlang(false), fory.WithCompatible(false)) + } + return frozenInner + }) + require.NoError(t, f.RegisterStructByName(registryFreezePooled{}, "test.RegistryFreezeReplay")) + + prepared, err := f.acquire() + require.NoError(t, err) + defer f.release(prepared) + _, err = f.Serialize(®istryFreezePooled{Value: 1}) + require.ErrorIs(t, err, fory.ErrRegistryFrozen) +} + +func TestRegistryFreezeOnFactoryPanic(t *testing.T) { + f := NewWithFactory(func() *fory.Fory { return nil }) + require.Panics(t, func() { + _, _ = f.Serialize(int32(1)) + }) + require.ErrorIs(t, + f.RegisterStructByName(registryFreezePooled{}, "test.RegistryFreezeFactory"), + fory.ErrRegistryFrozen) +} + +func TestRegistryFreezeRace(t *testing.T) { + const iterations = 100 + for i := 0; i < iterations; i++ { + f := New(fory.WithXlang(false), fory.WithCompatible(false)) + start := make(chan struct{}) + registrationResult := make(chan error, 1) + rootResult := make(chan error, 1) + var ready sync.WaitGroup + ready.Add(2) + + go func() { + ready.Done() + <-start + registrationResult <- f.RegisterStructByName(registryFreezeRace{}, "test.RegistryFreezeRace") + }() + go func() { + ready.Done() + <-start + _, err := f.Serialize(®istryFreezeRace{Value: int32(i)}) + rootResult <- err + }() + ready.Wait() + close(start) + + registrationErr := <-registrationResult + rootErr := <-rootResult + if registrationErr == nil { + require.NoError(t, rootErr) + } else { + require.ErrorIs(t, registrationErr, fory.ErrRegistryFrozen) + require.Error(t, rootErr) + } + } +} diff --git a/go/fory/type_resolver.go b/go/fory/type_resolver.go index ed82c94d34..0335845857 100644 --- a/go/fory/type_resolver.go +++ b/go/fory/type_resolver.go @@ -121,6 +121,16 @@ func joinRegisteredName(namespace, typeName string) string { return namespace + "." + typeName } +func (r *TypeResolver) validateNamedRegistration(namespace, typeName string) error { + if _, err := r.namespaceEncoder.EncodePackage(namespace); err != nil { + return fmt.Errorf("invalid type namespace: %w", err) + } + if _, err := r.typeNameEncoder.EncodeTypeName(typeName); err != nil { + return fmt.Errorf("invalid type name: %w", err) + } + return nil +} + type TypeInfo struct { Type reflect.Type FullNameBytes []byte @@ -472,6 +482,9 @@ func validateOptionalFields(type_ reflect.Type) error { if type_.Kind() != reflect.Struct { return nil } + if err := validateForyTags(type_); err != nil { + return err + } for i := 0; i < type_.NumField(); i++ { field := type_.Field(i) if field.PkgPath != "" { @@ -496,6 +509,9 @@ func validateOptionalFields(type_ reflect.Type) error { // RegisterStruct registers a type with a numeric user type ID for cross-language serialization. func (r *TypeResolver) RegisterStruct(type_ reflect.Type, typeID TypeId, userTypeID uint32) error { + if err := r.fory.checkRegistrationOpen(); err != nil { + return err + } // Check if already registered if info, ok := r.userTypeIdToTypeInfo[userTypeID]; ok { if info.Type == type_ { @@ -506,9 +522,6 @@ func (r *TypeResolver) RegisterStruct(type_ reflect.Type, typeID TypeId, userTyp switch type_.Kind() { case reflect.Struct: - if err := validateForyTags(type_); err != nil { - return err - } if err := validateOptionalFields(type_); err != nil { return err } @@ -556,6 +569,9 @@ func (r *TypeResolver) RegisterStruct(type_ reflect.Type, typeID TypeId, userTyp // RegisterUnion registers a union type with a numeric user type ID for cross-language serialization. func (r *TypeResolver) RegisterUnion(type_ reflect.Type, userTypeID uint32, serializer Serializer) error { + if err := r.fory.checkRegistrationOpen(); err != nil { + return err + } if serializer == nil { return fmt.Errorf("RegisterUnion requires a non-nil serializer") } @@ -591,6 +607,9 @@ func (r *TypeResolver) RegisterUnion(type_ reflect.Type, userTypeID uint32, seri // RegisterEnum registers an enum type (numeric type in Go) with a user type ID. func (r *TypeResolver) RegisterEnum(type_ reflect.Type, userTypeID uint32) error { + if err := r.fory.checkRegistrationOpen(); err != nil { + return err + } // Check if already registered if info, ok := r.userTypeIdToTypeInfo[userTypeID]; ok { return fmt.Errorf("type %s with id %d has been registered", info.Type, userTypeID) @@ -637,6 +656,9 @@ func (r *TypeResolver) registerEnumByName(type_ reflect.Type, namespace, typeNam if typeName == "" { return fmt.Errorf("typeName must be non-empty") } + if err := r.validateNamedRegistration(namespace, typeName); err != nil { + return err + } // Verify it's a numeric type switch type_.Kind() { @@ -674,7 +696,10 @@ func (r *TypeResolver) registerStructByName(type_ reflect.Type, namespace, typeN if typeName == "" { return fmt.Errorf("typeName must be non-empty") } - if err := validateForyTags(type_); err != nil { + if err := r.validateNamedRegistration(namespace, typeName); err != nil { + return err + } + if err := validateOptionalFields(type_); err != nil { return err } tag := joinRegisteredName(namespace, typeName) @@ -722,6 +747,9 @@ func (r *TypeResolver) registerUnionByName( if typeName == "" { return fmt.Errorf("typeName must be non-empty") } + if err := r.validateNamedRegistration(namespace, typeName); err != nil { + return err + } tag := joinRegisteredName(namespace, typeName) r.typeToSerializers[type_] = serializer r.typeToTypeInfo[type_] = "@" + tag @@ -758,6 +786,9 @@ func (r *TypeResolver) registerExtensionByName( if typeName == "" { return fmt.Errorf("typeName must be non-empty") } + if err := r.validateNamedRegistration(namespace, typeName); err != nil { + return err + } tag := joinRegisteredName(namespace, typeName) // Create adapter wrapping the user's ExtensionSerializer @@ -791,6 +822,9 @@ func (r *TypeResolver) RegisterExtension( userTypeID uint32, userSerializer ExtensionSerializer, ) error { + if err := r.fory.checkRegistrationOpen(); err != nil { + return err + } if userTypeID > maxUserTypeID { return fmt.Errorf("typeID must be in range [0, 0xfffffffe], got %d", userTypeID) } @@ -1197,12 +1231,18 @@ func (r *TypeResolver) registerType( } } - nsMeta, _ := r.namespaceEncoder.EncodePackage(namespace) + nsMeta, encodeErr := r.namespaceEncoder.EncodePackage(namespace) + if encodeErr != nil { + return nil, fmt.Errorf("invalid type namespace: %w", encodeErr) + } if nsBytes = r.metaStringResolver.GetMetaStrBytes(&nsMeta); nsBytes == nil { panic("failed to encode namespace") } - typeMeta, _ := r.typeNameEncoder.EncodeTypeName(typeName) + typeMeta, encodeErr := r.typeNameEncoder.EncodeTypeName(typeName) + if encodeErr != nil { + return nil, fmt.Errorf("invalid type name: %w", encodeErr) + } if typeBytes = r.metaStringResolver.GetMetaStrBytes(&typeMeta); typeBytes == nil { panic("failed to encode type name") } diff --git a/go/fory/writer.go b/go/fory/writer.go index dbdcf583b4..ada20827cc 100644 --- a/go/fory/writer.go +++ b/go/fory/writer.go @@ -78,6 +78,7 @@ func (c *WriteContext) Reset() { func (c *WriteContext) ResetState() { c.refWriter.Reset() c.depth = 0 + c.err = Error{} c.bufferCallback = nil c.outOfBand = false if c.refResolver != nil { From 826868a745fb1c60be269e61c8a465a41f2c3a56 Mon Sep 17 00:00:00 2001 From: chaokunyang Date: Fri, 28 Aug 2026 01:23:27 +0800 Subject: [PATCH 004/168] fix(java): freeze shared facades and clear root state --- .agents/languages/java.md | 11 + .../java/type-registration.md | 5 +- .../apache/fory/FacadeRegistrationGate.java | 70 ++++++ .../src/main/java/org/apache/fory/Fory.java | 18 +- .../java/org/apache/fory/ThreadLocalFory.java | 32 +-- .../java/org/apache/fory/ThreadSafeFory.java | 7 +- .../builder/StaticCompatibleCodecBuilder.java | 13 +- .../org/apache/fory/context/MapRefReader.java | 26 +- .../apache/fory/context/MetaReadContext.java | 15 ++ .../org/apache/fory/context/ReadContext.java | 2 +- .../exception/DeserializationException.java | 31 --- .../apache/fory/io/BlockedStreamUtils.java | 4 +- .../apache/fory/io/ForyReadableChannel.java | 30 +++ .../org/apache/fory/logging/LogOnceState.java | 2 + .../org/apache/fory/pool/ThreadPoolFory.java | 23 +- .../fory/resolver/AllowListChecker.java | 5 +- .../apache/fory/resolver/TypeResolver.java | 2 +- .../apache/fory/resolver/XtypeResolver.java | 3 +- .../serializer/AbstractObjectSerializer.java | 3 +- .../fory/serializer/CompatibleSerializer.java | 8 +- .../fory/serializer/ObjectSerializer.java | 8 +- .../org/apache/fory/util/ExceptionUtils.java | 15 +- .../test/java/org/apache/fory/ForyTest.java | 28 --- .../test/java/org/apache/fory/StreamTest.java | 51 ++++ .../org/apache/fory/ThreadSafeForyTest.java | 109 +++++++++ .../StaticCompatibleCodecBuilderTest.java | 226 +++++++++++++++++- .../apache/fory/context/MapRefReaderTest.java | 49 ++++ .../fory/context/MetaReadContextTest.java | 63 +++++ .../fory/resolver/ClassResolverTest.java | 86 ------- 29 files changed, 703 insertions(+), 242 deletions(-) create mode 100644 java/fory-core/src/main/java/org/apache/fory/FacadeRegistrationGate.java create mode 100644 java/fory-core/src/test/java/org/apache/fory/context/MapRefReaderTest.java create mode 100644 java/fory-core/src/test/java/org/apache/fory/context/MetaReadContextTest.java diff --git a/.agents/languages/java.md b/.agents/languages/java.md index a96d04b7f0..cfd7384e19 100644 --- a/.agents/languages/java.md +++ b/.agents/languages/java.md @@ -33,6 +33,14 @@ Load this file when changing anything under `java/` or when Java drives a cross- values; use qualified names only when a real name conflict requires it. - If you run temporary tests with `java -cp`, run `mvn -T16 install -DskipTests` first so local Fory jars are current. - `WriteContext`, `ReadContext`, and `CopyContext` must stay explicit. Do not reintroduce `ThreadLocal` or ambient runtime-context patterns. +- Java scoped meta-share TypeInfo occurrences are root-local, and the current table size is their + protocol visibility boundary. Reset tables of at most 8192 entries by setting the size to zero; + do not null retained slots. Only when the size exceeds 8192 may cleanup replace the backing array, + and it must restore a small eight-slot array instead of retaining or allocating 8192 slots. Keep + the normal root-cleanup path allocation-free, and do not add count-shape specializations. +- Deserialization failures must not copy or retain the active reference table or materialized graph + in the exception. Root reset owns releasing operation-local graph state; keep failure reporting + bounded independently of graph size. - Java root deserialization graph memory budgeting belongs to `ReadContext` and is initialized by `Fory` root APIs. Public config is `maxGraphMemoryBytes` with fixed `128 MiB` default. Positive explicit values override the default; @@ -125,6 +133,9 @@ Load this file when changing anything under `java/` or when Java drives a cross- unmatched named sender layers as data-only metadata, and does not route layer names through `ClassResolver.readClassInternal`. Inverse registration must not turn a missed input name into an accepted class. +- `warnOnce` keys live for the logger lifetime. Read-side resolver warnings selected by remote + names must use a fixed message and must not include a remote name or another + untrusted-cardinality value in the message arguments. - Keep JDK interface names that do not require explicit registration in `DefaultJdkClassAllowList`. `TypeResolver.loadClass` and `ClassResolver.isSecure` must both use this single owner. Keep custom `TypeChecker` and fixed disallowed-list checks on their existing diff --git a/docs/object-serialization/java/type-registration.md b/docs/object-serialization/java/type-registration.md index 774b27ca3f..46974d4e54 100644 --- a/docs/object-serialization/java/type-registration.md +++ b/docs/object-serialization/java/type-registration.md @@ -45,7 +45,10 @@ same classes in the same order. With explicit IDs, the order may differ, but eac same class on both sides. Complete class and serializer registration before the first `serialize`, `deserialize`, or `copy` -call. Later registration attempts are rejected. +call. Starting one of these operations permanently freezes registration even if the operation +fails. Calling `ThreadSafeFory#execute` also freezes registration before the callback runs. The +`Fory` instance passed to that callback is already frozen, including when the callback retains it. +Later registration attempts are rejected. `registerSerializer(Foo.class, ...)` is sufficient to use `Foo` when class registration is enabled. Use `registerSerializerAndType(Foo.class, ...)` when you also want Fory to assign a numeric type ID. diff --git a/java/fory-core/src/main/java/org/apache/fory/FacadeRegistrationGate.java b/java/fory-core/src/main/java/org/apache/fory/FacadeRegistrationGate.java new file mode 100644 index 0000000000..f13785413d --- /dev/null +++ b/java/fory-core/src/main/java/org/apache/fory/FacadeRegistrationGate.java @@ -0,0 +1,70 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.fory; + +import java.util.function.Function; +import java.util.function.Supplier; +import org.apache.fory.annotation.Internal; +import org.apache.fory.exception.ForyException; + +/** Owns the permanent registration freeze before a thread-safe facade's first root or callback. */ +@Internal +public final class FacadeRegistrationGate { + private final Object lock = new Object(); + private volatile boolean frozen; + + public void applyRegistration(Runnable action) { + synchronized (lock) { + if (frozen) { + throw new ForyException( + "Cannot register class/serializer after registration has been frozen. Please register " + + "all classes before invoking top-level `serialize/deserialize/copy` methods of " + + "ThreadSafeFory."); + } + action.run(); + } + } + + /** Initializes a child while registration callbacks cannot change. */ + public Fory initializeChild(Supplier initializer) { + synchronized (lock) { + return initializer.get(); + } + } + + public void freeze() { + if (!frozen) { + synchronized (lock) { + frozen = true; + } + } + } + + /** Freezes facade and child registration before invoking an action that can expose the child. */ + public R execute(Fory fory, Function action) { + synchronized (lock) { + // The callback may return or otherwise retain the raw child. Freeze before exposing it, + // because a later root through that escaped reference is invisible to the facade. + frozen = true; + fory.getTypeResolver().finishRegistration(); + } + return action.apply(fory); + } +} diff --git a/java/fory-core/src/main/java/org/apache/fory/Fory.java b/java/fory-core/src/main/java/org/apache/fory/Fory.java index a004eef527..5830c2aac0 100644 --- a/java/fory-core/src/main/java/org/apache/fory/Fory.java +++ b/java/fory-core/src/main/java/org/apache/fory/Fory.java @@ -442,7 +442,7 @@ public T deserialize(MemoryBuffer buffer, Class type) { jitContext.unlock(); } } catch (Throwable t) { - throw ExceptionUtils.handleReadFailed(this, t); + throw ExceptionUtils.handleReadFailed(t); } finally { readContext.reset(); } @@ -459,7 +459,11 @@ public T deserialize(ForyInputStream inputStream, Class type) { @Override public T deserialize(ForyReadableChannel channel, Class type) { - return deserialize(channel.getBuffer(), type); + try { + return deserialize(channel.getBuffer(), type); + } finally { + channel.compactBuffer(); + } } @Override @@ -517,7 +521,7 @@ public Object deserialize(MemoryBuffer buffer, Iterable outOfBandB jitContext.unlock(); } } catch (Throwable t) { - throw ExceptionUtils.handleReadFailed(this, t); + throw ExceptionUtils.handleReadFailed(t); } finally { readContext.reset(); } @@ -545,8 +549,12 @@ public Object deserialize(ForyReadableChannel channel) { @Override public Object deserialize(ForyReadableChannel channel, Iterable outOfBandBuffers) { - MemoryBuffer buf = channel.getBuffer(); - return deserialize(buf, outOfBandBuffers); + try { + MemoryBuffer buf = channel.getBuffer(); + return deserialize(buf, outOfBandBuffers); + } finally { + channel.compactBuffer(); + } } @SuppressWarnings("unchecked") diff --git a/java/fory-core/src/main/java/org/apache/fory/ThreadLocalFory.java b/java/fory-core/src/main/java/org/apache/fory/ThreadLocalFory.java index 57f0d947dc..7df396f136 100644 --- a/java/fory-core/src/main/java/org/apache/fory/ThreadLocalFory.java +++ b/java/fory-core/src/main/java/org/apache/fory/ThreadLocalFory.java @@ -47,7 +47,7 @@ public class ThreadLocalFory extends AbstractThreadSafeFory { private final ThreadLocal foryThreadLocal; private Consumer factoryCallback; private final Map allFory; - private final Object callbackLock = new Object(); + private final FacadeRegistrationGate registrationGate = new FacadeRegistrationGate(); public ThreadLocalFory(Function factory) { SharedRegistry sharedRegistry = new SharedRegistry(); @@ -63,32 +63,36 @@ public ThreadLocalFory(Function factory) { } private Fory newFory() { - synchronized (callbackLock) { - Fory fory = foryFactory.get(); - factoryCallback.accept(fory); - allFory.put(fory, null); - return fory; - } + return registrationGate.initializeChild( + () -> { + Fory fory = foryFactory.get(); + factoryCallback.accept(fory); + allFory.put(fory, null); + return fory; + }); } private Fory currentFory() { + registrationGate.freeze(); return foryThreadLocal.get(); } @Internal @Override public void registerCallback(Consumer callback) { - synchronized (callbackLock) { - synchronized (allFory) { - allFory.keySet().forEach(callback); - } - factoryCallback = factoryCallback.andThen(callback); - } + registrationGate.applyRegistration( + () -> { + synchronized (allFory) { + allFory.keySet().forEach(callback); + } + factoryCallback = factoryCallback.andThen(callback); + }); } @Override public R execute(Function action) { - return action.apply(currentFory()); + Fory fory = foryThreadLocal.get(); + return registrationGate.execute(fory, action); } @Override diff --git a/java/fory-core/src/main/java/org/apache/fory/ThreadSafeFory.java b/java/fory-core/src/main/java/org/apache/fory/ThreadSafeFory.java index 1f2c1f3f20..b92fc4729c 100644 --- a/java/fory-core/src/main/java/org/apache/fory/ThreadSafeFory.java +++ b/java/fory-core/src/main/java/org/apache/fory/ThreadSafeFory.java @@ -34,8 +34,11 @@ public interface ThreadSafeFory extends BaseFory { /** - * Provide a context to execution operations on {@link Fory} directly and return the executed - * result. + * Executes {@code action} with an underlying {@link Fory} instance and returns its result. + * + *

Calling this method permanently freezes registration before {@code action} runs. Complete + * all facade registration first; the supplied instance is already frozen and remains frozen if + * the callback returns or retains it. */ R execute(Function action); diff --git a/java/fory-core/src/main/java/org/apache/fory/builder/StaticCompatibleCodecBuilder.java b/java/fory-core/src/main/java/org/apache/fory/builder/StaticCompatibleCodecBuilder.java index 1069cf571a..4241f258a2 100644 --- a/java/fory-core/src/main/java/org/apache/fory/builder/StaticCompatibleCodecBuilder.java +++ b/java/fory-core/src/main/java/org/apache/fory/builder/StaticCompatibleCodecBuilder.java @@ -239,14 +239,19 @@ private String genRecordCompatibleRead() { Code.ExprCode newRecord = new Invoke(generatedObjectInstantiator(), "newInstanceWithArguments", OBJECT_TYPE, values) .genCode(ctx); + code.append("try {\n"); if (StringUtils.isNotBlank(newRecord.code())) { - code.append(newRecord.code()).append('\n'); + code.append(indent(newRecord.code(), 2)).append('\n'); } - code.append("Object _f_record = ").append(newRecord.value()).append(";\n"); + code.append(" Object _f_record = ") + .append(newRecord.value()) + .append(";\n") + .append(" return _f_record;\n") + .append("} finally {\n"); for (int i = 0; i < components.length; i++) { - code.append("_f_recordArgs[").append(i).append("] = null;\n"); + code.append(" _f_recordArgs[").append(i).append("] = null;\n"); } - code.append("return _f_record;"); + code.append("}"); return code.toString(); } diff --git a/java/fory-core/src/main/java/org/apache/fory/context/MapRefReader.java b/java/fory-core/src/main/java/org/apache/fory/context/MapRefReader.java index 59101b61ac..3fe0b66b87 100644 --- a/java/fory-core/src/main/java/org/apache/fory/context/MapRefReader.java +++ b/java/fory-core/src/main/java/org/apache/fory/context/MapRefReader.java @@ -34,8 +34,6 @@ public final class MapRefReader implements RefReader { private static final int DEFAULT_ARRAY_CAPACITY = 3; - private long readCounter; - private long readTotalObjectSize = 0; private final ObjectArray readObjects = new ObjectArray(DEFAULT_ARRAY_CAPACITY); private final IntArray readRefIds = new IntArray(DEFAULT_ARRAY_CAPACITY); private Object readObject; @@ -124,27 +122,15 @@ public void setReadRef(int id, Object object) { } } - /** Exposes the resolved read-reference table for debugging and focused tests. */ - public ObjectArray getReadRefs() { - return readObjects; - } - - /** Clears the current read state and keeps an approximate capacity for the next operation. */ + /** Clears the current read state and keeps capacity based on the most recent operation. */ @Override public void reset() { - long totalObjectSize = this.readTotalObjectSize + readObjects.size(); - long counter = this.readCounter + 1; - if (counter < 0 || totalObjectSize < 0) { - counter = 1; - totalObjectSize = readObjects.size(); - } - this.readCounter = counter; - this.readTotalObjectSize = totalObjectSize; - int avg = (int) (totalObjectSize / counter); - if (avg <= DEFAULT_ARRAY_CAPACITY) { - avg = DEFAULT_ARRAY_CAPACITY; + int nextCapacity = Math.max(readObjects.size(), DEFAULT_ARRAY_CAPACITY); + if (readObjects.objects.length > (long) nextCapacity * 4) { + readObjects.clearApproximate(nextCapacity); + } else { + readObjects.clear(); } - readObjects.clearApproximate(avg); readRefIds.clear(); readObject = null; } diff --git a/java/fory-core/src/main/java/org/apache/fory/context/MetaReadContext.java b/java/fory-core/src/main/java/org/apache/fory/context/MetaReadContext.java index e26e4ce0a3..119695aa7c 100644 --- a/java/fory-core/src/main/java/org/apache/fory/context/MetaReadContext.java +++ b/java/fory-core/src/main/java/org/apache/fory/context/MetaReadContext.java @@ -29,9 +29,24 @@ * type definitions announced by the peer remain available for later payloads. */ public class MetaReadContext { + private static final int MAX_RETAINED_TYPE_INFOS = 8192; + private static final int RESET_TYPE_INFO_CAPACITY = 8; + /** * Type infos announced by the peer, indexed by the protocol id assigned during the current or * shared meta-share session. */ public final ObjectArray readTypeInfos = new ObjectArray<>(); + + void reset() { + ObjectArray typeInfos = readTypeInfos; + int size = typeInfos.size; + // The current size is the protocol visibility boundary, so stale slots cannot be referenced + // by a later root. Keep bounded tables intact to make normal root cleanup allocation-free, and + // discard only an oversized backing array retained by an unusually metadata-heavy root. + typeInfos.size = 0; + if (size > MAX_RETAINED_TYPE_INFOS) { + typeInfos.objects = new Object[RESET_TYPE_INFO_CAPACITY]; + } + } } diff --git a/java/fory-core/src/main/java/org/apache/fory/context/ReadContext.java b/java/fory-core/src/main/java/org/apache/fory/context/ReadContext.java index 4d42074c7c..bf2624a88f 100644 --- a/java/fory-core/src/main/java/org/apache/fory/context/ReadContext.java +++ b/java/fory-core/src/main/java/org/apache/fory/context/ReadContext.java @@ -300,7 +300,7 @@ public void reset() { contextObjects.clear(); } if (scopedMetaShareEnabled) { - metaReadContext.readTypeInfos.size = 0; + metaReadContext.reset(); } else { metaReadContext = null; } diff --git a/java/fory-core/src/main/java/org/apache/fory/exception/DeserializationException.java b/java/fory-core/src/main/java/org/apache/fory/exception/DeserializationException.java index 5cd154c672..82556a9a45 100644 --- a/java/fory-core/src/main/java/org/apache/fory/exception/DeserializationException.java +++ b/java/fory-core/src/main/java/org/apache/fory/exception/DeserializationException.java @@ -19,13 +19,8 @@ package org.apache.fory.exception; -import java.util.List; - /** Exception thrown when a deserialization operation fails. */ public class DeserializationException extends ForyException { - - private transient List readObjects; - public DeserializationException(String message) { super(message); } @@ -37,30 +32,4 @@ public DeserializationException(Throwable cause) { public DeserializationException(String message, Throwable cause) { super(message, cause); } - - // if `readObjects` too big, generate message lazily to avoid big string creation cost. - public DeserializationException(List readObjects, Throwable cause) { - super(cause); - this.readObjects = readObjects; - } - - @Override - public String getMessage() { - if (readObjects == null) { - return super.getMessage(); - } else { - try { - return "Deserialize failed, read objects are: " + readObjects; - } catch (Throwable e) { - StringBuilder builder = - new StringBuilder("Deserialize failed, type of read objects are: ["); - for (Object readObject : readObjects) { - builder.append(readObject == null ? null : readObject.getClass()).append(", "); - } - builder.delete(builder.length() - 2, builder.length()); - builder.append("]"); - return builder.toString(); - } - } - } } diff --git a/java/fory-core/src/main/java/org/apache/fory/io/BlockedStreamUtils.java b/java/fory-core/src/main/java/org/apache/fory/io/BlockedStreamUtils.java index deb03160c2..5ac34e23f0 100644 --- a/java/fory-core/src/main/java/org/apache/fory/io/BlockedStreamUtils.java +++ b/java/fory-core/src/main/java/org/apache/fory/io/BlockedStreamUtils.java @@ -94,7 +94,7 @@ private static Object readFromChannel( readFrameBody(channel, buf, size); return action.apply(buf.slice(0, size)); } catch (Throwable t) { - throw ExceptionUtils.handleReadFailed(fory, t); + throw ExceptionUtils.handleReadFailed(t); } finally { fory.resetBuffer(); } @@ -158,7 +158,7 @@ private static Object deserializeFromStream( MemoryBuffer frame = readToBufferFromStream(inputStream, buf); return function.apply(frame); } catch (Throwable t) { - throw ExceptionUtils.handleReadFailed(fory, t); + throw ExceptionUtils.handleReadFailed(t); } finally { fory.resetBuffer(); } diff --git a/java/fory-core/src/main/java/org/apache/fory/io/ForyReadableChannel.java b/java/fory-core/src/main/java/org/apache/fory/io/ForyReadableChannel.java index ab15f90206..348129f1a1 100644 --- a/java/fory-core/src/main/java/org/apache/fory/io/ForyReadableChannel.java +++ b/java/fory-core/src/main/java/org/apache/fory/io/ForyReadableChannel.java @@ -24,6 +24,7 @@ import java.nio.channels.ReadableByteChannel; import java.nio.channels.SeekableByteChannel; import javax.annotation.concurrent.NotThreadSafe; +import org.apache.fory.annotation.Internal; import org.apache.fory.exception.DeserializationException; import org.apache.fory.memory.MemoryBuffer; import org.apache.fory.platform.AndroidSupport; @@ -292,6 +293,35 @@ public MemoryBuffer getBuffer() { return memoryBuffer; } + /** Discards consumed bytes while preserving unread bytes prefetched for the next root. */ + @Internal + public void compactBuffer() { + MemoryBuffer memoryBuf = memoryBuffer; + int readerIndex = memoryBuf.readerIndex(); + if (readerIndex == 0) { + return; + } + int unreadBytes = memoryBuf.remaining(); + // Retaining a smaller consumed prefix is cheaper than copying a larger unread suffix. Once the + // prefix reaches the suffix size, compaction keeps both buffer growth and byte movement + // amortized across roots. + if (readerIndex < unreadBytes) { + return; + } + ByteBuffer byteBuf = byteBuffer; + // A read method may compute its post-fill absolute cursor before invoking fillBuffer, so moving + // bytes during a fill invalidates that pending cursor. Root finalization is the safe owner for + // compaction and still preserves bytes prefetched from the following root. + int dataEnd = byteBuf.position(); + int dataStart = dataEnd - memoryBuf.size(); + byteBuf.limit(dataEnd); + byteBuf.position(dataStart + readerIndex); + byteBuf.compact(); + byteBuf.limit(unreadBytes); + memoryBuf.initByteBuffer(byteBuf, unreadBytes); + memoryBuf.readerIndex(0); + } + private void readFully(ByteBuffer dst, int length) throws IOException { int remaining = length; while (remaining > 0) { diff --git a/java/fory-core/src/main/java/org/apache/fory/logging/LogOnceState.java b/java/fory-core/src/main/java/org/apache/fory/logging/LogOnceState.java index 4878dceb76..f999c0c31a 100644 --- a/java/fory-core/src/main/java/org/apache/fory/logging/LogOnceState.java +++ b/java/fory-core/src/main/java/org/apache/fory/logging/LogOnceState.java @@ -28,6 +28,8 @@ final class LogOnceState { static final Object[] NO_ARGS = new Object[0]; + // Keys live as long as the logger. Read paths selected by untrusted names must use a fixed + // message without name-derived arguments so input cannot grow this set without bound. private final Set logged = Collections.newSetFromMap(new ConcurrentHashMap()); diff --git a/java/fory-core/src/main/java/org/apache/fory/pool/ThreadPoolFory.java b/java/fory-core/src/main/java/org/apache/fory/pool/ThreadPoolFory.java index 04971ae20b..e8d30f3fb0 100644 --- a/java/fory-core/src/main/java/org/apache/fory/pool/ThreadPoolFory.java +++ b/java/fory-core/src/main/java/org/apache/fory/pool/ThreadPoolFory.java @@ -29,6 +29,7 @@ import java.util.function.Supplier; import javax.annotation.concurrent.ThreadSafe; import org.apache.fory.AbstractThreadSafeFory; +import org.apache.fory.FacadeRegistrationGate; import org.apache.fory.Fory; import org.apache.fory.annotation.Internal; import org.apache.fory.config.ForyBuilder; @@ -52,7 +53,7 @@ public class ThreadPoolFory extends AbstractThreadSafeFory { private final Fory[] pooledFory; private final Semaphore waiterSignal = new Semaphore(0); private final AtomicInteger waitingBorrowers = new AtomicInteger(); - private final Object callbackLock = new Object(); + private final FacadeRegistrationGate registrationGate = new FacadeRegistrationGate(); public ThreadPoolFory(Function foryFactory, int poolSize) { if (poolSize <= 0) { @@ -73,6 +74,11 @@ public ThreadPoolFory(Function foryFactory, int poolSize) { } private PooledEntry acquire() { + registrationGate.freeze(); + return acquireEntry(); + } + + private PooledEntry acquireEntry() { int slotIndex = slotIndexForCurrentThread(); PooledEntry entry = tryBorrowPreferredSlots(slotIndex); if (entry != null) { @@ -147,18 +153,19 @@ private static int spread(int hash) { @Internal @Override public void registerCallback(Consumer callback) { - synchronized (callbackLock) { - for (Fory fory : pooledFory) { - callback.accept(fory); - } - } + registrationGate.applyRegistration( + () -> { + for (Fory fory : pooledFory) { + callback.accept(fory); + } + }); } @Override public R execute(Function action) { - PooledEntry entry = acquire(); + PooledEntry entry = acquireEntry(); try { - return action.apply(entry.fory); + return registrationGate.execute(entry.fory, action); } finally { release(entry); } diff --git a/java/fory-core/src/main/java/org/apache/fory/resolver/AllowListChecker.java b/java/fory-core/src/main/java/org/apache/fory/resolver/AllowListChecker.java index 235e58850e..d1cff26a1c 100644 --- a/java/fory-core/src/main/java/org/apache/fory/resolver/AllowListChecker.java +++ b/java/fory-core/src/main/java/org/apache/fory/resolver/AllowListChecker.java @@ -130,9 +130,8 @@ private boolean check(String className) { } if (!allowed) { LOG.warnOnce( - "Class {} not in allow list, please check whether objects of this class " - + "are allowed for serialization or deserialization.", - className); + "A class is not in the allow list. Check whether its objects are allowed for " + + "serialization or deserialization."); } return true; case STRICT: diff --git a/java/fory-core/src/main/java/org/apache/fory/resolver/TypeResolver.java b/java/fory-core/src/main/java/org/apache/fory/resolver/TypeResolver.java index 60b4e7e476..46bf8a4481 100644 --- a/java/fory-core/src/main/java/org/apache/fory/resolver/TypeResolver.java +++ b/java/fory-core/src/main/java/org/apache/fory/resolver/TypeResolver.java @@ -1498,7 +1498,7 @@ final Class loadClass( } catch (IllegalStateException e) { if (deserializeUnknownClass) { if (!config.suppressClassRegistrationWarnings()) { - LOG.warnOnce(e.getMessage()); + LOG.warnOnce("A class could not be loaded and will be read as an unknown class."); } return UnknownClass.getUnknowClass(className, isEnum, arrayDims, metaContextShareEnabled); } diff --git a/java/fory-core/src/main/java/org/apache/fory/resolver/XtypeResolver.java b/java/fory-core/src/main/java/org/apache/fory/resolver/XtypeResolver.java index 48e40ef728..e3e39199ae 100644 --- a/java/fory-core/src/main/java/org/apache/fory/resolver/XtypeResolver.java +++ b/java/fory-core/src/main/java/org/apache/fory/resolver/XtypeResolver.java @@ -1342,11 +1342,10 @@ private TypeInfo populateBytesToTypeInfo( compositeClassNameBytes2TypeInfo.put(typeNameBytes, typeInfo); return typeInfo; } - String msg = String.format("Class %s not registered", qualifiedName); Class type = null; if (config.deserializeUnknownClass()) { if (!config.suppressClassRegistrationWarnings()) { - LOG.warnOnce(msg); + LOG.warnOnce("A named type is not registered and will be read as an unknown class."); } switch (typeId) { case Types.NAMED_ENUM: diff --git a/java/fory-core/src/main/java/org/apache/fory/serializer/AbstractObjectSerializer.java b/java/fory-core/src/main/java/org/apache/fory/serializer/AbstractObjectSerializer.java index 422ccdaeeb..7a3f4e4124 100644 --- a/java/fory-core/src/main/java/org/apache/fory/serializer/AbstractObjectSerializer.java +++ b/java/fory-core/src/main/java/org/apache/fory/serializer/AbstractObjectSerializer.java @@ -931,11 +931,12 @@ private T copyRecord(CopyContext copyContext, T originObj) { fieldValues = RecordUtils.remapping(copyRecordInfo, fieldValues); try { T t = objectInstantiator.newInstanceWithArguments(fieldValues); - Arrays.fill(copyRecordInfo.getRecordComponents(), null); copyContext.reference(originObj, t); return t; } catch (Throwable e) { ExceptionUtils.throwException(e); + } finally { + Arrays.fill(copyRecordInfo.getRecordComponents(), null); } return originObj; } diff --git a/java/fory-core/src/main/java/org/apache/fory/serializer/CompatibleSerializer.java b/java/fory-core/src/main/java/org/apache/fory/serializer/CompatibleSerializer.java index c61d620299..c9d033bec9 100644 --- a/java/fory-core/src/main/java/org/apache/fory/serializer/CompatibleSerializer.java +++ b/java/fory-core/src/main/java/org/apache/fory/serializer/CompatibleSerializer.java @@ -253,9 +253,11 @@ public T read(ReadContext readContext) { readFields(readContext, fieldValues); } fieldValues = RecordUtils.remapping(recordInfo, fieldValues); - T t = objectInstantiator.newInstanceWithArguments(fieldValues); - Arrays.fill(recordInfo.getRecordComponents(), null); - return t; + try { + return objectInstantiator.newInstanceWithArguments(fieldValues); + } finally { + Arrays.fill(recordInfo.getRecordComponents(), null); + } } T targetObject = newInstance(); if (readContext.hasPreservedRefId()) { diff --git a/java/fory-core/src/main/java/org/apache/fory/serializer/ObjectSerializer.java b/java/fory-core/src/main/java/org/apache/fory/serializer/ObjectSerializer.java index d8d1f94aac..718d0e45e0 100644 --- a/java/fory-core/src/main/java/org/apache/fory/serializer/ObjectSerializer.java +++ b/java/fory-core/src/main/java/org/apache/fory/serializer/ObjectSerializer.java @@ -217,9 +217,11 @@ public T read(ReadContext readContext) { if (isRecord) { Object[] fields = readFields(readContext); fields = RecordUtils.remapping(recordInfo, fields); - T obj = objectInstantiator.newInstanceWithArguments(fields); - Arrays.fill(recordInfo.getRecordComponents(), null); - return obj; + try { + return objectInstantiator.newInstanceWithArguments(fields); + } finally { + Arrays.fill(recordInfo.getRecordComponents(), null); + } } T obj = newBean(); if (trackingRef) { diff --git a/java/fory-core/src/main/java/org/apache/fory/util/ExceptionUtils.java b/java/fory-core/src/main/java/org/apache/fory/util/ExceptionUtils.java index 2ffb7aae43..7d9589ff53 100644 --- a/java/fory-core/src/main/java/org/apache/fory/util/ExceptionUtils.java +++ b/java/fory-core/src/main/java/org/apache/fory/util/ExceptionUtils.java @@ -20,12 +20,6 @@ package org.apache.fory.util; import java.lang.reflect.Field; -import java.util.Arrays; -import java.util.List; -import org.apache.fory.Fory; -import org.apache.fory.collection.ObjectArray; -import org.apache.fory.context.MapRefReader; -import org.apache.fory.context.ReadContext; import org.apache.fory.exception.DeserializationException; import org.apache.fory.exception.ForyException; import org.apache.fory.platform.AndroidSupport; @@ -57,17 +51,10 @@ public static StackOverflowError trySetStackOverflowErrorMessage( } } - public static RuntimeException handleReadFailed(Fory fory, Throwable t) { + public static RuntimeException handleReadFailed(Throwable t) { if (t instanceof ForyException) { throw (ForyException) t; } - ReadContext readContext = fory.getReadContext(); - if (readContext.getRefReader() instanceof MapRefReader) { - ObjectArray readObjects = ((MapRefReader) readContext.getRefReader()).getReadRefs(); - // carry with read objects for better trouble shooting. - List objects = Arrays.asList(readObjects.objects).subList(0, readObjects.size); - throw new DeserializationException(objects, t); - } throw new DeserializationException("Failed to deserialize input", t); } diff --git a/java/fory-core/src/test/java/org/apache/fory/ForyTest.java b/java/fory-core/src/test/java/org/apache/fory/ForyTest.java index 62df5b3dc7..462c37d32f 100644 --- a/java/fory-core/src/test/java/org/apache/fory/ForyTest.java +++ b/java/fory-core/src/test/java/org/apache/fory/ForyTest.java @@ -770,34 +770,6 @@ public void testPkgAccessLevelParentClass() { serDeCheckSerializer(fory, table, "HashBasedTableSerializer"); } - @Data - static class PrintReadObject { - public PrintReadObject() { - throw new RuntimeException(); - } - - public PrintReadObject(boolean b) {} - } - - @Test - public void testPrintReadObjectsWhenFailed() { - Fory fory = - Fory.builder() - .withXlang(false) - .withRefTracking(true) - .withCodegen(false) - .requireClassRegistration(false) - .withCompatible(false) - .build(); - PrintReadObject o = new PrintReadObject(true); - try { - serDe(fory, ImmutableList.of(ImmutableList.of("a", "b"), o)); - Assert.fail(); - } catch (ForyException e) { - Assert.assertTrue(e.getMessage().contains("[a, b]")); - } - } - @Test public void testNullObjSerAndDe() { Fory fory = diff --git a/java/fory-core/src/test/java/org/apache/fory/StreamTest.java b/java/fory-core/src/test/java/org/apache/fory/StreamTest.java index ec8ead87d3..abdb2938fc 100644 --- a/java/fory-core/src/test/java/org/apache/fory/StreamTest.java +++ b/java/fory-core/src/test/java/org/apache/fory/StreamTest.java @@ -396,6 +396,57 @@ public void testStreamBufferGrowthIsGeometric() throws IOException { } } + @Test + public void testChannelPrefixCompaction() throws IOException { + Fory fory = builder().build(); + byte[] root = fory.serialize(12345); + int rootCount = 16; + byte[] roots = new byte[root.length * rootCount]; + for (int i = 0; i < rootCount; i++) { + System.arraycopy(root, 0, roots, i * root.length, root.length); + } + + try (ForyReadableChannel channel = + new ForyReadableChannel( + new ChunkedReadableByteChannel(roots, roots.length), + ByteBuffer.allocate(roots.length))) { + assertEquals(fory.deserialize(channel), 12345); + assertEquals(channel.getBuffer().readerIndex(), root.length); + assertEquals(channel.getBuffer().remaining(), root.length * (rootCount - 1)); + + for (int i = 1; i < rootCount / 2; i++) { + assertEquals(fory.deserialize(channel), 12345); + } + assertEquals(channel.getBuffer().readerIndex(), 0); + assertEquals(channel.getBuffer().remaining(), root.length * (rootCount / 2)); + + for (int i = rootCount / 2; i < rootCount; i++) { + assertEquals(fory.deserialize(channel), 12345); + } + assertEquals(channel.getBuffer().remaining(), 0); + } + + try (ForyReadableChannel channel = + new ForyReadableChannel( + new ChunkedReadableByteChannel(roots, roots.length), + ByteBuffer.allocateDirect(roots.length))) { + assertEquals(fory.deserialize(channel, Integer.class), 12345); + assertEquals(channel.getBuffer().readerIndex(), root.length); + assertEquals(channel.getBuffer().remaining(), root.length * (rootCount - 1)); + + for (int i = 1; i < rootCount / 2; i++) { + assertEquals(fory.deserialize(channel, Integer.class), 12345); + } + assertEquals(channel.getBuffer().readerIndex(), 0); + assertEquals(channel.getBuffer().remaining(), root.length * (rootCount / 2)); + + for (int i = rootCount / 2; i < rootCount; i++) { + assertEquals(fory.deserialize(channel, Integer.class), 12345); + } + assertEquals(channel.getBuffer().remaining(), 0); + } + } + private static void assertGeometricGrowth(MemoryBuffer buffer, int numBytes, String label) { int growCount = 0; Object lastBacking = backingBuffer(buffer); diff --git a/java/fory-core/src/test/java/org/apache/fory/ThreadSafeForyTest.java b/java/fory-core/src/test/java/org/apache/fory/ThreadSafeForyTest.java index 530a5429f3..938d3ca942 100644 --- a/java/fory-core/src/test/java/org/apache/fory/ThreadSafeForyTest.java +++ b/java/fory-core/src/test/java/org/apache/fory/ThreadSafeForyTest.java @@ -30,7 +30,9 @@ import java.util.concurrent.CountDownLatch; import java.util.concurrent.ExecutorService; import java.util.concurrent.Executors; +import java.util.concurrent.Future; import java.util.concurrent.TimeUnit; +import java.util.concurrent.TimeoutException; import java.util.concurrent.atomic.AtomicReference; import lombok.Data; import org.apache.fory.context.MetaReadContext; @@ -40,6 +42,7 @@ import org.apache.fory.exception.ForyException; import org.apache.fory.memory.MemoryBuffer; import org.apache.fory.pool.ThreadPoolFory; +import org.apache.fory.resolver.ClassResolver; import org.apache.fory.resolver.SharedRegistry; import org.apache.fory.resolver.TypeResolver; import org.apache.fory.serializer.Serializer; @@ -617,6 +620,112 @@ public void testPoolRegisterAfterSerializeThrows() { Assert.assertThrows(ForyException.class, () -> fory.register(BeanB.class)); } + @Test + public void testExecuteFreezesThreadLocal() throws Exception { + ThreadSafeFory fory = + Fory.builder() + .withXlang(false) + .requireClassRegistration(true) + .withCompatible(false) + .buildThreadLocalFory(); + fory.register(BeanA.class); + + Fory escaped = fory.execute(value -> value); + Assert.assertThrows(ForyException.class, () -> escaped.register(BeanB.class)); + assertNull(((ClassResolver) escaped.getTypeResolver()).getRegisteredClassId(BeanB.class)); + + ExecutorService executor = Executors.newSingleThreadExecutor(); + try { + Fory otherThreadFory = + executor.submit(() -> fory.execute(value -> value)).get(10, TimeUnit.SECONDS); + ClassResolver otherResolver = (ClassResolver) otherThreadFory.getTypeResolver(); + assertNotNull(otherResolver.getRegisteredClassId(BeanA.class)); + assertNull(otherResolver.getRegisteredClassId(BeanB.class)); + } finally { + executor.shutdownNow(); + } + } + + @Test + public void testRegistrationGateLinearization() throws Exception { + FacadeRegistrationGate gate = new FacadeRegistrationGate(); + CountDownLatch registrationEntered = new CountDownLatch(1); + CountDownLatch freezeEntered = new CountDownLatch(1); + CountDownLatch releaseRegistration = new CountDownLatch(1); + ExecutorService executor = Executors.newFixedThreadPool(2); + try { + Future registration = + executor.submit( + () -> + gate.applyRegistration( + () -> { + registrationEntered.countDown(); + awaitUnchecked(releaseRegistration); + })); + assertTrue(registrationEntered.await(10, TimeUnit.SECONDS)); + Future freeze = + executor.submit( + () -> { + freezeEntered.countDown(); + gate.freeze(); + }); + assertTrue(freezeEntered.await(10, TimeUnit.SECONDS)); + Assert.assertThrows(TimeoutException.class, () -> freeze.get(100, TimeUnit.MILLISECONDS)); + + releaseRegistration.countDown(); + registration.get(10, TimeUnit.SECONDS); + freeze.get(10, TimeUnit.SECONDS); + Assert.assertThrows( + ForyException.class, () -> gate.applyRegistration(() -> Assert.fail("must not run"))); + } finally { + releaseRegistration.countDown(); + executor.shutdownNow(); + } + } + + @Test + public void testExecuteFreezesPool() { + ThreadPoolFory fory = + (ThreadPoolFory) + Fory.builder() + .withXlang(false) + .requireClassRegistration(true) + .withCompatible(false) + .buildThreadSafeForyPool(2); + fory.register(BeanA.class); + + Fory escaped = fory.execute(value -> value); + Assert.assertThrows(ForyException.class, () -> escaped.register(BeanB.class)); + + Fory[] pooledFory = TestUtils.getFieldValue(fory, "pooledFory"); + for (Fory child : pooledFory) { + ClassResolver resolver = (ClassResolver) child.getTypeResolver(); + assertNotNull(resolver.getRegisteredClassId(BeanA.class)); + assertNull(resolver.getRegisteredClassId(BeanB.class)); + } + } + + @Test + public void testFailedRootFreezesFacade() { + ThreadSafeFory[] runtimes = + new ThreadSafeFory[] { + Fory.builder() + .withXlang(false) + .requireClassRegistration(true) + .withCompatible(false) + .buildThreadLocalFory(), + Fory.builder() + .withXlang(false) + .requireClassRegistration(true) + .withCompatible(false) + .buildThreadSafeForyPool(2) + }; + for (ThreadSafeFory fory : runtimes) { + Assert.assertThrows(RuntimeException.class, () -> fory.deserialize(new byte[0])); + Assert.assertThrows(ForyException.class, () -> fory.register(BeanB.class)); + } + } + private void assertConcurrentRoundTrip(ThreadSafeFory fory, BeanA beanA) throws InterruptedException { ExecutorService executorService = Executors.newFixedThreadPool(12); diff --git a/java/fory-core/src/test/java/org/apache/fory/builder/StaticCompatibleCodecBuilderTest.java b/java/fory-core/src/test/java/org/apache/fory/builder/StaticCompatibleCodecBuilderTest.java index 8bfc9fcf45..7c9e04fed3 100644 --- a/java/fory-core/src/test/java/org/apache/fory/builder/StaticCompatibleCodecBuilderTest.java +++ b/java/fory-core/src/test/java/org/apache/fory/builder/StaticCompatibleCodecBuilderTest.java @@ -47,10 +47,14 @@ import org.apache.fory.platform.GraalvmSupport; import org.apache.fory.reflect.TypeRef; import org.apache.fory.resolver.TypeResolver; +import org.apache.fory.serializer.AbstractObjectSerializer; +import org.apache.fory.serializer.CompatibleSerializer; import org.apache.fory.serializer.FieldGroups.FieldCodecCategory; +import org.apache.fory.serializer.ObjectSerializer; import org.apache.fory.serializer.Serializer; import org.apache.fory.serializer.StaticGeneratedStructSerializer; import org.apache.fory.serializer.StaticGeneratedStructSerializer.RemoteFieldInfo; +import org.apache.fory.util.record.RecordInfo; import org.testng.Assert; import org.testng.SkipException; import org.testng.annotations.DataProvider; @@ -254,33 +258,219 @@ public void testStaticCompatibleRecordSerializerConvertsRemoteField() throws Exc @Test public void testInaccessibleRecordInstantiator() throws Exception { assumeRecordSupport(); + String simpleName = "StaticCompatibleHiddenRecordFailure"; CompilationResult writerResult = compile( - "test.StaticCompatibleHiddenRecordPayload", - "package test;\n" - + "public class StaticCompatibleHiddenRecordPayload {\n" + "writer." + simpleName, + "package writer;\n" + + "public class " + + simpleName + + " {\n" + " public String id;\n" - + " public StaticCompatibleHiddenRecordPayload() {}\n" + + " public " + + simpleName + + "() {}\n" + "}\n"); + String readerName = "org.apache.fory.builder." + simpleName; CompilationResult readerResult = compile( - "test.StaticCompatibleHiddenRecordPayload", - "package test;\n" + "record StaticCompatibleHiddenRecordPayload(int id) {}\n"); + readerName, + "package org.apache.fory.builder;\n" + + "record " + + simpleName + + "(int id) {\n" + + " public static boolean fail;\n" + + " " + + simpleName + + " {\n" + + " if (fail) throw new IllegalStateException(\"expected\");\n" + + " }\n" + + "}\n"); Assert.assertTrue(writerResult.success, writerResult.diagnostics()); Assert.assertTrue(readerResult.success, readerResult.diagnostics()); - try (URLClassLoader writerLoader = writerResult.classLoader(); - URLClassLoader readerLoader = readerResult.classLoader()) { - Class writerType = writerLoader.loadClass("test.StaticCompatibleHiddenRecordPayload"); - Class readerType = readerLoader.loadClass("test.StaticCompatibleHiddenRecordPayload"); - Fory writer = compatibleFory(writerLoader, writerType, false, "hidden-record-writer"); - Fory reader = compatibleFory(readerLoader, readerType, false, "hidden-record-reader"); + try (URLClassLoader writerLoader = writerResult.classLoader()) { + Class writerType = writerLoader.loadClass("writer." + simpleName); + Class readerType = defineTestClass(readerResult, readerName); + Fory writer = compatibleFory(writerLoader, writerType, true, "hidden-record-writer"); + Fory reader = + compatibleFory( + StaticCompatibleCodecBuilderTest.class.getClassLoader(), + readerType, + true, + "hidden-record-reader"); TypeDef remoteTypeDef = TypeDef.buildTypeDef(writer.getTypeResolver(), writerType); String generatedSource = new StaticCompatibleCodecBuilder(TypeRef.of(readerType), reader, remoteTypeDef).genCode(); Assert.assertTrue(generatedSource.contains("newInstanceWithArguments")); Assert.assertTrue(generatedSource.contains("Object[] _f_recordArgs = this._f_recordArgs")); + Assert.assertTrue(generatedSource.contains("finally")); + Assert.assertTrue(generatedSource.contains("_f_recordArgs[0] = null")); Assert.assertFalse( - generatedSource.contains("return new test.StaticCompatibleHiddenRecordPayload")); + generatedSource.contains("return new org.apache.fory.builder." + simpleName)); + + Class staticSerializerClass = + CodecUtils.loadOrGenStaticCompatibleCodecClass( + reader.getTypeResolver(), cast(readerType), remoteTypeDef); + // Construction installs this TypeDef-specific serializer in the reader resolver. The + // deserialization below verifies that the installed instance owns record-argument cleanup. + staticSerializerClass + .getConstructor(TypeResolver.class, Class.class, TypeDef.class) + .newInstance(reader.getTypeResolver(), readerType, remoteTypeDef); + + Object writerValue = writerType.getConstructor().newInstance(); + setField(writerType, writerValue, "id", "73"); + writer.setMetaWriteContext(new MetaWriteContext()); + byte[] bytes = writer.serialize(writerValue); + + setField(readerType, null, "fail", true); + MetaReadContext metaReadContext = new MetaReadContext(); + reader.setMetaReadContext(metaReadContext); + Assert.assertThrows(RuntimeException.class, () -> reader.deserialize(bytes)); + + Serializer serializer = metaReadContext.readTypeInfos.get(0).getSerializer(); + Assert.assertTrue( + serializer instanceof GeneratedStaticCompatibleSerializer, + serializer.getClass().getName()); + Field recordArgsField = serializer.getClass().getDeclaredField("_f_recordArgs"); + recordArgsField.setAccessible(true); + Assert.assertEquals(recordArgsField.get(serializer), new Object[] {null}); + + setField(readerType, null, "fail", false); + reader.setMetaReadContext(new MetaReadContext()); + Assert.assertEquals(invoke(readerType, reader.deserialize(bytes), "id"), 73); + } + } + + @Test + public void testCompatibleRecordClearsArgs() throws Exception { + assumeRecordSupport(); + CompilationResult writerResult = + compile( + "test.CompatibleFailingRecord", + "package test;\n" + + "public class CompatibleFailingRecord {\n" + + " public String value;\n" + + " public CompatibleFailingRecord() {}\n" + + "}\n"); + CompilationResult readerResult = + compile( + "test.CompatibleFailingRecord", + "package test;\n" + + "public record CompatibleFailingRecord(String value) {\n" + + " public static boolean fail;\n" + + " public CompatibleFailingRecord {\n" + + " if (fail) throw new IllegalStateException(\"expected\");\n" + + " }\n" + + "}\n"); + Assert.assertTrue(writerResult.success, writerResult.diagnostics()); + Assert.assertTrue(readerResult.success, readerResult.diagnostics()); + try (URLClassLoader writerLoader = writerResult.classLoader(); + URLClassLoader readerLoader = readerResult.classLoader()) { + Class writerType = writerLoader.loadClass("test.CompatibleFailingRecord"); + Class readerType = readerLoader.loadClass("test.CompatibleFailingRecord"); + Fory writer = compatibleFory(writerLoader, writerType, false, "failing-record-writer", false); + Fory reader = compatibleFory(readerLoader, readerType, false, "failing-record-reader", false); + Object writerValue = writerType.getConstructor().newInstance(); + setField(writerType, writerValue, "value", "retained-value"); + writer.setMetaWriteContext(new MetaWriteContext()); + byte[] bytes = writer.serialize(writerValue); + + setField(readerType, null, "fail", true); + MetaReadContext metaReadContext = new MetaReadContext(); + reader.setMetaReadContext(metaReadContext); + Assert.assertThrows(RuntimeException.class, () -> reader.deserialize(bytes)); + + Serializer serializer = metaReadContext.readTypeInfos.get(0).getSerializer(); + Assert.assertTrue(serializer instanceof CompatibleSerializer); + Field recordInfoField = CompatibleSerializer.class.getDeclaredField("recordInfo"); + recordInfoField.setAccessible(true); + RecordInfo recordInfo = (RecordInfo) recordInfoField.get(serializer); + Assert.assertEquals(recordInfo.getRecordComponents(), new Object[] {null}); + } + } + + @Test + public void testRecordFailureClearsArgs() throws Exception { + assumeRecordSupport(); + CompilationResult result = + compile( + "test.FailingRecord", + "package test;\n" + + "public record FailingRecord(String value) {\n" + + " public static boolean fail;\n" + + " public FailingRecord {\n" + + " if (fail) throw new IllegalStateException(\"expected\");\n" + + " }\n" + + "}\n"); + Assert.assertTrue(result.success, result.diagnostics()); + try (URLClassLoader loader = result.classLoader()) { + Class type = loader.loadClass("test.FailingRecord"); + Fory fory = + Fory.builder() + .withClassLoader(loader) + .withXlang(false) + .withRefTracking(true) + .withCodegen(false) + .requireClassRegistration(false) + .build(); + Object value = type.getConstructor(String.class).newInstance("retained-value"); + byte[] bytes = fory.serialize(value); + + setField(type, null, "fail", true); + Assert.assertThrows(RuntimeException.class, () -> fory.deserialize(bytes, cast(type))); + + Serializer serializer = fory.getTypeResolver().getSerializer(type); + Assert.assertTrue(serializer instanceof ObjectSerializer); + Field recordInfoField = ObjectSerializer.class.getDeclaredField("recordInfo"); + recordInfoField.setAccessible(true); + RecordInfo recordInfo = (RecordInfo) recordInfoField.get(serializer); + Assert.assertEquals(recordInfo.getRecordComponents(), new Object[] {null}); + + setField(type, null, "fail", false); + Assert.assertEquals( + invoke(type, fory.deserialize(bytes, cast(type)), "value"), "retained-value"); + } + } + + @Test + public void testRecordCopyClearsArgs() throws Exception { + assumeRecordSupport(); + CompilationResult result = + compile( + "test.CopyFailingRecord", + "package test;\n" + + "public record CopyFailingRecord(String value) {\n" + + " public static boolean fail;\n" + + " public CopyFailingRecord {\n" + + " if (fail) throw new IllegalStateException(\"expected\");\n" + + " }\n" + + "}\n"); + Assert.assertTrue(result.success, result.diagnostics()); + try (URLClassLoader loader = result.classLoader()) { + Class type = loader.loadClass("test.CopyFailingRecord"); + Fory fory = + Fory.builder() + .withClassLoader(loader) + .withXlang(false) + .withRefTracking(true) + .withRefCopy(true) + .withCodegen(false) + .requireClassRegistration(false) + .build(); + Object value = type.getConstructor(String.class).newInstance("retained-value"); + Serializer serializer = fory.getTypeResolver().getSerializer(type); + Assert.assertTrue(serializer instanceof ObjectSerializer); + + setField(type, null, "fail", true); + Assert.assertThrows(RuntimeException.class, () -> fory.copy(value)); + + Field copyRecordInfoField = AbstractObjectSerializer.class.getDeclaredField("copyRecordInfo"); + copyRecordInfoField.setAccessible(true); + RecordInfo recordInfo = (RecordInfo) copyRecordInfoField.get(serializer); + Assert.assertEquals(recordInfo.getRecordComponents(), new Object[] {null}); + + setField(type, null, "fail", false); + Assert.assertEquals(invoke(type, fory.copy(value), "value"), "retained-value"); } } @@ -774,6 +964,16 @@ private static CompilationResult compile(String typeName, String source) throws } } + private static Class defineTestClass(CompilationResult result, String typeName) + throws Exception { + Path classFile = result.classRoot.resolve(typeName.replace('.', '/') + ".class"); + byte[] classBytes = Files.readAllBytes(classFile); + java.lang.reflect.Method defineClass = + java.lang.invoke.MethodHandles.Lookup.class.getMethod("defineClass", byte[].class); + return (Class) + defineClass.invoke(java.lang.invoke.MethodHandles.lookup(), (Object) classBytes); + } + private static void assumeRecordSupport() { if (javaSpecificationVersion() < 16) { throw new SkipException("Record source tests require JDK 16 or newer"); diff --git a/java/fory-core/src/test/java/org/apache/fory/context/MapRefReaderTest.java b/java/fory-core/src/test/java/org/apache/fory/context/MapRefReaderTest.java new file mode 100644 index 0000000000..7852402b06 --- /dev/null +++ b/java/fory-core/src/test/java/org/apache/fory/context/MapRefReaderTest.java @@ -0,0 +1,49 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.fory.context; + +import org.apache.fory.TestUtils; +import org.apache.fory.collection.ObjectArray; +import org.testng.Assert; +import org.testng.annotations.Test; + +public class MapRefReaderTest { + @Test + public void testResetUsesRecentSize() { + MapRefReader reader = new MapRefReader(); + for (int i = 0; i < 100; i++) { + reader.preserveRefId(); + reader.reference(i); + } + ObjectArray readObjects = TestUtils.getFieldValue(reader, "readObjects"); + Object[] proportionateTable = readObjects.objects; + reader.reset(); + Assert.assertSame(readObjects.objects, proportionateTable); + for (int i = 0; i < 100; i++) { + Assert.assertNull(proportionateTable[i]); + } + + reader.preserveRefId(); + reader.reference("small"); + reader.reset(); + Assert.assertEquals(readObjects.objects.length, 3); + Assert.assertNull(readObjects.objects[0]); + } +} diff --git a/java/fory-core/src/test/java/org/apache/fory/context/MetaReadContextTest.java b/java/fory-core/src/test/java/org/apache/fory/context/MetaReadContextTest.java new file mode 100644 index 0000000000..1d228de3a1 --- /dev/null +++ b/java/fory-core/src/test/java/org/apache/fory/context/MetaReadContextTest.java @@ -0,0 +1,63 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.fory.context; + +import static org.testng.Assert.assertEquals; +import static org.testng.Assert.assertNotSame; +import static org.testng.Assert.assertNull; +import static org.testng.Assert.assertSame; + +import org.apache.fory.resolver.TypeInfo; +import org.testng.annotations.Test; + +public class MetaReadContextTest { + @Test + public void testRetainedOccurrenceTableReset() { + MetaReadContext context = new MetaReadContext(); + TypeInfo typeInfo = new TypeInfo(Object.class, null); + for (int i = 0; i < 8192; i++) { + context.readTypeInfos.add(typeInfo); + } + Object[] objects = context.readTypeInfos.objects; + + context.reset(); + + assertEquals(context.readTypeInfos.size, 0); + assertSame(context.readTypeInfos.objects, objects); + assertSame(objects[0], typeInfo); + } + + @Test + public void testLargeOccurrenceTableReset() { + MetaReadContext context = new MetaReadContext(); + TypeInfo typeInfo = new TypeInfo(Object.class, null); + for (int i = 0; i < 8193; i++) { + context.readTypeInfos.add(typeInfo); + } + Object[] objects = context.readTypeInfos.objects; + + context.reset(); + + assertEquals(context.readTypeInfos.size, 0); + assertEquals(context.readTypeInfos.objects.length, 8); + assertNotSame(context.readTypeInfos.objects, objects); + assertNull(context.readTypeInfos.objects[0]); + } +} diff --git a/java/fory-core/src/test/java/org/apache/fory/resolver/ClassResolverTest.java b/java/fory-core/src/test/java/org/apache/fory/resolver/ClassResolverTest.java index ac48a604d9..d7112d2e4f 100644 --- a/java/fory-core/src/test/java/org/apache/fory/resolver/ClassResolverTest.java +++ b/java/fory-core/src/test/java/org/apache/fory/resolver/ClassResolverTest.java @@ -28,11 +28,8 @@ import com.google.common.collect.ImmutableMap; import com.google.common.primitives.Primitives; -import java.io.ByteArrayOutputStream; -import java.io.PrintStream; import java.io.Serializable; import java.lang.reflect.Constructor; -import java.lang.reflect.InvocationTargetException; import java.lang.reflect.Method; import java.nio.charset.StandardCharsets; import java.util.ArrayList; @@ -64,14 +61,11 @@ import org.apache.fory.context.WriteContext; import org.apache.fory.exception.ForyException; import org.apache.fory.exception.InsecureException; -import org.apache.fory.logging.LogLevel; import org.apache.fory.logging.Logger; import org.apache.fory.logging.LoggerFactory; import org.apache.fory.memory.MemoryBuffer; import org.apache.fory.memory.MemoryUtils; import org.apache.fory.meta.ClassSpec; -import org.apache.fory.meta.EncodedMetaString; -import org.apache.fory.meta.Encoders; import org.apache.fory.meta.FieldTypes; import org.apache.fory.meta.TypeDef; import org.apache.fory.reflect.TypeRef; @@ -302,33 +296,6 @@ public void testGetSerializerClass() throws ClassNotFoundException { MapSerializers.DefaultJavaMapSerializer.class); } - @Test - public void testSuppressXtypeWarnings() throws Exception { - String suppressed = - captureOutput( - () -> - resolveMissingXtype( - Fory.builder() - .withXlang(true) - .withMetaShare(true) - .withDeserializeUnknownClass(true) - .suppressClassRegistrationWarnings(true) - .build())); - assertEquals(count(suppressed, "Class missing.pkg.MissingType not registered"), 0); - - String unsuppressed = - captureOutput( - () -> - resolveMissingXtype( - Fory.builder() - .withXlang(true) - .withMetaShare(true) - .withDeserializeUnknownClass(true) - .suppressClassRegistrationWarnings(false) - .build())); - assertEquals(count(unsuppressed, "Class missing.pkg.MissingType not registered"), 1); - } - @Test public void testSharedRegistrySharesTypeDefCachesAcrossForyInstances() { ForyBuilder builder = @@ -1941,57 +1908,4 @@ public int hashCode() { return Objects.hash(codes); } } - - private static String captureOutput(Runnable action) throws Exception { - int previousLogLevel = LoggerFactory.getLogLevel(); - PrintStream previousOut = System.out; - ByteArrayOutputStream out = new ByteArrayOutputStream(); - try { - LoggerFactory.setLogLevel(LogLevel.WARN_LEVEL); - System.setOut(new PrintStream(out, true, StandardCharsets.UTF_8.name())); - action.run(); - } finally { - System.setOut(previousOut); - LoggerFactory.setLogLevel(previousLogLevel); - } - return out.toString(StandardCharsets.UTF_8.name()); - } - - private static void resolveMissingXtype(Fory fory) { - try { - Method method = - XtypeResolver.class.getDeclaredMethod( - "loadBytesToTypeInfoWithTypeId", - int.class, - EncodedMetaString.class, - EncodedMetaString.class); - method.setAccessible(true); - method.invoke( - fory.getTypeResolver(), - Types.NAMED_STRUCT, - Encoders.PACKAGE_ENCODER.encodeBinary("missing.pkg"), - Encoders.TYPE_NAME_ENCODER.encodeBinary("MissingType")); - } catch (InvocationTargetException e) { - Throwable cause = e.getCause(); - if (!(cause instanceof IllegalStateException) - || !cause.getMessage().contains("missing.pkg.MissingType")) { - throw new AssertionError(e); - } - } catch (ReflectiveOperationException e) { - throw new AssertionError(e); - } - } - - private static int count(String text, String pattern) { - int count = 0; - int from = 0; - while (true) { - int index = text.indexOf(pattern, from); - if (index < 0) { - return count; - } - count++; - from = index + pattern.length(); - } - } } From 6a3546e0f4ae0b057c5d3298fc6b0c60bbfaf212 Mon Sep 17 00:00:00 2001 From: chaokunyang Date: Fri, 28 Aug 2026 01:23:35 +0800 Subject: [PATCH 005/168] fix(javascript): freeze registries and clear root state --- .agents/languages/javascript.md | 4 + .../javascript/type-registration.md | 4 + javascript/packages/core/lib/context.ts | 54 ++++- javascript/packages/core/lib/fory.ts | 24 ++- javascript/packages/core/lib/typeResolver.ts | 13 ++ javascript/packages/core/lib/writer/index.ts | 12 +- javascript/test/array.test.ts | 28 +-- javascript/test/depthLimit.test.ts | 13 +- javascript/test/enum.test.ts | 14 +- javascript/test/fory.test.ts | 71 ++++++ javascript/test/protocol/struct.test.ts | 2 +- javascript/test/rootReadCleanup.test.ts | 203 ++++++++++++++++++ javascript/test/typemeta.test.ts | 28 +-- 13 files changed, 398 insertions(+), 72 deletions(-) create mode 100644 javascript/test/rootReadCleanup.test.ts diff --git a/.agents/languages/javascript.md b/.agents/languages/javascript.md index 534e49481b..db27e82f0c 100644 --- a/.agents/languages/javascript.md +++ b/.agents/languages/javascript.md @@ -11,6 +11,10 @@ Load this file when changing `javascript/`. - Preserve generated serializer hot paths that bind writer, reader, ref, resolver, and metadata locals in outer closures; do not replace them with per-call context lookups without a measured reason. - Do not add parallel header-low/header-high slot caches in TypeMeta hot paths to chase benchmark gaps. Header-cache hits must use the concrete checked cache owner directly; if a small hit hint is needed, cache TypeMeta objects themselves and compare `TypeMeta.headerHash`, not separate low/high header fields or benchmark-pattern state. - JavaScript TypeMeta header cache hits should compare the 52-bit TypeMeta header hash directly. The hash is precise in JS `Number` and already includes the low header bits as hash input; do not add extra low-bit fields, sentinel state, nullable accepted headers, or parallel slot arrays around it. +- Root-local read metadata occurrence arrays and write metadata owner arrays use an 8192-entry + retention boundary. Clear the logical length at or below that boundary and replace the array + only above it. Root serializers must release reference and metadata owner state in `finally` + after both successful and failed writes. - Runtime value carriers such as decimal or reduced-precision numeric types belong under the core `types/` ownership boundary, with imports, exports, and codegen externals updated together. - Keep `TypeInfo` as schema metadata. Compatibility-sensitive decisions belong on `TypeResolver` or explicit operations, not as retained resolver state on metadata objects. - Normalize optional boolean config values at config construction; do not carry `null` through runtime paths when it means `false`. diff --git a/docs/object-serialization/javascript/type-registration.md b/docs/object-serialization/javascript/type-registration.md index a0c471c075..e2b4a3a109 100644 --- a/docs/object-serialization/javascript/type-registration.md +++ b/docs/object-serialization/javascript/type-registration.md @@ -117,6 +117,10 @@ fory.register(Type.enum("example.status", Status)); Registration is per `Fory` instance. If you create two instances, you need to register schemas in both. +Register every type and custom serializer before the first root serialization or deserialization +attempt. Starting that first root operation permanently closes registration for the instance, even +if the operation fails. Create a new `Fory` instance when you need a different registration set. + ## What `register` Returns `fory.register(schema)` returns a bound serializer pair: diff --git a/javascript/packages/core/lib/context.ts b/javascript/packages/core/lib/context.ts index d6875f0ffb..6f19837a29 100644 --- a/javascript/packages/core/lib/context.ts +++ b/javascript/packages/core/lib/context.ts @@ -276,6 +276,8 @@ export class RefReader { } export class MetaStringWriter { + private static readonly MAX_RETAINED_META_STRING_OWNERS = 8192; + private disposeMetaStringBytes: MetaStringBytes[] = []; private dynamicNameId = 0; private namespaceEncoder = new MetaStringEncoder(".", "_"); @@ -306,14 +308,22 @@ export class MetaStringWriter { } reset() { - this.disposeMetaStringBytes.forEach((item) => { - item.dynamicWriteStringId = -1; - }); + const names = this.disposeMetaStringBytes; + for (let i = 0; i < names.length; i++) { + names[i].dynamicWriteStringId = -1; + } this.dynamicNameId = 0; + if (names.length > MetaStringWriter.MAX_RETAINED_META_STRING_OWNERS) { + this.disposeMetaStringBytes = []; + } else { + names.length = 0; + } } } export class MetaStringReader { + private static readonly MAX_RETAINED_NAMES = 8192; + private names: string[] = []; private namespaceDecoder = new MetaStringDecoder(".", "_"); private typenameDecoder = new MetaStringDecoder("$", "_"); @@ -351,11 +361,17 @@ export class MetaStringReader { } reset() { - this.names = []; + if (this.names.length > MetaStringReader.MAX_RETAINED_NAMES) { + this.names = []; + } else { + this.names.length = 0; + } } } export class WriteContext { + private static readonly MAX_RETAINED_TYPE_META_OWNERS = 8192; + readonly writer: BinaryWriter; readonly refWriter: RefWriter; readonly metaStringWriter: MetaStringWriter; @@ -376,10 +392,15 @@ export class WriteContext { this.writer.reset(); this.refWriter.reset(); this.metaStringWriter.reset(); - this.disposeTypeMetaOwners.forEach((owner) => { - owner.dynamicTypeId = -1; - }); - this.disposeTypeMetaOwners = []; + const owners = this.disposeTypeMetaOwners; + for (let i = 0; i < owners.length; i++) { + owners[i].dynamicTypeId = -1; + } + if (owners.length > WriteContext.MAX_RETAINED_TYPE_META_OWNERS) { + this.disposeTypeMetaOwners = []; + } else { + owners.length = 0; + } this.dynamicTypeId = 0; } @@ -548,6 +569,7 @@ export class WriteContext { export class ReadContext { private static readonly MIN_REMOTE_TYPE_META_LIMIT = 8192; private static readonly MAX_REMOTE_TYPE_KEYS = 8192; + private static readonly MAX_RETAINED_TYPE_META = 8192; readonly reader: BinaryReader; readonly refReader: RefReader; @@ -584,15 +606,25 @@ export class ReadContext { this.reader.reset(bytes); this.refReader.reset(); this.metaStringReader.reset(); - this.typeMeta = []; + if (this.typeMeta.length !== 0) { + this.typeMeta.length = 0; + } this._depth = 0; this.remainingGraphMemoryBytes = this.maxGraphMemoryBytes; this.remainingUnbackedContainerItems = this.maxUnbackedContainerItems; } - resetReadDepth() { - // Root reads call this in finally; nested readers retain depth when a child throws. + resetRootState() { + // Root reads own failure cleanup; nested readers retain their live state when a child throws. + this.refReader.reset(); + this.metaStringReader.reset(); + if (this.typeMeta.length > ReadContext.MAX_RETAINED_TYPE_META) { + this.typeMeta = []; + } else { + this.typeMeta.length = 0; + } this._depth = 0; + this.remainingGraphMemoryBytes = 0; this.remainingUnbackedContainerItems = 0; } diff --git a/javascript/packages/core/lib/fory.ts b/javascript/packages/core/lib/fory.ts index 8eff8df042..4314faff00 100644 --- a/javascript/packages/core/lib/fory.ts +++ b/javascript/packages/core/lib/fory.ts @@ -147,6 +147,9 @@ export default class Fory { deserialize(bytes: Uint8Array): InstanceType | null; }; register(constructor: any, customSerializer?: CustomSerializer) { + // Root codegen captures resolver state in generated closures and checked metadata caches. + // Freezing permanently at the first attempt keeps every later root on that same registry. + this.typeResolver.ensureRegistrationOpen(); let serializer: Serializer; if (constructor.prototype?.[ForyTypeInfoSymbol]) { const typeInfo: TypeInfo = (constructor.prototype[ForyTypeInfoSymbol] as WithForyClsInfo) @@ -173,6 +176,7 @@ export default class Fory { } deserialize(bytes: Uint8Array, serializer: Serializer = this.anySerializer): T | null { + this.typeResolver.freezeRegistration(); this.readContext.reset(bytes); try { const reader = this.readContext.reader; @@ -182,7 +186,7 @@ export default class Fory { } return serializer.readRef(); } finally { - this.readContext.resetReadDepth(); + this.readContext.resetRootState(); } } @@ -206,11 +210,16 @@ export default class Fory { const writer = writeContext.writer; const rootHeader = ConfigFlags.isCrossLanguageFlag; rootSerializer = (data: any) => { - writeContext.reset(); - writer.writeUint8(rootHeader); - writer.reserve(serializer.fixedSize); - serializer.writeRef(data); - return writer.dump(); + this.typeResolver.freezeRegistration(); + try { + writer.writeUint8(rootHeader); + writer.reserve(serializer.fixedSize); + serializer.writeRef(data); + return writer.dump(); + } finally { + // dump() returns an owned copy, so cleanup cannot invalidate a successful result. + writeContext.reset(); + } }; this.rootSerializers.set(serializer, rootSerializer); return rootSerializer; @@ -228,6 +237,7 @@ export default class Fory { : this.anySerializer; const rootHeader = ConfigFlags.isCrossLanguageFlag; rootDeserializer = (bytes: Uint8Array) => { + this.typeResolver.freezeRegistration(); readContext.reset(bytes); try { const bitmap = reader.readUint8(); @@ -236,7 +246,7 @@ export default class Fory { } return rootSerializer.readRef(); } finally { - readContext.resetReadDepth(); + readContext.resetRootState(); } }; this.rootDeserializers.set(serializer, rootDeserializer); diff --git a/javascript/packages/core/lib/typeResolver.ts b/javascript/packages/core/lib/typeResolver.ts index 30d27d54c0..65c5a98ddf 100644 --- a/javascript/packages/core/lib/typeResolver.ts +++ b/javascript/packages/core/lib/typeResolver.ts @@ -97,6 +97,7 @@ export default class TypeResolver { readonly trackingRef: boolean; private internalSerializer: Serializer[] = new Array(300); private customSerializer: Map = new Map(); + private registrationFrozen = false; private writeContext!: WriteContext; private readContext!: ReadContext; @@ -270,7 +271,18 @@ export default class TypeResolver { this.initInternalSerializer(); } + freezeRegistration() { + this.registrationFrozen = true; + } + + ensureRegistrationOpen() { + if (this.registrationFrozen) { + throw new Error("types and serializers must be registered before the first root operation"); + } + } + registerSerializer(typeInfo: TypeInfo, serializer: Serializer = uninitSerialize) { + this.ensureRegistrationOpen(); const typeId = this.computeTypeId(typeInfo); if (!TypeId.isNamedType(typeId)) { if (TypeId.needsUserTypeId(typeId) && typeInfo.userTypeId !== -1) { @@ -312,6 +324,7 @@ export default class TypeResolver { } regenerateReadSerializer(typeInfo: TypeInfo) { + this.ensureRegistrationOpen(); const serializer = this.generateReadSerializer(typeInfo); return this.registerSerializer(typeInfo, { readDataAlwaysAdvances: serializer.readDataAlwaysAdvances, diff --git a/javascript/packages/core/lib/writer/index.ts b/javascript/packages/core/lib/writer/index.ts index f82cad30f3..e3c199966d 100644 --- a/javascript/packages/core/lib/writer/index.ts +++ b/javascript/packages/core/lib/writer/index.ts @@ -59,7 +59,7 @@ export class BinaryWriter { hps?: Hps; } = {}, ) { - this.initPoll(); + this.initPool(); this.config = config; this.hpsEnable = Boolean(config?.hps); this.internalStringDetector = getInternalStringDetector(); @@ -72,7 +72,7 @@ export class BinaryWriter { } } - private initPoll() { + private initPool() { this.byteLength = 1024 * 100; this.platformBuffer = alloc(this.byteLength); this.dataView = new DataView(this.platformBuffer.buffer, this.platformBuffer.byteOffset); @@ -95,6 +95,8 @@ export class BinaryWriter { } this.cursor = 0; this.reserved = 0; + // Successful dumps already release a large buffer; this also covers aborted roots. + this.releaseLargeBuffer(); } bool(bool: boolean) { @@ -492,16 +494,16 @@ export class BinaryWriter { this.platformBuffer[this.cursor++] = Number(val & 255n); } - tryFreePool() { + private releaseLargeBuffer() { if (this.byteLength > MAX_POOL_SIZE) { - this.initPoll(); + this.initPool(); } } dump() { const result = alloc(this.cursor); this.platformBuffer.copy(result, 0, 0, this.cursor); - this.tryFreePool(); + this.releaseLargeBuffer(); return result; } diff --git a/javascript/test/array.test.ts b/javascript/test/array.test.ts index 5637d0f78b..9ef09290a8 100644 --- a/javascript/test/array.test.ts +++ b/javascript/test/array.test.ts @@ -284,7 +284,21 @@ describe("array", () => { values: Type.uint16Array(), }, ); + const float16Type = Type.struct( + { typeName: "example.float16array" }, + { + values: Type.float16Array(), + }, + ); + const bfloat16Type = Type.struct( + { typeName: "example.bfloat16array" }, + { + values: Type.bfloat16Array(), + }, + ); const uint16Serializer = fory.register(uint16Type).serializer; + const float16Serializer = fory.register(float16Type).serializer; + const bfloat16Serializer = fory.register(bfloat16Type).serializer; const uint16Bytes = fory.serialize( { values: new Uint16Array([0x1234, 0xabcd]), @@ -293,13 +307,6 @@ describe("array", () => { ); expect(containsBytes(uint16Bytes, [0x34, 0x12, 0xcd, 0xab])).toBe(true); - const float16Type = Type.struct( - { typeName: "example.float16array" }, - { - values: Type.float16Array(), - }, - ); - const float16Serializer = fory.register(float16Type).serializer; const float16Bytes = fory.serialize( { values: new ForyFloat16Array([1, -2]), @@ -308,13 +315,6 @@ describe("array", () => { ); expect(containsBytes(float16Bytes, [0x00, 0x3c, 0x00, 0xc0])).toBe(true); - const bfloat16Type = Type.struct( - { typeName: "example.bfloat16array" }, - { - values: Type.bfloat16Array(), - }, - ); - const bfloat16Serializer = fory.register(bfloat16Type).serializer; const bfloat16Bytes = fory.serialize( { values: new BFloat16Array([1, -2]), diff --git a/javascript/test/depthLimit.test.ts b/javascript/test/depthLimit.test.ts index 5a0eb029fb..2b2a40f5c3 100644 --- a/javascript/test/depthLimit.test.ts +++ b/javascript/test/depthLimit.test.ts @@ -275,6 +275,11 @@ describe("depth-limit", () => { readerFory.register(readerChild); const writer = writerFory.register(writerRoot); const reader = readerFory.register(readerRoot); + const shallowType = Type.struct(7403, { + value: Type.int32().setId(1), + }); + const shallowWriter = writerFory.register(shallowType); + const shallowReader = readerFory.register(shallowType); const malformedDepth = writer.serialize({ child: { grandchild: { value: "7" }, @@ -288,18 +293,13 @@ describe("depth-limit", () => { ); expect(readerFory.readContext.depth).toBe(0); - const shallowType = Type.struct(7403, { - value: Type.int32().setId(1), - }); - const shallowWriter = writerFory.register(shallowType); - const shallowReader = readerFory.register(shallowType); expect(shallowReader.deserialize(shallowWriter.serialize({ value: 10 }))).toEqual({ value: 10, }); expect(readerFory.readContext.depth).toBe(0); }); - test("should reset depth at start of each deserialization", () => { + test("should reset depth after each deserialization", () => { const fory = new Fory({ compatible: false, maxDepth: 50 }); const typeInfo = Type.struct( { @@ -313,7 +313,6 @@ describe("depth-limit", () => { const { serialize, deserialize } = fory.register(typeInfo); deserialize(serialize({ a: 1 })); - // Depth will be reset at the start of resetRead() call expect(fory.readContext.depth).toBe(0); deserialize(serialize({ a: 2 })); diff --git a/javascript/test/enum.test.ts b/javascript/test/enum.test.ts index b9852e8803..3ca3a6d75c 100644 --- a/javascript/test/enum.test.ts +++ b/javascript/test/enum.test.ts @@ -79,13 +79,6 @@ describe("enum", () => { const enumSerializer = fory.register(enumType); expect(enumSerializer.serializer.needToWriteRef()).toBe(false); - const rootBytes = enumSerializer.serialize(Foo.first); - const reader = new BinaryReader({}); - reader.reset(rootBytes); - expect(reader.readUint8()).toBe(ConfigFlags.isCrossLanguageFlag); - expect(reader.readInt8()).toBe(RefFlags.NotNullValueFlag); - expect(reader.readUint8()).toBe(TypeId.ENUM); - const nodeType = Type.struct(102, { value: Type.int32(), }); @@ -96,6 +89,13 @@ describe("enum", () => { second: nodeType.clone().setTrackingRef(true).setId(3), }), ); + const rootBytes = enumSerializer.serialize(Foo.first); + const reader = new BinaryReader({}); + reader.reset(rootBytes); + expect(reader.readUint8()).toBe(ConfigFlags.isCrossLanguageFlag); + expect(reader.readInt8()).toBe(RefFlags.NotNullValueFlag); + expect(reader.readUint8()).toBe(TypeId.ENUM); + const shared = { value: 7 }; const result = sequenceSerializer.deserialize( sequenceSerializer.serialize({ diff --git a/javascript/test/fory.test.ts b/javascript/test/fory.test.ts index 50b9b51948..9c38b3ba63 100644 --- a/javascript/test/fory.test.ts +++ b/javascript/test/fory.test.ts @@ -20,6 +20,7 @@ import Fory, { TypeInfo, Type } from "../packages/core/index"; import { describe, expect, test } from "@jest/globals"; import { fromUint8Array } from "../packages/core/lib/platformBuffer"; +import { TypeId } from "../packages/core/lib/type"; describe("fory", () => { test("defaults to compatible mode unless explicitly set", () => { @@ -84,6 +85,76 @@ describe("fory", () => { testTypeInfo(typeinfo8, "123"); }); + test.each(["serialize", "deserialize"] as const)( + "freezes registration when %s starts and fails", + (operation) => { + const fory = new Fory({ compatible: false }); + fory.register(Type.struct(8101, {})); + + if (operation === "serialize") { + expect(() => fory.serialize(Symbol("unsupported"))).toThrow(); + } else { + expect(() => fory.deserialize(new Uint8Array([0]))).toThrow(); + } + + expect(() => fory.register(Type.struct(8102, {}))).toThrow(); + }, + ); + + test("freezes direct resolver registration", () => { + const fory = new Fory({ compatible: false }); + fory.serialize(1); + + expect(() => fory.typeResolver.registerSerializer(Type.struct(8105, {}))).toThrow(); + expect(fory.typeResolver.getSerializerById(TypeId.STRUCT, 8105)).toBeUndefined(); + }); + + test("keeps rejected descriptor mutable", () => { + const fory = new Fory({ compatible: false }); + const typeInfo = Type.struct(8106, {}); + fory.serialize(1); + + expect(() => fory.register(typeInfo)).toThrow(); + typeInfo.setNullable(true); + expect(typeInfo.nullable).toBe(true); + }); + + test("rejects regeneration before codegen", () => { + let generated = 0; + const fory = new Fory({ + compatible: false, + hooks: { + afterCodeGenerated(code) { + generated++; + return code; + }, + }, + }); + fory.serialize(1); + const generatedBefore = generated; + + expect(() => fory.typeResolver.regenerateReadSerializer(Type.struct(8107, {}))).toThrow(); + expect(generated).toBe(generatedBefore); + }); + + test.each(["serialize", "deserialize"] as const)( + "freezes registration after registered %s succeeds", + (operation) => { + const typeInfo = Type.struct(8103, {}); + const source = new Fory({ compatible: false }).register(typeInfo.clone()); + const fory = new Fory({ compatible: false }); + const registered = fory.register(typeInfo); + + if (operation === "serialize") { + registered.serialize({}); + } else { + registered.deserialize(source.serialize({})); + } + + expect(() => fory.register(Type.struct(8104, {}))).toThrow(); + }, + ); + function testTypeInfo(typeinfo: TypeInfo, input: any, expected?: any) { const fory = new Fory({ compatible: false }); const serialize = fory.register(typeinfo); diff --git a/javascript/test/protocol/struct.test.ts b/javascript/test/protocol/struct.test.ts index 95cb26e2ca..4bbe8db3e6 100644 --- a/javascript/test/protocol/struct.test.ts +++ b/javascript/test/protocol/struct.test.ts @@ -60,7 +60,6 @@ describe("protocol", () => { }, ); const nonNullableSer = fory.register(nonNullable); - expect(() => nonNullableSer.serialize({ a: null })).toThrow(/Field "a" is not nullable/); // 2) nullable not specified => keep old behavior (null allowed) const nullableUnspecified = Type.struct( @@ -72,6 +71,7 @@ describe("protocol", () => { }, ); const { serialize, deserialize } = fory.register(nullableUnspecified); + expect(() => nonNullableSer.serialize({ a: null })).toThrow(); expect(deserialize(serialize({ a: null }))).toEqual({ a: null }); }); diff --git a/javascript/test/rootReadCleanup.test.ts b/javascript/test/rootReadCleanup.test.ts new file mode 100644 index 0000000000..9c64a4adf8 --- /dev/null +++ b/javascript/test/rootReadCleanup.test.ts @@ -0,0 +1,203 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +import Fory, { Type } from "../packages/core/index"; +import { TypeMeta } from "../packages/core/lib/meta/TypeMeta"; +import { describe, expect, test } from "@jest/globals"; + +function expectRootStateCleared(readContext: any) { + expect(readContext.refReader.readObjects).toHaveLength(0); + expect(readContext.metaStringReader.names).toHaveLength(0); + expect(readContext.typeMeta).toHaveLength(0); +} + +function populateLogicalTables(readContext: any, typeMeta: TypeMeta) { + readContext.metaStringReader.names.push("stale"); + readContext.typeMeta.push(typeMeta); +} + +describe.each([ + { + name: "Fory.deserialize", + invoke: (fory: Fory, registered: ReturnType, bytes: Uint8Array) => + fory.deserialize(bytes, registered.serializer), + }, + { + name: "registered deserialize", + invoke: (_fory: Fory, registered: ReturnType, bytes: Uint8Array) => + registered.deserialize(bytes), + }, +])("$name root cleanup", ({ invoke }) => { + test.each(["success", "failure"] as const)("clears generated root state after %s", (outcome) => { + const writerFory = new Fory({ compatible: true, ref: true }); + const readerFory = new Fory({ compatible: true, ref: true }); + const writer = writerFory.register( + Type.struct(7601, { + value: Type.int32().setId(1), + }), + ); + const reader = readerFory.register( + Type.struct(7601, { + value: Type.int32().setId(1), + }), + ); + const bytes = writer.serialize({ value: 7 }); + const input = outcome === "failure" ? bytes.subarray(0, bytes.length - 1) : bytes; + const read = () => invoke(readerFory, reader, input); + + if (outcome === "failure") { + expect(read).toThrow(); + } else { + expect(read()).toEqual({ value: 7 }); + } + + const readContext = (readerFory as any).readContext; + expectRootStateCleared(readContext); + }); + + test.each(["success", "failure"] as const)("clears logical tables after %s", (outcome) => { + const fory = new Fory({ compatible: true, ref: true }); + const registered = fory.register(Type.struct(7602, {})); + const readContext = (fory as any).readContext; + const typeMeta = TypeMeta.fromTypeInfo(Type.struct(7603, {})); + const headerHash = typeMeta.getHash(); + readContext.typeMetaCache.set(headerHash, typeMeta); + populateLogicalTables(readContext, typeMeta); + + registered.serializer.readRef = () => { + expectRootStateCleared(readContext); + populateLogicalTables(readContext, typeMeta); + if (outcome === "failure") { + throw new Error("root read failed"); + } + return 7; + }; + + const read = () => invoke(fory, registered, new Uint8Array([1])); + if (outcome === "failure") { + expect(read).toThrow(); + } else { + expect(read()).toBe(7); + } + + expectRootStateCleared(readContext); + expect(readContext.typeMetaCache.get(headerHash)).toBe(typeMeta); + }); +}); + +test("retains bounded read metadata", () => { + const fory = new Fory({ compatible: true }); + const readContext = (fory as any).readContext; + const typeMeta = TypeMeta.fromTypeInfo(Type.struct(7604, {})); + + readContext.typeMeta.push(...new Array(8192).fill(typeMeta)); + const bounded = readContext.typeMeta; + readContext.metaStringReader.names.push(...new Array(8192).fill("stale")); + const boundedNames = readContext.metaStringReader.names; + readContext.resetRootState(); + expect(readContext.typeMeta).toBe(bounded); + expect(readContext.metaStringReader.names).toBe(boundedNames); + expect(readContext.typeMeta).toHaveLength(0); + expect(readContext.metaStringReader.names).toHaveLength(0); + + readContext.typeMeta.push(...new Array(8193).fill(typeMeta)); + const oversized = readContext.typeMeta; + readContext.metaStringReader.names.push(...new Array(8193).fill("stale")); + const oversizedNames = readContext.metaStringReader.names; + readContext.resetRootState(); + expect(readContext.typeMeta).not.toBe(oversized); + expect(readContext.metaStringReader.names).not.toBe(oversizedNames); + expect(readContext.typeMeta).toHaveLength(0); + expect(readContext.metaStringReader.names).toHaveLength(0); +}); + +test("retains bounded write metadata", () => { + const fory = new Fory({ compatible: true }); + const writeContext = (fory as any).writeContext; + const typeMetaOwners = Array.from({ length: 8192 }, (_, dynamicTypeId) => ({ + dynamicTypeId, + })); + const metaStringOwners = Array.from({ length: 8192 }, (_, dynamicWriteStringId) => ({ + dynamicWriteStringId, + })); + + writeContext.disposeTypeMetaOwners.push(...typeMetaOwners); + const bounded = writeContext.disposeTypeMetaOwners; + writeContext.metaStringWriter.disposeMetaStringBytes.push(...metaStringOwners); + const boundedNames = writeContext.metaStringWriter.disposeMetaStringBytes; + writeContext.reset(); + expect(writeContext.disposeTypeMetaOwners).toBe(bounded); + expect(writeContext.metaStringWriter.disposeMetaStringBytes).toBe(boundedNames); + + writeContext.disposeTypeMetaOwners.push(...typeMetaOwners, { dynamicTypeId: 8192 }); + const oversized = writeContext.disposeTypeMetaOwners; + writeContext.metaStringWriter.disposeMetaStringBytes.push(...metaStringOwners, { + dynamicWriteStringId: 8192, + }); + const oversizedNames = writeContext.metaStringWriter.disposeMetaStringBytes; + writeContext.reset(); + expect(writeContext.disposeTypeMetaOwners).not.toBe(oversized); + expect(writeContext.metaStringWriter.disposeMetaStringBytes).not.toBe(oversizedNames); +}); + +test.each(["success", "failure"] as const)("clears root write state after %s", (outcome) => { + const fory = new Fory({ compatible: true, ref: true }); + const registered = fory.register(Type.struct(7606, {})); + const writeContext = (fory as any).writeContext; + const typeMeta = TypeMeta.fromTypeInfo(Type.struct(7607, {})); + const name = writeContext.metaStringWriter.encodeTypeName("FailedRoot"); + const value = {}; + + registered.serializer.writeRef = () => { + writeContext.refWriter.writeRef(value); + writeContext.metaStringWriter.writeBytes(writeContext.writer, name); + writeContext.writeTypeMeta(typeMeta, typeMeta.toBytes()); + if (outcome === "failure") { + throw new Error("root write failed"); + } + }; + + if (outcome === "failure") { + expect(() => registered.serialize(value)).toThrow("root write failed"); + } else { + expect(registered.serialize(value)).toBeDefined(); + } + expect(writeContext.refWriter.writeObjects.size).toBe(0); + expect(writeContext.metaStringWriter.disposeMetaStringBytes).toHaveLength(0); + expect(writeContext.disposeTypeMetaOwners).toHaveLength(0); + expect(name.dynamicWriteStringId).toBe(-1); + expect(typeMeta.dynamicTypeId).toBe(-1); + expect(writeContext.writer.writeGetCursor()).toBe(0); + expect(fory.serialize(7)).toBeDefined(); +}); + +test("releases a failed root write buffer", () => { + const fory = new Fory({ compatible: true }); + const registered = fory.register(Type.struct(7608, {})); + const writer = (fory as any).writeContext.writer; + + registered.serializer.writeRef = () => { + writer.buffer(new Uint8Array(4 * 1024 * 1024)); + throw new Error("root write failed"); + }; + + expect(() => registered.serialize({})).toThrow("root write failed"); + expect(writer.getPlatformBuffer().byteLength).toBeLessThan(4 * 1024 * 1024); + expect(writer.writeGetCursor()).toBe(0); +}); diff --git a/javascript/test/typemeta.test.ts b/javascript/test/typemeta.test.ts index effd162278..e0d0d555e4 100644 --- a/javascript/test/typemeta.test.ts +++ b/javascript/test/typemeta.test.ts @@ -337,7 +337,6 @@ describe("typemeta", () => { const writer = writerFory.register(writerRoot); const reader = readerFory.register(readerRoot); const childTypeMeta = TypeMeta.fromTypeInfo(writerChild, (writerFory as any).typeResolver); - const rootTypeMeta = TypeMeta.fromTypeInfo(writerRoot, (writerFory as any).typeResolver); const value = { child: { value: 9 } }; const valid = writer.serialize(value); const overwritten = replaceFirstBytes( @@ -347,11 +346,7 @@ describe("typemeta", () => { ); const readContext = (readerFory as any).readContext; - expect(() => reader.deserialize(overwritten)).toThrow( - "Invalid new TypeMeta index 0; expected 1", - ); - expect(readContext.typeMeta).toHaveLength(1); - expect(readContext.typeMeta[0].getHash()).toBe(rootTypeMeta.getHash()); + expect(() => reader.deserialize(overwritten)).toThrow(); expect(readContext.typeMetaCache.has(childTypeMeta.getHash())).toBe(false); expect(reader.deserialize(valid)).toEqual(value); }); @@ -514,7 +509,6 @@ describe("typemeta", () => { readContext.typeMetaCache.set(localTypeMeta.getHash(), cachedTypeMeta); expect(readerFory.deserialize(bytes, reader.serializer)).toBe(Color.Red); - expect(readContext.typeMeta[0]).toBe(localTypeMeta); expect(readContext.typeMetaCache.get(localTypeMeta.getHash())).toBe(cachedTypeMeta); expect(readContext.totalAcceptedSchemaVersions).toBe(0); }); @@ -815,7 +809,6 @@ describe("typemeta", () => { readContext.typeMetaCache.set(localTypeMeta.getHash(), cachedTypeMeta); expect(reader.deserialize(bytes)).toEqual({}); - expect(readContext.typeMeta[0]).toBe(localTypeMeta); expect(readContext.typeMetaCache.get(localTypeMeta.getHash())).toBe(cachedTypeMeta); expect(readContext.totalAcceptedSchemaVersions).toBe(0); }); @@ -1166,8 +1159,7 @@ describe("typemeta", () => { ); const readContext = (readerFory as any).readContext; - expect(() => reader.deserialize(wrongBytes)).toThrow("Compatible TypeMeta owner mismatch"); - expect(readContext.typeMeta).toHaveLength(1); + expect(() => reader.deserialize(wrongBytes)).toThrow(); expect(readContext.typeMetaCache.has(writerChildMeta.getHash())).toBe(false); expect(readContext.compatibleReadSerializers.has(writerChildMeta.getHash())).toBe(false); @@ -1175,8 +1167,7 @@ describe("typemeta", () => { value: 8, }); expect(readContext.typeMetaCache.has(writerChildMeta.getHash())).toBe(false); - expect(() => reader.deserialize(wrongBytes)).toThrow("Compatible TypeMeta owner mismatch"); - expect(readContext.typeMeta).toHaveLength(1); + expect(() => reader.deserialize(wrongBytes)).toThrow(); expect(readContext.compatibleReadSerializers.has(writerChildMeta.getHash())).toBe(false); const localChildType = Type.struct(readerChildId, { @@ -1222,12 +1213,7 @@ describe("typemeta", () => { first: { value: 1 }, second: { value: 2 }, }); - const readContext = (readerFory as any).readContext; - - expect(() => reader.deserialize(wrongBytes)).toThrow("Compatible TypeMeta owner mismatch"); - expect(readContext.typeMeta).toHaveLength(2); - expect(() => reader.deserialize(wrongBytes)).toThrow("Compatible TypeMeta owner mismatch"); - expect(readContext.typeMeta).toHaveLength(2); + expect(() => reader.deserialize(wrongBytes)).toThrow(); localWriterFory.register(Type.struct(writerChildId, childProps)); localWriterFory.register(Type.struct(readerChildId, childProps)); @@ -2177,16 +2163,18 @@ describe("typemeta", () => { expect(Array.from(result.values as Int32Array)).toEqual([0, 1, -1]); + const taggedWriterFory = new Fory({ compatible: true }); + const taggedReaderFory = new Fory({ compatible: true }); const taggedWriterType = Type.struct(7219, { values: Type.list(Type.int64({ encoding: "tagged" })).setId(1), }); const taggedReaderType = Type.struct(7219, { values: Type.int64Array().setId(1), }); - const taggedBytes = writerFory.register(taggedWriterType).serialize({ + const taggedBytes = taggedWriterFory.register(taggedWriterType).serialize({ values: [0n, 1n, -1n], }); - const taggedResult = readerFory.register(taggedReaderType).deserialize(taggedBytes); + const taggedResult = taggedReaderFory.register(taggedReaderType).deserialize(taggedBytes); expect(Array.from(taggedResult.values as BigInt64Array)).toEqual([0n, 1n, -1n]); }); From 8e8ee7367135864110dd0d72b999cf267a5b5b9f Mon Sep 17 00:00:00 2001 From: chaokunyang Date: Fri, 28 Aug 2026 01:23:43 +0800 Subject: [PATCH 006/168] fix(python): freeze registries and finalize configured types --- .agents/languages/python.md | 7 + .../python/basic-serialization.md | 26 +- .../python/configuration.md | 43 +- .../python/functions-classes-methods.md | 81 +-- docs/object-serialization/python/index.md | 51 +- docs/object-serialization/python/native.md | 33 +- .../python/numpy-integration.md | 76 +- .../python/out-of-band.md | 106 ++- .../python/schema-evolution.md | 20 +- docs/object-serialization/python/security.md | 23 +- .../python/serialization-hooks.md | 19 +- .../python/troubleshooting.md | 77 ++- .../python/type-registration.md | 7 + python/README.md | 640 ++++------------- python/pyfory/_fory.py | 48 +- python/pyfory/context.pxi | 14 +- python/pyfory/registry.py | 353 ++++++---- python/pyfory/serialization.pyx | 40 +- python/pyfory/serializer.py | 18 +- python/pyfory/struct.pxi | 14 +- python/pyfory/struct.py | 11 +- python/pyfory/tests/test_class_serializer.py | 65 +- python/pyfory/tests/test_collection_safety.py | 8 + python/pyfory/tests/test_function.py | 82 ++- .../pyfory/tests/test_graph_memory_budget.py | 22 +- .../pyfory/tests/test_metastring_resolver.py | 13 +- python/pyfory/tests/test_method.py | 61 +- python/pyfory/tests/test_pickle_buffer.py | 47 -- python/pyfory/tests/test_policy.py | 62 +- python/pyfory/tests/test_reduce_serializer.py | 36 +- python/pyfory/tests/test_ref_tracking.py | 2 + python/pyfory/tests/test_serializer.py | 647 +++++++++++++++++- .../tests/test_stateful_reproduction.py | 152 ---- .../pyfory/tests/test_stateful_serializer.py | 10 +- python/pyfory/tests/test_struct.py | 9 + 35 files changed, 1652 insertions(+), 1271 deletions(-) delete mode 100644 python/pyfory/tests/test_stateful_reproduction.py diff --git a/.agents/languages/python.md b/.agents/languages/python.md index aed074dfbf..7f55ae65ef 100644 --- a/.agents/languages/python.md +++ b/.agents/languages/python.md @@ -11,6 +11,13 @@ Load this file when changing `python/`, Cython serialization, or Python xlang be - Python mode is the pure-Python xlang implementation and is mainly for debugging and testing. - Cython mode is the default high-performance implementation. - Cython mode owns the hot runtime path. Do not duplicate core runtime types between Python and Cython, tunnel Python facade methods into hidden Cython internals, or keep dead shims unless the user explicitly needs a compatibility module path. +- Python `TypeResolver` owns registry freeze and finalization state. Its Cython companion may cache + completion of the one Python-owner dispatch needed to populate native resolver tables, but the + `Fory` facade must not mirror that state. Cython roots call the resolver owner directly. +- In non-strict native mode, public unqualified `register_type` for a built-in native carrier uses + the same reserved type identity as pre-root discovery. Ordinary application classes and + dataclasses retain their struct registration identity. Configure both through public registration; + do not prewarm private resolver state or enumerate version-specific transitive object shapes. - Use explicit Cython fields and methods for fixed hot-path shapes. Avoid `__getattr__`, generic `object` fields, public bridge internals, or `Fory` backreferences where ownership can stay explicit. - Keep Python and Cython context/ref-tracking branch conditions and stack mutations semantically aligned unless a documented intentional difference exists. - Root deserialization graph memory budget state belongs to pure-Python and Cython `ReadContext`. diff --git a/docs/object-serialization/python/basic-serialization.md b/docs/object-serialization/python/basic-serialization.md index b7245315d6..1c39ea5da8 100644 --- a/docs/object-serialization/python/basic-serialization.md +++ b/docs/object-serialization/python/basic-serialization.md @@ -41,6 +41,19 @@ print(obj) # {'name': 'Alice', 'age': 30, 'scores': [95, 87, 92]} **Note**: `dumps()`/`loads()` are aliases for `serialize()`/`deserialize()`. Both APIs are identical, use whichever feels more intuitive. +## Registration Lifecycle + +Register every application type before the first root serialization or deserialization attempt. In +Python native mode, also register any callable, class, method, state, or reduction carrier that can +appear in the graph. The first root serialization or deserialization attempt permanently freezes +the instance's registry, even when the operation fails. `strict=False` does not enable late type +discovery or registration. + +If the first operation fails because registration is incomplete or invalid, create a new instance, +register the complete type surface, and retry with that instance. A fully configured instance can +process a later root after a failure while reading input data or serializing a value. See +[Type Registration](type-registration.md) for the complete lifecycle. + ## Custom Class Serialization Use dataclasses and type annotations for stable xlang payloads: @@ -82,15 +95,14 @@ result = f.deserialize(data) assert result[0] is result[1] ``` -For arbitrary Python object graphs, local classes, functions, and methods, use +For configured Python-native object graphs, local classes, functions, and methods, use [Native Serialization](native.md). ## Performance Tips 1. **Disable `ref=True` if not needed**: Reference tracking has overhead -2. **Use type_id instead of name**: Integer IDs are faster than string names -3. **Reuse Fory instances**: Create once, use many times -4. **Enable Cython**: Make sure `ENABLE_FORY_CYTHON_SERIALIZATION=1` +2. **Reuse Fory instances**: Create once, use many times +3. **Enable Cython**: Make sure `ENABLE_FORY_CYTHON_SERIALIZATION=1` ```python # Good: Reuse instance @@ -288,7 +300,11 @@ Fory row-format schemas. ### Differences from Python Native Mode -The binary protocol and API are similar to `pyfory`'s Python native mode, but Python native mode can serialize any Python object—including global functions, local functions, lambdas, local classes, and types with custom serialization using `__getstate__/__reduce__/__reduce_ex__`, which are **not allowed** in xlang mode. +The binary protocol and API are similar to `pyfory`'s Python native mode, but native mode supports a +configured Python-only type surface that may include global functions, local functions, lambdas, +local classes, and types with custom serialization using `__getstate__`, `__reduce__`, or +`__reduce_ex__`. Register those application and carrier types before the first root attempt. These +Python-specific values are **not allowed** in xlang mode. ### Specifications and References diff --git a/docs/object-serialization/python/configuration.md b/docs/object-serialization/python/configuration.md index 04652322e7..5bc7f579ab 100644 --- a/docs/object-serialization/python/configuration.md +++ b/docs/object-serialization/python/configuration.md @@ -65,7 +65,7 @@ class ThreadSafeFory: | -------------------------------------- | ------------------------------- | ----------- | -------------------------------------------------------------------------------------------------------------------------------------------------------- | | `xlang` | `bool` | `True` | Use xlang mode. Set `False` for Python native mode. | | `ref` | `bool` | `False` | Enable reference tracking for shared/circular references. Disable for better performance if your data has no shared references. | -| `strict` | `bool` | `True` | Require registration before loading application classes. Compatible unknown Structs use the data-only `UnknownStruct` carrier. | +| `strict` | `bool` | `True` | Require registration before loading application classes. Compatible unknown Structs use `UnknownStruct`. `False` does not permit late discovery. | | `compatible` | `bool \| None` | `None` | Schema evolution mode. `None` enables compatible mode in both xlang and native mode. Set `False` only when every reader and writer uses the same schema. | | `max_depth` | `int` | `50` | Maximum deserialization depth for security, preventing stack overflow attacks. | | `max_type_fields` | `int` | `512` | Maximum fields accepted in one received remote struct metadata body. | @@ -99,17 +99,24 @@ fory.register(MyClass, name="my.package.MyClass") fory.register(MyClass, name="my.package.MyClass", serializer=custom_serializer) ``` +Complete registration before the first root serialization or deserialization attempt. That first +attempt permanently freezes registration, even if it fails. If the first operation exposes an +incomplete or invalid registration, create and configure a new instance before retrying. A fully +configured instance can process a later root after a failure while reading input data or serializing +a value. See +[Type Registration](type-registration.md) for the complete lifecycle. + ## Xlang And Native Mode Comparison -| Feature | Native mode (`xlang=False`) | Xlang mode (default) | -| ------------------- | ---------------------------------------------- | -------------------------------------------------------------------------------- | -| Use case | Python-only applications | Multi-language systems | -| Compatibility | Python only | Java, C++, Go, Rust, JavaScript/TypeScript, C#, Swift, Dart, Scala, Kotlin, etc. | -| Supported types | Python object surface | Cross-language compatible types | -| Functions/lambdas | Supported with trusted dynamic deserialization | Not allowed | -| Local classes | Supported with trusted dynamic deserialization | Not allowed | -| Dynamic classes | Supported with trusted dynamic deserialization | Not allowed | -| Schema mode default | Compatible | Compatible | +| Feature | Native mode (`xlang=False`) | Xlang mode (default) | +| ------------------- | -------------------------------------------- | -------------------------------------------------------------------------------- | +| Use case | Python-only applications | Multi-language systems | +| Compatibility | Python only | Java, C++, Go, Rust, JavaScript/TypeScript, C#, Swift, Dart, Scala, Kotlin, etc. | +| Supported types | Configured Python object surface | Cross-language compatible types | +| Functions/lambdas | Supported when their carriers are registered | Not allowed | +| Local classes | Supported when their carriers are registered | Not allowed | +| Class objects | Supported when their carriers are registered | Not allowed | +| Schema mode default | Compatible | Compatible | ## Xlang Mode @@ -128,15 +135,18 @@ Use `compatible=False` for xlang payloads only when every reader and writer alwa ## Native Mode ```python +import types + import pyfory fory = pyfory.Fory(xlang=False, ref=True, strict=False) +fory.register_type(types.FunctionType) ``` Native mode supports Python-specific object features such as functions, local classes, methods, -`__reduce__`, and `__getstate__`. Compatible mode is still enabled by default. Set -`compatible=False` only when every reader and writer always uses the same Python -class schema and you want faster serialization and smaller size. +`__reduce__`, and `__getstate__` when their application and carrier types are registered before the +first root attempt. Compatible mode is still enabled by default. Set `compatible=False` only when +every reader and writer always uses the same Python class schema. ## Compatible Mode @@ -163,9 +173,11 @@ fory = pyfory.Fory( fory.register(UserModel, name="example.User") ``` -### Native Mode With Dynamic Types +### Native Mode With Configured Python Types ```python +import types + import pyfory fory = pyfory.Fory( @@ -174,9 +186,12 @@ fory = pyfory.Fory( strict=False, max_depth=1000, ) + +fory.register_type(types.FunctionType) ``` Use `strict=False` only for trusted data, preferably with a `policy=` deserialization policy. +Register every application and Python-native carrier type before the first root attempt. ## Security diff --git a/docs/object-serialization/python/functions-classes-methods.md b/docs/object-serialization/python/functions-classes-methods.md index c4295b4442..e95f39554a 100644 --- a/docs/object-serialization/python/functions-classes-methods.md +++ b/docs/object-serialization/python/functions-classes-methods.md @@ -23,6 +23,10 @@ Python native mode serializes Python-specific callable and type values that are type system. Use `strict=False` only for trusted payloads and apply a deserialization policy when the accepted dynamic surface must be restricted. +Register every callable carrier and application type before the first root operation. The first +serialization or deserialization attempt permanently freezes the registry, even when it fails; +`strict=False` does not enable late discovery or registration. + ## Serialize Global Functions Capture and serialize functions defined at module level. Fory deserializes and returns the same @@ -30,12 +34,14 @@ function object: ```python import pyfory +import types fory = pyfory.Fory(xlang=False, ref=True, strict=False) def my_global_function(x): return 10 * x +fory.register_type(types.FunctionType) data = fory.dumps(my_global_function) print(fory.loads(data)(10)) # 100 ``` @@ -47,6 +53,7 @@ automatically: ```python import pyfory +import types fory = pyfory.Fory(xlang=False, ref=True, strict=False) @@ -57,6 +64,7 @@ def my_function(): return x * local_var return local_func +fory.register_type(types.FunctionType) data = fory.dumps(my_function()) print(fory.loads(data)(10)) # 100 @@ -65,80 +73,37 @@ data = fory.dumps(lambda x: 10 * x) print(fory.loads(data)(10)) # 100 ``` -## Serialize Global Classes/Methods +## Serialize Methods -Serialize class objects, instance methods, class methods, and static methods: +Register the method carriers and receiver class before serializing bound, class, or static methods: ```python -from dataclasses import dataclass import pyfory -fory = pyfory.Fory(xlang=False, ref=True, strict=False) +import types -@dataclass -class Person: - name: str - age: int +fory = pyfory.Fory(xlang=False, ref=True, strict=False) - def f(self, x): - return self.age * x +class Calculator: + def scale(self, x): + return 3 * x @classmethod - def g(cls, x): + def ten(cls, x): return 10 * x @staticmethod - def h(x): - return 10 * x + def double(x): + return 2 * x -# Serialize global class -print(fory.loads(fory.dumps(Person))("Bob", 25)) # Person(name='Bob', age=25) +for carrier in (type, types.FunctionType, types.MethodType, staticmethod, classmethod, Calculator): + fory.register_type(carrier) # Serialize instance method -print(fory.loads(fory.dumps(Person("Bob", 20).f))(10)) # 200 +print(fory.loads(fory.dumps(Calculator().scale))(10)) # 30 # Serialize class method -print(fory.loads(fory.dumps(Person.g))(10)) # 100 +print(fory.loads(fory.dumps(Calculator.ten))(10)) # 100 # Serialize static method -print(fory.loads(fory.dumps(Person.h))(10)) # 100 -``` - -## Serialize Local Classes/Methods - -Serialize classes defined inside functions along with their methods: - -```python -from dataclasses import dataclass -import pyfory -fory = pyfory.Fory(xlang=False, ref=True, strict=False) - -def create_local_class(): - class LocalClass: - def f(self, x): - return 10 * x - - @classmethod - def g(cls, x): - return 10 * x - - @staticmethod - def h(x): - return 10 * x - return LocalClass - -# Serialize local class -data = fory.dumps(create_local_class()) -print(fory.loads(data)().f(10)) # 100 - -# Serialize local class instance method -data = fory.dumps(create_local_class()().f) -print(fory.loads(data)(10)) # 100 - -# Serialize local class method -data = fory.dumps(create_local_class().g) -print(fory.loads(data)(10)) # 100 - -# Serialize local class static method -data = fory.dumps(create_local_class().h) -print(fory.loads(data)(10)) # 100 +print(fory.loads(fory.dumps(Calculator.double))(10)) # 20 ``` diff --git a/docs/object-serialization/python/index.md b/docs/object-serialization/python/index.md index cf05c7510a..6c7bad945c 100644 --- a/docs/object-serialization/python/index.md +++ b/docs/object-serialization/python/index.md @@ -19,7 +19,8 @@ license: | limitations under the License. --- -**Apache Fory™** is a blazing fast multi-language serialization framework powered by **JIT compilation** and **zero-copy** techniques, providing up to **ultra-fast performance** while maintaining ease of use and safety. +**Apache Fory™** is a multi-language serialization framework with Python-native and +cross-language object serialization modes. `pyfory` provides the Python implementation of Apache Fory™, offering xlang mode for cross-language payloads and native mode for Python-only object serialization. @@ -28,28 +29,28 @@ license: | ### Flexible Serialization Modes - **Xlang mode**: Default cross-language wire format with compatible schema evolution -- **Python native mode**: Same-language mode and drop-in replacement for pickle/cloudpickle +- **Python native mode**: Same-language mode for a configured Python type surface ### Versatile Serialization Features - **Reference tracking** for shared xlang schema objects and Python native-mode circular graphs -- **Polymorphism support** for customized types with automatic type dispatching +- **Polymorphism support** for registered customized types - **Schema evolution** support for backward/forward compatibility when using dataclasses in xlang mode -- **Out-of-band buffer support** for zero-copy serialization of large data structures like NumPy arrays and Pandas DataFrames, compatible with pickle protocol 5 +- **Out-of-band buffer support** for NumPy ndarrays and `pickle.PickleBuffer` values -### Blazing Fast Performance +### Python Runtime Support -- **Extremely fast performance** compared to other serialization frameworks -- **Runtime code generation** and **Cython-accelerated** core implementation for optimal performance +- **Runtime code generation** for registered data models +- **Cython-accelerated** core implementation ### Compact Data Size -- **Compact object graph protocol** with minimal space overhead—up to 3× size reduction compared to pickle/cloudpickle +- **Compact object graph protocol** - **Meta packing and sharing** to minimize type forward/backward compatibility space overhead ### Security & Safety -- **Strict mode** prevents deserialization of untrusted types by type registration and checks. +- **Strict mode** requires application type registration. - **Reference tracking** for handling circular references safely ## Installation @@ -82,10 +83,11 @@ pip install -e ".[dev]" `pyfory` provides `ThreadSafeFory` for thread-safe serialization using a pooled wrapper: ```python -import pyfory import threading from dataclasses import dataclass +import pyfory + @dataclass class Person: name: str @@ -111,8 +113,9 @@ for t in threads: t.join() - **Instance Pool**: Maintains a pool of `Fory` instances protected by a lock for thread safety - **Shared Configuration**: All registrations must be done upfront and are applied to all instances -- **Same API**: Drop-in replacement for `Fory` class with identical methods -- **Registration Safety**: Prevents registration after first use to ensure consistency +- **Matching Operations**: Exposes the corresponding root and registration methods +- **Registration Safety**: The first root attempt permanently freezes registration, even if the + operation fails **When to Use:** @@ -123,9 +126,10 @@ for t in threads: t.join() ## Quick Start ```python -import pyfory from dataclasses import dataclass +import pyfory + @dataclass class Person: name: str @@ -141,11 +145,28 @@ result = fory.deserialize(data) print(result) # Person(name='Alice', age=30) ``` +## Registration Lifecycle + +Register every application type before the first root serialization or deserialization attempt. In +native mode, also register callable, class, method, state, and reduction carrier types that can +appear in the object graph. The first root attempt permanently freezes the instance's registry, +including when that attempt fails. Setting `strict=False` does not permit discovery or registration +after that point. + +If the first operation exposes an incomplete or invalid registration, create a new instance and +register the complete type surface before retrying. A fully configured instance can process a later +root after a failure while reading input data or serializing a value. See +[Type Registration](type-registration.md) for the complete lifecycle. + ## Xlang Mode And Native Mode Use xlang mode for cross-language payloads and dataclass schemas shared with other Fory implementations. Xlang mode is the default Python wire mode, and Python examples that use it set `xlang=True` explicitly so the mode choice is visible. -Use native mode for Python-only traffic. Native mode is selected with `xlang=False` and owns pickle/cloudpickle-style behavior such as functions, lambdas, classes, methods, `__reduce__`, `__getstate__`, and out-of-band pickle protocol 5 buffers. It is optimized for Python's type system and supports a broader Python object surface than xlang mode, so use it when replacing pickle or cloudpickle. Compatible mode is enabled by default. Set `compatible=False` only when every reader and writer uses the same Python class schema and you want faster serialization and smaller size. +Use native mode for Python-only traffic. Native mode is selected with `xlang=False` and supports a +configured surface that may include functions, lambdas, classes, methods, `__reduce__`, +`__getstate__`, NumPy ndarrays, and out-of-band buffers. Register application types and Python-native +carrier types before the first root attempt. Compatible mode is enabled by default. Set +`compatible=False` only when every reader and writer uses the same Python class schema. See [Native Serialization](native.md) for Python-only serialization details and [Cross-Language Interoperability](basic-serialization.md#cross-language-interoperability) for Python xlang registration and interoperability rules. @@ -156,7 +177,7 @@ See [Native Serialization](native.md) for Python-only serialization details and - [Configuration](configuration.md) - Fory parameters, modes, and security - [Type Registration](type-registration.md) - User-defined type registration - [Custom Serializers](custom-serializers.md) - Extend serialization behavior -- [Row Format](../../row-format/python.md) - Zero-copy row format +- [Row Format](../../row-format/python.md) - Row-format APIs - [gRPC Support](../../grpc/python.md) - Fory payloads over grpcio ## Links diff --git a/docs/object-serialization/python/native.md b/docs/object-serialization/python/native.md index e77e24bbea..d781c9e176 100644 --- a/docs/object-serialization/python/native.md +++ b/docs/object-serialization/python/native.md @@ -50,9 +50,11 @@ import pyfory fory = pyfory.Fory(xlang=False, ref=False, strict=True) ``` -Keep `strict=True` for registered, trusted type surfaces. Use `strict=False` only when native-mode -payloads need dynamic Python types such as functions, local classes, or objects reconstructed by -reduction hooks. +Keep `strict=True` for registered, trusted type surfaces. Use `strict=False` only when the configured +surface includes native carriers such as functions, local classes, or objects reconstructed by +reduction hooks. In either mode, register every application type and native carrier before the first +root serialization or deserialization attempt. The first attempt permanently freezes the instance's +registry even when it fails; `strict=False` does not enable late discovery or registration. ## Common Usage @@ -61,9 +63,6 @@ import pyfory fory = pyfory.Fory(xlang=False, ref=True, strict=False) -data = fory.dumps({"name": "Alice", "age": 30, "scores": [95, 87, 92]}) -print(fory.loads(data)) - from dataclasses import dataclass @dataclass @@ -71,6 +70,11 @@ class Person: name: str age: int +fory.register_type(Person) + +data = fory.dumps({"name": "Alice", "age": 30, "scores": [95, 87, 92]}) +print(fory.loads(data)) + person = Person("Bob", 25) data = fory.dumps(person) print(fory.loads(data)) # Person(name='Bob', age=25) @@ -86,9 +90,10 @@ deserialization. Treat untrusted native-mode bytes the same way you would treat bytes. - Keep `strict=True` when deserializing data that should contain only registered or built-in types. -- Use `strict=False` only for trusted payloads that require dynamic Python classes or functions. -- Provide a `policy=` deserialization policy when dynamic types are required but the accepted type - surface should still be restricted. +- Use `strict=False` only for trusted payloads that require configured Python class or function + carriers. +- Provide a `policy=` deserialization policy when native carriers are required but the accepted + type surface should still be restricted. - Do not use xlang/native mode choice as a security control. Apply strict mode, policies, registration, and resource limits based on the payload source. @@ -144,9 +149,9 @@ Use this when the payload stays in Python and large buffers should avoid extra c | ------------------------------------------ | ------------------------ | ----------------------- | | Python-only payloads | Yes | Optional | | Non-Python readers or writers | No | Yes | -| Functions, lambdas, local classes | Yes | No | +| Registered functions, methods, and classes | Yes | No | | `__reduce__` / `__getstate__` object hooks | Yes | No | -| Pickle/cloudpickle replacement | Yes | No | +| Configured pickle-style workloads | Yes | No | | Portable type mapping across languages | No | Yes | ## Performance Comparison @@ -172,8 +177,10 @@ on every peer, and avoid Python-only values such as lambdas or local classes. ### A dynamic class or function fails to deserialize -Use `strict=False` for trusted payloads and provide a deserialization `policy=` when only selected -dynamic types should be accepted. +Before the first root operation, register the application type and the native carrier types needed +by the payload. Use `strict=False` for trusted payloads and provide a deserialization `policy=` when +only selected dynamic types should be accepted. Create a new configured `Fory` instance if an +already-used instance needs a different type surface. ### A cycle does not round-trip diff --git a/docs/object-serialization/python/numpy-integration.md b/docs/object-serialization/python/numpy-integration.md index 8188696ace..f0656fd584 100644 --- a/docs/object-serialization/python/numpy-integration.md +++ b/docs/object-serialization/python/numpy-integration.md @@ -1,5 +1,5 @@ --- -title: NumPy & Pandas +title: NumPy sidebar_position: 12 id: numpy-integration license: | @@ -19,85 +19,61 @@ license: | limitations under the License. --- -Fory natively supports numpy arrays and pandas DataFrame with optimized serialization. +Python native mode supports NumPy ndarrays as built-in values. ## NumPy Array Serialization -Large arrays use zero-copy when possible: +Serialize and deserialize an ndarray directly: ```python -import pyfory import numpy as np +import pyfory -f = pyfory.Fory(xlang=False) +fory = pyfory.Fory(xlang=False) # Numpy arrays are supported natively arrays = { - 'matrix': np.random.rand(1000, 1000), - 'vector': np.arange(10000), - 'bool_mask': np.random.choice([True, False], size=5000) + "matrix": np.arange(12, dtype=np.float64).reshape(3, 4), + "vector": np.arange(10, dtype=np.int64), + "bool_mask": np.array([True, False, True]), } -data = f.serialize(arrays) -result = f.deserialize(data) +data = fory.serialize(arrays) +result = fory.deserialize(data) -# Zero-copy for compatible array types -assert np.array_equal(arrays['matrix'], result['matrix']) +assert np.array_equal(arrays["matrix"], result["matrix"]) ``` -## Pandas DataFrames - -Fory can serialize Pandas DataFrames efficiently: - -```python -import pyfory -import pandas as pd -import numpy as np - -f = pyfory.Fory(xlang=False, ref=False, strict=False) +The ndarray carrier itself is available when the instance is created. Register application types +that contain ndarrays, plus every custom type that can appear inside an object-dtype ndarray, before +the first root attempt. The first root attempt permanently freezes registration, including when it +fails. `strict=False` does not permit late discovery or registration. -df = pd.DataFrame({ - 'a': np.arange(1000, dtype=np.float64), - 'b': np.arange(1000, dtype=np.int64), - 'c': ['text'] * 1000 -}) +## Out-of-Band Buffers -data = f.serialize(df) -result = f.deserialize(data) - -assert df.equals(result) -``` - -## Zero-Copy with Out-of-Band Buffers - -For maximum performance with large arrays, use out-of-band serialization: +Use a buffer callback to transport ndarray storage separately from the root bytes: ```python -import pyfory import numpy as np +import pyfory -f = pyfory.Fory(xlang=False, ref=False, strict=False) +fory = pyfory.Fory(xlang=False, ref=False) -# Large array -array = np.random.rand(10000, 1000) +array = np.arange(10000, dtype=np.float64).reshape(100, 100) -# Out-of-band for zero-copy buffer_objects = [] -data = f.serialize(array, buffer_callback=buffer_objects.append) +data = fory.serialize(array, buffer_callback=buffer_objects.append) buffers = [obj.getbuffer() for obj in buffer_objects] -result = f.deserialize(data, buffers=buffers) +result = fory.deserialize(data, buffers=buffers) assert np.array_equal(array, result) ``` -## Supported Array Types - -- `np.ndarray` (all dtypes) -- `np.matrix` -- Structured arrays -- Record arrays +For a contiguous ndarray, `getbuffer()` can expose the existing storage as a `memoryview`. A +non-contiguous ndarray may be copied to create a contiguous transport buffer. The application must +send all collected buffers with the root bytes and provide them to `deserialize` in the same order. ## Related Topics -- [Out-of-Band Serialization](out-of-band.md) - Zero-copy buffers +- [Out-of-Band Serialization](out-of-band.md) - Buffer callback APIs - [Basic Serialization](basic-serialization.md) - Standard usage diff --git a/docs/object-serialization/python/out-of-band.md b/docs/object-serialization/python/out-of-band.md index 8db222fecb..ab7034a3ba 100644 --- a/docs/object-serialization/python/out-of-band.md +++ b/docs/object-serialization/python/out-of-band.md @@ -19,24 +19,31 @@ license: | limitations under the License. --- -Fory supports pickle5-compatible out-of-band buffer serialization for efficient zero-copy handling of large data structures. +Fory can separate supported binary storage from the main serialized bytes through an out-of-band +buffer callback. Python native mode supports this flow for NumPy ndarrays and +`pickle.PickleBuffer` values. ## Overview -Out-of-band serialization separates metadata from the actual data buffers, allowing for: +Out-of-band serialization separates the Fory root bytes from selected buffers: -- **Zero-copy transfers** when sending data over networks or IPC using `memoryview` -- **Improved performance** for large datasets -- **Pickle5 compatibility** using `pickle.PickleBuffer` -- **Flexible stream support** - write to any writable object (files, BytesIO, sockets, etc.) +- `BufferObject.getbuffer()` exposes a `memoryview`; contiguous NumPy storage can be exposed without + an additional copy. +- The application transports the root bytes and out-of-band buffers together and in order. +- `BufferObject.write_to()` writes a selected buffer to a writable stream. + +`numpy.ndarray` and `pickle.PickleBuffer` are built-in native types. If an application wrapper or +an object-dtype ndarray can contain custom values, register every application and Python-native +carrier type before the first root attempt. That first attempt permanently freezes registration, +including when it fails; `strict=False` does not permit late discovery. ## Basic Out-of-Band Serialization ```python -import pyfory import numpy as np +import pyfory -fory = pyfory.Fory(xlang=False, ref=False, strict=False) +fory = pyfory.Fory(xlang=False, ref=False) # Large numpy array array = np.arange(10000, dtype=np.float64) @@ -45,9 +52,7 @@ array = np.arange(10000, dtype=np.float64) buffer_objects = [] serialized_data = fory.serialize(array, buffer_callback=buffer_objects.append) -# Convert buffer objects to memoryview for zero-copy transmission -# For contiguous buffers (bytes, numpy arrays), this is zero-copy -# For non-contiguous data, a copy may be created to ensure contiguity +# Convert collected buffer objects to memoryviews for transport. buffers = [obj.getbuffer() for obj in buffer_objects] # Deserialize with out-of-band buffers (accepts memoryview, bytes, or Buffer) @@ -56,73 +61,48 @@ deserialized_array = fory.deserialize(serialized_data, buffers=buffers) assert np.array_equal(array, deserialized_array) ``` -## Out-of-Band with Pandas DataFrames - -```python -import pyfory -import pandas as pd -import numpy as np - -fory = pyfory.Fory(xlang=False, ref=False, strict=False) - -# Create a DataFrame with numeric columns -df = pd.DataFrame({ - 'a': np.arange(1000, dtype=np.float64), - 'b': np.arange(1000, dtype=np.int64), - 'c': ['text'] * 1000 -}) - -# Serialize with out-of-band buffers -buffer_objects = [] -serialized_data = fory.serialize(df, buffer_callback=buffer_objects.append) -buffers = [obj.getbuffer() for obj in buffer_objects] - -# Deserialize -deserialized_df = fory.deserialize(serialized_data, buffers=buffers) - -assert df.equals(deserialized_df) -``` - ## Selective Out-of-Band Serialization Control which buffers go out-of-band by providing a callback that returns `True` to keep data in-band or `False` to send it out-of-band: ```python -import pyfory import numpy as np +import pyfory -fory = pyfory.Fory(xlang=False, ref=True, strict=False) +fory = pyfory.Fory(xlang=False, ref=True) arr1 = np.arange(1000, dtype=np.float64) arr2 = np.arange(2000, dtype=np.float64) data = [arr1, arr2] buffer_objects = [] -counter = 0 def selective_callback(buffer_object): - global counter - counter += 1 - # Only send even-numbered buffers out-of-band - if counter % 2 == 0: + # Send buffers of at least 12,000 bytes out-of-band. + if buffer_object.total_bytes() >= 12_000: buffer_objects.append(buffer_object) - return False # Out-of-band - return True # In-band + return False + return True serialized = fory.serialize(data, buffer_callback=selective_callback) buffers = [obj.getbuffer() for obj in buffer_objects] deserialized = fory.deserialize(serialized, buffers=buffers) + +assert np.array_equal(arr1, deserialized[0]) +assert np.array_equal(arr2, deserialized[1]) ``` -## Pickle5 Compatibility +## `pickle.PickleBuffer` Values -Fory's out-of-band serialization is fully compatible with pickle protocol 5: +Python native mode accepts `pickle.PickleBuffer` as a built-in value. The outer bytes remain Fory +native bytes; they are not Pickle wire data. ```python -import pyfory import pickle -fory = pyfory.Fory(xlang=False, ref=False, strict=False) +import pyfory + +fory = pyfory.Fory(xlang=False, ref=False) # PickleBuffer objects are automatically supported data = b"Large binary data" @@ -138,16 +118,17 @@ deserialized = fory.deserialize(serialized, buffers=buffers) assert bytes(deserialized.raw()) == data ``` -## Writing Buffers to Different Streams +## Writing A Buffer To A Stream -The `BufferObject.write_to()` method accepts any writable stream object: +The `BufferObject.write_to()` method accepts a writable stream object: ```python -import pyfory -import numpy as np import io -fory = pyfory.Fory(xlang=False, ref=False, strict=False) +import numpy as np +import pyfory + +fory = pyfory.Fory(xlang=False, ref=False) array = np.arange(1000, dtype=np.float64) @@ -155,22 +136,17 @@ array = np.arange(1000, dtype=np.float64) buffer_objects = [] serialized = fory.serialize(array, buffer_callback=buffer_objects.append) -# Write to different stream types +# Write to an in-memory stream and obtain a memoryview. for buffer_obj in buffer_objects: - # Write to BytesIO (in-memory stream) bytes_stream = io.BytesIO() buffer_obj.write_to(bytes_stream) - - # Write to file - with open('/tmp/buffer_data.bin', 'wb') as f: - buffer_obj.write_to(f) - - # Get zero-copy memoryview (for contiguous buffers) + assert bytes_stream.getvalue() == array.tobytes() mv = buffer_obj.getbuffer() assert isinstance(mv, memoryview) ``` -**Note**: For contiguous memory buffers (like bytes, numpy arrays), `getbuffer()` returns a zero-copy `memoryview`. For non-contiguous data, a copy may be created to ensure contiguity. +For a contiguous NumPy ndarray, `getbuffer()` can expose the existing storage. A non-contiguous +array may be copied to produce a contiguous transport buffer. ## Related Topics diff --git a/docs/object-serialization/python/schema-evolution.md b/docs/object-serialization/python/schema-evolution.md index a8927c92c0..8ebf0ba2f2 100644 --- a/docs/object-serialization/python/schema-evolution.md +++ b/docs/object-serialization/python/schema-evolution.md @@ -60,25 +60,27 @@ class SlotMessage: import pyfory from dataclasses import dataclass -# Version 1: Original class +# Version 1: Writer schema @dataclass -class User: +class UserV1: name: str age: pyfory.Int32 -f = pyfory.Fory(xlang=True) -f.register(User, name="User") -data = f.dumps(User("Alice", 30)) +writer = pyfory.Fory(xlang=True) +writer.register(UserV1, name="User") +data = writer.dumps(UserV1("Alice", 30)) -# Version 2: Add new field (backward compatible) +# Version 2: Reader schema with a new field @dataclass -class User: +class UserV2: name: str age: pyfory.Int32 email: str = "unknown@example.com" # New field with default -# Can still deserialize old data -user = f.loads(data) +# Register the reader schema on a separate instance. +reader = pyfory.Fory(xlang=True) +reader.register(UserV2, name="User") +user = reader.loads(data) print(user.email) # "unknown@example.com" ``` diff --git a/docs/object-serialization/python/security.md b/docs/object-serialization/python/security.md index df3c5cba6b..0942c0196d 100644 --- a/docs/object-serialization/python/security.md +++ b/docs/object-serialization/python/security.md @@ -31,13 +31,15 @@ Before deserialization: - Enforce request or file size, timeout, and concurrency limits outside Fory. - Register only the application types the endpoint accepts and configure the reader before its first root operation. +- In native mode, register every callable, class, method, state, or reduction carrier that an + accepted graph may contain. - Validate the deserialized value against application authorization and domain rules before use. ## Built-in safeguards Treat native-mode bytes from untrusted sources the same way you would treat untrusted pickle bytes. -Native mode can reconstruct Python objects, import modules, invoke reduction hooks, and rebuild -dynamic classes or functions when `strict=False`. +Within the configured type surface, native mode can reconstruct Python objects, import modules, +invoke reduction hooks, and rebuild classes or functions when `strict=False`. ### Production Configuration @@ -63,9 +65,11 @@ fory.register(UserModel, name="example.User") fory.register(OrderModel, name="example.Order") ``` -Use dynamic native-mode deserialization (`strict=False`) only for trusted Python-only payloads: +Use native-mode deserialization with `strict=False` only for trusted Python-only payloads: ```python +import types + import pyfory fory = pyfory.Fory( @@ -74,8 +78,15 @@ fory = pyfory.Fory( strict=False, max_depth=100, ) + +fory.register_type(types.FunctionType) ``` +The first root attempt permanently freezes registration, including when that attempt fails. +`strict=False` does not allow late discovery or registration. If that first operation exposes an +incomplete or invalid registration, create and configure a new instance before retrying. A fully +configured reader can process a later root after a malformed-data failure. + Received remote metadata is also limited: - `max_type_fields` limits the number of fields accepted in one received struct metadata body. @@ -99,8 +110,8 @@ schema-evolution semantics. ### DeserializationPolicy -When `strict=False` is necessary, use `DeserializationPolicy` to restrict the dynamic types and -hooks accepted during deserialization: +When `strict=False` is necessary, use `DeserializationPolicy` to restrict the types and hooks +accepted during deserialization: ```python import pyfory @@ -146,7 +157,7 @@ unchanged. ### Security Checklist - Keep `strict=True` for untrusted data. -- Register all expected application types before deserialization. +- Register all expected application and Python-native carrier types before the first root attempt. - Use `DeserializationPolicy` when `strict=False` is necessary. - Keep `max_depth` low enough to reject unexpectedly deep payloads. - Keep `max_graph_memory_bytes` at the fixed `128 MiB` default for most inputs, or set a positive diff --git a/docs/object-serialization/python/serialization-hooks.md b/docs/object-serialization/python/serialization-hooks.md index 009f3a41ad..4eeda78200 100644 --- a/docs/object-serialization/python/serialization-hooks.md +++ b/docs/object-serialization/python/serialization-hooks.md @@ -20,14 +20,15 @@ license: | --- Python native mode honors Python object customization protocols while writing Fory native bytes. -It does not emit Pickle wire data. Use this page when replacing pickle or cloudpickle or when a -class controls its reduction, construction, or state restoration. +It does not emit Pickle wire data. Use this page when a class controls its reduction, construction, +or state restoration. -## Pickle And Cloudpickle Replacement +## When To Use Native Mode -Native mode is the Python mode to choose when the existing boundary uses `pickle` or -`cloudpickle`. It supports richer Python values than JSON and xlang mode, including Python -functions, local classes, closures, and reduction hooks. +Native mode supports a configured Python-only type surface that may include Python functions, local +classes, closures, and reduction hooks. Register every application type and Python-native carrier +before the first root attempt. The first attempt permanently freezes registration, even if it +fails, and `strict=False` does not permit late discovery. Use xlang mode instead when the payload crosses language boundaries or the data model should be a portable schema shared with other Fory implementations. @@ -49,13 +50,15 @@ class SessionToken: def __setstate__(self, state): self.value = state["value"] -fory = pyfory.Fory(xlang=False, strict=False) +fory = pyfory.Fory(xlang=False, ref=True, strict=False, compatible=False) +fory.register_type(SessionToken) token = fory.loads(fory.dumps(SessionToken("abc"))) print(token.value) # abc ``` Use these hooks for Python-only payloads. For xlang payloads, model the data as dataclasses with -portable field annotations instead. +portable field annotations instead. Complete all registration before the first root operation, as +described in [Type Registration](type-registration.md). ## Protocol 5 buffers diff --git a/docs/object-serialization/python/troubleshooting.md b/docs/object-serialization/python/troubleshooting.md index 11928d08c5..e96196e265 100644 --- a/docs/object-serialization/python/troubleshooting.md +++ b/docs/object-serialization/python/troubleshooting.md @@ -62,16 +62,23 @@ object identity or cycles matter: f = pyfory.Fory(ref=True) ``` -For arbitrary Python object graphs with circular references, use Python native mode: +For configured Python-native object graphs with circular references, use Python native mode: ```python +from dataclasses import dataclass +from typing import Optional + +import pyfory + f = pyfory.Fory(xlang=False, ref=True, strict=False) # Example with circular reference +@dataclass class Node: - def __init__(self, value): - self.value = value - self.next = None + value: int + next: Optional["Node"] = pyfory.field(ref=True, nullable=True, default=None) + +f.register_type(Node) node1 = Node(1) node2 = Node(2) @@ -86,27 +93,30 @@ assert result.next.next is result # Circular reference preserved ### Schema Evolution Not Working ```python -# Keep compatible mode enabled. This is the default. -f = pyfory.Fory() +from dataclasses import dataclass + +import pyfory # Version 1: Original class @dataclass -class User: +class UserV1: name: str age: pyfory.Int32 -f.register(User, name="User") -data = f.dumps(User("Alice", 30)) +writer = pyfory.Fory(xlang=True) +writer.register(UserV1, name="example.User") +data = writer.dumps(UserV1("Alice", 30)) # Version 2: Add new field (backward compatible) @dataclass -class User: +class UserV2: name: str age: pyfory.Int32 email: str = "unknown@example.com" # New field with default -# Can still deserialize old data -user = f.loads(data) +reader = pyfory.Fory(xlang=True) +reader.register(UserV2, name="example.User") +user = reader.loads(data) print(user.email) # "unknown@example.com" ``` @@ -120,10 +130,16 @@ f = pyfory.Fory(strict=True) f.register(MyClass, type_id=100) f.register(AnotherClass, type_id=101) -# Or disable strict mode (NOT recommended for production) -f = pyfory.Fory(strict=False) # Use only in trusted environments +# Native mode may use strict=False only for trusted data, but application +# and Python-native carrier types still must be registered before use. +native_fory = pyfory.Fory(xlang=False, strict=False) +native_fory.register_type(MyClass) ``` +The first root attempt permanently freezes registration, even when it fails. Do not register a +missing type and retry on that same instance. Create a new instance, register the complete type +surface, and retry with the new instance. + ## Debug Mode Set environment variable BEFORE importing pyfory to disable Cython for debugging: @@ -144,28 +160,33 @@ import pyfory # Now uses pure Python implementation Handle common serialization errors gracefully: ```python +from dataclasses import dataclass + import pyfory -from pyfory.error import TypeUnregisteredError, TypeNotCompatibleError +from pyfory.error import TypeUnregisteredError -fory = pyfory.Fory(strict=True) +@dataclass +class Message: + text: str +message = Message("hello") +unconfigured = pyfory.Fory(xlang=False, strict=True, compatible=False) try: - data = fory.dumps(my_object) + unconfigured.dumps(message) except TypeUnregisteredError as e: print(f"Type not registered: {e}") - # Register the type and retry - fory.register(type(my_object), type_id=100) - data = fory.dumps(my_object) -except Exception as e: - print(f"Serialization failed: {e}") + # The failed instance is already frozen. Configure a new one. + fory = pyfory.Fory(xlang=False, strict=True, compatible=False) + fory.register_type(Message, type_id=100) + data = fory.dumps(message) try: - obj = fory.loads(data) -except TypeNotCompatibleError as e: - print(f"Schema mismatch: {e}") - # Handle version mismatch -except Exception as e: - print(f"Deserialization failed: {e}") + fory.loads(b"") +except Exception: + pass + +# Root cleanup makes the configured instance reusable after the failed read. +assert fory.loads(data) == message ``` ## Development Setup diff --git a/docs/object-serialization/python/type-registration.md b/docs/object-serialization/python/type-registration.md index 917537c767..48379553ab 100644 --- a/docs/object-serialization/python/type-registration.md +++ b/docs/object-serialization/python/type-registration.md @@ -79,6 +79,13 @@ classes. Register application classes before serializing or deserializing payloads, and keep the same registration IDs or names on every peer that shares those payloads. +The first root serialization or deserialization attempt permanently freezes the +instance's registry, including when that attempt fails. `strict=False` relaxes +the deserialization policy for registered Python-native carriers; it does not +permit late discovery or registration. Register callable, class, method, and +reduction carriers together with the application types that can appear in the +graph before the first root operation. + Compatible metadata has one data-only exception: when a remote Struct has no local registration, deserialization returns the fixed framework `pyfory.UnknownStruct` value instead of loading or generating the sender's diff --git a/python/README.md b/python/README.md index b26faf5195..593aed98a9 100644 --- a/python/README.md +++ b/python/README.md @@ -7,34 +7,33 @@ [![Slack Channel](https://img.shields.io/badge/slack-join-3f0e40?logo=slack&style=for-the-badge)](https://join.slack.com/t/fory-project/shared_invite/zt-36g0qouzm-kcQSvV_dtfbtBKHRwT5gsw) [![X](https://img.shields.io/badge/@ApacheFory-follow-blue?logo=x&style=for-the-badge)](https://x.com/ApacheFory) -**Apache Fory™** is a blazing fast multi-language serialization framework powered by **JIT compilation** and **zero-copy** techniques, providing up to **ultra-fast performance** while maintaining ease of use and safety. - -`pyfory` provides the Python implementation of Apache Fory™, offering both high-performance object serialization and advanced row-format capabilities for data processing tasks. +`pyfory` is the Python implementation of Apache Fory™. It provides Python-native and cross-language +object serialization together with row-format APIs for analytical data. ## Key Features ### **Flexible Serialization Modes** - **Xlang mode**: Default cross-language wire format with compatible schema evolution -- **Python native mode**: Same-language mode and drop-in replacement for pickle/cloudpickle -- **Row Format**: Zero-copy row format for analytics workloads +- **Python native mode**: Same-language mode for configured Python type surfaces +- **Row Format**: Random and partial access to analytical row data ### Versatile Serialization Features - **Shared/circular reference support** for complex object graphs in both Python native and xlang modes - **Polymorphism support** for customized types with automatic type dispatching - **Schema evolution** support for backward/forward compatibility when using dataclasses in xlang mode -- **Out-of-band buffer support** for zero-copy serialization of large data structures like NumPy arrays and Pandas DataFrames, compatible with pickle protocol 5 +- **Out-of-band buffer support** for NumPy arrays and `pickle.PickleBuffer` values - **Reduced-precision xlang types** use reserved `pyfory.Float16` and `pyfory.BFloat16` annotations and native Python `float` values; dense array payloads use public wrappers such as `Float16Array` and `BFloat16Array` -### Blazing Fast Performance +### Python Runtime Implementation -- **Extremely fast performance** compared to other serialization frameworks -- **Runtime code generation** and **Cython-accelerated** core implementation for optimal performance +- **Runtime code generation** for supported Python classes +- **Cython-accelerated** core implementation ### Compact Data Size -- **Compact object graph protocol** with minimal space overhead—up to 3× size reduction compared to pickle/cloudpickle +- **Compact object graph protocol** for Python-native and cross-language payloads - **Meta packing and sharing** to minimize type forward/backward compatibility space overhead ### **Security & Safety** @@ -71,11 +70,12 @@ pip install -e ".[dev,format]" ## Python Native Serialization -`pyfory` provides a Python native mode for Python-only payloads. It is optimized for Python's type -system and offers the same object surface as pickle/cloudpickle, but with **significantly better -performance, smaller data size, and enhanced security features**. +`pyfory` provides a Python native mode for configured Python-only payloads, with support for +functions, methods, dataclasses, stateful types, and reduction hooks. -The binary protocol and API are similar to Fory's xlang mode, but Python native mode can serialize any Python object—including global functions, local functions, lambdas, local classes and types with custom serialization using `__getstate__/__reduce__/__reduce_ex__`, which are not allowed in xlang mode. +Register every application type and native carrier before the first root operation. The first +serialization or deserialization attempt permanently freezes that `Fory` instance's registry, +including when the attempt fails. To use Python native mode, create `Fory` with `xlang=False`. Use this mode when replacing pickle or cloudpickle for pure Python applications: @@ -131,18 +131,26 @@ result = fory.deserialize(data) print(result) # Person(name='Bob', age=25, ...) ``` -## Drop-in Replacement for Pickle/Cloudpickle +## Pickle-Style Python-Native Serialization -`pyfory` can serialize any Python object with the following configuration: +`pyfory` can serialize a configured Python type surface with the following options: - **For circular references**: Set `ref=True` to enable reference tracking -- **For functions/classes**: Set `strict=False` to allow deserialization of dynamic types +- **For Python-native carriers**: Set `strict=False` only for trusted payloads + +Register every application type and native carrier type before the first root serialization or +deserialization call. The first root operation permanently freezes that `Fory` instance's registry, +even when the operation fails. `strict=False` relaxes deserialization policy; it does not permit late +type discovery or registration. If the first operation exposes incomplete or invalid registration, +configure a new instance before retrying. -**Security Warning**: When `strict=False`, Fory will deserialize arbitrary types, which can pose security risks if data comes from untrusted sources. Only use `strict=False` in controlled environments where you trust the data source completely. If you do need to use `strict=False`, please configure a `DeserializationPolicy` when creating fory using `policy=your_policy` to controlling deserialization behavior. +**Security Warning**: Configured native carriers can import modules and construct Python objects +when `strict=False`. Use this mode only with trusted payloads, and provide a +`DeserializationPolicy` through `policy=` when the accepted surface must be restricted. ### Common Usage -Serialize common Python objects including dicts, lists, and custom classes without any registration: +Built-in containers require no registration. Register custom classes before the first root: ```python import pyfory @@ -150,11 +158,6 @@ import pyfory # Create Fory instance fory = pyfory.Fory(xlang=False, ref=True, strict=False) -# serialize common Python objects -data = fory.dumps({"name": "Alice", "age": 30, "scores": [95, 87, 92]}) -print(fory.loads(data)) - -# serialize custom objects from dataclasses import dataclass @dataclass @@ -162,6 +165,13 @@ class Person: name: str age: int +fory.register_type(Person) + +# serialize common Python objects +data = fory.dumps({"name": "Alice", "age": 30, "scores": [95, 87, 92]}) +print(fory.loads(data)) + +# serialize custom objects person = Person("Bob", 25) data = fory.dumps(person) print(fory.loads(data)) # Person(name='Bob', age=25) @@ -173,6 +183,7 @@ Capture and get functions defined at module level. Fory deserialize and return s ```python import pyfory +import types # Create Fory instance fory = pyfory.Fory(xlang=False, ref=True, strict=False) @@ -181,6 +192,7 @@ fory = pyfory.Fory(xlang=False, ref=True, strict=False) def my_global_function(x): return 10 * x +fory.register_type(types.FunctionType) data = fory.dumps(my_global_function) print(fory.loads(data)(10)) # 100 ``` @@ -191,6 +203,7 @@ Serialize functions with closures and lambda expressions. Fory captures the clos ```python import pyfory +import types # Create Fory instance fory = pyfory.Fory(xlang=False, ref=True, strict=False) @@ -202,6 +215,7 @@ def my_function(): return x * local_var return local_func +fory.register_type(types.FunctionType) data = fory.dumps(my_function()) print(fory.loads(data)(10)) # 100 @@ -210,239 +224,54 @@ data = fory.dumps(lambda x: 10 * x) print(fory.loads(data)(10)) # 100 ``` -#### Serialize Global Classes/Methods +#### Serialize Methods -Serialize class objects, instance methods, class methods, and static methods. All method types are supported: +Register method carriers and receiver classes before serializing bound, class, or static methods: ```python -from dataclasses import dataclass import pyfory -fory = pyfory.Fory(xlang=False, ref=True, strict=False) +import types -# serialize global class -@dataclass -class Person: - name: str - age: int +fory = pyfory.Fory(xlang=False, ref=True, strict=False) - def f(self, x): - return self.age * x +class Calculator: + def scale(self, x): + return 3 * x @classmethod - def g(cls, x): + def ten(cls, x): return 10 * x @staticmethod - def h(x): - return 10 * x + def double(x): + return 2 * x -print(fory.loads(fory.dumps(Person))("Bob", 25)) # Person(name='Bob', age=25) -# serialize global class instance method -print(fory.loads(fory.dumps(Person("Bob", 20).f))(10)) # 200 -# serialize global class class method -print(fory.loads(fory.dumps(Person.g))(10)) # 100 -# serialize global class static method -print(fory.loads(fory.dumps(Person.h))(10)) # 100 -``` +for carrier in (type, types.FunctionType, types.MethodType, staticmethod, classmethod, Calculator): + fory.register_type(carrier) -#### Serialize Local Classes/Methods - -Serialize classes defined inside functions along with their methods. Useful for dynamic class creation: - -```python -from dataclasses import dataclass -import pyfory -fory = pyfory.Fory(xlang=False, ref=True, strict=False) - -def create_local_class(): - class LocalClass: - def f(self, x): - return 10 * x - - @classmethod - def g(cls, x): - return 10 * x - - @staticmethod - def h(x): - return 10 * x - return LocalClass - -# serialize local class -data = fory.dumps(create_local_class()) -print(fory.loads(data)().f(10)) # 100 - -# serialize local class instance method -data = fory.dumps(create_local_class()().f) -print(fory.loads(data)(10)) # 100 - -# serialize local class method -data = fory.dumps(create_local_class().g) -print(fory.loads(data)(10)) # 100 - -# serialize local class static method -data = fory.dumps(create_local_class().h) -print(fory.loads(data)(10)) # 100 +print(fory.loads(fory.dumps(Calculator().scale))(10)) # 30 +print(fory.loads(fory.dumps(Calculator.ten))(10)) # 100 +print(fory.loads(fory.dumps(Calculator.double))(10)) # 20 ``` ### Out-of-Band Buffer Serialization -Fory supports pickle5-compatible out-of-band buffer serialization for efficient zero-copy handling of large data structures. This is particularly useful for NumPy arrays, Pandas DataFrames, and other objects with large memory footprints. - -Out-of-band serialization separates metadata from the actual data buffers, allowing for: - -- **Zero-copy transfers** when sending data over networks or IPC using `memoryview` -- **Improved performance** for large datasets -- **Pickle5 compatibility** using `pickle.PickleBuffer` -- **Flexible stream support** - write to any writable object (files, BytesIO, sockets, etc.) - -#### Basic Out-of-Band Serialization - -```python -import pyfory -import numpy as np - -fory = pyfory.Fory(xlang=False, ref=False, strict=False) - -# Large numpy array -array = np.arange(10000, dtype=np.float64) - -# Serialize with out-of-band buffers -buffer_objects = [] -serialized_data = fory.serialize(array, buffer_callback=buffer_objects.append) - -# Convert buffer objects to memoryview for zero-copy transmission -# For contiguous buffers (bytes, numpy arrays), this is zero-copy -# For non-contiguous data, a copy may be created to ensure contiguity -buffers = [obj.getbuffer() for obj in buffer_objects] - -# Deserialize with out-of-band buffers (accepts memoryview, bytes, or Buffer) -deserialized_array = fory.deserialize(serialized_data, buffers=buffers) - -assert np.array_equal(array, deserialized_array) -``` - -#### Out-of-Band with Pandas DataFrames - -```python -import pyfory -import pandas as pd -import numpy as np - -fory = pyfory.Fory(xlang=False, ref=False, strict=False) - -# Create a DataFrame with numeric columns -df = pd.DataFrame({ - 'a': np.arange(1000, dtype=np.float64), - 'b': np.arange(1000, dtype=np.int64), - 'c': ['text'] * 1000 -}) - -# Serialize with out-of-band buffers -buffer_objects = [] -serialized_data = fory.serialize(df, buffer_callback=buffer_objects.append) -buffers = [obj.getbuffer() for obj in buffer_objects] - -# Deserialize -deserialized_df = fory.deserialize(serialized_data, buffers=buffers) - -assert df.equals(deserialized_df) -``` - -#### Selective Out-of-Band Serialization - -You can control which buffers go out-of-band by providing a callback that returns `True` to keep data in-band or `False` (and appending to a list) to send it out-of-band: - -```python -import pyfory -import numpy as np - -fory = pyfory.Fory(xlang=False, ref=True, strict=False) - -arr1 = np.arange(1000, dtype=np.float64) -arr2 = np.arange(2000, dtype=np.float64) -data = [arr1, arr2] - -buffer_objects = [] -counter = 0 - -def selective_callback(buffer_object): - global counter - counter += 1 - # Only send even-numbered buffers out-of-band - if counter % 2 == 0: - buffer_objects.append(buffer_object) - return False # Out-of-band - return True # In-band - -serialized = fory.serialize(data, buffer_callback=selective_callback) -buffers = [obj.getbuffer() for obj in buffer_objects] -deserialized = fory.deserialize(serialized, buffers=buffers) -``` - -#### Pickle5 Compatibility - -Fory's out-of-band serialization is fully compatible with pickle protocol 5. When objects implement `__reduce_ex__(protocol)`, Fory automatically uses protocol 5 to enable `pickle.PickleBuffer` support: - -```python -import pyfory -import pickle - -fory = pyfory.Fory(xlang=False, ref=False, strict=False) - -# PickleBuffer objects are automatically supported -data = b"Large binary data" -pickle_buffer = pickle.PickleBuffer(data) - -# Serialize with buffer callback for out-of-band handling -buffer_objects = [] -serialized = fory.serialize(pickle_buffer, buffer_callback=buffer_objects.append) -buffers = [obj.getbuffer() for obj in buffer_objects] - -# Deserialize with buffers -deserialized = fory.deserialize(serialized, buffers=buffers) -assert bytes(deserialized.raw()) == data -``` - -#### Writing Buffers to Different Streams - -The `BufferObject.write_to()` method accepts any writable stream object, making it flexible for various use cases: - -```python -import pyfory -import numpy as np -import io - -fory = pyfory.Fory(xlang=False, ref=False, strict=False) - -array = np.arange(1000, dtype=np.float64) - -# Collect out-of-band buffers -buffer_objects = [] -serialized = fory.serialize(array, buffer_callback=buffer_objects.append) +Python native mode can separate supported NumPy ndarray and `pickle.PickleBuffer` storage from the +root bytes through `buffer_callback`. Transport the collected buffers with the root bytes and pass +them to `deserialize` in the same order. For contiguous storage, `BufferObject.getbuffer()` can +expose a `memoryview` without an additional source-side copy; non-contiguous storage may be copied. +This does not promise copy-free transport or decoding. -# Write to different stream types -for buffer_obj in buffer_objects: - # Write to BytesIO (in-memory stream) - bytes_stream = io.BytesIO() - buffer_obj.write_to(bytes_stream) - - # Write to file - with open('/tmp/buffer_data.bin', 'wb') as f: - buffer_obj.write_to(f) - - # Get zero-copy memoryview (for contiguous buffers) - mv = buffer_obj.getbuffer() - assert isinstance(mv, memoryview) -``` - -**Note**: For contiguous memory buffers (like bytes, numpy arrays), `getbuffer()` returns a zero-copy `memoryview`. For non-contiguous data, a copy may be created to ensure contiguity. +See [Out-of-Band Buffers](../docs/object-serialization/python/out-of-band.md) for the callback, +transport, and stream APIs. ## Cross-Language Object Graph Serialization `pyfory` supports cross-language object graph serialization, allowing you to serialize data in Python and deserialize it in Java, Go, Rust, or other supported languages. -The binary protocol and API are similar to `pyfory`'s Python native mode, but Python native mode can serialize any Python object—including global functions, local functions, lambdas, local classes, and types with custom serialization using `__getstate__/__reduce__/__reduce_ex__`, which are not allowed in xlang mode. +The binary protocol and API are similar to `pyfory`'s Python native mode. Python-specific callable, +stateful, and reduction carriers are available only in native mode and must be registered before +the first root operation. Xlang mode is the default. Set `xlang=True` explicitly in cross-language examples so the mode choice is visible: @@ -499,9 +328,11 @@ fory.register(Person.class, "example.Person"); Person person = (Person) fory.deserialize(binaryData); ``` -## Row Format - Zero-Copy Processing +## Row Format -Apache Fory™ provides a random-access row format that enables reading nested fields from binary data without full deserialization. This drastically reduces overhead when working with large objects where only partial data access is needed. The format also supports memory-mapped files for ultra-low memory footprint. +Row Format provides random and partial access to trusted analytical data without reconstructing the +complete object graph. See the [Python Row Format guide](../docs/row-format/python.md) for supported +types, schema requirements, and APIs. ### Basic Row Format Usage @@ -541,7 +372,7 @@ foo = Foo( # Encode to row format binary: bytes = encoder.to_row(foo).to_bytes() -# Zero-copy access - no full deserialization needed! +# Access selected fields without full deserialization. foo_row = pyfory.RowData(encoder.schema, binary) print(foo_row.f2[100000]) # Access 100,000th element directly print(foo_row.f4[100000].f1) # Access nested field directly @@ -586,7 +417,7 @@ foo.f4 = bars; // Encode to row format (cross-language compatible with Python) BinaryRow binaryRow = encoder.toRow(foo); -// Zero-copy random access without full deserialization +// Random access without full deserialization BinaryArray f2Array = binaryRow.getArray(1); // Access f2 list BinaryArray f4Array = binaryRow.getArray(3); // Access f4 list BinaryRow bar10 = f4Array.getStruct(10); // Access 11th Bar @@ -644,7 +475,7 @@ fory::row::encoder::RowEncoder encoder; encoder.encode(foo); auto row = encoder.get_writer().to_row(); -// Zero-copy random access without full deserialization +// Random access without full deserialization auto f2_array = row->get_array(1); // Access f2 list auto f4_array = row->get_array(3); // Access f4 list auto bar10 = f4_array->get_struct(10); // Access 11th Bar @@ -654,11 +485,9 @@ std::string str = bar10->get_string(0); // Access bar.f1 ### Key Benefits -- **Zero-Copy Access**: Read nested fields without deserializing the entire object -- **Memory Efficiency**: Memory-map large datasets directly from disk -- **Cross-Language**: Binary format is compatible between Python, Java, and other Fory implementations -- **Partial Deserialization**: Deserialize only the specific elements you need -- **High Performance**: Skip unnecessary data parsing for analytics and big data workloads +- **Random access**: Read nested fields without deserializing the entire object +- **Cross-language layout**: Share Standard Row Format data between supported runtimes +- **Partial deserialization**: Deserialize only the elements the application needs ## Core API Reference @@ -728,8 +557,8 @@ for t in threads: t.join() - **Instance Pool**: Maintains a pool of `Fory` instances protected by a lock for thread safety - **Shared Configuration**: All registrations must be done upfront and are applied to all instances -- **Same API**: Drop-in replacement for `Fory` class with identical methods -- **Registration Safety**: Prevents registration after first use to ensure consistency +- **Same root API**: Provides the same serialization and deserialization methods as `Fory` +- **Registration Safety**: The first root attempt permanently freezes registration, even if it fails **When to Use:** @@ -767,44 +596,36 @@ fory.register(MyClass, name="my.package.MyClass", serializer=custom_serializer) ### Xlang And Native Mode Comparison -| Feature | Native mode (`xlang=False`) | Xlang mode (default) | -| ------------------- | ---------------------------------------------- | ------------------------------------- | -| Use case | Pure Python applications | Multi-language systems | -| Compatibility | Python only | Java, Go, Rust, C++, JavaScript, etc. | -| Supported types | Python object surface | Cross-language compatible types | -| Functions/lambdas | Supported with trusted dynamic deserialization | Not allowed | -| Local classes | Supported with trusted dynamic deserialization | Not allowed | -| Dynamic classes | Supported with trusted dynamic deserialization | Not allowed | -| Schema mode default | Compatible | Compatible | +| Feature | Native mode (`xlang=False`) | Xlang mode (default) | +| ------------------- | --------------------------------------------- | ------------------------------------- | +| Use case | Pure Python applications | Multi-language systems | +| Compatibility | Python only | Java, Go, Rust, C++, JavaScript, etc. | +| Supported types | Configured Python type surface | Cross-language compatible types | +| Functions/lambdas | Supported with registered native carriers | Not allowed | +| Methods | Supported with registered carriers and owners | Not allowed | +| Stateful/reduce | Supported with registered application types | Not allowed | +| Schema mode default | Compatible | Compatible | #### Native Mode (`xlang=False`) -Python native mode supports Python-specific objects including functions, classes, and closures. Use it for Python-only applications: +Python native mode supports Python-specific objects such as functions and closures. Configure the +complete type surface before the first root operation: ```python import pyfory +import types # Python native mode fory = pyfory.Fory(xlang=False, ref=True, strict=False) +fory.register_type(types.FunctionType) -# Supports ALL Python objects: +# Every carrier is registered before this first root operation. data = fory.dumps({ - 'function': lambda x: x * 2, # Functions and lambdas - 'class': type('Dynamic', (), {}), # Dynamic classes - 'method': str.upper, # Methods - 'nested': {'circular_ref': None} # Circular references (when ref=True) + 'function': lambda x: x * 2, + 'values': [1, 2, 3], }) - -# Drop-in replacement for pickle/cloudpickle -import pickle -obj = [1, 2, {"nested": [3, 4]}] -assert fory.loads(fory.dumps(obj)) == pickle.loads(pickle.dumps(obj)) - -# Significantly faster and more compact than pickle -import timeit -obj = {f"key{i}": f"value{i}" for i in range(10000)} -print(f"Fory: {timeit.timeit(lambda: fory.dumps(obj), number=1000):.3f}s") -print(f"Pickle: {timeit.timeit(lambda: pickle.dumps(obj), number=1000):.3f}s") +result = fory.loads(data) +assert result['function'](4) == 8 ``` #### Xlang Mode @@ -854,101 +675,23 @@ assert result.children[0].parent is result # Reference preserved ### Type Registration -In strict mode, Fory loads and instantiates only registered application types. -Compatible metadata for an unregistered remote Struct returns the fixed -data-only `pyfory.UnknownStruct` carrier; it does not load or generate the -sender-named class. This prevents arbitrary class materialization: - -```python -import pyfory - -# Strict mode (recommended for production) -f = pyfory.Fory(xlang=False, strict=True) - -class SafeClass: - def __init__(self, data): - self.data = data - -# Must register types in strict mode -f.register(SafeClass, name="com.example.SafeClass") - -# Now serialization works -obj = SafeClass("safe data") -data = f.serialize(obj) -result = f.deserialize(data) - -# Unregistered types will raise an exception -class UnsafeClass: - pass - -# This will fail in strict mode -try: - f.serialize(UnsafeClass()) -except Exception as e: - print("Security protection activated!") -``` +Register the complete application type surface before the first root operation. See +[Type Registration](../docs/object-serialization/python/type-registration.md) for registration +identity, strict-mode behavior, and the frozen registry lifecycle. See +[Python Security](../docs/object-serialization/python/security.md) before accepting untrusted input. ### Custom Serializers -Implement custom serialization logic for specialized types with a single `write/read` API: +Custom serializers implement the serializer-owned `write` and `read` operations and are registered +before the first root operation. See +[Custom Serializers](../docs/object-serialization/python/custom-serializers.md) for the supported +constructor and context APIs. -```python -import pyfory -from pyfory.serializer import Serializer -from dataclasses import dataclass - -@dataclass -class Foo: - f1: int - f2: str - -class FooSerializer(Serializer): - def __init__(self, fory, cls): - super().__init__(fory, cls) - - def write(self, buffer, obj: Foo): - # Custom serialization logic - buffer.write_varint32(obj.f1) - buffer.write_string(obj.f2) - - def read(self, buffer): - # Custom deserialization logic - f1 = buffer.read_varint32() - f2 = buffer.read_string() - return Foo(f1, f2) - -f = pyfory.Fory(xlang=False) -f.register(Foo, type_id=100, serializer=FooSerializer(f, Foo)) - -# Now Foo uses your custom serializer -data = f.dumps(Foo(42, "hello")) -result = f.loads(data) -print(result) # Foo(f1=42, f2='hello') -``` - -### Numpy & Scientific Computing - -Fory natively supports numpy arrays with optimized serialization. Large arrays use zero-copy when possible: - -```python -import pyfory -import numpy as np - -f = pyfory.Fory(xlang=False) - -# Numpy arrays are supported natively -arrays = { - 'matrix': np.random.rand(1000, 1000), - 'vector': np.arange(10000), - 'bool_mask': np.random.choice([True, False], size=5000) -} - -data = f.serialize(arrays) -result = f.deserialize(data) +### NumPy & Scientific Computing -# Zero-copy for compatible array types -assert np.array_equal(arrays['matrix'], result['matrix']) -``` +Python native mode supports NumPy ndarrays, including multidimensional and object-dtype arrays. See +[NumPy Integration](../docs/object-serialization/python/numpy-integration.md) for supported behavior +and out-of-band transport. ## Best Practices @@ -975,14 +718,16 @@ fory.register(ProductModel, type_id=102) ### Performance Tips -Optimize serialization speed and memory usage with these guidelines: +Use these configuration rules before measuring an application workload: 1. **Disable `ref=True` if not needed**: Reference tracking has overhead -2. **Use type_id instead of name**: Integer IDs are faster than string names -3. **Reuse Fory instances**: Create once, use many times -4. **Use `compatible=False` only for same-schema data**: Disable compatible mode only when every reader and writer always uses the same Python class schema and you want faster serialization and smaller size -5. **Enable Cython**: Make sure `ENABLE_FORY_CYTHON_SERIALIZATION=1`, should be enabled by default -6. **Use row format for large arrays**: Zero-copy access for analytics +2. **Reuse configured Fory instances**: Create once, use many times; use `ThreadSafeFory` when an + instance must be shared across threads +3. **Use `compatible=False` only for same-schema data**: Every reader and writer must use the same + Python class schema +4. **Use Row Format for partial reads**: Choose it when applications need random access to trusted + analytical row data instead of object reconstruction; see the + [Python Row Format guide](../docs/row-format/python.md) ```python # Good: Reuse instance @@ -998,53 +743,17 @@ for obj in objects: ### Type Registration Patterns -Choose the right registration approach for your use case: - -```python -# Pattern 1: Simple registration -fory.register(MyClass, type_id=100) - -# Pattern 2: Cross-language with name -fory.register(MyClass, name="com.example.MyClass") - -# Pattern 3: With custom serializer -fory.register(MyClass, type_id=100, serializer=MySerializer(fory, MyClass)) - -# Pattern 4: Batch registration -type_id = 100 -for model_class in [User, Order, Product, Invoice]: - fory.register(model_class, type_id=type_id) - type_id += 1 -``` +Use stable names for shared xlang schemas and numeric IDs for Python-native type identity. See +[Type Registration](../docs/object-serialization/python/type-registration.md) for the supported +patterns, including custom serializers and batch registration. ### Error Handling -Handle common serialization errors gracefully. Catch specific exceptions for better error recovery: - -```python -import pyfory -from pyfory.error import TypeUnregisteredError, TypeNotCompatibleError - -fory = pyfory.Fory(strict=True) - -try: - data = fory.dumps(my_object) -except TypeUnregisteredError as e: - print(f"Type not registered: {e}") - # Register the type and retry - fory.register(type(my_object), type_id=100) - data = fory.dumps(my_object) -except Exception as e: - print(f"Serialization failed: {e}") - -try: - obj = fory.loads(data) -except TypeNotCompatibleError as e: - print(f"Schema mismatch: {e}") - # Handle version mismatch -except Exception as e: - print(f"Deserialization failed: {e}") -``` +A failed root never reopens the registry. Create and fully configure a new instance after a missing +or invalid registration failure. A fully configured instance can process another root after a +failure while reading input data or serializing a value. See +[Error Handling](../docs/object-serialization/python/troubleshooting.md#error-handling) for a +complete example. ## Security Best Practices @@ -1081,9 +790,11 @@ if os.getenv('ENV') == 'development': fory = pyfory.Fory( xlang=False, ref=True, - strict=False, # Allow any type for development + strict=False, # Use only with trusted development payloads max_depth=1000 # Higher limit for development ) + for model_class in [UserModel, ProductModel, OrderModel]: + fory.register_type(model_class) else: # Production configuration (security hardened) fory = pyfory.Fory( @@ -1098,65 +809,10 @@ else: ### DeserializationPolicy -When `strict=False` is necessary (e.g., deserializing functions/lambdas), use `DeserializationPolicy` to implement fine-grained security controls during deserialization. This provides protection similar to `pickle.Unpickler.find_class()` but with more comprehensive hooks. - -**Why use DeserializationPolicy?** - -- Block dangerous classes/modules (e.g., `subprocess.Popen`) -- Intercept and validate `__reduce__` callables before invocation -- Sanitize sensitive data during `__setstate__` -- Replace or reject deserialized objects based on custom rules - -**Example: Blocking Dangerous Classes** - -```python -import pyfory -from pyfory import DeserializationPolicy - -dangerous_modules = {'subprocess', 'os', '__builtin__'} - -class SafeDeserializationPolicy(DeserializationPolicy): - """Block potentially dangerous classes during deserialization.""" - - def validate_class(self, cls, is_local, **kwargs): - # Block dangerous modules - if cls.__module__ in dangerous_modules: - raise ValueError(f"Blocked dangerous class: {cls.__module__}.{cls.__name__}") - - def intercept_reduce_call(self, callable_obj, args, **kwargs): - # Block specific callable invocations during __reduce__ - if getattr(callable_obj, '__name__', "") == 'Popen': - raise ValueError("Blocked attempt to invoke subprocess.Popen") - return None - - def intercept_setstate(self, obj, state, **kwargs): - # Sanitize sensitive data - if isinstance(state, dict) and 'password' in state: - state['password'] = '***REDACTED***' - return None - -# Create Fory with custom security policy -policy = SafeDeserializationPolicy() -fory = pyfory.Fory(xlang=False, ref=True, strict=False, policy=policy) - -# Now deserialization is protected by your custom policy -data = fory.serialize(my_object) -result = fory.deserialize(data) # Policy hooks will be invoked -``` - -**Available Policy Hooks:** - -- Reference validation hooks reject by raising exceptions and otherwise leave deserialized references unchanged. -- `validate_class(cls, is_local)` - Validate/block class types during deserialization -- `validate_module(module_name, is_local)` - Validate/block module imports -- `validate_function(func, is_local)` - Validate/block function references -- `validate_method(method, is_local)` - Validate/block method references -- `intercept_reduce_call(callable_obj, args)` - Intercept `__reduce__` invocations -- `inspect_reduced_object(obj)` - Inspect/replace objects created via `__reduce__` -- `intercept_setstate(obj, state)` - Sanitize state before `__setstate__` -- `authorize_instantiation(cls, args, kwargs)` - Control class instantiation - -**See also:** `pyfory/policy.py` contains detailed documentation and examples for each hook. +When `strict=False` is necessary for trusted native-mode payloads, configure a +`DeserializationPolicy` before the first root operation to restrict accepted types and object hooks. +See [Python Security](../docs/object-serialization/python/security.md#deserializationpolicy) for the +supported policy hooks and configuration example. ## Troubleshooting @@ -1206,7 +862,8 @@ object identity or cycles matter: f = pyfory.Fory(ref=True) ``` -For arbitrary Python object graphs with circular references, use Python native mode: +For configured Python object graphs with circular references, use native mode and register every +application type before the first root: ```python f = pyfory.Fory(xlang=False, ref=True, strict=False) @@ -1222,6 +879,7 @@ node2 = Node(2) node1.next = node2 node2.next = node1 # Circular reference +f.register_type(Node) data = f.dumps(node1) result = f.loads(data) assert result.next.next is result # Circular reference preserved @@ -1243,30 +901,9 @@ import pyfory # Now uses pure Python implementation **Q: Schema evolution not working** -```python -# A: Xlang mode defaults to compatible schema evolution. -f = pyfory.Fory(xlang=True) - -# Version 1: Original class -@dataclass -class User: - name: str - age: int - -f.register(User, name="User") -data = f.dumps(User("Alice", 30)) - -# Version 2: Add new field (backward compatible) -@dataclass -class User: - name: str - age: int - email: str = "unknown@example.com" # New field with default - -# Can still deserialize old data -user = f.loads(data) -print(user.email) # "unknown@example.com" -``` +Xlang mode defaults to compatible schema evolution. Configure writer and reader schemas on separate +instances because each instance's registry freezes on its first root operation. See +[Schema Evolution](../docs/object-serialization/python/schema-evolution.md) for a complete example. **Q: Type registration errors in strict mode** @@ -1278,8 +915,9 @@ f = pyfory.Fory(strict=True) f.register(MyClass, type_id=100) f.register(AnotherClass, type_id=101) -# Or disable strict mode (NOT recommended for production) -f = pyfory.Fory(strict=False) # Use only in trusted environments +# Native carriers still require pre-registration when strict mode is disabled. +f = pyfory.Fory(xlang=False, strict=False) # Use only with trusted payloads +f.register_type(MyClass) ``` ## Contributing @@ -1299,10 +937,6 @@ Apache Fory™ is an open-source project under the Apache Software Foundation. W Apache License 2.0. See [LICENSE](https://github.com/apache/fory/blob/main/LICENSE) for details. ---- - -**Apache Fory™** - Blazing fast, secure, and versatile serialization for modern applications. - ## Links - **Documentation**: https://fory.apache.org/docs/object-serialization/python/ diff --git a/python/pyfory/_fory.py b/python/pyfory/_fory.py index a8ff222759..9444bf5ffe 100644 --- a/python/pyfory/_fory.py +++ b/python/pyfory/_fory.py @@ -88,9 +88,10 @@ class Fory: objects and cross-language reference metadata; Python native mode handles the broader Python object graph surface, including circular Python objects. - In Python native mode (xlang=False), Fory can serialize all Python objects - including dataclasses, classes with custom serialization methods, and local - functions/classes, making it a drop-in replacement for pickle. + In Python native mode (xlang=False), Fory can serialize a configured Python + type surface including dataclasses, classes with custom serialization methods, + and local functions/classes. Register application types and native carriers + before the first root operation, which permanently freezes the registry. In xlang mode, the default, Fory serializes objects in a format that can be deserialized by other Fory-supported languages (Java, Go, Rust, C++, etc.). @@ -152,9 +153,9 @@ def __init__( Args: xlang: Enable xlang mode. When False, uses - Python native mode supporting all Python objects (dataclasses, __reduce__, - local functions/classes). With ref=True and strict=False, serves as a - drop-in replacement for pickle. When True, uses the xlang wire format + Python native mode supporting configured Python objects (dataclasses, + __reduce__, local functions/classes). Register all application types and + native carriers before the first root operation. When True, uses the xlang wire format compatible with other Fory languages (Java, Go, Rust, etc), but Python- specific features like functions and __reduce__ methods are not supported. @@ -165,8 +166,9 @@ def __init__( strict: Require registration before loading or instantiating application classes (default: True). Compatible metadata for an unregistered remote Struct uses the fixed data-only UnknownStruct carrier instead of loading - or generating the sender-named class. When strict mode is disabled, - dynamic application types can be deserialized, which may be insecure if + or generating the sender-named class. Disabling strict mode authorizes + configured native carriers but does not permit discovery or registration + after the first root. Dynamic application types can be insecure if malicious code exists in __new__/__init__/__eq__/__hash__ methods. **WARNING**: Only disable in trusted environments. When disabling strict mode, you should provide a custom `policy` parameter to control which types @@ -300,6 +302,7 @@ def register( >>> fory.register(Person, type_id=100) >>> >>> # Register with name (more flexible) + >>> fory = Fory(xlang=True) >>> fory.register(Person, name="com.example.Person") >>> >>> # Python native mode (no cross-language matching needed) @@ -346,6 +349,7 @@ def register_type( >>> fory.register_type(Person, type_id=100) >>> >>> # Register with name (more flexible) + >>> fory = Fory(xlang=True) >>> fory.register_type(Person, name="com.example.Person") >>> >>> # Python native mode (no cross-language matching needed) @@ -390,10 +394,18 @@ def register_serializer(self, cls: type, serializer): Example: >>> fory = Fory(xlang=False) - >>> fory.register_serializer(MyClass, MyCustomSerializer()) + >>> fory.register_type(MyClass) + >>> serializer = MyCustomSerializer(fory.type_resolver, MyClass) + >>> fory.register_serializer(MyClass, serializer) """ self.type_resolver.register_serializer(cls, serializer) + def _freeze_registry(self): + # The resolver remains authoritative because callers may register through + # it directly. Avoid entering its finalization method after the first root. + if not self.type_resolver._registry_frozen: + self.type_resolver._freeze_registry() + def dumps( self, obj, @@ -421,6 +433,7 @@ def dump(self, obj, stream): the passed object (or a view of it) is unsupported. If your sink needs retention, copy bytes inside ``write``. """ + self._freeze_registry() try: self.buffer.set_writer_index(0) output_stream = Buffer.wrap_output_stream(stream) @@ -434,7 +447,7 @@ def dump(self, obj, stream): self.force_flush() finally: self.buffer.bind_output_stream(None) - self.reset_write() + self.write_context.reset() def loads( self, @@ -476,6 +489,7 @@ def serialize( >>> print(type(data)) """ + self._freeze_registry() try: write_buffer = self._serialize( obj, @@ -489,7 +503,7 @@ def serialize( return write_buffer return write_buffer.to_bytes(0, write_buffer.get_writer_index()) finally: - self.reset_write() + self.write_context.reset() def _serialize( self, @@ -553,10 +567,11 @@ def deserialize( >>> print(obj) {'key': 'value'} """ + self._freeze_registry() try: return self._deserialize(buffer, buffers, unsupported_objects) finally: - self.reset_read() + self.read_context.reset() def _deserialize( self, @@ -610,7 +625,8 @@ def reset(self): Reset both write and read state. Clears all per-operation state including buffers and reference tracking. - Use this to ensure a clean state before reusing a Fory instance. + Use this to ensure clean operation state before reusing a Fory instance. + Reset does not reopen registration after the first root attempt. """ self.reset_write() self.reset_read() @@ -625,9 +641,9 @@ class ThreadSafeFory: needs to serialize or deserialize data, it acquires an instance from the pool, uses it, and returns it for reuse by other threads. - All type registrations must be performed before any serialization operations to ensure - consistency across all pooled instances. Attempting to register types after the first - serialization will raise a RuntimeError. + All type registrations must be performed before the first root serialization or + deserialization attempt to ensure consistency across all pooled instances. Registration + remains closed even when that first operation fails. Args: xlang (bool): Whether to enable xlang mode. Defaults to True. diff --git a/python/pyfory/context.pxi b/python/pyfory/context.pxi index b701185bcd..8609fd5b5d 100644 --- a/python/pyfory/context.pxi +++ b/python/pyfory/context.pxi @@ -126,7 +126,7 @@ cdef class RefWriter: return True return False - cpdef inline reset(self): + cpdef inline void reset(self) noexcept: cdef PyObject *item if not self.track_ref: return @@ -328,7 +328,7 @@ cdef class MetaStringWriter: return buffer.write_var_uint32(((deref(entry).second + 1) << 1) | 1) - cpdef inline reset(self): + cpdef inline void reset(self) noexcept: cdef PyObject *item self._written_encoded_meta_strings.clear() for item in self._written_objects: @@ -504,7 +504,7 @@ cdef class MetaStringReader: cdef class MetaShareWriteContext: cdef flat_hash_map[uint64_t, int32_t] class_map - cpdef inline reset(self): + cpdef inline void reset(self) noexcept: self.class_map.clear() @@ -565,7 +565,7 @@ cdef class WriteContext: self.buffer_callback = buffer_callback self.unsupported_callback = unsupported_callback - cpdef inline reset(self): + cpdef inline void reset(self) noexcept: self.ref_writer.reset() self.meta_string_writer.reset() if self.meta_share_context is not None: @@ -574,8 +574,10 @@ cdef class WriteContext: self.context_objects.clear() self.buffer = None self.c_buffer = NULL - self.buffer_callback = None - self.unsupported_callback = None + if self.buffer_callback is not None: + self.buffer_callback = None + if self.unsupported_callback is not None: + self.unsupported_callback = None cpdef inline add_context_object(self, key, obj): self.context_objects[id(key)] = obj diff --git a/python/pyfory/registry.py b/python/pyfory/registry.py index 9cf00a4432..a148d6a4c8 100644 --- a/python/pyfory/registry.py +++ b/python/pyfory/registry.py @@ -372,6 +372,8 @@ class TypeResolver: "meta_share", "_internal_py_serializer_map", "_actual_type_resolver", + "_registry_frozen", + "_registry_finalizing", ) def __init__(self, config, *, shared_registry): @@ -412,6 +414,37 @@ def __init__(self, config, *, shared_registry): self.meta_share = config.meta_share self._internal_py_serializer_map = {} self._actual_type_resolver = self + # Fory exposes this resolver, so the resolver must own the root-use gate; + # facade-only state would leave direct registration methods mutable. + self._registry_frozen = False + self._registry_finalizing = False + + def _check_registry_mutable(self): + if self._registry_frozen: + raise RuntimeError("Cannot register types or serializers after the first root operation has started") + + def _needs_registration_finalization(self, type_info): + if type_info.serializer is None: + return True + if is_struct_type(type_info.type_id): + from pyfory.struct import DataClassStubSerializer + + if isinstance(type_info.serializer, DataClassStubSerializer): + return True + return self.meta_share and type_info.type_def is None and TypeId.is_type_share_meta(type_info.type_id) + + def _freeze_registry(self): + if self._registry_frozen: + return False + self._registry_frozen = True + self._registry_finalizing = True + try: + for type_info in self._types_info.values(): + if self._needs_registration_finalization(type_info): + self._set_type_info(type_info) + finally: + self._registry_finalizing = False + return True def _set_actual_resolver(self, type_resolver): # Cython mode injects the compiled companion before initialize() so all @@ -425,6 +458,7 @@ def initialize(self): self._initialize_py() else: self._initialize_xlang() + self._get_nonexist_enum_type_info() def _initialize_py(self): register = functools.partial(self._register_type, internal=True) @@ -578,6 +612,7 @@ def register_type( name: str = None, serializer=None, ): + self._check_registry_mutable() namespace, typename = _split_registration_name(name) return self._register_type( cls, @@ -595,17 +630,17 @@ def register_union( name: str = None, serializer=None, ): + self._check_registry_mutable() + cls = normalize_fory_type(cls) + if cls in self._types_info: + raise TypeError(f"{cls} registered already") namespace, typename = _split_registration_name(name) if serializer is None: raise TypeError("register_union requires a serializer") - if serializer is not None and not isinstance(serializer, Serializer): - serializer = _construct_serializer( - serializer, - self._actual_type_resolver, - cls, - ) if typename is not None and type_id is not None: raise TypeError(f"type name {typename} and id {type_id} should not be set at the same time") + previous_type_id_counter = self._type_id_counter + automatic_type_id = typename is None and type_id is None if typename is None and type_id is None: type_id = self._next_type_id() if type_id not in {0, None}: @@ -614,15 +649,20 @@ def register_union( else: user_type_id = NO_USER_TYPE_ID type_id = TypeId.NAMED_UNION - return self.__register_type( - cls, - type_id=type_id, - user_type_id=user_type_id, - namespace=namespace, - typename=typename, - serializer=serializer, - internal=False, - ) + try: + return self.__register_type( + cls, + type_id=type_id, + user_type_id=user_type_id, + namespace=namespace, + typename=typename, + serializer=serializer, + internal=False, + ) + except BaseException: + if automatic_type_id and self._type_id_counter == user_type_id: + self._type_id_counter = previous_type_id_counter + raise def _register_type( self, @@ -641,40 +681,37 @@ def _register_type( if internal: if type_id is not None and type_id >= 0 and type_id > 0xFF: raise ValueError(f"Internal type id overflow: {type_id}") - else: - if user_type_id not in {None, NO_USER_TYPE_ID} and (user_type_id < 0 or user_type_id > 0xFFFFFFFE): - raise ValueError(f"user_type_id must be in range [0, 0xfffffffe], got {user_type_id}") - if serializer is not None and not isinstance(serializer, Serializer): - serializer = _construct_serializer( - serializer, - self._actual_type_resolver, - cls, - ) - if ( - cls in self._types_info - and type_id is None - and typename is None - and namespace is None - and serializer is None - and user_type_id in {None, NO_USER_TYPE_ID} - ): - return self._types_info[cls] + if cls in self._types_info: + if type_id is None and typename is None and namespace is None and serializer is None and user_type_id in {None, NO_USER_TYPE_ID}: + return self._types_info[cls] + raise TypeError(f"{cls} registered already") + if not internal and not self.xlang and not self.strict and type_id is None and typename is None and namespace is None and serializer is None: + # Native carriers keep their reserved discovery identity when users + # configure them explicitly; application classes retain struct registration. + typeinfo = self._register_inferred_type(cls, native_only=True) + if typeinfo is not None: + return typeinfo n_params = len({typename, type_id, None}) - 1 + previous_type_id_counter = self._type_id_counter + automatic_type_id = n_params == 0 and typename is None if n_params == 0 and typename is None: type_id = self._next_type_id() if n_params == 2: raise TypeError(f"type name {typename} and id {type_id} should not be set at the same time") - if cls in self._types_info: - raise TypeError(f"{cls} registered already") - return self._register_xtype( - cls, - type_id=type_id, - user_type_id=user_type_id, - namespace=namespace, - typename=typename, - serializer=serializer, - internal=internal, - ) + try: + return self._register_xtype( + cls, + type_id=type_id, + user_type_id=user_type_id, + namespace=namespace, + typename=typename, + serializer=serializer, + internal=internal, + ) + except BaseException: + if automatic_type_id and self._type_id_counter == type_id: + self._type_id_counter = previous_type_id_counter + raise def _register_xtype( self, @@ -693,7 +730,6 @@ def _register_xtype( evolving = object_meta.evolving if serializer is None: if issubclass(cls, enum.Enum): - serializer = EnumSerializer(self._actual_type_resolver, cls) if type_id is None: type_id = TypeId.NAMED_ENUM user_type_id = NO_USER_TYPE_ID @@ -746,6 +782,38 @@ def __register_type( internal: bool = False, ): dynamic_type = type_id is not None and type_id < 0 + namespace_metastr = None + typename_metastr = None + if typename is not None: + if namespace is None: + splits = typename.rsplit(".", 1) + if len(splits) == 2: + namespace, typename = splits + else: + namespace = "" + else: + namespace = namespace or "" + if not typename: + raise ValueError("type name must not be empty") + if not internal and needs_user_type_id(type_id): + if not isinstance(user_type_id, int) or isinstance(user_type_id, bool) or user_type_id < 0 or user_type_id > 0xFFFFFFFE: + raise ValueError(f"user_type_id must be an integer in range [0, 0xfffffffe], got {user_type_id}") + self._preflight_registration( + cls, + type_id=type_id, + user_type_id=user_type_id, + namespace=namespace, + typename=typename, + ) + if typename is not None: + namespace_metastr = self.namespace_encoder.encode(namespace or "") + typename_metastr = self.typename_encoder.encode(typename) + if serializer is not None and not isinstance(serializer, Serializer): + serializer = _construct_serializer( + serializer, + self._actual_type_resolver, + cls, + ) # In metashare mode, for struct types, we want to keep serializer=None # so that _set_type_info will be called to create the TypeDef-based serializer # This applies to both types registered by name and by ID @@ -759,27 +827,12 @@ def __register_type( if typename is None: typeinfo = TypeInfo(cls, type_id, user_type_id, serializer, None, None, dynamic_type) else: - if namespace is None: - splits = typename.rsplit(".", 1) - if len(splits) == 2: - namespace, typename = splits - else: - namespace = "" # Use empty string for consistency with lookup - if not typename: - raise ValueError("type name must not be empty") - ns_metastr = self.namespace_encoder.encode(namespace or "") - ns_meta_bytes = self.shared_registry.get_encoded_meta_string(ns_metastr) - type_metastr = self.typename_encoder.encode(typename) - type_meta_bytes = self.shared_registry.get_encoded_meta_string(type_metastr) + ns_meta_bytes = self.shared_registry.get_encoded_meta_string(namespace_metastr) + type_meta_bytes = self.shared_registry.get_encoded_meta_string(typename_metastr) typeinfo = TypeInfo(cls, type_id, user_type_id, serializer, ns_meta_bytes, type_meta_bytes, dynamic_type) self._named_type_to_type_info[(namespace, typename)] = typeinfo self._ns_type_to_type_info[(ns_meta_bytes, type_meta_bytes)] = typeinfo - self._types_info[cls] = typeinfo if type_id is not None and type_id != 0: - if needs_user_type_id(type_id) and user_type_id not in {None, NO_USER_TYPE_ID}: - existing = self._user_type_id_to_type_info.get(user_type_id) - if existing is not None and existing.cls is not cls: - raise TypeError(f"user_type_id {user_type_id} already registered for {existing.cls}") if needs_user_type_id(type_id) and user_type_id not in {None, NO_USER_TYPE_ID}: if user_type_id not in self._user_type_id_to_type_info or not internal: self._user_type_id_to_type_info[user_type_id] = typeinfo @@ -791,6 +844,27 @@ def __register_type( self._index_python_type(cls) return typeinfo + def _preflight_registration( + self, + cls, + *, + type_id, + user_type_id, + namespace, + typename, + ): + if typename is not None: + existing = self._named_type_to_type_info.get((namespace, typename)) + if existing is not None and existing.cls is not cls: + raise TypeError(f"type name {(namespace, typename)!r} already registered for {existing.cls}") + if needs_user_type_id(type_id) and user_type_id not in { + None, + NO_USER_TYPE_ID, + }: + existing = self._user_type_id_to_type_info.get(user_type_id) + if existing is not None and existing.cls is not cls: + raise TypeError(f"user_type_id {user_type_id} already registered for {existing.cls}") + def _index_python_type(self, cls): if self._python_name_to_type is None or not isinstance(cls, type): return @@ -813,6 +887,7 @@ def _next_type_id(self): return type_id def register_serializer(self, cls, serializer): + self._check_registry_mutable() cls = normalize_fory_type(cls) assert isinstance(cls, type) or type(cls) is int, cls if cls not in self._types_info: @@ -822,7 +897,7 @@ def register_serializer(self, cls, serializer): prev_user_type_id = typeinfo.user_type_id if needs_user_type_id(prev_type_id) and prev_user_type_id not in {None, NO_USER_TYPE_ID}: self._user_type_id_to_type_info.pop(prev_user_type_id, None) - else: + elif not TypeId.is_namespaced_type(prev_type_id): self._type_id_to_type_info.pop(prev_type_id, None) if typeinfo.serializer is not serializer: if typeinfo.typename_bytes is not None: @@ -831,9 +906,10 @@ def register_serializer(self, cls, serializer): else: typeinfo.type_id = TypeId.EXT typeinfo.serializer = serializer + typeinfo.type_def = None if needs_user_type_id(typeinfo.type_id) and typeinfo.user_type_id not in {None, NO_USER_TYPE_ID}: self._user_type_id_to_type_info[typeinfo.user_type_id] = typeinfo - else: + elif not TypeId.is_namespaced_type(typeinfo.type_id): self._type_id_to_type_info[typeinfo.type_id] = typeinfo def get_serializer(self, cls: type): @@ -851,34 +927,41 @@ def get_type_info(self, cls, create=True): type_info = self._types_info.get(cls) if type_info is not None: if type_info.serializer is None: - self._set_type_info(type_info) + self._finalize_type_info(type_info) return type_info elif not create: return None - if cls is NonExistEnum: - return self._get_nonexist_enum_type_info() if self.require_registration and not issubclass(cls, Enum): raise TypeUnregisteredError(f"{cls} not registered") + self._check_registry_mutable() + if cls is NonExistEnum: + return self._get_nonexist_enum_type_info() logger.info("Type %s not registered", cls) + return self._register_inferred_type(cls) + + def _register_inferred_type(self, cls, native_only=False): serializer = self._create_serializer(cls) - type_id = None - if not self.xlang: - if isinstance(serializer, EnumSerializer): - type_id = TypeId.NAMED_ENUM - elif isinstance(serializer, (ObjectSerializer, StatefulSerializer)): - type_id = TypeId.NAMED_EXT - elif self._internal_py_serializer_map.get(type(serializer)) is not None: - type_id = self._internal_py_serializer_map.get(type(serializer))[1] - if not self.require_registration: + native_registration = self._internal_py_serializer_map.get(type(serializer)) + if native_registration is not None: + type_id = native_registration[1] + elif native_only: + return None + elif not self.xlang and isinstance(serializer, EnumSerializer): + type_id = TypeId.NAMED_ENUM + elif not self.xlang and isinstance(serializer, (ObjectSerializer, StatefulSerializer)): + type_id = TypeId.NAMED_EXT + else: + type_id = None + if not self.xlang and not self.require_registration: from pyfory import struct as struct_module data_class_types = tuple( - cls - for cls in ( + serializer_type + for serializer_type in ( getattr(struct_module, "DataClassSerializer", None), getattr(struct_module, "DataClassStubSerializer", None), ) - if cls is not None + if serializer_type is not None ) if data_class_types and isinstance(serializer, data_class_types): type_id = TypeId.NAMED_STRUCT @@ -892,37 +975,53 @@ def get_type_info(self, cls, create=True): serializer=serializer, ) + def _finalize_type_info(self, typeinfo): + if not self._registry_finalizing: + self._check_registry_mutable() + return self._set_type_info(typeinfo) + def _set_type_info(self, typeinfo): serializer_type_resolver = self._actual_type_resolver type_id = typeinfo.type_id - if is_struct_type(type_id): - from pyfory.struct import DataClassSerializer, DataClassStubSerializer - - # Set a stub serializer FIRST to break recursion for self-referencing types. - # get_type_info() only calls _set_type_info when serializer is None, - # so setting stub first prevents re-entry for circular type references. - typeinfo.serializer = DataClassStubSerializer(serializer_type_resolver, typeinfo.cls) - - if self.meta_share: - type_def = encode_typedef(serializer_type_resolver, typeinfo.cls) - if type_def is not None: - typeinfo.serializer = type_def.create_serializer(serializer_type_resolver) - typeinfo.type_def = type_def - else: - typeinfo.serializer = DataClassSerializer(serializer_type_resolver, typeinfo.cls) + previous_serializer = typeinfo.serializer + previous_type_def = typeinfo.type_def + try: + if is_struct_type(type_id): + from pyfory.struct import DataClassSerializer, DataClassStubSerializer + + if typeinfo.serializer is None or isinstance(typeinfo.serializer, DataClassStubSerializer): + # Publish the stub only for recursive construction. If later + # work fails, restore the pre-finalization state so a frozen + # registry cannot retain a replaceable partial descriptor. + typeinfo.serializer = DataClassStubSerializer(serializer_type_resolver, typeinfo.cls) + + if self.meta_share: + type_def = encode_typedef(serializer_type_resolver, typeinfo.cls) + if type_def is not None: + typeinfo.serializer = type_def.create_serializer(serializer_type_resolver) + typeinfo.type_def = type_def + else: + typeinfo.serializer = DataClassSerializer(serializer_type_resolver, typeinfo.cls) + else: + typeinfo.serializer = DataClassSerializer(serializer_type_resolver, typeinfo.cls) + elif self.meta_share and typeinfo.type_def is None and TypeId.is_type_share_meta(type_id): + typeinfo.type_def = encode_typedef(serializer_type_resolver, typeinfo.cls) else: - typeinfo.serializer = DataClassSerializer(serializer_type_resolver, typeinfo.cls) - else: - typeinfo.serializer = self._create_serializer(typeinfo.cls) - if ( - self.meta_share - and typeinfo.type_def is None - and ( - TypeId.is_namespaced_type(type_id) - or (needs_user_type_id(type_id) and typeinfo.user_type_id is not None and typeinfo.user_type_id != NO_USER_TYPE_ID) - ) - ): - typeinfo.type_def = encode_typedef(serializer_type_resolver, typeinfo.cls) + if typeinfo.serializer is None: + typeinfo.serializer = self._create_serializer(typeinfo.cls) + if ( + self.meta_share + and typeinfo.type_def is None + and ( + TypeId.is_namespaced_type(type_id) + or (needs_user_type_id(type_id) and typeinfo.user_type_id is not None and typeinfo.user_type_id != NO_USER_TYPE_ID) + ) + ): + typeinfo.type_def = encode_typedef(serializer_type_resolver, typeinfo.cls) + except BaseException: + typeinfo.serializer = previous_serializer + typeinfo.type_def = previous_type_def + raise return typeinfo @@ -1048,15 +1147,13 @@ def _load_metabytes_to_type_info(self, ns_metabytes, type_metabytes): typename = type_metabytes.decode(self.typename_decoder) # the hash computed between languages may be different. typeinfo = self._named_type_to_type_info.get((ns, typename)) - if typeinfo is None and typename and not self.strict: - alt_typename = typename[0].upper() + typename[1:] - typeinfo = self._named_type_to_type_info.get((ns, alt_typename)) if typeinfo is not None: self._cache_wire_type_info(ns_metabytes, type_metabytes, typeinfo) return typeinfo if self.strict: name = ns + "." + typename if ns else typename raise TypeUnregisteredError(f"{name} not registered") + self._check_registry_mutable() cls = load_class(ns + "#" + typename, policy=self.policy) typeinfo = self.get_type_info(cls) self._cache_wire_type_info(ns_metabytes, type_metabytes, typeinfo) @@ -1107,35 +1204,10 @@ def read_type_info(self, read_context, expected_typeinfo=None): ) ns_metabytes = read_context.meta_string_reader.read_encoded_meta_string(buffer) type_metabytes = read_context.meta_string_reader.read_encoded_meta_string(buffer) - typeinfo = self._ns_type_to_type_info.get((ns_metabytes, type_metabytes)) - if typeinfo is None: - ns = ns_metabytes.decode(self.namespace_decoder) - typename = type_metabytes.decode(self.typename_decoder) - typeinfo = self._named_type_to_type_info.get((ns, typename)) - if typeinfo is None and self.strict: - name = ns + "." + typename if ns else typename - raise TypeUnregisteredError(f"{name} not registered") - if typeinfo is None and typename: - alt_typename = typename[0].upper() + typename[1:] - typeinfo = self._named_type_to_type_info.get((ns, alt_typename)) - if typeinfo is not None: - self._cache_wire_type_info(ns_metabytes, type_metabytes, typeinfo) - return typeinfo - if not ns and "." in typename: - split_ns, split_typename = typename.rsplit(".", 1) - typeinfo = self._named_type_to_type_info.get((split_ns, split_typename)) - if typeinfo is not None: - self._cache_wire_type_info(ns_metabytes, type_metabytes, typeinfo) - return typeinfo - typename = split_typename - ns = split_ns - if typename and not self.strict: - matches = [info for (reg_ns, reg_typename), info in self._named_type_to_type_info.items() if reg_typename == typename] - if len(matches) == 1: - return matches[0] - name = ns + "." + typename if ns else typename - raise TypeUnregisteredError(f"{name} not registered") - return typeinfo + return self._load_metabytes_to_type_info( + ns_metabytes, + type_metabytes, + ) if type_id in {TypeId.ENUM, TypeId.STRUCT, TypeId.EXT, TypeId.TYPED_UNION}: user_type_id = buffer.read_var_uint32() return self.get_type_info_by_id(type_id, user_type_id=user_type_id) @@ -1165,6 +1237,7 @@ def _get_nonexist_enum_type_info(self): typeinfo = self._types_info.get(NonExistEnum) if typeinfo is None: + self._check_registry_mutable() serializer = NonExistEnumSerializer(self._actual_type_resolver) typeinfo = TypeInfo(NonExistEnum, TypeId.ENUM, NO_USER_TYPE_ID, serializer, None, None, False) self._types_info[NonExistEnum] = typeinfo @@ -1200,7 +1273,7 @@ def write_shared_type_meta(self, write_context, typeinfo): buffer.write_var_uint32(index << 1) type_def = typeinfo.type_def if type_def is None: - self._set_type_info(typeinfo) + self._finalize_type_info(typeinfo) type_def = typeinfo.type_def buffer.write_bytes(type_def.encoded) @@ -1383,7 +1456,7 @@ def _read_uncached_type_info(self, buffer, header, expected_typeinfo=None): raise TypeError("Type metadata owner does not match the declared type") if local_type_info is not None: if local_type_info.type_def is None: - self._set_type_info(local_type_info) + self._finalize_type_info(local_type_info) if local_type_info.type_def is not None: local_header = int.from_bytes(local_type_info.type_def.encoded[:8], "little", signed=True) if _typedef_hash_key(local_header) == hash_key: diff --git a/python/pyfory/serialization.pyx b/python/pyfory/serialization.pyx index 169e42fd9d..477c65316c 100644 --- a/python/pyfory/serialization.pyx +++ b/python/pyfory/serialization.pyx @@ -269,6 +269,9 @@ cdef class TypeResolver: cdef flat_hash_map[uint32_t, PyObject *] _c_user_type_id_to_type_info cdef flat_hash_map[uint64_t, PyObject *] _c_types_info cdef flat_hash_map[pair[int64_t, int64_t], PyObject *] _c_meta_hash_to_type_info + # The Python resolver owns registry mutability. This monotonic native cache only + # avoids re-entering Python after that owner has completed its first freeze attempt. + cdef bint registry_freeze_complete def __init__(self, Config config, *, shared_registry): """ @@ -300,6 +303,7 @@ cdef class TypeResolver: self._ns_type_to_type_info = resolver._ns_type_to_type_info self._local_type_info_by_hash = resolver._local_type_info_by_hash self._meta_shared_type_info = resolver._meta_shared_type_info + self.registry_freeze_complete = False for typeinfo in resolver._types_info.values(): self._populate_type_info(typeinfo) @@ -310,6 +314,14 @@ cdef class TypeResolver: for typeinfo in self.resolver._types_info.values(): self._populate_type_info(typeinfo) + cdef inline void _freeze_registry(self): + cdef object typeinfo + if not self.registry_freeze_complete: + if self.resolver._freeze_registry(): + for typeinfo in self.resolver._types_info.values(): + self._populate_type_info(typeinfo) + self.registry_freeze_complete = True + def register_type( self, cls, @@ -348,11 +360,14 @@ cdef class TypeResolver: cdef TypeInfo typeinfo cdef uint8_t previous_type_id cdef uint32_t previous_user_type_id - typeinfo = self.resolver.get_type_info(cls) + self.resolver._check_registry_mutable() + typeinfo = self._types_info.get(normalize_fory_type(cls)) + if typeinfo is None: + self.resolver.register_serializer(cls, serializer) + return previous_type_id = typeinfo.type_id previous_user_type_id = typeinfo.user_type_id self.resolver.register_serializer(cls, serializer) - typeinfo = self.resolver.get_type_info(cls) if previous_type_id != typeinfo.type_id or previous_user_type_id != typeinfo.user_type_id: if ( previous_type_id == TypeId.ENUM @@ -566,7 +581,7 @@ cdef class TypeResolver: write_context.write_var_uint32(index << 1) type_def = typeinfo.type_def if type_def is None: - self.resolver._set_type_info(typeinfo) + self.resolver._finalize_type_info(typeinfo) type_def = typeinfo.type_def write_context.write_bytes(type_def.encoded) @@ -709,7 +724,7 @@ cdef class TypeResolver: raise TypeError("Type metadata owner does not match the declared type") if typeinfo is not None: if typeinfo.type_def is None: - self.resolver._set_type_info(typeinfo) + self.resolver._finalize_type_info(typeinfo) if typeinfo.type_def is not None: local_header = Buffer(typeinfo.type_def.encoded).read_int64() if _typedef_hash_key(local_header) == hash_key: @@ -1224,6 +1239,7 @@ cdef class Fory: ) def dump(self, obj, stream): + self.type_resolver._freeze_registry() try: self.buffer.set_writer_index(0) self.buffer.bind_output_stream(Buffer.wrap_output_stream(stream)) @@ -1236,7 +1252,7 @@ cdef class Fory: self.force_flush() finally: self.buffer.bind_output_stream(None) - self.reset_write() + self.write_context.reset() def loads(self, buffer, buffers=None, unsupported_objects=None): return self.deserialize( @@ -1247,6 +1263,7 @@ cdef class Fory: def serialize(self, obj, Buffer buffer=None, buffer_callback=None, unsupported_callback=None): cdef Buffer write_buffer + self.type_resolver._freeze_registry() try: write_buffer = self._serialize( obj, @@ -1260,7 +1277,7 @@ cdef class Fory: return write_buffer return write_buffer.to_bytes(0, write_buffer.get_writer_index()) finally: - self.reset_write() + self.write_context.reset() cdef Buffer _serialize(self, obj, Buffer buffer=None, buffer_callback=None, unsupported_callback=None): cdef WriteContext write_context = self.write_context @@ -1273,8 +1290,12 @@ cdef class Fory: # so it should not pay an extra method call just to bind the active buffer. write_context.buffer = buffer write_context.c_buffer = buffer.c_buffer - write_context.buffer_callback = buffer_callback - write_context.unsupported_callback = unsupported_callback + # Root cleanup clears prior callbacks, so the common None path needs no + # object assignment or reference-count traffic here. + if buffer_callback is not None: + write_context.buffer_callback = buffer_callback + if unsupported_callback is not None: + write_context.unsupported_callback = unsupported_callback mask_index = buffer.get_writer_index() buffer.grow(1) buffer.set_writer_index(mask_index + 1) @@ -1286,6 +1307,7 @@ cdef class Fory: return buffer def deserialize(self, buffer, buffers=None, unsupported_objects=None): + self.type_resolver._freeze_registry() try: return self._deserialize( buffer, @@ -1293,7 +1315,7 @@ cdef class Fory: unsupported_objects=unsupported_objects, ) finally: - self.reset_read() + self.read_context.reset() cdef object _deserialize(self, buffer, buffers=None, unsupported_objects=None): cdef ReadContext read_context = self.read_context diff --git a/python/pyfory/serializer.py b/python/pyfory/serializer.py index cf025ab283..d1b8ed8320 100644 --- a/python/pyfory/serializer.py +++ b/python/pyfory/serializer.py @@ -563,7 +563,7 @@ def write(self, write_context, value): else: write_context.write_int8(NOT_NULL_VALUE_FLAG) write_context.write_no_ref(step) - write_context.write_ref(value.dtype) + write_context.write_string(value.dtype.str) write_context.write_ref(value.name) def read(self, read_context): @@ -579,7 +579,7 @@ def read(self, read_context): step = None else: step = read_context.read_no_ref() - dtype = read_context.read_ref() + dtype = np.dtype(read_context.read_string()) name = read_context.read_ref() return self.type_(start, stop, step, dtype=dtype, name=name) @@ -1347,10 +1347,17 @@ def write(self, write_context, value): elif len(reduce_result) == 4: # Case 4: (callable, args, state, listitems) callable_obj, args, state, listitems = reduce_result + # Reduce item iterators carry contents, not runtime iterator identity. + if listitems is not None: + listitems = list(listitems) reduce_data = (1, callable_obj, args, state, listitems) elif len(reduce_result) == 5: # Case 5: (callable, args, state, listitems, dictitems) callable_obj, args, state, listitems, dictitems = reduce_result + if listitems is not None: + listitems = list(listitems) + if dictitems is not None: + dictitems = list(dictitems) reduce_data = ( 1, callable_obj, @@ -1799,12 +1806,13 @@ def _deserialize_function(self, read_context): freevars.append(read_context.read_string()) globals_dict = read_context.read_ref() + if type(globals_dict) is not dict: + raise ValueError("function globals must be a dict") # Create a globals dictionary with module's globals as the base func_global_entries = len(mod.__dict__) if mod else 0 - if isinstance(globals_dict, dict): - func_global_entries = max(func_global_entries, len(globals_dict)) - has_builtins = (mod is not None and "__builtins__" in mod.__dict__) or (isinstance(globals_dict, dict) and "__builtins__" in globals_dict) + func_global_entries = max(func_global_entries, len(globals_dict)) + has_builtins = (mod is not None and "__builtins__" in mod.__dict__) or "__builtins__" in globals_dict if not has_builtins: func_global_entries += 1 read_context.reserve_graph_memory(_DICT_OWNER_BYTES + func_global_entries * 2 * _REFERENCE_BYTES) diff --git a/python/pyfory/struct.pxi b/python/pyfory/struct.pxi index 184ed07b41..76f6df55e0 100644 --- a/python/pyfory/struct.pxi +++ b/python/pyfory/struct.pxi @@ -632,8 +632,8 @@ cdef class DataClassSerializer(Serializer): @cython.final cdef class DataClassStubSerializer(Serializer): - # Keep a lazy stub so recursive dataclass registration can install the real - # serializer on first use without re-entering construction. + # Keep a delegate stub so recursive dataclass construction can refer to the + # canonical serializer without re-entering construction. cpdef write(self, WriteContext write_context, value): self._replace().write(write_context, value) @@ -641,6 +641,10 @@ cdef class DataClassStubSerializer(Serializer): return self._replace().read(read_context) cpdef object _replace(self): - cdef TypeInfo typeinfo = self.type_resolver.get_type_info(self.type_) - typeinfo.serializer = DataClassSerializer(self.type_resolver, self.type_) - return typeinfo.serializer + cdef TypeInfo typeinfo = self.type_resolver.get_type_info(self.type_, create=False) + cdef object serializer = None if typeinfo is None else typeinfo.serializer + # Root-entry finalization must commit the canonical serializer before + # use; this stub must never repair registry state after the freeze. + if serializer is None or isinstance(serializer, DataClassStubSerializer): + raise RuntimeError(f"Serializer finalization incomplete for {self.type_}") + return serializer diff --git a/python/pyfory/struct.py b/python/pyfory/struct.py index 4cac1a32d3..b2256a85f7 100644 --- a/python/pyfory/struct.py +++ b/python/pyfory/struct.py @@ -972,9 +972,14 @@ def read(self, read_context): return self._replace().read(read_context) def _replace(self): - typeinfo = self.type_resolver.get_type_info(self.type_) - typeinfo.serializer = DataClassSerializer(self.type_resolver, self.type_) - return typeinfo.serializer + typeinfo = self.type_resolver.get_type_info(self.type_, create=False) + serializer = None if typeinfo is None else typeinfo.serializer + # Recursive serializers may retain this stub while the canonical + # serializer is being built. Root-entry finalization must commit that + # serializer before use; the stub must never repair registry state later. + if serializer is None or isinstance(serializer, DataClassStubSerializer): + raise RuntimeError(f"Serializer finalization incomplete for {self.type_}") + return serializer basic_types = { diff --git a/python/pyfory/tests/test_class_serializer.py b/python/pyfory/tests/test_class_serializer.py index f96095aac8..52364b28d9 100644 --- a/python/pyfory/tests/test_class_serializer.py +++ b/python/pyfory/tests/test_class_serializer.py @@ -15,8 +15,22 @@ # specific language governing permissions and limitations # under the License. -from pyfory import Fory from dataclasses import dataclass +import types + +from pyfory import Fory + + +def register_class_types(fory, *classes): + for cls in ( + type, + types.FunctionType, + types.MethodType, + staticmethod, + classmethod, + *classes, + ): + fory.register_type(cls) def test_local_class_serialization(): @@ -42,6 +56,7 @@ def __eq__(self, other): # Test basic serialization of the class type itself fory = Fory(xlang=False, ref=True, strict=False, compatible=False) + register_class_types(fory, LocalClass) # Serialize the class type serialized = fory.serialize(LocalClass) @@ -79,6 +94,7 @@ def get_multiplied_value(self): LocalClassWithClosure = create_local_class_with_closure(3) fory = Fory(xlang=False, ref=True, strict=False, compatible=False) + register_class_types(fory, LocalClassWithClosure) # Serialize the class type serialized = fory.serialize(LocalClassWithClosure) @@ -116,6 +132,7 @@ def get_value(self): LocalClass = create_local_class_with_inheritance() fory = Fory(xlang=False, ref=True, strict=False, compatible=False) + register_class_types(fory, LocalClass) # Serialize and deserialize the class serialized = fory.serialize(LocalClass) @@ -131,7 +148,7 @@ def get_value(self): assert instance2.base_method() == "base" -def test_local_class_with_class_variables(): +def test_local_class_variables(): """Test local class with class variables""" def create_class_with_vars(): @@ -157,6 +174,7 @@ def get_info(self): LocalClass = create_class_with_vars() fory = Fory(xlang=False, ref=True, strict=False, compatible=False) + register_class_types(fory, LocalClass) # Create some instances to modify class state LocalClass(1) # This increments the counter @@ -202,6 +220,7 @@ def create_inner(self, inner_val): return self.InnerGlobalClass(inner_val) fory = Fory(xlang=False, ref=True, strict=False, compatible=False) + register_class_types(fory, OuterGlobalClass, OuterGlobalClass.InnerGlobalClass) # Test serializing the outer class serialized_outer = fory.serialize(OuterGlobalClass) @@ -259,10 +278,10 @@ def inner_method(self): return OuterLocalClass - fory = Fory(xlang=False, ref=True, strict=False, compatible=False) - # Create complex local class with nested closures ComplexLocalClass = create_complex_local_scenario(5) + fory = Fory(xlang=False, ref=True, strict=False, compatible=False) + register_class_types(fory, ComplexLocalClass) # Serialize and deserialize serialized = fory.serialize(ComplexLocalClass) @@ -283,10 +302,10 @@ def inner_method(self): assert inner2.inner_method() == 17 # 12 + 5 -def test_local_class_with_multiple_inheritance(): +def test_local_multiple_inheritance(): """Test local class with multiple inheritance""" - def create_local_class_with_multiple_inheritance(): + def create_local_multiple_inheritance(): class MixinA: def method_a(self): return "A" @@ -304,9 +323,9 @@ def combined_method(self): return LocalMultipleInheritanceClass + LocalClass = create_local_multiple_inheritance() fory = Fory(xlang=False, ref=True, strict=False, compatible=False) - - LocalClass = create_local_class_with_multiple_inheritance() + register_class_types(fory, LocalClass) # Serialize and deserialize serialized = fory.serialize(LocalClass) @@ -341,29 +360,9 @@ def h(x): def test_dataclass_serialize(): fory = Fory(xlang=False, ref=True, strict=False, compatible=False) + register_class_types(fory, Person) - # serialize global class - @dataclass - class LocalPerson: - name: str - age: int - - def f(self, x): - return self.age * x - - @classmethod - def g(cls, x): - return 10 * x - - @staticmethod - def h(x): - return 10 * x - - for cls in [LocalPerson, LocalPerson]: - assert str(fory.loads(fory.dumps(cls))("Bob", 25)) == str(cls("Bob", 25)) - # serialize global class instance method - assert fory.loads(fory.dumps(cls("Bob", 20).f))(10) == 200 - # serialize global class class method - assert fory.loads(fory.dumps(cls.g))(10) == 100 - # serialize global class static method - assert fory.loads(fory.dumps(cls.h))(10) == 100 + assert str(fory.loads(fory.dumps(Person))("Bob", 25)) == str(Person("Bob", 25)) + assert fory.loads(fory.dumps(Person("Bob", 20).f))(10) == 200 + assert fory.loads(fory.dumps(Person.g))(10) == 100 + assert fory.loads(fory.dumps(Person.h))(10) == 100 diff --git a/python/pyfory/tests/test_collection_safety.py b/python/pyfory/tests/test_collection_safety.py index 70ec36e3d7..dfaee720ee 100644 --- a/python/pyfory/tests/test_collection_safety.py +++ b/python/pyfory/tests/test_collection_safety.py @@ -15,6 +15,8 @@ # specific language governing permissions and limitations # under the License. +import types + import pytest import pyfory @@ -120,6 +122,8 @@ def __reduce__(self): def test_published_list_has_valid_slots(): fory = pyfory.Fory(xlang=False, ref=True, strict=False, compatible=False) + fory.register_type(type) + fory.register_type(ListCapture) outer = [] outer.append(ListCapture(outer)) @@ -130,6 +134,8 @@ def test_published_list_has_valid_slots(): def test_published_list_failure_cleanup(): fory = pyfory.Fory(xlang=False, ref=True, strict=False, compatible=False) + fory.register_type(FailingListCapture) + fory.register_type(types.FunctionType) outer = [] outer.append(FailingListCapture(outer)) data = fory.serialize(outer) @@ -141,6 +147,8 @@ def test_published_list_failure_cleanup(): def test_reentrant_list_clear(): fory = pyfory.Fory(xlang=False, ref=True, strict=False, compatible=False) + fory.register_type(ClearingListCapture) + fory.register_type(types.FunctionType) outer = [] outer.append(ClearingListCapture(outer)) data = fory.serialize(outer) diff --git a/python/pyfory/tests/test_function.py b/python/pyfory/tests/test_function.py index 46ab15b55f..02100a62bf 100644 --- a/python/pyfory/tests/test_function.py +++ b/python/pyfory/tests/test_function.py @@ -15,7 +15,82 @@ # specific language governing permissions and limitations # under the License. +import marshal +import types + +import pytest + import pyfory +from pyfory.policy import DEFAULT_POLICY +from pyfory.serialization import Buffer +from pyfory.serializer import FunctionSerializer + + +def test_function_globals_carrier(): + def local_func(): + return None + + writer = pyfory.Fory(xlang=False) + buffer = Buffer.allocate(256) + try: + writer.write_context.prepare(buffer) + buffer.write_int8(2) + buffer.write_string(local_func.__module__) + buffer.write_string(local_func.__qualname__) + buffer.write_bytes_and_size(marshal.dumps(local_func.__code__)) + buffer.write_bool(False) + buffer.write_bool(False) + buffer.write_var_uint32(0) + buffer.write_var_uint32(0) + writer.write_context.write_ref([]) + writer.write_context.write_ref({}) + data = buffer.to_bytes(0, buffer.get_writer_index()) + finally: + writer.reset_write() + + reader = pyfory.Fory(xlang=False, strict=False) + serializer = FunctionSerializer(reader.type_resolver, types.FunctionType) + try: + reader.read_context.prepare(Buffer(data)) + with pytest.raises(Exception): + reader.read_context.read_non_ref(serializer) + finally: + reader.reset_read() + + class DictSubclass(dict): + def __len__(self): + raise AssertionError("dict subclass operations must not run") + + class FunctionReadContext: + policy = DEFAULT_POLICY + + def __init__(self): + self._strings = iter((local_func.__module__, local_func.__qualname__)) + + def read_int8(self): + return 2 + + def read_string(self): + return next(self._strings) + + def read_bytes_and_size(self): + return marshal.dumps(local_func.__code__) + + def reserve_graph_memory(self, _size): + pass + + def read_bool(self): + return False + + def read_var_uint32(self): + return 0 + + def read_ref(self): + return DictSubclass() + + with pytest.raises(Exception) as failure: + serializer._deserialize_function(FunctionReadContext()) + assert not isinstance(failure.value, AssertionError) def test_lambda_functions_serialization(): @@ -23,7 +98,6 @@ def test_lambda_functions_serialization(): fory = pyfory.Fory( xlang=False, strict=False, - compatible=False, ) test_input = 5 @@ -65,13 +139,13 @@ def complex_function(a, b, c=10): # Test regular function fory.register_type(type(add_one)) + # Registry contents are finalized by the first root operation. + fory.register_type(tuple) + fory.register_type(list) serialized = fory.serialize(add_one) deserialized = fory.deserialize(serialized) assert add_one(test_input) == deserialized(test_input) - # Register the necessary types for complex functions - fory.register_type(tuple) - fory.register_type(list) # dict is already registered by default with MapSerializer # Test complex function diff --git a/python/pyfory/tests/test_graph_memory_budget.py b/python/pyfory/tests/test_graph_memory_budget.py index 5acb047443..8b42749f0f 100644 --- a/python/pyfory/tests/test_graph_memory_budget.py +++ b/python/pyfory/tests/test_graph_memory_budget.py @@ -19,6 +19,7 @@ import dataclasses import struct import sys +import types from typing import Any, List import pytest @@ -374,16 +375,19 @@ def test_reduce_object_budget(): value = BudgetReduceObject() writer = new_fory(xlang=False) writer.register_type(BudgetReduceObject) + writer.register_type(type) data = writer.serialize(value) reduce_args_budget = tuple_memory(0) + PY_OBJECT_OWNER_BYTES with pytest.raises(ValueError, match="Estimated graph memory budget exceeded"): reader = new_fory(reduce_args_budget - 1, xlang=False) reader.register_type(BudgetReduceObject) + reader.register_type(type) reader.deserialize(data) reader = new_fory(reduce_args_budget, xlang=False) reader.register_type(BudgetReduceObject) + reader.register_type(type) assert isinstance(reader.deserialize(data), BudgetReduceObject) @@ -394,6 +398,7 @@ def local_func(value=5): return value + captured writer = new_fory(xlang=False) + writer.register_type(types.FunctionType) data = writer.serialize(local_func) module_entries = len(sys.modules[local_func.__module__].__dict__) @@ -408,9 +413,13 @@ def local_func(value=5): + map_memory(0) ) with pytest.raises(ValueError, match="Estimated graph memory budget exceeded"): - new_fory(budget - PY_OBJECT_OWNER_BYTES, xlang=False).deserialize(data) + reader = new_fory(budget - PY_OBJECT_OWNER_BYTES, xlang=False) + reader.register_type(types.FunctionType) + reader.deserialize(data) - restored = new_fory(budget, xlang=False).deserialize(data) + reader = new_fory(budget, xlang=False) + reader.register_type(types.FunctionType) + restored = reader.deserialize(data) assert restored() == local_func() @@ -423,15 +432,20 @@ class LocalBudgetClass: cls = make_class() writer = new_fory(xlang=False) + writer.register_type(type) data = writer.serialize(cls) class_attrs = {name: value for name, value in cls.__dict__.items() if name not in SKIP_CLASS_ATTR_NAMES} class_attr_value_budget = sum(collection_memory(len(value)) for value in class_attrs.values() if isinstance(value, tuple)) budget = collection_memory(1) + PY_OBJECT_OWNER_BYTES + map_memory(len(class_attrs)) + class_attr_value_budget with pytest.raises(ValueError, match="Estimated graph memory budget exceeded"): - new_fory(collection_memory(1), xlang=False).deserialize(data) + reader = new_fory(collection_memory(1), xlang=False) + reader.register_type(type) + reader.deserialize(data) - restored = new_fory(budget, xlang=False).deserialize(data) + reader = new_fory(budget, xlang=False) + reader.register_type(type) + restored = reader.deserialize(data) assert restored.__name__ == cls.__name__ assert restored.__bases__ == cls.__bases__ diff --git a/python/pyfory/tests/test_metastring_resolver.py b/python/pyfory/tests/test_metastring_resolver.py index 0e49cd53da..87b7008b4e 100644 --- a/python/pyfory/tests/test_metastring_resolver.py +++ b/python/pyfory/tests/test_metastring_resolver.py @@ -276,7 +276,7 @@ def validate_module(self, module_name, *, is_local, **kwargs): ENABLE_FORY_CYTHON_SERIALIZATION, reason="pure TypeResolver regression", ) -def test_namespace_alias_not_cached(): +def test_namespace_alias_rejected(): config = Fory(xlang=True, compatible=False, strict=False).config resolver = TypeResolver(config, shared_registry=SharedRegistry()) resolver.initialize() @@ -297,7 +297,8 @@ def test_namespace_alias_not_cached(): meta_string_reader=MetaStringReader(resolver.shared_registry), ) - assert resolver.read_type_info(read_context) is typeinfo + with pytest.raises(Exception): + resolver.read_type_info(read_context) assert (namespace, typename) not in resolver._ns_type_to_type_info assert ( typeinfo.namespace_bytes, @@ -305,16 +306,13 @@ def test_namespace_alias_not_cached(): ) in resolver._ns_type_to_type_info -def test_wire_type_alias_cache_is_bounded(): +def test_wire_type_alias_rejected(): fory = Fory(xlang=True, compatible=False, strict=False) resolver = fory.type_resolver typeinfo = resolver.register_type( NamespaceAliasType, name="trusted.NamespaceAliasType", ) - for i in range(MAX_CACHED_ENCODED_META_STRINGS): - resolver._ns_type_to_type_info[(i, i)] = typeinfo - namespace = resolver.shared_registry.get_encoded_meta_string(MetaStringEncoder(".", "_").encode("trusted")) typename = resolver.shared_registry.get_encoded_meta_string(MetaStringEncoder("$", "_").encode("namespaceAliasType")) buffer = Buffer.allocate(128) @@ -325,7 +323,8 @@ def test_wire_type_alias_cache_is_bounded(): buffer.set_reader_index(0) try: fory.read_context.prepare(buffer) - assert resolver.read_type_info(fory.read_context) is typeinfo + with pytest.raises(Exception): + resolver.read_type_info(fory.read_context) assert (namespace, typename) not in resolver._ns_type_to_type_info finally: fory.reset_read() diff --git a/python/pyfory/tests/test_method.py b/python/pyfory/tests/test_method.py index 41b21b2962..20933ee8ab 100644 --- a/python/pyfory/tests/test_method.py +++ b/python/pyfory/tests/test_method.py @@ -15,9 +15,23 @@ # specific language governing permissions and limitations # under the License. +import types + import pyfory +def register_method_types(fory, *classes): + for cls in ( + type, + types.FunctionType, + types.MethodType, + staticmethod, + classmethod, + *classes, + ): + fory.register_type(cls) + + # Global classes for testing global class method serialization class GlobalTestClass: """Global test class for method serialization.""" @@ -86,6 +100,7 @@ def instance_method(self): obj = TestClass(5) method = obj.instance_method + register_method_types(fory, TestClass) # Test serialization/deserialization serialized = fory.serialize(method) @@ -106,6 +121,7 @@ def class_method(cls): return cls.class_var method = TestClass.class_method + register_method_types(fory, TestClass) # Test serialization/deserialization serialized = fory.serialize(method) @@ -124,6 +140,7 @@ def static_method(): return "static_result" method = TestClass.static_method + register_method_types(fory, TestClass) # Test serialization/deserialization serialized = fory.serialize(method) @@ -152,6 +169,7 @@ def subtract(a, b): return a - b obj = TestClass(10) + register_method_types(fory, TestClass) # Test instance method instance_method = obj.add @@ -174,7 +192,7 @@ def subtract(a, b): assert static_method(10, 3) == deserialized(10, 3) assert static_method(10, 3) == 7 - def test_nested_class_method_serialization(self): + def test_nested_class_method(self): """Test serialization of methods from nested classes.""" fory = pyfory.Fory(xlang=False, strict=False, ref=True, compatible=False) @@ -185,6 +203,7 @@ def inner_class_method(cls): return "inner_result" method = OuterClass.InnerClass.inner_class_method + register_method_types(fory, OuterClass, OuterClass.InnerClass) # Test serialization/deserialization serialized = fory.serialize(method) @@ -208,6 +227,7 @@ def g(): return A method = A.f + register_method_types(fory, A) serialized = fory.serialize(method) deserialized = fory.deserialize(serialized) @@ -223,29 +243,13 @@ def g(): assert original_result == deserialized_result -def test_staticmethod_serialization(): - """Standalone test for staticmethod serialization.""" - fory = pyfory.Fory(xlang=False, strict=False, ref=True, compatible=False) - - class A: - @staticmethod - def g(): - return "static_result" - - method = A.g - serialized = fory.serialize(method) - deserialized = fory.deserialize(serialized) - - assert method() == deserialized() - assert method() == "static_result" - - # Global class method tests -def test_global_classmethod_serialization(): +def test_global_classmethod(): """Test serialization of global class methods.""" fory = pyfory.Fory(xlang=False, strict=False, ref=True, compatible=False) method = GlobalTestClass.class_method + register_method_types(fory, GlobalTestClass) serialized = fory.serialize(method) deserialized = fory.deserialize(serialized) @@ -259,6 +263,7 @@ def test_global_classmethod_with_args(): fory = pyfory.Fory(xlang=False, strict=False, ref=True, compatible=False) method = GlobalTestClass.class_method_with_args + register_method_types(fory, GlobalTestClass) serialized = fory.serialize(method) deserialized = fory.deserialize(serialized) @@ -267,11 +272,12 @@ def test_global_classmethod_with_args(): assert deserialized(*args) == "class_global_class_value_arg1_arg2" -def test_global_staticmethod_serialization(): +def test_global_staticmethod(): """Test serialization of global static methods.""" fory = pyfory.Fory(xlang=False, strict=False, ref=True, compatible=False) method = GlobalTestClass.static_method + register_method_types(fory, GlobalTestClass) serialized = fory.serialize(method) deserialized = fory.deserialize(serialized) @@ -284,6 +290,7 @@ def test_global_staticmethod_with_args(): fory = pyfory.Fory(xlang=False, strict=False, ref=True, compatible=False) method = GlobalTestClass.static_method_with_args + register_method_types(fory, GlobalTestClass) serialized = fory.serialize(method) deserialized = fory.deserialize(serialized) @@ -292,12 +299,13 @@ def test_global_staticmethod_with_args(): assert deserialized(*args) == "static_test1_test2" -def test_global_instance_method_serialization(): +def test_global_instance_method(): """Test serialization of global instance methods.""" fory = pyfory.Fory(xlang=False, strict=False, ref=True, compatible=False) obj = GlobalTestClass("test_value") method = obj.instance_method + register_method_types(fory, GlobalTestClass) serialized = fory.serialize(method) deserialized = fory.deserialize(serialized) @@ -312,6 +320,7 @@ def test_multiple_global_classes(): # Test methods from different global classes method1 = GlobalTestClass.class_method method2 = AnotherGlobalClass.another_class_method + register_method_types(fory, GlobalTestClass, AnotherGlobalClass) serialized1 = fory.serialize(method1) serialized2 = fory.serialize(method2) @@ -331,6 +340,7 @@ def test_global_class_inheritance(): # Test inherited class method method = GlobalClassWithInheritance.inherited_class_method + register_method_types(fory, GlobalTestClass, GlobalClassWithInheritance) serialized = fory.serialize(method) deserialized = fory.deserialize(serialized) @@ -346,12 +356,13 @@ def test_global_class_inheritance(): assert deserialized_parent() == "class_inherited_value" # Uses child's class_variable -def test_global_methods_without_ref_tracking(): +def test_global_methods_without_refs(): """Test serialization of global class methods without reference tracking.""" fory = pyfory.Fory(xlang=False, strict=False, ref=False, compatible=False) # Global classes should work even without track_ref method = GlobalTestClass.class_method + register_method_types(fory, GlobalTestClass) serialized = fory.serialize(method) deserialized = fory.deserialize(serialized) @@ -368,6 +379,7 @@ def test_global_method_collection(): GlobalTestClass.static_method, AnotherGlobalClass.another_class_method, ] + register_method_types(fory, GlobalTestClass, AnotherGlobalClass) serialized = fory.serialize(methods) deserialized = fory.deserialize(serialized) @@ -386,13 +398,14 @@ def test_global_method_in_dict(): "static_method": GlobalTestClass.static_method, "another_method": AnotherGlobalClass.another_class_method, } + register_method_types(fory, GlobalTestClass, AnotherGlobalClass) serialized = fory.serialize(method_dict) deserialized = fory.deserialize(serialized) assert len(deserialized) == len(method_dict) - for key in method_dict: - assert method_dict[key]() == deserialized[key]() + for key, method in method_dict.items(): + assert method() == deserialized[key]() if __name__ == "__main__": diff --git a/python/pyfory/tests/test_pickle_buffer.py b/python/pyfory/tests/test_pickle_buffer.py index a073c084b5..d571b40a22 100644 --- a/python/pyfory/tests/test_pickle_buffer.py +++ b/python/pyfory/tests/test_pickle_buffer.py @@ -25,11 +25,6 @@ except ImportError: np = None -try: - import pandas as pd -except ImportError: - pd = None - def test_pickle_buffer_serialization(): fory = Fory(xlang=False, ref=False, strict=False, compatible=False) @@ -60,28 +55,6 @@ def test_numpy_out_of_band_serialization(): np.testing.assert_array_equal(arr, deserialized) -@pytest.mark.skipif(pd is None, reason="Requires pandas") -def test_pandas_out_of_band_serialization(): - fory = Fory(xlang=False, ref=False, strict=False, compatible=False) - - df = pd.DataFrame( - { - "a": np.arange(1000, dtype=np.float64), - "b": np.arange(1000, dtype=np.int64), - "c": ["text"] * 1000, - } - ) - - buffer_objects = [] - serialized = fory.serialize(df, buffer_callback=buffer_objects.append) - - buffers = [o.getbuffer() for o in buffer_objects] - - deserialized = fory.deserialize(serialized, buffers=buffers) - - pd.testing.assert_frame_equal(df, deserialized) - - @pytest.mark.skipif(np is None, reason="Requires numpy") def test_numpy_multiple_arrays_out_of_band(): fory = Fory(xlang=False, ref=True, strict=False, compatible=False) @@ -127,26 +100,6 @@ def test_numpy_with_mixed_types(): np.testing.assert_array_equal(arr, deserialized["array"]) -@pytest.mark.skipif(pd is None or np is None, reason="Requires numpy and pandas") -def test_mixed_numpy_pandas_out_of_band(): - fory = Fory(xlang=False, ref=True, strict=False, compatible=False) - - arr = np.arange(500, dtype=np.float64) - df = pd.DataFrame({"x": np.arange(500, dtype=np.int64), "y": np.arange(500, dtype=np.float32)}) - - data = {"array": arr, "dataframe": df} - - buffer_objects = [] - serialized = fory.serialize(data, buffer_callback=buffer_objects.append) - - buffers = [o.getbuffer() for o in buffer_objects] - - deserialized = fory.deserialize(serialized, buffers=buffers) - - np.testing.assert_array_equal(arr, deserialized["array"]) - pd.testing.assert_frame_equal(df, deserialized["dataframe"]) - - @pytest.mark.skipif(np is None, reason="Requires numpy") def test_selective_out_of_band_serialization(): fory = Fory(xlang=False, ref=True, strict=False, compatible=False) diff --git a/python/pyfory/tests/test_policy.py b/python/pyfory/tests/test_policy.py index 9db9cff94d..8bef3148f1 100644 --- a/python/pyfory/tests/test_policy.py +++ b/python/pyfory/tests/test_policy.py @@ -267,6 +267,7 @@ class UnsafeClass: policy = BlockClassPolicy(blocked_class_names=["UnsafeClass"]) fory = Fory(xlang=False, ref=True, strict=False, policy=policy, compatible=False) + fory.register_type(type) # Serialize and deserialize the class type itself (not an instance) safe_data = fory.serialize(SafeClass) @@ -291,6 +292,9 @@ def __reduce__(self): policy = BlockReduceCallPolicy(blocked_names=["ReducibleClass"]) fory = Fory(xlang=False, ref=True, strict=False, policy=policy, compatible=False) + fory.register_type(type) + fory.register_type(types.FunctionType) + fory.register_type(ReducibleClass) data = fory.serialize(ReducibleClass(42)) with pytest.raises(ValueError, match="ReducibleClass is blocked"): @@ -309,6 +313,9 @@ def __reduce__(self): policy = ReplaceObjectPolicy(replacement_value="REPLACED") fory = Fory(xlang=False, ref=True, strict=False, policy=policy, compatible=False) + fory.register_type(type) + fory.register_type(types.FunctionType) + fory.register_type(ReducibleClass) data = fory.serialize(ReducibleClass(42)) result = fory.deserialize(data) @@ -331,6 +338,7 @@ def __setstate__(self, state): policy = SanitizeStatePolicy() fory = Fory(xlang=False, ref=False, strict=False, policy=policy, compatible=False) + fory.register_type(SecretHolder) data = fory.serialize(SecretHolder("admin", "secret123")) result = fory.deserialize(data) @@ -364,6 +372,9 @@ def __setstate__(self, state): policy = CountingSanitizePolicy() fory = Fory(xlang=False, ref=True, strict=False, policy=policy, compatible=False) + fory.register_type(type) + fory.register_type(types.FunctionType) + fory.register_type(SecretReduceHolder) data = fory.serialize(SecretReduceHolder()) result = fory.deserialize(data) @@ -387,6 +398,8 @@ def intercept_setstate(self, obj, state, **kwargs): policy=BlockSetStatePolicy(), compatible=False, ) + fory.register_type(FalseyState) + fory.register_type(FalseyStatePayload) data = fory.serialize(FalseyStatePayload()) with pytest.raises(ValueError, match="state blocked"): @@ -434,6 +447,7 @@ class LocalClass: policy = BlockClassPolicy(blocked_class_names=["LocalClass"]) fory = Fory(xlang=False, ref=True, strict=False, policy=policy, compatible=False) + fory.register_type(type) # Serialize the local class type data = fory.serialize(LocalCls) @@ -454,6 +468,9 @@ def __reduce__(self): policy = BlockReduceCallPolicy(blocked_names=["ReducibleClass"]) fory = Fory(xlang=False, ref=True, strict=False, policy=policy, compatible=False) + fory.register_type(type) + fory.register_type(types.FunctionType) + fory.register_type(ReducibleClass) data = fory.serialize(ReducibleClass(42)) @@ -500,6 +517,9 @@ def __reduce__(self): policy = MultiHookPolicy() fory = Fory(xlang=False, ref=True, strict=False, policy=policy, compatible=False) + fory.register_type(type) + fory.register_type(types.FunctionType) + fory.register_type(TestClass) data = fory.serialize(TestClass(42)) result = fory.deserialize(data) @@ -529,6 +549,10 @@ def __reduce__(self): policy = BlockReduceCallPolicy(blocked_names=["Inner"]) fory = Fory(xlang=False, ref=True, strict=False, policy=policy, compatible=False) + fory.register_type(type) + fory.register_type(types.FunctionType) + fory.register_type(Inner) + fory.register_type(Outer) data = fory.serialize(Outer(Inner(42))) @@ -561,6 +585,7 @@ def authorize_instantiation(self, cls, **kwargs): policy = BlockInstantiationPolicy() fory = Fory(xlang=False, ref=True, strict=False, policy=policy, compatible=False) + fory.register_type(StatefulPayload) with pytest.raises(ValueError, match="StatefulPayload blocked"): fory.deserialize(fory.serialize(StatefulPayload())) assert policy.authorize_instantiation_calls == 1 @@ -590,6 +615,9 @@ def authorize_instantiation(self, cls, **kwargs): policy = BlockInstantiationPolicy() fory = Fory(xlang=False, ref=True, strict=False, policy=policy, compatible=False) + fory.register_type(type) + fory.register_type(ReducePayload) + fory.register_type(ReduceTarget) with pytest.raises(ValueError, match="ReduceTarget blocked"): fory.deserialize(fory.serialize(ReducePayload())) assert policy.reduce_target_calls == 1 @@ -774,6 +802,8 @@ def validate_method(self, method, is_local, **kwargs): policy = ReturnPolicy() fory = Fory(xlang=False, ref=True, strict=False, policy=policy, compatible=False) + fory.register_type(type) + fory.register_type(types.FunctionType) assert fory.deserialize(fory.serialize(json)) is json assert fory.deserialize(fory.serialize(PolicyGlobalClass)) is PolicyGlobalClass assert fory.deserialize(fory.serialize(policy_global_function)) is policy_global_function @@ -808,6 +838,11 @@ def validate_class(self, cls, is_local, **kwargs): policy=ReturnClassPolicy(), compatible=False, ) + fory.register_type(type) + fory.register_type(types.FunctionType) + fory.register_type(types.MethodType) + fory.register_type(staticmethod) + fory.register_type(classmethod) decoded = fory.deserialize(fory.serialize(make_payload_class())) assert decoded is not SafeClass assert decoded.run() == "payload" @@ -832,6 +867,7 @@ def validate_module(self, module_name, is_local, **kwargs): policy = BlockModulePolicy() fory = Fory(xlang=False, ref=True, strict=False, policy=policy, compatible=False) + fory.register_type(type) with pytest.raises(ValueError, match="subprocess blocked"): fory.deserialize(fory.serialize(subprocess.Popen)) assert policy.validate_module_calls == 1 @@ -856,6 +892,7 @@ def validate_function(self, func, is_local, **kwargs): policy = BlockMethodPolicy() fory = Fory(xlang=False, ref=True, strict=False, policy=policy, compatible=False) + fory.register_type(type(abs)) with pytest.raises(ValueError, match="method blocked"): fory.deserialize(fory.serialize([].append)) @@ -890,6 +927,9 @@ def validate_method(self, method, is_local, **kwargs): policy=BlockMethodPolicy(), compatible=False, ) + fory.register_type(types.FunctionType) + fory.register_type(types.MethodType) + fory.register_type(GuardedMethod) data = fory.serialize(method) GuardedMethod.getattribute_called = False @@ -1267,7 +1307,10 @@ def test_default_global_round_trips(): import time fory = Fory(xlang=False, ref=True, strict=False, compatible=False) - for value in (PolicyGlobalClass, policy_global_function, time.time, policy_reduce_global): + values = (PolicyGlobalClass, policy_global_function, time.time, policy_reduce_global) + for value in values: + fory.register_type(type(value)) + for value in values: assert fory.deserialize(fory.serialize(value)) is value @@ -1397,6 +1440,16 @@ def validate_method(self, method, is_local, **kwargs): writer = Fory(xlang=False, ref=True, strict=False, compatible=False) policy = ClassMethodPolicy() reader = Fory(xlang=False, ref=True, strict=False, policy=policy, compatible=False) + writer.register_type(type) + writer.register_type(types.FunctionType) + writer.register_type(types.MethodType) + writer.register_type(staticmethod) + writer.register_type(classmethod) + reader.register_type(type) + reader.register_type(types.FunctionType) + reader.register_type(types.MethodType) + reader.register_type(staticmethod) + reader.register_type(classmethod) data = writer.serialize(make_local_class()) with pytest.raises(ValueError, match="classmethod blocked"): @@ -1620,6 +1673,7 @@ def validate_module(self, module_name, is_local, **kwargs): policy = BlockModulePolicy() fory = Fory(xlang=False, ref=True, strict=False, policy=policy, compatible=False) + fory.register_type(types.FunctionType) with pytest.raises(ValueError, match="function module blocked"): fory.deserialize(fory.serialize(policy_global_function)) assert policy.validate_module_calls == 1 @@ -1646,6 +1700,7 @@ def validate_module(self, module_name, is_local, **kwargs): policy = BlockModulePolicy() fory = Fory(xlang=False, ref=True, strict=False, policy=policy, compatible=False) + fory.register_type(types.FunctionType) with pytest.raises(ValueError, match="local function module blocked"): fory.deserialize(fory.serialize(local_function)) assert policy.validate_module_calls == 1 @@ -1695,6 +1750,7 @@ def validate_module(self, module_name, is_local, **kwargs): policy = BlockModulePolicy() fory = Fory(xlang=False, ref=True, strict=False, policy=policy, compatible=False) + fory.register_type(type(abs)) with pytest.raises(ValueError, match="time blocked"): fory.deserialize(fory.serialize(time.time)) assert policy.validate_module_calls == 1 @@ -1781,6 +1837,7 @@ def validate_module(self, module_name, is_local, **kwargs): policy = BlockModulePolicy() fory = Fory(xlang=False, ref=True, strict=False, policy=policy, compatible=False) + fory.register_type(GlobalNamePayload) with pytest.raises(ValueError, match="subprocess blocked"): fory.deserialize(fory.serialize(GlobalNamePayload())) assert policy.validate_module_calls == 1 @@ -1812,6 +1869,7 @@ def validate_class(self, cls, is_local, **kwargs): policy = BlockClassPolicy() fory = Fory(xlang=False, ref=True, strict=False, policy=policy, compatible=False) + fory.register_type(GlobalNamePayload) with pytest.raises(ValueError, match="subprocess.Popen blocked"): fory.deserialize(fory.serialize(GlobalNamePayload())) assert policy.validate_module_calls == 1 @@ -1843,6 +1901,7 @@ def validate_function(self, func, is_local, **kwargs): policy = BlockFunctionPolicy() fory = Fory(xlang=False, ref=True, strict=False, policy=policy, compatible=False) + fory.register_type(GlobalNamePayload) with pytest.raises(ValueError, match="eval blocked"): fory.deserialize(fory.serialize(GlobalNamePayload())) assert policy.validate_module_calls == 1 @@ -1877,6 +1936,7 @@ def validate_function(self, func, is_local, **kwargs): policy = MethodPolicy() fory = Fory(xlang=False, ref=True, strict=False, policy=policy, compatible=False) + fory.register_type(GlobalNamePayload) with pytest.raises(ValueError, match="method blocked"): fory.deserialize(fory.serialize(GlobalNamePayload())) assert policy.validate_module_calls == 1 diff --git a/python/pyfory/tests/test_reduce_serializer.py b/python/pyfory/tests/test_reduce_serializer.py index 8442fcf5b9..49e170f8d0 100644 --- a/python/pyfory/tests/test_reduce_serializer.py +++ b/python/pyfory/tests/test_reduce_serializer.py @@ -26,6 +26,11 @@ from pyfory.serializer import ReduceSerializer +def register_reduce_types(fory, *classes): + for cls in (type, types.BuiltinFunctionType, *classes): + fory.register_type(cls) + + _reduce_factory_calls = [] _class_callable_calls = [] _function_factory_calls = [] @@ -295,6 +300,7 @@ def validate_function(self, func, is_local, **kwargs): def test_nonstrict_reduce_global(): _reduce_factory_calls.clear() fory = Fory(xlang=False, ref=True, strict=False, compatible=False) + fory.register_type(NestedGlobalReduce) assert fory.deserialize(fory.serialize(NestedGlobalReduce("outer", "allowed"))) == NestedGlobalReduce("result", "allowed") assert _reduce_factory_calls == ["allowed"] @@ -389,6 +395,8 @@ def validate_function(self, func, is_local, **kwargs): def test_nonstrict_reduce_function(): fory = Fory(xlang=False, ref=True, strict=False, compatible=False) + fory.register_type(types.FunctionType) + fory.register_type(FunctionCallableReduce) _function_factory_calls.clear() value = FunctionCallableReduce("allowed") @@ -457,6 +465,7 @@ def test_basic_reduce_object(): fory = Fory(xlang=False, ref=True, strict=False, compatible=False) obj = BasicReduceObject(42, 3) + register_reduce_types(fory, BasicReduceObject) # Verify ReduceSerializer is used serializer = fory.type_resolver.get_serializer(BasicReduceObject) @@ -476,6 +485,7 @@ def test_reduce_with_state_object(): fory = Fory(xlang=False, ref=True, strict=False, compatible=False) obj = ReduceWithStateObject("test", {"key": "value"}) + register_reduce_types(fory, ReduceWithStateObject) # Verify ReduceSerializer is used serializer = fory.type_resolver.get_serializer(ReduceWithStateObject) @@ -496,6 +506,7 @@ def test_reduce_ex_object(): fory = Fory(xlang=False, ref=True, strict=False, compatible=False) obj = ReduceExObject(5, 7) + register_reduce_types(fory, ReduceExObject) # Verify ReduceSerializer is used serializer = fory.type_resolver.get_serializer(ReduceExObject) @@ -516,6 +527,7 @@ def test_reduce_with_list_items(): fory = Fory(xlang=False, ref=True, strict=False, compatible=False) obj = ReduceWithListItems([1, 2, 3, 4]) + register_reduce_types(fory, ReduceWithListItems) # Verify ReduceSerializer is used serializer = fory.type_resolver.get_serializer(ReduceWithListItems) @@ -535,6 +547,7 @@ def test_reduce_with_dict_items(): fory = Fory(xlang=False, ref=True, strict=False, compatible=False) obj = ReduceWithDictItems({"a": 1, "b": 2}) + register_reduce_types(fory, ReduceWithDictItems) # Verify ReduceSerializer is used serializer = fory.type_resolver.get_serializer(ReduceWithDictItems) @@ -549,11 +562,12 @@ def test_reduce_with_dict_items(): assert deserialized.name == "dict_obj" -def test_reduce_precedence_over_stateful(): +def test_reduce_precedes_stateful(): """Test that ReduceSerializer has higher precedence than StatefulSerializer""" fory = Fory(xlang=False, ref=True, strict=False, compatible=False) obj = BothReduceAndStateful(100) + register_reduce_types(fory, BothReduceAndStateful) # Verify ReduceSerializer is used, not StatefulSerializer serializer = fory.type_resolver.get_serializer(BothReduceAndStateful) @@ -576,6 +590,7 @@ def test_reference_tracking(): obj1 = BasicReduceObject(42) obj2 = BasicReduceObject(42) container = [obj1, obj1, obj2] # obj1 appears twice + register_reduce_types(fory, BasicReduceObject) serialized = fory.serialize(container) deserialized = fory.deserialize(serialized) @@ -595,6 +610,7 @@ def test_nested_reduce_objects(): inner = BasicReduceObject(10, 2) outer = ReduceWithStateObject("outer", {"inner": inner}) + register_reduce_types(fory, BasicReduceObject, ReduceWithStateObject) serialized = fory.serialize(outer) deserialized = fory.deserialize(serialized) @@ -604,21 +620,3 @@ def test_nested_reduce_objects(): assert deserialized.data["inner"] == inner assert deserialized.data["inner"].value == 10 assert deserialized.data["inner"].multiplier == 2 - - -def test_cross_language_compatibility(): - """Test cross-language compatibility""" - fory = Fory(xlang=False, ref=True, strict=False, compatible=False) - - obj = BasicReduceObject(123, 4) - - # Serialize with Python - serialized = fory.serialize(obj) - - # Should be able to deserialize (basic test) - deserialized = fory.deserialize(serialized) - assert deserialized == obj - - # The serialized data should use Fory's native format, not pickle - # This is verified by the fact that we're using write_ref/read_ref - # in the ReduceSerializer implementation diff --git a/python/pyfory/tests/test_ref_tracking.py b/python/pyfory/tests/test_ref_tracking.py index f5dd689b98..e3f74c512f 100644 --- a/python/pyfory/tests/test_ref_tracking.py +++ b/python/pyfory/tests/test_ref_tracking.py @@ -123,6 +123,7 @@ def test_collection_tuple_shared_reference_python_mode(): def test_collection_set_element_alias_with_outer_reference_python_mode(): fory = pyfory.Fory(xlang=False, ref=True, strict=False, compatible=False) token = HashKey("shared-key") + fory.register_type(HashKey) payload = [{token}, token] restored = _roundtrip(fory, payload) @@ -155,6 +156,7 @@ def test_map_self_cycle_and_shared_submap_python_mode(): def test_map_key_alias_with_outer_reference_python_mode(): fory = pyfory.Fory(xlang=False, ref=True, strict=False, compatible=False) key = HashKey("k") + fory.register_type(HashKey) payload = [{key: "value"}, key] restored = _roundtrip(fory, payload) diff --git a/python/pyfory/tests/test_serializer.py b/python/pyfory/tests/test_serializer.py index 8815610988..c9639f0cc3 100644 --- a/python/pyfory/tests/test_serializer.py +++ b/python/pyfory/tests/test_serializer.py @@ -22,6 +22,7 @@ import io import os import pickle +import types import weakref from collections.abc import MutableSequence from enum import Enum, IntEnum @@ -35,6 +36,7 @@ import pytest import pyfory +import pyfory.registry as registry_module from pyfory.serialization import Buffer, _bfloat16_from_bits, _bfloat16_to_bits, _float16_from_bits, _float16_to_bits from pyfory import Fory, EnumSerializer from pyfory.serializer import ( @@ -46,6 +48,7 @@ Numpy1DArraySerializer, ) from pyfory.types import TypeId +from pyfory.union import UnionSerializer from pyfory.utils import lazy_import pa = lazy_import("pyarrow") @@ -685,6 +688,8 @@ def test_ref_cleanup(): fory = Fory(xlang=False, ref=True, strict=False, compatible=False) o1 = RefTestClass1() o2 = RefTestClass2(f1=o1) + fory.register_type(RefTestClass1) + fory.register_type(RefTestClass2) pickle.loads(pickle.dumps(o2)) ref1 = weakref.ref(o1) ref2 = weakref.ref(o2) @@ -792,6 +797,97 @@ def __init__(self, f1=None): self.f1 = f1 +class FrozenRegistration: + pass + + +class RejectedRegistration: + pass + + +@dataclass +class FrozenChild: + value: int + + +@dataclass +class FrozenParent: + child: FrozenChild + + +@dataclass +class BrokenFinalization: + value: int + + +@dataclass +class FrozenRecursive: + child: Optional["FrozenRecursive"] = None + + +class FrozenMetadataEnum(Enum): + VALUE = 1 + + +@dataclass +class FrozenExt: + value: int + + +@dataclass +class FrozenSecondExt: + value: int + + +class FrozenExtSerializer(pyfory.Serializer): + def write(self, write_context, value): + write_context.write_int32(value.value) + + def read(self, read_context): + return self.type_(read_context.read_int32()) + + +class FrozenUnion: + def __init__(self, case_id, value): + self._case_id = case_id + self._value = value + + def case_id(self): + return self._case_id + + @staticmethod + def _from_case_id(case_id, value): + return FrozenUnion(case_id, value) + + def __eq__(self, other): + return isinstance(other, FrozenUnion) and (self._case_id, self._value) == (other._case_id, other._value) + + +def registration_state(fory): + resolver = fory.type_resolver + maps = tuple( + (name, dict(getattr(resolver, name))) + for name in ( + "_types_info", + "_type_id_to_type_info", + "_user_type_id_to_type_info", + "_ns_type_to_type_info", + "_named_type_to_type_info", + "_local_type_info_by_hash", + "_meta_shared_type_info", + ) + if hasattr(resolver, name) + ) + used_type_ids = getattr(resolver, "_used_user_type_ids", None) + return ( + maps, + None if used_type_ids is None else set(used_type_ids), + getattr(resolver, "_type_id_counter", None), + dict(resolver.shared_registry._metastr_to_bytes), + dict(resolver.shared_registry._encoded_metastrings), + ) + + def test_register_py_serializer(): fory = Fory(xlang=False, ref=True, strict=False, compatible=False) @@ -800,12 +896,12 @@ def write(self, write_context, value): write_context.write_int32(value.f1) def read(self, read_context): - a = A() - a.f1 = read_context.read_int32() - return a + return self.type_(read_context.read_int32()) - fory.register_type(A, serializer=Serializer(fory.type_resolver, RegisterClass)) - assert fory.deserialize(fory.serialize(RegisterClass(100))).f1 == 100 + fory.register_type(RegisterClass, serializer=Serializer(fory.type_resolver, RegisterClass)) + value = fory.deserialize(fory.serialize(RegisterClass(100))) + assert isinstance(value, RegisterClass) + assert value.f1 == 100 @pytest.mark.parametrize("registration", ["id", "name"]) @@ -902,19 +998,524 @@ def test_register_type_name_exclusive(): fory.register_type(A, type_id=100, name="example.A") -def test_np_types(): - fory = Fory(xlang=False, ref=True, strict=False, compatible=False) - o1 = [1, True, np.dtype(np.int32)] - data1 = fory.serialize(o1) - new_o1 = fory.deserialize(data1) - assert o1 == new_o1 +@pytest.mark.parametrize("root", ["serialize", "deserialize", "dump"]) +def test_registry_freezes_at_root(root): + fory = Fory(xlang=True, compatible=False) + fory.register_type(FrozenRegistration, type_id=701) + if root == "serialize": + fory.serialize(None) + elif root == "deserialize": + with pytest.raises(Exception): + fory.deserialize(b"") + else: + fory.dump(None, io.BytesIO()) + + registrations = ( + lambda: fory.register(RejectedRegistration, type_id=702), + lambda: fory.register_type(RejectedRegistration, name="test.Rejected"), + lambda: fory.register_union( + RejectedRegistration, + name="test.RejectedUnion", + serializer=object(), + ), + lambda: fory.register_serializer(FrozenRegistration, object()), + lambda: fory.type_resolver.register_type(RejectedRegistration, type_id=702), + lambda: fory.type_resolver.register_union( + RejectedRegistration, + name="test.RejectedUnion", + serializer=object(), + ), + lambda: fory.type_resolver.register_serializer(FrozenRegistration, object()), + ) + for registration in registrations: + with pytest.raises(Exception): + registration() + assert fory.type_resolver.get_type_info(RejectedRegistration, create=False) is None + assert fory.type_resolver.get_type_info(FrozenRegistration).type_id == TypeId.STRUCT + + +def test_frozen_serializer_lookup(monkeypatch): + fory = Fory(xlang=False, strict=False, compatible=False) + fory.serialize(None) + serializer_count = 0 + serializer_type = registry_module._DefaultPolicyObjectSerializer + + class CountingSerializer(serializer_type): + def __init__(self, type_resolver, cls): + nonlocal serializer_count + serializer_count += 1 + super().__init__(type_resolver, cls) + + monkeypatch.setattr( + registry_module, + "_DefaultPolicyObjectSerializer", + CountingSerializer, + ) + registry_sizes = tuple( + len(getattr(fory.type_resolver, attr)) + for attr in ( + "_types_info", + "_type_id_to_type_info", + "_user_type_id_to_type_info", + "_ns_type_to_type_info", + ) + ) -def test_pandas_dataframe(): - fory = Fory(xlang=False, ref=True, strict=False, compatible=False) - df = pd.DataFrame({"a": list(range(10))}) - df2 = fory.deserialize(fory.serialize(df)) - assert df2.equals(df) + with pytest.raises(Exception): + fory.type_resolver.register_serializer(RejectedRegistration, object()) + with pytest.raises(Exception): + fory.type_resolver.get_type_info(RejectedRegistration) + + assert serializer_count == 0 + assert registry_sizes == tuple( + len(getattr(fory.type_resolver, attr)) + for attr in ( + "_types_info", + "_type_id_to_type_info", + "_user_type_id_to_type_info", + "_ns_type_to_type_info", + ) + ) + assert fory.type_resolver.get_type_info(RejectedRegistration, create=False) is None + + +def test_frozen_named_lookup(monkeypatch): + writer = Fory(xlang=True, compatible=False) + writer.register_type(FrozenRegistration, name="test.FrozenWireType") + data = writer.serialize(FrozenRegistration()) + + reader = Fory(xlang=True, strict=False, compatible=False) + loaded = False + + def reject_load(*_args, **_kwargs): + nonlocal loaded + loaded = True + raise AssertionError("frozen named lookup must not load a class") + + monkeypatch.setattr(registry_module, "load_class", reject_load) + with pytest.raises(Exception): + reader.deserialize(data) + assert not loaded + + +def test_registered_types_finalize(): + fory = Fory(xlang=True, compatible=True) + parent_info = fory.register_type(FrozenParent, name="test.FrozenParent") + child_info = fory.register_type(FrozenChild, name="test.FrozenChild") + assert parent_info.serializer is None + assert child_info.serializer is None + + value = FrozenParent(FrozenChild(7)) + data = fory.serialize(value) + + assert parent_info.serializer is not None + assert child_info.serializer is not None + assert parent_info.type_def is not None + assert child_info.type_def is not None + assert fory.deserialize(data) == value + + +def test_lazy_dataclass_finalizes(): + from pyfory.struct import DataClassStubSerializer + + fory = Fory(xlang=False, strict=False, compatible=False) + type_info = fory.register_type(BrokenFinalization) + assert isinstance(type_info.serializer, DataClassStubSerializer) + + value = BrokenFinalization(7) + data = fory.serialize(value) + + assert not isinstance(type_info.serializer, DataClassStubSerializer) + assert fory.deserialize(data) == value + + +def test_recursive_serializer_stable(): + fory = Fory(xlang=True, compatible=True, ref=True) + type_info = fory.register_type(FrozenRecursive, name="test.FrozenRecursive") + value = FrozenRecursive(FrozenRecursive()) + + first = fory.serialize(value) + serializer = type_info.serializer + maps = tuple( + dict(getattr(fory.type_resolver, attr)) + for attr in ( + "_types_info", + "_type_id_to_type_info", + "_user_type_id_to_type_info", + "_ns_type_to_type_info", + ) + ) + + assert fory.deserialize(first) == value + assert fory.deserialize(fory.serialize(value)) == value + assert type_info.serializer is serializer + assert maps == tuple( + dict(getattr(fory.type_resolver, attr)) + for attr in ( + "_types_info", + "_type_id_to_type_info", + "_user_type_id_to_type_info", + "_ns_type_to_type_info", + ) + ) + + +def test_named_metadata_finalizes(): + enum_fory = Fory(xlang=True, compatible=True) + enum_info = enum_fory.register_type(FrozenMetadataEnum, name="test.FrozenMetadataEnum") + enum_serializer = enum_info.serializer + enum_data = enum_fory.serialize(FrozenMetadataEnum.VALUE) + assert enum_fory.deserialize(enum_data) is FrozenMetadataEnum.VALUE + assert enum_info.serializer is enum_serializer + + ext_fory = Fory(xlang=True, compatible=True) + ext_serializer = FrozenExtSerializer(ext_fory.type_resolver, FrozenExt) + ext_info = ext_fory.register_type(FrozenExt, name="test.FrozenExt", serializer=ext_serializer) + ext_data = ext_fory.serialize(FrozenExt(7)) + assert ext_fory.deserialize(ext_data) == FrozenExt(7) + assert ext_info.serializer is ext_serializer + + union_fory = Fory(xlang=True, compatible=True) + union_serializer = UnionSerializer(union_fory.type_resolver, FrozenUnion, {0: str}) + union_info = union_fory.register_union( + FrozenUnion, + name="test.FrozenUnion", + serializer=union_serializer, + ) + union_data = union_fory.serialize(FrozenUnion(0, "value")) + assert union_fory.deserialize(union_data) == FrozenUnion(0, "value") + assert union_info.serializer is union_serializer + + for type_info in (enum_info, ext_info, union_info): + assert type_info.type_def is not None + + +def test_pre_root_serializer_rebind(): + fory = Fory(xlang=True, compatible=True) + type_info = fory.register_type(FrozenExt, name="test.ReboundExt") + fory.type_resolver.get_serializer(FrozenExt) + old_header = Buffer(type_info.type_def.encoded).read_int64() + id_map = dict(fory.type_resolver._type_id_to_type_info) + + serializer = FrozenExtSerializer(fory.type_resolver, FrozenExt) + fory.register_serializer(FrozenExt, serializer) + assert type_info.type_def is None + assert fory.type_resolver._type_id_to_type_info == id_map + + data = fory.serialize(FrozenExt(9)) + assert fory.deserialize(data) == FrozenExt(9) + assert type_info.serializer is serializer + assert type_info.type_id == TypeId.NAMED_EXT + new_header = Buffer(type_info.type_def.encoded).read_int64() + assert new_header != old_header + + +def test_rebind_skips_default_build(monkeypatch): + fory = Fory(xlang=True, compatible=True) + type_info = fory.register_type(BrokenFinalization, name="test.ReboundPending") + serializer = FrozenExtSerializer(fory.type_resolver, BrokenFinalization) + + def reject_default_build(*_args): + raise AssertionError("custom serializer registration must not build the default serializer") + + monkeypatch.setattr(registry_module, "encode_typedef", reject_default_build) + fory.register_serializer(BrokenFinalization, serializer) + + assert type_info.serializer is serializer + assert type_info.type_def is None + + +def test_named_serializer_id_map(): + fory = Fory(xlang=True, compatible=True) + first = fory.register_type(FrozenExt, name="test.NamedFirst") + second = fory.register_type(FrozenSecondExt, name="test.NamedSecond") + for cls in (FrozenExt, FrozenSecondExt): + fory.type_resolver.get_serializer(cls) + id_map = dict(fory.type_resolver._type_id_to_type_info) + wire_map = dict(fory.type_resolver._ns_type_to_type_info) + named_infos = ( + fory.type_resolver.get_type_info_by_name("test", "NamedFirst"), + fory.type_resolver.get_type_info_by_name("test", "NamedSecond"), + ) + + first_serializer = FrozenExtSerializer(fory.type_resolver, FrozenExt) + second_serializer = FrozenExtSerializer(fory.type_resolver, FrozenSecondExt) + fory.register_serializer(FrozenExt, first_serializer) + fory.register_serializer(FrozenSecondExt, second_serializer) + + assert fory.type_resolver._type_id_to_type_info == id_map + assert TypeId.NAMED_EXT not in fory.type_resolver._type_id_to_type_info + assert fory.type_resolver._ns_type_to_type_info == wire_map + assert named_infos == ( + fory.type_resolver.get_type_info_by_name("test", "NamedFirst"), + fory.type_resolver.get_type_info_by_name("test", "NamedSecond"), + ) + assert fory.deserialize(fory.serialize(FrozenExt(3))) == FrozenExt(3) + assert first.serializer is first_serializer + assert second.serializer is second_serializer + + +def test_failed_finalization_freezes(monkeypatch): + writer = Fory(xlang=True, compatible=True) + writer.register_type(FrozenChild, name="test.PendingFinalization") + pending_data = writer.serialize(FrozenChild(7)) + + fory = Fory(xlang=True, compatible=True) + type_info = fory.register_type( + BrokenFinalization, + name="test.BrokenFinalization", + ) + pending_info = fory.register_type( + FrozenChild, + name="test.PendingFinalization", + ) + encode_type_def = registry_module.encode_typedef + + class FinalizationAbort(BaseException): + pass + + def fail_finalization(resolver, cls): + if cls is type_info.cls: + assert type_info.serializer is not None + raise FinalizationAbort + return encode_type_def(resolver, cls) + + monkeypatch.setattr( + registry_module, + "encode_typedef", + fail_finalization, + ) + + with pytest.raises(FinalizationAbort): + fory.serialize(None) + assert type_info.serializer is None + assert type_info.type_def is None + assert pending_info.serializer is None + assert pending_info.type_def is None + + with pytest.raises(Exception): + fory.deserialize(pending_data) + assert pending_info.serializer is None + assert pending_info.type_def is None + + with pytest.raises(Exception): + fory.type_resolver.get_type_info(type_info.cls) + assert type_info.serializer is None + assert type_info.type_def is None + with pytest.raises(Exception): + fory.register_type(RejectedRegistration, name="test.Rejected") + assert fory.type_resolver.get_type_info(RejectedRegistration, create=False) is None + + +def test_native_carrier_registration(): + writer = Fory(xlang=False, strict=False, compatible=False) + reader = Fory(xlang=False, strict=False, compatible=False) + discovered = Fory(xlang=False, strict=False, compatible=False) + + writer_info = writer.register_type(types.FunctionType) + reader_info = reader.register_type(types.FunctionType) + discovered_info = discovered.type_resolver.get_type_info(types.FunctionType) + + assert writer_info.type_id == reader_info.type_id == discovered_info.type_id + value = lambda number: number + 1 # noqa: E731 + assert reader.deserialize(writer.serialize(value))(4) == 5 + + +def test_native_application_type(): + class PlainValue: + pass + + @dataclass + class DataValue: + value: int + + fory = Fory(xlang=False, strict=False, compatible=False) + plain_info = fory.register_type(PlainValue) + data_info = fory.register_type(DataValue) + + assert plain_info.type_id == TypeId.STRUCT + assert data_info.type_id == TypeId.STRUCT + + +def test_registration_conflicts(): + class First: + pass + + class Second: + pass + + class DifferentKind(Enum): + VALUE = 1 + + fory = Fory(xlang=True, compatible=False) + first = fory.register_type(First, name="SameName") + with pytest.raises(Exception): + fory.register_type(Second, name=".SameName") + with pytest.raises(Exception): + fory.register_type(DifferentKind, name="SameName") + assert fory.type_resolver.get_type_info_by_name("", "SameName") is first + assert fory.type_resolver.get_type_info(Second, create=False) is None + assert fory.type_resolver.get_type_info(DifferentKind, create=False) is None + + numeric = Fory(xlang=True, compatible=False) + first = numeric.register_type(First, type_id=703) + with pytest.raises(Exception): + numeric.register_type(Second, type_id=703) + assert ( + numeric.type_resolver.get_type_info_by_id( + TypeId.STRUCT, + user_type_id=703, + ) + is first + ) + assert numeric.type_resolver.get_type_info(Second, create=False) is None + + for registration in ("name", "id"): + union_fory = Fory(xlang=True, compatible=False) + serializer = UnionSerializer(union_fory.type_resolver, FrozenUnion, {0: str}) + if registration == "name": + union_fory.register_union( + FrozenUnion, + name="test.FirstUnion", + serializer=serializer, + ) + + def duplicate(): + union_fory.register_union( + FrozenUnion, + name="test.SecondUnion", + serializer=serializer, + ) + + else: + union_fory.register_union(FrozenUnion, type_id=704, serializer=serializer) + + def duplicate(): + union_fory.register_union( + FrozenUnion, + type_id=705, + serializer=serializer, + ) + + state = ( + dict(union_fory.type_resolver._types_info), + dict(union_fory.type_resolver._ns_type_to_type_info), + dict(union_fory.type_resolver._user_type_id_to_type_info), + ) + with pytest.raises(Exception): + duplicate() + assert state == ( + dict(union_fory.type_resolver._types_info), + dict(union_fory.type_resolver._ns_type_to_type_info), + dict(union_fory.type_resolver._user_type_id_to_type_info), + ) + + +@pytest.mark.parametrize("registration", ["type", "union"]) +@pytest.mark.parametrize("type_id", [-1, 0xFFFFFFFF, 0x100000000, 701.5, "701", True]) +def test_registration_id_range(registration, type_id): + fory = Fory(xlang=True, compatible=False) + before = registration_state(fory) + if registration == "type": + + def register(): + fory.register_type(RejectedRegistration, type_id=type_id) + + else: + serializer = UnionSerializer(fory.type_resolver, FrozenUnion, {0: str}) + + def register(): + fory.register_union( + FrozenUnion, + type_id=type_id, + serializer=serializer, + ) + + with pytest.raises(Exception): + register() + assert registration_state(fory) == before + + +@pytest.mark.parametrize( + "name", + [ + pytest.param(f"{'n' * 32768}.Value", id="namespace"), + pytest.param(f"scope.{'V' * 32768}", id="typename"), + ], +) +def test_registration_name_preflight(name): + class NamedValue: + pass + + serializer_count = 0 + + class CountingSerializer(pyfory.Serializer): + def __init__(self, type_resolver, cls): + nonlocal serializer_count + serializer_count += 1 + super().__init__(type_resolver, cls) + + fory = Fory(xlang=True, compatible=False) + before = registration_state(fory) + with pytest.raises(Exception): + fory.register_type(NamedValue, name=name, serializer=CountingSerializer) + assert serializer_count == 0 + assert registration_state(fory) == before + + +@pytest.mark.parametrize("registration", ["type", "union"]) +def test_failed_registration_keeps_id(registration): + class FailedValue: + pass + + class NextValue: + pass + + class BrokenSerializer: + def __init__(self, *_args): + raise ValueError("serializer construction failed") + + fory = Fory(xlang=True, compatible=False) + before = registration_state(fory) + with pytest.raises(Exception): + if registration == "type": + fory.register_type(FailedValue, serializer=BrokenSerializer) + else: + fory.register_union(FailedValue, serializer=BrokenSerializer) + assert registration_state(fory) == before + + actual = fory.register_type(NextValue) + expected_fory = Fory(xlang=True, compatible=False) + expected = expected_fory.register_type(NextValue) + assert actual.user_type_id == expected.user_type_id + + +def test_duplicate_type_keeps_id(): + class FirstValue: + pass + + class NextValue: + pass + + serializer_count = 0 + + class CountingSerializer(pyfory.Serializer): + def __init__(self, type_resolver, cls): + nonlocal serializer_count + serializer_count += 1 + super().__init__(type_resolver, cls) + + fory = Fory(xlang=True, compatible=False) + first = fory.register_type(FirstValue) + before = registration_state(fory) + with pytest.raises(Exception): + fory.register_type(FirstValue, serializer=CountingSerializer) + assert serializer_count == 0 + assert registration_state(fory) == before + + next_value = fory.register_type(NextValue) + assert next_value.user_type_id == first.user_type_id + 1 def test_unsupported_callback(): @@ -929,6 +1530,7 @@ def f2(x): return x + x obj1 = [1, True, f1, f2, {1: 2}] + fory.register_type(type(f1)) unsupported_objects = [] binary1 = fory.serialize(obj1, unsupported_callback=unsupported_objects.append) # Functions are now properly supported, so unsupported_objects should be empty @@ -984,6 +1586,7 @@ class SparseIntEnum(IntEnum): def test_enum(): fory = Fory(xlang=False, ref=True, compatible=False) + fory.register_type(EnumClass) assert ser_de(fory, EnumClass.E1) == EnumClass.E1 assert ser_de(fory, EnumClass.E2) == EnumClass.E2 assert ser_de(fory, EnumClass.E3) == EnumClass.E3 @@ -1000,6 +1603,7 @@ def test_xlang_enum_uses_sparse_integer_values(): def test_duplicate_serialize(): fory = Fory(xlang=False, ref=True, compatible=False) + fory.register_type(EnumClass) assert ser_de(fory, EnumClass.E1) == EnumClass.E1 assert ser_de(fory, EnumClass.E2) == EnumClass.E2 assert ser_de(fory, EnumClass.E4) == EnumClass.E4 @@ -1013,7 +1617,7 @@ def test_pandas_range_index(): fory.register_type(pd.RangeIndex, serializer=pyfory.serializer.PandasRangeIndexSerializer(fory.type_resolver)) index = pd.RangeIndex(1, 100, 2, name="a") new_index = ser_de(fory, index) - pd.testing.assert_index_equal(new_index, new_index) + pd.testing.assert_index_equal(new_index, index) @dataclass(unsafe_hash=True) @@ -1035,6 +1639,7 @@ def test_py_serialize_dataclass(track_ref): strict=False, compatible=False, ) + fory.register_type(PyDataClass1) obj1 = PyDataClass1(f1=1, f2=-2.0, f3="abc", f4=True, f5="xyz", f6=[1, 2], f7={"k1": "v1"}) assert ser_de(fory, obj1) == obj1 obj2 = PyDataClass1(f1=None, f2=-2.0, f3="abc", f4=None, f5="xyz", f6=None, f7=None) @@ -1095,6 +1700,7 @@ def test_function(track_ref): strict=False, compatible=False, ) + fory.register_type(types.FunctionType) c = fory.deserialize(fory.serialize(lambda x: x * 2)) assert c(2) == 4 @@ -1104,10 +1710,6 @@ def func(x): c = fory.deserialize(fory.serialize(func)) assert c(2) == 4 - df = pd.DataFrame({"a": list(range(10))}) - df_sum = fory.deserialize(fory.serialize(df.sum)) - assert df_sum().equals(df.sum()) - @dataclass(unsafe_hash=True) class MapFields: @@ -1163,6 +1765,8 @@ def __eq__(self, other): map_fields_object.dict_with_custom_obj = dict_with_custom_obj map_fields_object.single_key_dict = single_key_dict + fory.register_type(MapFields) + fory.register_type(CustomClass) serialized = fory.serialize(map_fields_object) deserialized = fory.deserialize(serialized) @@ -1216,6 +1820,7 @@ def test_py_serialize_object(track_ref): def test_py_serialize_empty_object(track_ref): fory = Fory(xlang=False, ref=track_ref, strict=False, compatible=False) obj = object() + fory.register_type(object) result = ser_de(fory, obj) assert type(result) is object diff --git a/python/pyfory/tests/test_stateful_reproduction.py b/python/pyfory/tests/test_stateful_reproduction.py deleted file mode 100644 index 2e82bf1efd..0000000000 --- a/python/pyfory/tests/test_stateful_reproduction.py +++ /dev/null @@ -1,152 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. - -from pyfory import Fory - - -# Test class with __getstate__ and __setstate__ -class StatefulObject: - def __init__(self, value, secret=None): - self.value = value - self.secret = secret or "default_secret" - self.computed = self.value * 2 - - def __getstate__(self): - # Only serialize value, not secret or computed - return {"value": self.value} - - def __setstate__(self, state): - self.value = state["value"] - self.secret = "restored_secret" - self.computed = self.value * 2 - - def __eq__(self, other): - return isinstance(other, StatefulObject) and self.value == other.value and self.computed == other.computed - # Note: secret is expected to be different after deserialization - - def __repr__(self): - return f"StatefulObject(value={self.value}, secret={self.secret}, computed={self.computed})" - - -# Test class with getnewargs_ex -class ImmutableWithArgs: - def __init__(self, x, y, name="default"): - self._x = x - self._y = y - self._name = name - - def __getnewargs_ex__(self): - return (self._x, self._y), {"name": self._name} - - def __getstate__(self): - return {"extra_data": "some_extra"} - - def __setstate__(self, state): - self._extra = state.get("extra_data", "none") - - def __eq__(self, other): - return ( - isinstance(other, ImmutableWithArgs) - and self._x == other._x - and self._y == other._y - and self._name == other._name - and getattr(self, "_extra", None) == getattr(other, "_extra", None) - ) - - def __repr__(self): - return f"ImmutableWithArgs(x={self._x}, y={self._y}, name={self._name}, extra={getattr(self, '_extra', None)})" - - -# Test class with getnewargs (older style) -class ImmutableOldStyle: - def __init__(self, a, b): - self._a = a - self._b = b - - def __getnewargs__(self): - return self._a, self._b - - def __getstate__(self): - return {"metadata": "old_style"} - - def __setstate__(self, state): - self._metadata = state.get("metadata", "none") - - def __eq__(self, other): - return ( - isinstance(other, ImmutableOldStyle) - and self._a == other._a - and self._b == other._b - and getattr(self, "_metadata", None) == getattr(other, "_metadata", None) - ) - - def __repr__(self): - return f"ImmutableOldStyle(a={self._a}, b={self._b}, metadata={getattr(self, '_metadata', None)})" - - -def test_current_behavior(): - print("Testing current behavior with stateful objects...") - - fory = Fory(xlang=False, ref=True, strict=False, compatible=False) - - # Test basic stateful object - obj1 = StatefulObject(42, "original_secret") - print(f"Original: {obj1}") - - serialized = fory.serialize(obj1) - deserialized = fory.deserialize(serialized) - print(f"Deserialized: {deserialized}") - print(f"Equal: {obj1 == deserialized}") - print() - - # Test with getnewargs_ex - obj2 = ImmutableWithArgs(10, 20, "test") - print(f"Original: {obj2}") - - serialized2 = fory.serialize(obj2) - deserialized2 = fory.deserialize(serialized2) - print(f"Deserialized attributes: {dir(deserialized2)}") - print(f"Deserialized vars: {vars(deserialized2)}") - try: - print(f"Deserialized: {deserialized2}") - print(f"Equal: {obj2 == deserialized2}") - except Exception as e: - print(f"Error in repr/comparison: {e}") - print() - - # Test with getnewargs (old style) - obj3 = ImmutableOldStyle(100, 200) - print(f"Original: {obj3}") - - serialized3 = fory.serialize(obj3) - deserialized3 = fory.deserialize(serialized3) - print(f"Deserialized: {deserialized3}") - print(f"Equal: {obj3 == deserialized3}") - print() - - # Check what serializer is being used - serializer1 = fory.type_resolver.get_serializer(StatefulObject) - serializer2 = fory.type_resolver.get_serializer(ImmutableWithArgs) - serializer3 = fory.type_resolver.get_serializer(ImmutableOldStyle) - - print(f"StatefulObject serializer: {type(serializer1)}") - print(f"ImmutableWithArgs serializer: {type(serializer2)}") - print(f"ImmutableOldStyle serializer: {type(serializer3)}") - - -if __name__ == "__main__": - test_current_behavior() diff --git a/python/pyfory/tests/test_stateful_serializer.py b/python/pyfory/tests/test_stateful_serializer.py index c0cd6f859d..1cb9d4c95d 100644 --- a/python/pyfory/tests/test_stateful_serializer.py +++ b/python/pyfory/tests/test_stateful_serializer.py @@ -149,6 +149,7 @@ def test_basic_stateful_object(): fory = Fory(xlang=False, ref=True, strict=False, compatible=False) obj = BasicStatefulObject(42, "original_secret") + fory.register_type(BasicStatefulObject) serialized = fory.serialize(obj) deserialized = fory.deserialize(serialized) @@ -168,6 +169,7 @@ def test_immutable_with_getnewargs_ex(): fory = Fory(xlang=False, ref=True, strict=False, compatible=False) obj = ImmutableWithArgsEx(10, 20, "test") + fory.register_type(ImmutableWithArgsEx) # Simulate the state that would be set by __setstate__ for comparison obj._extra = "some_extra" @@ -191,6 +193,7 @@ def test_immutable_with_getnewargs(): fory = Fory(xlang=False, ref=True, strict=False, compatible=False) obj = ImmutableWithArgs(100, 200) + fory.register_type(ImmutableWithArgs) # Simulate the state that would be set by __setstate__ for comparison obj._metadata = "old_style" @@ -213,6 +216,7 @@ def test_stateful_only_object(): fory = Fory(xlang=False, ref=True, strict=False, compatible=False) obj = StatefulOnlyObject("test_data") + fory.register_type(StatefulOnlyObject) # Simulate the state that would be set by __setstate__ for comparison obj.processed = "restored_test_data" @@ -234,6 +238,7 @@ def test_complex_state_object(): fory = Fory(xlang=False, ref=True, strict=False, compatible=False) obj = ComplexStateObject("test", [1, 2, 3, {"nested": "value"}]) + fory.register_type(ComplexStateObject) # Simulate the state that would be set by __setstate__ for comparison obj.extra_info = {"serialized_at": "test_time"} @@ -257,6 +262,7 @@ def test_reference_tracking(): fory = Fory(xlang=False, ref=True, strict=False, compatible=False) obj = BasicStatefulObject(42) + fory.register_type(BasicStatefulObject) # Create a list with the same object referenced twice container = [obj, obj, {"ref": obj}] @@ -275,6 +281,8 @@ def test_nested_stateful_objects(): inner = BasicStatefulObject(10) outer = ComplexStateObject("outer", [inner, BasicStatefulObject(20)]) + fory.register_type(BasicStatefulObject) + fory.register_type(ComplexStateObject) serialized = fory.serialize(outer) deserialized = fory.deserialize(serialized) @@ -288,7 +296,7 @@ def test_nested_stateful_objects(): assert deserialized.items[1].value == 20 -def test_cross_language_compatibility(): +def test_registered_stateful_roundtrip(): """Test that StatefulSerializer works with type registration""" fory = Fory(xlang=False, ref=True, strict=True, compatible=False) diff --git a/python/pyfory/tests/test_struct.py b/python/pyfory/tests/test_struct.py index fc49cd8058..490da93337 100644 --- a/python/pyfory/tests/test_struct.py +++ b/python/pyfory/tests/test_struct.py @@ -663,6 +663,7 @@ def test_inheritance(): print(type_hints) assert type_hints.keys() == {"f1", "f2", "f3"} fory = Fory(xlang=False, ref=True, strict=False, compatible=False) + fory.register_type(ChildClass1) obj = ChildClass1(f1="a", f2=-10, f3={"a": -10.0, "b": 1 / 3}) assert ser_de(fory, obj) == obj assert type(fory.type_resolver.get_serializer(ChildClass1)) is pyfory.DataClassSerializer @@ -814,6 +815,7 @@ class TemporalNumberClass: ) def test_bool_field_coercion(value, expected): fory = Fory(xlang=False, ref=True, strict=False, compatible=False) + fory.register_type(BoolCoercionObject) result = ser_de(fory, BoolCoercionObject(value)) assert result.b is expected @@ -821,6 +823,7 @@ def test_bool_field_coercion(value, expected): def test_bool_field_coercion_numpy_bool(): np = pytest.importorskip("numpy") fory = Fory(xlang=False, ref=True, strict=False, compatible=False) + fory.register_type(BoolCoercionObject) result_true = ser_de(fory, BoolCoercionObject(np.bool_(True))) assert result_true.b is True @@ -917,6 +920,7 @@ def test_data_class_serializer_xlang(): @pytest.mark.parametrize("track_ref", [False, True]) def test_dataclass_with_typed_tuple_field(track_ref): fory = Fory(xlang=False, ref=track_ref, strict=False, compatible=False) + fory.register_type(TupleFieldObject) obj = TupleFieldObject(bar=("a", 1)) assert ser_de(fory, obj) == obj @@ -1131,6 +1135,8 @@ def test_optional_fields(xlang, compatible): fory = Fory(xlang=xlang, ref=True, compatible=compatible, strict=False) if xlang: fory.register_type(OptionalFieldsObject, name="example.OptionalFieldsObject") + else: + fory.register_type(OptionalFieldsObject) obj_with_none = OptionalFieldsObject(f1=None, f2=None, f3=None, f4=42, f5="test") result = ser_de(fory, obj_with_none) @@ -1171,6 +1177,9 @@ def test_nested_optional_fields(xlang, compatible): if xlang: fory.register_type(ComplexObject, name="example.ComplexObject") fory.register_type(NestedOptionalObject, name="example.NestedOptionalObject") + else: + fory.register_type(ComplexObject) + fory.register_type(NestedOptionalObject) obj_with_none = NestedOptionalObject(f1=None, f2=None, f3="test") result = ser_de(fory, obj_with_none) From ba3b5f7c5b6d8c7463301df8b300ced72ce6dfcc Mon Sep 17 00:00:00 2001 From: chaokunyang Date: Fri, 28 Aug 2026 01:23:53 +0800 Subject: [PATCH 007/168] fix(jvm): gate generated serializer registration --- .agents/languages/kotlin.md | 7 ++ .agents/languages/scala.md | 9 ++ .../fory/kotlin/xlang/KotlinXlangPeer.kt | 62 +++++++++++++ .../serializer/kotlin/KotlinSerializers.java | 43 +++++++-- .../kotlin/BuiltinClassSerializerTests.kt | 13 +++ .../serializer/scala/ScalaEnumSerializer.java | 6 ++ .../serializer/scala/ScalaSerializers.java | 46 +++++++--- .../apache/fory/scala/ForySerializer.scala | 71 ++++++++++----- .../scala/ForySerializerDerivationTest.scala | 91 ++++++++++++++++++- .../fory/serializer/scala/ScalaEnumTest.scala | 27 ++++++ 10 files changed, 332 insertions(+), 43 deletions(-) diff --git a/.agents/languages/kotlin.md b/.agents/languages/kotlin.md index 49b2444389..a7f0e4465f 100644 --- a/.agents/languages/kotlin.md +++ b/.agents/languages/kotlin.md @@ -14,6 +14,13 @@ Load this file when changing `kotlin/` or compiler code that generates Kotlin so Fory. Do not auto-install a new serializer for an existing type-registered Kotlin class unless the wire format matches the previous serializer family and old-payload/new-runtime compatibility is tested. +- Public registration helpers must check the registry freeze before constructing a serializer, + enum serializer, or union serializer. A registered-type serializer replacement must check again + after generated serializer construction and before the explicit replacement because + `TypeResolver.setSerializer` remains available for lazy internal resolution. +- Combined generated-struct registration must publish the canonical type before constructing its + serializer because generated construction resolves the canonical `TypeInfo`. Do not move that + construction before type registration or add rollback, staging, or a parallel registration path. - When adding Kotlin gRPC service companions, emit Kotlin source only. Reuse the generated schema module's `ThreadSafeFory` and KSP-generated schema serializers, and keep grpc-java/grpc-kotlin dependencies application-owned instead of adding them as hard `fory-kotlin` dependencies. diff --git a/.agents/languages/scala.md b/.agents/languages/scala.md index ecfd3b07e1..08ad71d746 100644 --- a/.agents/languages/scala.md +++ b/.agents/languages/scala.md @@ -9,6 +9,15 @@ Load this file when changing `scala/`. - Scala supports the JVM and GraalVM Native Image, not Android. Do not add Android-specific Scala sources, tests, resources, R8 metadata, compiler plugins, macros, dependencies, or compatibility design. +- Public registration helpers must check the registry freeze before invoking generated serializer + construction or enum discovery. Registered-type replacement must check again after + `ForySerializer` callbacks and before mutation; Scala enum registration must likewise recheck + after companion-driven value discovery and reuse the values already owned by the serializer. +- Combined generated-struct registration must publish the canonical type before constructing its + serializer because generated construction resolves the canonical `TypeInfo`. Do not move that + construction before type registration or add rollback, staging, or a parallel registration path. + Union construction is the exception because it does not require canonical registration: finish + its serializer-owned callbacks and recheck the freeze before publishing the union type. ## Commands diff --git a/kotlin/fory-kotlin-tests/src/main/kotlin/org/apache/fory/kotlin/xlang/KotlinXlangPeer.kt b/kotlin/fory-kotlin-tests/src/main/kotlin/org/apache/fory/kotlin/xlang/KotlinXlangPeer.kt index 92e1904118..cd9e820043 100644 --- a/kotlin/fory-kotlin-tests/src/main/kotlin/org/apache/fory/kotlin/xlang/KotlinXlangPeer.kt +++ b/kotlin/fory-kotlin-tests/src/main/kotlin/org/apache/fory/kotlin/xlang/KotlinXlangPeer.kt @@ -305,6 +305,7 @@ private fun staticSerializerRoundTrip(dataFile: String) { compatibleScalarContainerRefs() compatibleDenseUIntList() trackedDenseArrayRefs() + serializerRegistrationFreezes() val fory = newFory() fory.register("kotlin.KotlinUser") @@ -753,6 +754,67 @@ private fun trackedDenseArrayRefs() { check(noRefDecoded.added == "reader-default") } +private fun serializerRegistrationFreezes() { + val registeredFory = newFory() + registeredFory.register("kotlin.KotlinUser") + registeredFory.serialize(KotlinUser(1u, "freeze", 2L)) + val serializer = registeredFory.getSerializer(KotlinUser::class.java) + val typeId = registeredFory.typeResolver.getTypeInfo(KotlinUser::class.java).typeId + check( + runCatching { KotlinSerializers.registerSerializer(registeredFory, KotlinUser::class.java) } + .isFailure + ) + check(registeredFory.getSerializer(KotlinUser::class.java) === serializer) + check(registeredFory.typeResolver.getTypeInfo(KotlinUser::class.java).typeId == typeId) + + val unregisteredFory = newFory() + unregisteredFory.serialize("freeze") + check(!unregisteredFory.typeResolver.isRegistered(KotlinUser::class.java)) + check( + runCatching { KotlinSerializers.registerSerializer(unregisteredFory, KotlinUser::class.java) } + .isFailure + ) + check(!unregisteredFory.typeResolver.isRegistered(KotlinUser::class.java)) + + val failedRootFory = newFory() + check(runCatching { failedRootFory.deserialize(byteArrayOf()) }.isFailure) + for (frozenFory in listOf(unregisteredFory, failedRootFory)) { + val resolver = frozenFory.typeResolver + val cacheField = resolver.sharedRegistry.javaClass.getDeclaredField("objectInstantiatorCache") + cacheField.isAccessible = true + val cache = cacheField.get(resolver.sharedRegistry) as Map<*, *> + check(KotlinPet::class.java !in cache) + check( + runCatching { + KotlinSerializers.registerUnion( + frozenFory, + KotlinPet::class.java, + "kotlin.LateKotlinPet", + ) + } + .isFailure + ) + check(KotlinPet::class.java !in cache) + check(!resolver.isRegistered(KotlinPet::class.java)) + } + + val compatibleFory = newCompatibleFory() + KotlinSerializers.registerType( + compatibleFory, + KotlinUser::class.java, + "kotlin.KotlinUserCompatible", + ) + check( + compatibleFory.typeResolver.getTypeInfo(KotlinUser::class.java).typeId == + Types.NAMED_COMPATIBLE_STRUCT + ) + KotlinSerializers.registerSerializer(compatibleFory, KotlinUser::class.java) + check( + compatibleFory.typeResolver.getTypeInfo(KotlinUser::class.java).typeId == + Types.NAMED_COMPATIBLE_STRUCT + ) +} + private fun checkUnionListBudget(values: List) { val writer = newFory() writer.register("kotlin.KotlinUser") diff --git a/kotlin/fory-kotlin/src/main/java/org/apache/fory/serializer/kotlin/KotlinSerializers.java b/kotlin/fory-kotlin/src/main/java/org/apache/fory/serializer/kotlin/KotlinSerializers.java index d8edd694dc..696dd3459b 100644 --- a/kotlin/fory-kotlin/src/main/java/org/apache/fory/serializer/kotlin/KotlinSerializers.java +++ b/kotlin/fory-kotlin/src/main/java/org/apache/fory/serializer/kotlin/KotlinSerializers.java @@ -57,6 +57,8 @@ public static void registerSerializers(ThreadSafeFory fory) { } public static void registerSerializers(Fory fory) { + TypeResolver resolver = fory.getTypeResolver(); + checkRegistrationOpen(resolver); synchronized (INSTALLED_FORY) { if (INSTALLED_FORY.containsKey(fory)) { return; @@ -65,7 +67,6 @@ public static void registerSerializers(Fory fory) { } try { DefaultValueUtils.setKotlinDefaultValueSupport(new KotlinDefaultValueSupport()); - TypeResolver resolver = fory.getTypeResolver(); if (resolver.isCrossLanguage()) { return; } @@ -217,27 +218,27 @@ public static void registerType(Fory fory, Class cls, String namespace, Strin public static void register(Fory fory, Class cls) { fory.register(cls); - registerSerializer(fory, cls); + registerSerializerAfterType(fory, cls); } public static void register(Fory fory, Class cls, long typeId) { registerType(fory, cls, typeId); - registerSerializer(fory, cls); + registerSerializerAfterType(fory, cls); } public static void register(Fory fory, Class cls, String name) { registerType(fory, cls, name); - registerSerializer(fory, cls); + registerSerializerAfterType(fory, cls); } public static void register(Fory fory, Class cls, String namespace, String typeName) { registerType(fory, cls, namespace, typeName); - registerSerializer(fory, cls); + registerSerializerAfterType(fory, cls); } public static void registerSerializer(Fory fory, Class cls) { TypeResolver resolver = fory.getTypeResolver(); - Serializer serializer = newGeneratedSerializer(resolver, cls); + Serializer serializer = newRegistrationSerializer(resolver, cls); if (resolver.isRegistered(cls)) { resolver.setSerializer(cls, serializer); } else { @@ -245,14 +246,40 @@ public static void registerSerializer(Fory fory, Class cls) { } } + private static Serializer newRegistrationSerializer(TypeResolver resolver, Class cls) { + checkRegistrationOpen(resolver); + Serializer serializer = newGeneratedSerializer(resolver, cls); + checkRegistrationOpen(resolver); + return serializer; + } + + private static void registerSerializerAfterType(Fory fory, Class cls) { + TypeResolver resolver = fory.getTypeResolver(); + Serializer serializer = newGeneratedSerializer(resolver, cls); + resolver.setSerializer(cls, serializer); + } + + private static void checkRegistrationOpen(TypeResolver resolver) { + // Resolver setSerializer remains available for lazy internal resolution, so this facade owns + // the public freeze checks around construction and before the final replacement. + if (resolver.isRegistrationFinished()) { + throw new ForyException( + "Cannot register class/serializer after registration has been frozen. Please register " + + "all classes before invoking top-level `serialize/deserialize/copy` methods of " + + "Fory."); + } + } + public static void registerEnum(Fory fory, Class cls, long typeId) { TypeResolver resolver = fory.getTypeResolver(); + checkRegistrationOpen(resolver); resolver.registerEnum(cls, typeId, new EnumSerializer(resolver.getConfig(), enumClass(cls))); } public static void registerEnum(Fory fory, Class cls, String namespace, String typeName) { checkTypeName(typeName); TypeResolver resolver = fory.getTypeResolver(); + checkRegistrationOpen(resolver); resolver.registerEnum( cls, namespace, typeName, new EnumSerializer(resolver.getConfig(), enumClass(cls))); } @@ -260,12 +287,14 @@ public static void registerEnum(Fory fory, Class cls, String namespace, Strin public static void registerEnum(Fory fory, Class cls, String name) { TypeResolver resolver = fory.getTypeResolver(); String[] parts = splitName(name); + checkRegistrationOpen(resolver); resolver.registerEnum( cls, parts[0], parts[1], new EnumSerializer(resolver.getConfig(), enumClass(cls))); } public static void registerUnion(Fory fory, Class cls, long typeId) { TypeResolver resolver = fory.getTypeResolver(); + checkRegistrationOpen(resolver); resolver.registerUnion(cls, typeId, newGeneratedSerializer(resolver, cls)); registerCaseAliases(fory, cls); } @@ -273,6 +302,7 @@ public static void registerUnion(Fory fory, Class cls, long typeId) { public static void registerUnion(Fory fory, Class cls, String namespace, String typeName) { checkTypeName(typeName); TypeResolver resolver = fory.getTypeResolver(); + checkRegistrationOpen(resolver); resolver.registerUnion(cls, namespace, typeName, newGeneratedSerializer(resolver, cls)); registerCaseAliases(fory, cls); } @@ -280,6 +310,7 @@ public static void registerUnion(Fory fory, Class cls, String namespace, Stri public static void registerUnion(Fory fory, Class cls, String name) { TypeResolver resolver = fory.getTypeResolver(); String[] parts = splitName(name); + checkRegistrationOpen(resolver); resolver.registerUnion(cls, parts[0], parts[1], newGeneratedSerializer(resolver, cls)); registerCaseAliases(fory, cls); } diff --git a/kotlin/fory-kotlin/src/test/kotlin/org/apache/fory/serializer/kotlin/BuiltinClassSerializerTests.kt b/kotlin/fory-kotlin/src/test/kotlin/org/apache/fory/serializer/kotlin/BuiltinClassSerializerTests.kt index 4ffa61fa15..bdc1e807bc 100644 --- a/kotlin/fory-kotlin/src/test/kotlin/org/apache/fory/serializer/kotlin/BuiltinClassSerializerTests.kt +++ b/kotlin/fory-kotlin/src/test/kotlin/org/apache/fory/serializer/kotlin/BuiltinClassSerializerTests.kt @@ -34,10 +34,23 @@ import kotlin.time.Duration.Companion.seconds import kotlin.uuid.ExperimentalUuidApi import kotlin.uuid.Uuid import org.apache.fory.Fory +import org.apache.fory.exception.ForyException import org.apache.fory.kotlin.ForyKotlin +import org.apache.fory.util.DefaultValueUtils import org.testng.Assert +import org.testng.Assert.assertThrows class BuiltinClassSerializerTests { + @Test + fun testLateBootstrapIsReadOnly() { + val fory = ForyKotlin.builder().withXlang(false).requireClassRegistration(true).build() + fory.serialize(1) + val defaultValueSupport = DefaultValueUtils.getKotlinDefaultValueSupport() + + assertThrows(ForyException::class.java) { KotlinSerializers.registerSerializers(fory) } + Assert.assertSame(DefaultValueUtils.getKotlinDefaultValueSupport(), defaultValueSupport) + } + @Test fun testSerializePair() { val fory: Fory = ForyKotlin.builder().withXlang(false).requireClassRegistration(true).build() diff --git a/scala/fory-scala/src/main/java/org/apache/fory/serializer/scala/ScalaEnumSerializer.java b/scala/fory-scala/src/main/java/org/apache/fory/serializer/scala/ScalaEnumSerializer.java index f6f6e3a7a7..c8f3909d5c 100644 --- a/scala/fory-scala/src/main/java/org/apache/fory/serializer/scala/ScalaEnumSerializer.java +++ b/scala/fory-scala/src/main/java/org/apache/fory/serializer/scala/ScalaEnumSerializer.java @@ -138,6 +138,12 @@ static Object[] loadValues(Class cls) { } } + // Registration reuses the values already discovered here so enum companion code cannot run + // after the canonical type has been published. + Object[] getEnumConstants() { + return enumConstants; + } + static boolean canSerialize(Class cls) { Class enumClass = ScalaTypes.resolveScalaEnumClass(cls); if (enumClass == null) { diff --git a/scala/fory-scala/src/main/java/org/apache/fory/serializer/scala/ScalaSerializers.java b/scala/fory-scala/src/main/java/org/apache/fory/serializer/scala/ScalaSerializers.java index 595d2c7885..0799360b2e 100644 --- a/scala/fory-scala/src/main/java/org/apache/fory/serializer/scala/ScalaSerializers.java +++ b/scala/fory-scala/src/main/java/org/apache/fory/serializer/scala/ScalaSerializers.java @@ -19,6 +19,9 @@ package org.apache.fory.serializer.scala; +import static org.apache.fory.serializer.scala.ToFactorySerializers.IterableToFactoryClass; +import static org.apache.fory.serializer.scala.ToFactorySerializers.MapToFactoryClass; + import java.util.Collections; import java.util.Map; import java.util.Objects; @@ -27,13 +30,11 @@ import org.apache.fory.ThreadSafeFory; import org.apache.fory.annotation.Internal; import org.apache.fory.config.Config; +import org.apache.fory.exception.ForyException; import org.apache.fory.resolver.TypeResolver; import scala.collection.immutable.NumericRange; import scala.collection.immutable.Range; -import static org.apache.fory.serializer.scala.ToFactorySerializers.IterableToFactoryClass; -import static org.apache.fory.serializer.scala.ToFactorySerializers.MapToFactoryClass; - public class ScalaSerializers { private static final Map INSTALLED_FORY = Collections.synchronizedMap(new WeakHashMap<>()); @@ -43,13 +44,14 @@ public static void registerSerializers(ThreadSafeFory fory) { } public static void registerSerializers(Fory fory) { + TypeResolver resolver = fory.getTypeResolver(); + checkRegistrationOpen(resolver); synchronized (INSTALLED_FORY) { if (INSTALLED_FORY.containsKey(fory)) { return; } INSTALLED_FORY.put(fory, Boolean.TRUE); } - TypeResolver resolver = fory.getTypeResolver(); try { fory.registerSerializerFactory(new ScalaSerializerFactory()); if (resolver.isCrossLanguage()) { @@ -193,8 +195,12 @@ public static void registerSerializers(Fory fory) { public static void registerEnum(Fory fory, Class cls, long typeId) { TypeResolver resolver = fory.getTypeResolver(); - resolver.registerEnum(cls, typeId, new ScalaEnumSerializer(resolver, cls)); - registerEnumRuntimeAliases(fory, cls); + checkRegistrationOpen(resolver); + ScalaEnumSerializer serializer = new ScalaEnumSerializer(resolver, cls); + checkRegistrationOpen(resolver); + Object[] values = serializer.getEnumConstants(); + resolver.registerEnum(cls, typeId, serializer); + registerEnumRuntimeAliases(fory, cls, values); } private static String[] splitName(String name) { @@ -222,15 +228,23 @@ private static void checkTypeName(String typeName) { public static void registerEnum(Fory fory, Class cls, String name) { TypeResolver resolver = fory.getTypeResolver(); String[] parts = splitName(name); - resolver.registerEnum(cls, parts[0], parts[1], new ScalaEnumSerializer(resolver, cls)); - registerEnumRuntimeAliases(fory, cls); + checkRegistrationOpen(resolver); + ScalaEnumSerializer serializer = new ScalaEnumSerializer(resolver, cls); + checkRegistrationOpen(resolver); + Object[] values = serializer.getEnumConstants(); + resolver.registerEnum(cls, parts[0], parts[1], serializer); + registerEnumRuntimeAliases(fory, cls, values); } public static void registerEnum(Fory fory, Class cls, String namespace, String typeName) { checkTypeName(typeName); TypeResolver resolver = fory.getTypeResolver(); - resolver.registerEnum(cls, namespace, typeName, new ScalaEnumSerializer(resolver, cls)); - registerEnumRuntimeAliases(fory, cls); + checkRegistrationOpen(resolver); + ScalaEnumSerializer serializer = new ScalaEnumSerializer(resolver, cls); + checkRegistrationOpen(resolver); + Object[] values = serializer.getEnumConstants(); + resolver.registerEnum(cls, namespace, typeName, serializer); + registerEnumRuntimeAliases(fory, cls, values); } @Internal @@ -239,8 +253,8 @@ public static void registerRuntimeTypeAlias( fory.getTypeResolver().registerRuntimeTypeAlias(runtimeClass, canonicalClass); } - private static void registerEnumRuntimeAliases(Fory fory, Class cls) { - for (Object value : ScalaEnumSerializer.loadValues(cls)) { + private static void registerEnumRuntimeAliases(Fory fory, Class cls, Object[] values) { + for (Object value : values) { Class runtimeClass = value.getClass(); if (runtimeClass != cls) { registerRuntimeTypeAlias(fory, runtimeClass, cls); @@ -248,4 +262,12 @@ private static void registerEnumRuntimeAliases(Fory fory, Class cls) { } } + private static void checkRegistrationOpen(TypeResolver resolver) { + if (resolver.isRegistrationFinished()) { + throw new ForyException( + "Cannot register class/serializer after registration has been frozen. Please register " + + "all classes before invoking top-level `serialize/deserialize/copy` methods of " + + "Fory."); + } + } } diff --git a/scala/fory-scala/src/main/scala-3/org/apache/fory/scala/ForySerializer.scala b/scala/fory-scala/src/main/scala-3/org/apache/fory/scala/ForySerializer.scala index bf839fc1bb..31dfc87d8c 100644 --- a/scala/fory-scala/src/main/scala-3/org/apache/fory/scala/ForySerializer.scala +++ b/scala/fory-scala/src/main/scala-3/org/apache/fory/scala/ForySerializer.scala @@ -21,6 +21,7 @@ package org.apache.fory.scala import org.apache.fory.{BaseFory, Fory, ForyModule, ThreadSafeFory} import org.apache.fory.annotation.Internal +import org.apache.fory.exception.ForyException import org.apache.fory.meta.TypeDef import org.apache.fory.resolver.TypeResolver import org.apache.fory.serializer.Serializer @@ -61,6 +62,16 @@ object ForySerializer { } } + private def checkRegistrationOpen(resolver: TypeResolver): Unit = { + // Public generated registration must freeze with the root facade. Resolver serializer + // mutation stays available for lazy internal resolution after registration has finished. + if resolver.isRegistrationFinished then { + throw new ForyException( + "Cannot register class/serializer after registration has been frozen. Please register " + + "all classes before invoking top-level `serialize/deserialize/copy` methods of Fory.") + } + } + def register[T](fory: Fory, cls: Class[T])(using serializer: ForySerializer[T]): Unit = { register(fory, cls, null, null) } @@ -106,11 +117,16 @@ object ForySerializer { @Internal def registerSerializer[T](fory: Fory, cls: Class[T])(using serializer: ForySerializer[T]): Unit = { - if serializer.isUnion then { + val resolver = fory.getTypeResolver + checkRegistrationOpen(resolver) + val union = serializer.isUnion + checkRegistrationOpen(resolver) + if union then { throw new IllegalArgumentException("Use ForySerializer.register for Scala union serializers") } - val resolver = fory.getTypeResolver - resolver.setSerializer(cls, serializer.createSerializer(resolver)) + val generatedSerializer = serializer.createSerializer(resolver) + checkRegistrationOpen(resolver) + resolver.setSerializer(cls, generatedSerializer) } private def register[T]( @@ -123,27 +139,34 @@ object ForySerializer { checkTypeName(typeName) } val resolver = fory.getTypeResolver - serializer match { - case _ if serializer.isUnion => - val unionSerializer = serializer.createSerializer(resolver) - if typeId != null then { - resolver.registerUnion(cls, typeId.longValue(), unionSerializer) - } else { - val unionNamespace = - if namespace != null then namespace else Option(cls.getPackage).map(_.getName).orNull - val unionTypeName = if typeName != null then typeName else cls.getSimpleName - fory.registerUnion( - cls, - if unionNamespace == null then "" else unionNamespace, - unionTypeName, - unionSerializer) - } - serializer.handledRuntimeClasses(cls).foreach { runtimeClass => - ScalaSerializers.registerRuntimeTypeAlias(fory, runtimeClass, cls) - } - case _ => - registerType(fory, cls, typeId, namespace, typeName) - resolver.setSerializer(cls, serializer.createSerializer(resolver)) + checkRegistrationOpen(resolver) + val union = serializer.isUnion + checkRegistrationOpen(resolver) + if union then { + // Union construction does not require canonical registration, so finish the remaining user + // methods before publishing any type state. + val generatedSerializer = serializer.createSerializer(resolver) + checkRegistrationOpen(resolver) + val runtimeClasses = serializer.handledRuntimeClasses(cls) + checkRegistrationOpen(resolver) + if typeId != null then { + resolver.registerUnion(cls, typeId.longValue(), generatedSerializer) + } else { + val unionNamespace = + if namespace != null then namespace else Option(cls.getPackage).map(_.getName).orNull + val unionTypeName = if typeName != null then typeName else cls.getSimpleName + fory.registerUnion( + cls, + if unionNamespace == null then "" else unionNamespace, + unionTypeName, + generatedSerializer) + } + runtimeClasses.foreach { runtimeClass => + ScalaSerializers.registerRuntimeTypeAlias(fory, runtimeClass, cls) + } + } else { + registerType(fory, cls, typeId, namespace, typeName) + resolver.setSerializer(cls, serializer.createSerializer(resolver)) } } diff --git a/scala/fory-scala/src/test/scala-3/org/apache/fory/serializer/scala/ForySerializerDerivationTest.scala b/scala/fory-scala/src/test/scala-3/org/apache/fory/serializer/scala/ForySerializerDerivationTest.scala index 49d731882a..305ce069a9 100644 --- a/scala/fory-scala/src/test/scala-3/org/apache/fory/serializer/scala/ForySerializerDerivationTest.scala +++ b/scala/fory-scala/src/test/scala-3/org/apache/fory/serializer/scala/ForySerializerDerivationTest.scala @@ -31,7 +31,7 @@ import org.apache.fory.annotation.{ UInt8Type } import org.apache.fory.config.Int64Encoding -import org.apache.fory.exception.InsecureException +import org.apache.fory.exception.{ForyException, InsecureException} import org.apache.fory.memory.MemoryBuffer import org.apache.fory.meta.TypeDef import org.apache.fory.reflect.{FieldAccessor, ObjectInstantiators} @@ -323,6 +323,95 @@ class ForySerializerDerivationTest extends AnyWordSpec with Matchers { fory.deserialize(fory.serialize(fixed)) shouldEqual fixed } + "freeze public registration helpers" in { + var personUnionCheckCalled = false + given ForySerializer[Person] with { + override def isUnion: Boolean = { + personUnionCheckCalled = true + false + } + + override def createSerializer( + typeResolver: org.apache.fory.resolver.TypeResolver, + typeDef: TypeDef): org.apache.fory.serializer.Serializer[Person] = + throw new IllegalStateException("Late person serializer creation") + } + + var searchTargetUnionCheckCalled = false + var searchTargetSerializerCreated = false + given ForySerializer[SearchTarget] with { + override def isUnion: Boolean = { + searchTargetUnionCheckCalled = true + true + } + + override def createSerializer( + typeResolver: org.apache.fory.resolver.TypeResolver, + typeDef: TypeDef): org.apache.fory.serializer.Serializer[SearchTarget] = { + searchTargetSerializerCreated = true + throw new IllegalStateException("Late union serializer creation") + } + } + + Seq( + () => { + val runtime = xlangFory() + runtime.serialize(Person("Ada", 36, None)) + runtime + }, + () => { + val runtime = xlangFory() + intercept[RuntimeException] { + runtime.deserialize(Array.emptyByteArray) + } + runtime + }).foreach { runtime => + val frozenRuntime = runtime() + val serializer = frozenRuntime.getSerializer(classOf[Person]) + intercept[RuntimeException] { + ForySerializer.registerSerializer(frozenRuntime, classOf[Person]) + } + personUnionCheckCalled shouldBe false + frozenRuntime.getSerializer(classOf[Person]) shouldBe theSameInstanceAs(serializer) + frozenRuntime.getTypeResolver.isRegistered(classOf[StoredState]) shouldBe false + intercept[RuntimeException] { + ForySerializer.register( + frozenRuntime, + classOf[StoredState], + "scala_test.LateStoredState") + } + frozenRuntime.getTypeResolver.isRegistered(classOf[StoredState]) shouldBe false + intercept[RuntimeException] { + ForySerializer.register( + frozenRuntime, + classOf[SearchTarget], + "scala_test.LateSearchTarget") + } + searchTargetUnionCheckCalled shouldBe false + searchTargetSerializerCreated shouldBe false + } + } + + "reject serializer replacement frozen during creation" in { + val runtime = xlangFory() + val originalSerializer = runtime.getSerializer(classOf[Person]) + val replacementSerializer = + summon[ForySerializer[Person]].createSerializer(runtime.getTypeResolver) + val reentrantSerializer = new ForySerializer[Person] { + override def createSerializer( + typeResolver: org.apache.fory.resolver.TypeResolver, + typeDef: TypeDef): org.apache.fory.serializer.Serializer[Person] = { + runtime.serialize(Person("Ada", 36, None)) + replacementSerializer + } + } + + intercept[ForyException] { + ForySerializer.registerSerializer(runtime, classOf[Person])(using reentrantSerializer) + } + runtime.getSerializer(classOf[Person]) shouldBe theSameInstanceAs(originalSerializer) + } + "serialize derived case classes with Scala collection fields" in { val fory = xlangFory() val box = CollectionBox(List("a", "b"), Set("x", "y"), Map("a" -> 1, "b" -> 2)) diff --git a/scala/fory-scala/src/test/scala-3/org/apache/fory/serializer/scala/ScalaEnumTest.scala b/scala/fory-scala/src/test/scala-3/org/apache/fory/serializer/scala/ScalaEnumTest.scala index 91c9b33df6..b77139476c 100644 --- a/scala/fory-scala/src/test/scala-3/org/apache/fory/serializer/scala/ScalaEnumTest.scala +++ b/scala/fory-scala/src/test/scala-3/org/apache/fory/serializer/scala/ScalaEnumTest.scala @@ -22,6 +22,7 @@ package org.apache.fory.serializer.scala import org.apache.fory.Fory import org.apache.fory.scala.ForyScala import org.apache.fory.annotation.ForyEnumId +import org.apache.fory.exception.ForyException import org.scalatest.matchers.should.Matchers import org.scalatest.wordspec.AnyWordSpec @@ -42,6 +43,16 @@ object ScalaEnumTest { case Red } + object EnumDiscoveryProbe { + var initialized: Int = 0 + } + + enum CountingEnum { case Value } + + object CountingEnum { + EnumDiscoveryProbe.initialized += 1 + } + case class Colors(set: Set[ColorEnum]) } @@ -82,5 +93,21 @@ class ScalaEnumTest extends AnyWordSpec with Matchers { reader.deserialize(writer.serialize(StableColorV1.Green)) shouldBe StableColorV2.Green } + "reject late bootstrap before enum discovery" in { + val frozen = ForyScala.builder() + .withXlang(false) + .requireClassRegistration(false) + .build() + frozen.serialize(1) + + intercept[ForyException] { + ScalaSerializers.registerSerializers(frozen) + } + EnumDiscoveryProbe.initialized = 0 + intercept[ForyException] { + ScalaSerializers.registerEnum(frozen, classOf[CountingEnum], 712L) + } + EnumDiscoveryProbe.initialized shouldBe 0 + } } } From 29c86a054342721286302b0fc685ea58ad2ce256 Mon Sep 17 00:00:00 2001 From: chaokunyang Date: Fri, 28 Aug 2026 01:24:00 +0800 Subject: [PATCH 008/168] fix(swift): retain registry finalization failures --- .agents/languages/swift.md | 4 ++ .../swift/polymorphism.md | 5 ++ .../swift/type-registration.md | 6 +- swift/Sources/Fory/TypeResolver.swift | 25 +++++++-- swift/Sources/ForyMacro/ForyObjectMacro.swift | 30 +++++++++- .../ExternalTypeSerializationTests.swift | 56 ++++++++++++++----- swift/Tests/ForyTests/ForySwiftTests.swift | 39 +++++++++++++ 7 files changed, 141 insertions(+), 24 deletions(-) diff --git a/.agents/languages/swift.md b/.agents/languages/swift.md index 1df5b89d3d..af1755ca16 100644 --- a/.agents/languages/swift.md +++ b/.agents/languages/swift.md @@ -37,6 +37,10 @@ Load this file when changing `swift/` or Swift xlang behavior. ignored declaration fields are budget-only and must not enter target access, construction, metadata, or wire code. Omitted large value storage must be declared explicitly and ignored. +- `@ForyStruct` supports protocol conformances but rejects every superclass during registration + finalization because macros cannot inspect inherited storage. SwiftSyntax represents both in one + inheritance clause, and Swift provides no public superclass query for arbitrary Swift classes; + keep the minimal `_getSuperclass` check finalization-owned and out of root hot paths. - Direct `Any` and `AnyObject` root overloads remain disfavored forwarding facades over `DynamicSerializer` and `DynamicSerializer`, including their Data-buffer forms. Arbitrary protocol roots explicitly select `DynamicSerializer`. Do not add an unconstrained diff --git a/docs/object-serialization/swift/polymorphism.md b/docs/object-serialization/swift/polymorphism.md index f862dfd399..1350f628be 100644 --- a/docs/object-serialization/swift/polymorphism.md +++ b/docs/object-serialization/swift/polymorphism.md @@ -169,6 +169,11 @@ var animal: AnimalBase Register every concrete subclass that may appear. +`@ForyStruct` cannot be applied to a class with any superclass because Swift +macros cannot inspect inherited storage. Protocol conformances remain supported. +Use a custom serializer for each concrete subclass, then select it through the +dynamic field serializer. + ## Dynamic `Any` Fields ```swift diff --git a/docs/object-serialization/swift/type-registration.md b/docs/object-serialization/swift/type-registration.md index 6ceea8cd0a..ce96000613 100644 --- a/docs/object-serialization/swift/type-registration.md +++ b/docs/object-serialization/swift/type-registration.md @@ -92,8 +92,10 @@ Keep registration mapping consistent across peers: - Do not mix ID and name mapping for the same logical type across services - Register only one serializer for each target type on a `Fory` instance -Registration closes after the first root serialization or deserialization. -Complete all registrations before the first root operation. +Registration closes permanently when the first root serialization or +deserialization attempt begins, even if that operation fails. Complete all +registrations before the first root operation; reusing the same `Fory` instance +does not reopen registration. ## Dynamic Types and Registration diff --git a/swift/Sources/Fory/TypeResolver.swift b/swift/Sources/Fory/TypeResolver.swift index 3f90359b32..a520d5da05 100644 --- a/swift/Sources/Fory/TypeResolver.swift +++ b/swift/Sources/Fory/TypeResolver.swift @@ -622,7 +622,8 @@ final class TypeResolver { private static let maxRemoteTypeMetaKeys = 8192 private let trackRef: Bool - private var registrationFinished = false + private var registryFrozen = false + private var registrationFinalized = false private var bySerializerType = UInt64Map(initialCapacity: 64) private var byTargetType = UInt64Map(initialCapacity: 64) @@ -634,6 +635,7 @@ final class TypeResolver { private var typeInfoByHeaderHash = UInt64Map(initialCapacity: 64) private var remoteSchemaVersionsByType: [String: Int] = [:] private var totalAcceptedSchemaVersions = 0 + private var registrationFailure: (any Error)? init(trackRef: Bool = false) { self.trackRef = trackRef @@ -833,7 +835,7 @@ final class TypeResolver { @inline(__always) func finishRegistration() throws { - if registrationFinished { + if registrationFinalized { return } try finishRegistrationSlow() @@ -841,10 +843,21 @@ final class TypeResolver { @inline(never) private func finishRegistrationSlow() throws { - for typeInfo in registeredTypeInfos { - try typeInfo.finalizeTypeMeta(resolver: self) + // Freezing and finalization are separate states: the first root permanently closes + // registration, while a partial builder failure must never be mistaken for success. + if let registrationFailure { + throw registrationFailure + } + registryFrozen = true + do { + for typeInfo in registeredTypeInfos { + try typeInfo.finalizeTypeMeta(resolver: self) + } + registrationFinalized = true + } catch { + registrationFailure = error + throw error } - registrationFinished = true } func register(_ type: T.Type, id: UInt32) throws { @@ -1323,7 +1336,7 @@ final class TypeResolver { } private func ensureRegistrationAllowed() throws { - guard !registrationFinished else { + guard !registryFrozen else { throw ForyError.invalidData( "cannot register more types after top-level serialize/deserialize has frozen registration" ) diff --git a/swift/Sources/ForyMacro/ForyObjectMacro.swift b/swift/Sources/ForyMacro/ForyObjectMacro.swift index ee0244ecab..4f342e0e57 100644 --- a/swift/Sources/ForyMacro/ForyObjectMacro.swift +++ b/swift/Sources/ForyMacro/ForyObjectMacro.swift @@ -62,6 +62,9 @@ public struct ForyStructMacro: MemberMacro, ExtensionMacro { objectConfig.targetType ?? declaration.as(ClassDeclSyntax.self)?.name.text ?? "Self" + // SwiftSyntax uses the same inheritance clause for a superclass and protocol + // conformances, so semantic superclass validation must happen after registration. + let needsSuperclassValidation = declaration.as(ClassDeclSyntax.self)?.inheritanceClause != nil let successBodyAttribute = objectConfig.targetType != nil && parsed.fields.contains(where: { @@ -102,7 +105,11 @@ public struct ForyStructMacro: MemberMacro, ExtensionMacro { let schemaHashDecl: DeclSyntax = DeclSyntax(stringLiteral: try buildSchemaHashDecl(fields: parsed.fields)) let compatibleTypeMetaDecl: DeclSyntax = DeclSyntax( - stringLiteral: buildCompatibleTypeMetaFieldsDecl(sortedFields: sortedFields, accessPrefix: accessPrefix) + stringLiteral: buildCompatibleTypeMetaFieldsDecl( + sortedFields: sortedFields, + accessPrefix: accessPrefix, + needsSuperclassValidation: needsSuperclassValidation + ) ) let defaultDecl: DeclSyntax = DeclSyntax( stringLiteral: buildDefaultDecl( @@ -2540,10 +2547,28 @@ private func buildSchemaHashDecl(fields: [ParsedField]) throws -> String { """ } -private func buildCompatibleTypeMetaFieldsDecl(sortedFields: [ParsedField], accessPrefix: String) -> String { +private func buildCompatibleTypeMetaFieldsDecl( + sortedFields: [ParsedField], + accessPrefix: String, + needsSuperclassValidation: Bool +) -> String { let disabledExpr = compatibleTypeMetaFieldsExpr(sortedFields: sortedFields, trackRefExpression: "false") let enabledExpr = compatibleTypeMetaFieldsExpr(sortedFields: sortedFields, trackRefExpression: "true") let resolvedBody = resolvedTypeMetaFieldsBody(sortedFields: sortedFields) + let superclassValidation: String + if needsSuperclassValidation { + // Swift has no public API for querying an arbitrary Swift class's superclass. + // Keep the underscored query in this registration-finalization check only. + superclassValidation = """ + if _getSuperclass(Self.self) != nil { + throw ForyError.encodingError( + "@ForyStruct classes cannot inherit from a superclass because macros cannot inspect inherited storage" + ) + } + """ + } else { + superclassValidation = "" + } return """ private static let __foryFieldsInfoTrackRefDisabled: [TypeMeta.FieldInfo] = \(disabledExpr) private static let __foryFieldsInfoTrackRefEnabled: [TypeMeta.FieldInfo] = \(enabledExpr) @@ -2556,6 +2581,7 @@ private func buildCompatibleTypeMetaFieldsDecl(sortedFields: [ParsedField], acce trackRef: Bool, resolveSerializerTypeId: (Any.Type) throws -> TypeId ) throws -> [TypeMeta.FieldInfo] { + \(superclassValidation) \(resolvedBody) } """ diff --git a/swift/Tests/ForyTests/ExternalTypeSerializationTests.swift b/swift/Tests/ForyTests/ExternalTypeSerializationTests.swift index a37a23859b..7a40ef2e18 100644 --- a/swift/Tests/ForyTests/ExternalTypeSerializationTests.swift +++ b/swift/Tests/ForyTests/ExternalTypeSerializationTests.swift @@ -164,6 +164,19 @@ private final class LocalNode { required init() {} } +private class SuperclassBase { + required init() {} +} + +@ForyStruct +private final class SuperclassChild: SuperclassBase { + var local: Int32 = 0 + + required init() { + super.init() + } +} + @ForyStruct private struct LocalNamedValue: NamedValue, Equatable { var name: String @@ -525,7 +538,7 @@ func externalIgnoredFieldBudget() throws { ) ) try limited.register(NodeSerializer.self, id: 142) - #expect(throws: ForyError.self) { + #expect(throws: (any Error).self) { let _: Node = try limited.deserialize( bytes, with: NodeSerializer.self @@ -719,7 +732,7 @@ func customCarrierEnforcesBudget() throws { ) try reader.register(UserSerializer.self, id: 138) try reader.register(UserArrayCustomSerializer.self, id: 139) - #expect(throws: ForyError.self) { + #expect(throws: (any Error).self) { let _: [User] = try reader.deserialize( bytes, with: UserArrayCustomSerializer.self @@ -1076,7 +1089,7 @@ func directExternalDynamicRoot() throws { func dynamicTargetFailures() throws { let unregistered = Fory() let unregisteredValue: Any = User(name: "unregistered", age: 1) - #expect(throws: ForyError.self) { + #expect(throws: (any Error).self) { _ = try unregistered.serialize( unregisteredValue, with: DynamicSerializer.self @@ -1088,13 +1101,13 @@ func dynamicTargetFailures() throws { let value: Any = Key(value: "not-named") let bytes = try fory.serialize(value, with: DynamicSerializer.self) - #expect(throws: ForyError.self) { + #expect(throws: (any Error).self) { let _: any NamedValue = try fory.deserialize( bytes, with: DynamicSerializer.self ) } - #expect(throws: ForyError.self) { + #expect(throws: (any Error).self) { let _: AnyObject = try fory.deserialize( bytes, with: DynamicSerializer.self @@ -1130,33 +1143,33 @@ func dynamicReferenceEnvelopeBytes() throws { @Test func registrationRejectsInvalidOwnership() throws { let carrier = Fory() - #expect(throws: ForyError.self) { + #expect(throws: (any Error).self) { try carrier.register(ArraySerializer.self, id: 126) } let wrapper = Fory() - #expect(throws: ForyError.self) { + #expect(throws: (any Error).self) { try wrapper.register(OptionalSerializer.self, id: 132) } let dynamic = Fory() - #expect(throws: ForyError.self) { + #expect(throws: (any Error).self) { try dynamic.register(DynamicSerializer.self, id: 133) } let duplicateTarget = Fory() try duplicateTarget.register(UserSerializer.self, id: 127) - #expect(throws: ForyError.self) { + #expect(throws: (any Error).self) { try duplicateTarget.register(AlternateUserSerializer.self, id: 128) } let builtinTarget = Fory() - #expect(throws: ForyError.self) { + #expect(throws: (any Error).self) { try builtinTarget.register(StringCustomSerializer.self, id: 129) } let valueDeclaration = Fory() - #expect(throws: ForyError.self) { + #expect(throws: (any Error).self) { try valueDeclaration.register(ValueNodeSerializer.self, id: 130) } } @@ -1167,16 +1180,31 @@ func hiddenCarrierAliasIsRejected() throws { try fory.register(UserSerializer.self, id: 78) try fory.register(HiddenCarrierHolder.self, id: 79) - #expect(throws: ForyError.self) { + #expect(throws: (any Error).self) { _ = try fory.serialize(HiddenCarrierHolder(users: [])) } + #expect(throws: (any Error).self) { + _ = try fory.serialize(Int32(1)) + } + #expect(throws: (any Error).self) { + try fory.register(KeySerializer.self, id: 80) + } +} + +@Test +func superclassIsRejected() throws { + let fory = Fory() + try fory.register(SuperclassChild.self, id: 133) + #expect(throws: (any Error).self) { + _ = try fory.serialize(SuperclassChild()) + } } @Test func numericIDConflictIsAtomic() throws { let fory = Fory() try fory.register(UserSerializer.self, id: 80) - #expect(throws: ForyError.self) { + #expect(throws: (any Error).self) { try fory.register(KeySerializer.self, id: 80) } try fory.register(KeySerializer.self, id: 81) @@ -1199,7 +1227,7 @@ func numericIDConflictIsAtomic() throws { func registrationFreezesAtFirstRoot() throws { let fory = Fory() _ = try fory.serialize(Int32(1)) - #expect(throws: ForyError.self) { + #expect(throws: (any Error).self) { try fory.register(UserSerializer.self, id: 131) } } diff --git a/swift/Tests/ForyTests/ForySwiftTests.swift b/swift/Tests/ForyTests/ForySwiftTests.swift index d7478d1a5e..1a82f5a48f 100644 --- a/swift/Tests/ForyTests/ForySwiftTests.swift +++ b/swift/Tests/ForyTests/ForySwiftTests.swift @@ -202,6 +202,27 @@ struct LateMetaExt: Serializer, Equatable { } } +private final class RegistrationFinalizationError: Error {} + +private struct FailingRegistrationSerializer: StructSerializer { + typealias Target = Self + + static let failure = RegistrationFinalizationError() + static var staticTypeId: TypeId { .structType } + + static func defaultValue(_: ReadContext) throws -> Self { Self() } + static func writeData(_: Self, _: WriteContext) throws {} + static func readData(_: ReadContext) throws -> Self { Self() } + static func readCompatible(_: ReadContext, typeInfo _: TypeInfo) throws -> Self { Self() } + + static func foryFieldsInfo( + trackRef _: Bool, + resolveSerializerTypeId _: (Any.Type) throws -> TypeId + ) throws -> [TypeMeta.FieldInfo] { + throw failure + } +} + @ForyStruct struct LateMetaHolder: Equatable { var ext: LateMetaExt @@ -1180,6 +1201,24 @@ func registrationIsRejectedAfterFirstTopLevelUse() throws { } } +@Test +func finalizationPreservesFailure() throws { + let fory = Fory() + try fory.register(FailingRegistrationSerializer.self, id: 701) + + for _ in 0..<2 { + do { + _ = try fory.serialize(FailingRegistrationSerializer()) + Issue.record("expected registration finalization failure") + } catch { + #expect( + (error as? RegistrationFinalizationError) + === FailingRegistrationSerializer.failure + ) + } + } +} + @Test func serializeToAppendsRoots() throws { let fory = Fory() From 811cc40cd283ee2b856d57d2b6d258510465f504 Mon Sep 17 00:00:00 2001 From: chaokunyang Date: Fri, 28 Aug 2026 01:24:15 +0800 Subject: [PATCH 009/168] docs: define root registry lifecycle ownership --- AGENTS.md | 19 +++++++++++++++++++ docs/object-serialization/core-concepts.md | 3 ++- docs/security/deserialization.md | 22 ++++++++++++++++++++++ 3 files changed, 43 insertions(+), 1 deletion(-) diff --git a/AGENTS.md b/AGENTS.md index 10f82994a0..fe05399eeb 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -152,6 +152,19 @@ This is the entry point for AI guidance in Apache Fory. Read this file first, th than conflating them with `readData`. - For remote TypeDef/TypeMeta reads, the checked metadata cache is the only owner of remote "already validated" state. Cache hit means the header was previously parsed, body/hash-validated, policy-checked, and published by that cache, so the hot path must skip the body and use cached metadata without extra validation, hashing, limit checks, exact-local checks, allocation, or policy work. The protocol-defined 52-bit TypeDef/TypeMeta header hash is the unique schema identity, so a known expected local header/hash match is a local-schema hit and must not recompare field arrays or metadata bodies. The low 12 header bits belong only to the current frame; on a hit, use its current size and optional extension for bounds and skip, but do not validate its reserved or compression flags. A local hit uses the local TypeInfo/TypeMeta without schema-version counting or cache publish. Cache miss is the only path that parses and validates non-local metadata, including low flags, and enforces limits. If the local header becomes available only after that first parse, compare its 52-bit hash with the validated received hash; equality selects the local owner without a second byte or field comparison. Only a non-local miss publishes remote metadata to the cache. Do not add nullable accepted-header fields, sentinel headers, per-TypeInfo markers, pending metadata state, parallel header-low/header-high slots, or parallel acceptance state for this decision. If a runtime needs a metadata hit hint, cache the concrete checked metadata owner object, such as the TypeInfo, TypeDef, or TypeMeta used by that runtime, and compare its validated header identity directly. - Checked MetaString caches follow the same rule: validate and publish only on cache miss; on cache hit, skip the encoded body and use the cached value without rehashing, comparing body bytes, or repeating validation. The protocol-defined wire hash alone is the MetaString cache identity; the current frame length is used only for bounds checking and advancing the reader, and must not participate in hit selection. Do not add hit-time byte or length comparison or parallel acceptance state for MetaString caches. +- Java scoped meta-share TypeInfo occurrences are root-local, and their current table size is the + protocol visibility boundary. Root cleanup must reset tables of at most 8192 entries by setting + only the size to zero; do not clear retained slots. When the size exceeds 8192, replace the + backing array with an eight-slot array. Do not add per-count or benchmark-shape specializations + to this path. +- JavaScript root-local read metadata occurrence arrays and write metadata owner arrays use the + same 8192-entry retention boundary. Reset a bounded array by clearing its logical length and + replace it only after the root exceeds 8192 entries; ordinary compatible roots must not allocate + replacement arrays during cleanup. Root serializers clear reference and metadata owner state in + `finally` after success or failure. +- Root failure exceptions must not copy or retain the operation reference table or materialized + object graph for diagnostics. Root cleanup owns releasing that graph, and failure reporting must + remain bounded independently of graph size. - When a user corrects a non-obvious invariant, encode it in the nearest source comment before continuing, and also update `AGENTS.md`, `.agents/**`, docs, or specs when the rule is reusable beyond one file. Do not rely only on chat history, task notes, commit messages, or benchmark logs for corrections that protect security, protocol behavior, ownership, naming, or hot-path performance. - Reject semantic hacks. Do not bypass broken semantics by deleting cases, simplifying callers, adding coercion hooks, or using workaround fallbacks; fix the underlying bug and prove it with focused tests. - Protect hot paths. Avoid per-call allocations, callback objects, result tuples or records, unnecessary runtime branches, and wrapper-class substitutions in hot codec/runtime paths; prefer conditional imports and allocation-free concrete implementations where they fit the language. @@ -172,6 +185,9 @@ This is the entry point for AI guidance in Apache Fory. Read this file first, th serializer rebinding, metadata rebuilding, or other late-registration machinery. Registration-order finalization before the first root operation remains registration-owned and must not create a runtime invalidation path. +- Python `TypeResolver` is the sole registry freeze and finalization owner. Its Cython resolver + companion may cache completion of the Python-owner dispatch needed to populate native tables, + but the `Fory` facade must not mirror that state; Cython roots call the resolver owner directly. - Use semantic naming only. Name things after protocol or domain concepts, not history, runtime origin, or workaround style; avoid vague names such as `Internal`, `java_style_*`, `Runtime`, `Session`, `Plan`, `Payload`, or `Binding` when they do not name the real concept. Keep class, method, function, and variable names concise; do not encode the whole scenario or implementation history into one identifier. Never name a class or method with a `Plan` suffix; use the real domain concept instead. For Fory codec/read APIs, do not use generic `payload` naming; name the exact owner and data shape, such as bytes, body, frame, field, string, list, map, compressed bytes, or primitive-array encoding. - Keep one implementation path. Do not keep parallel helpers, serializers, harnesses, wrappers, or registration flows for the same concept; extend the existing owner path instead of inventing another one. - Follow current scope exactly. The latest explicit user instruction overrides earlier plans, and when scope narrows, remove leaked out-of-scope edits immediately. @@ -188,6 +204,7 @@ This is the entry point for AI guidance in Apache Fory. Read this file first, th - Do not allow implementation drift from the design document. - Do not compromise design decisions to make implementation easier. - Do not leave workaround code behind. +- Do not introduce unnecessary abstractions or concepts. - All code must have a clean owner model; the wrong owner model or abstraction is unacceptable. - Do not leave ugly or temporary code behind. - Do not leave legacy, dead, useless, or stale code, tests, or docs behind. @@ -198,6 +215,8 @@ This is the entry point for AI guidance in Apache Fory. Read this file first, th - Do not preserve legacy, dead, or useless code, tests, or docs unless the user explicitly requests it. - Ignore internal API compatibility unless the user explicitly requests it. Do not keep shims, wrappers, or transitional paths only to preserve internal call sites. - Performance is the top priority. Do not introduce regressions without explicit justification. +- Do not add object allocation to hot paths. Breaking internal compatibility is acceptable; remove + obsolete code, tests, and docs instead of preserving compatibility-only paths. - "Refactor" means changing structure, ownership, naming, or API shape without changing behavior, wire format, or implementation strategy unless the user explicitly asks for those changes. - Do not make design tradeoffs the user did not request. If a refactor appears to require a behavior, logic, protocol, or performance tradeoff, stop and ask. - Treat existing low-level or optimized code as deliberate by default. During a refactor, preserve the current implementation strategy unless the user explicitly asks to redesign or optimize it. diff --git a/docs/object-serialization/core-concepts.md b/docs/object-serialization/core-concepts.md index bdb8e9b877..e87ba2632f 100644 --- a/docs/object-serialization/core-concepts.md +++ b/docs/object-serialization/core-concepts.md @@ -39,7 +39,8 @@ only values. Use [Row Format](../row-format/index.md) for trusted analytical row A Fory instance owns its mode, schema behavior, reference settings, registered types, custom serializers, and read limits. Configure and register the instance before its first root serialization or deserialization operation, then reuse it. Registration is frozen after the first -root operation so the same instance always resolves a type in the same way. +root attempt, even when that operation fails, so the same instance always resolves a type in the +same way. Thread-safety differs by Fory implementation. Some implementations provide a thread-safe wrapper or pool; others use one instance per thread or task. Follow the selected language guide instead of sharing an ordinary diff --git a/docs/security/deserialization.md b/docs/security/deserialization.md index 0e6fcd1f31..3d3cf073fd 100644 --- a/docs/security/deserialization.md +++ b/docs/security/deserialization.md @@ -640,6 +640,18 @@ Metadata readers should: entry so input cannot make the JVM derive an unbounded family of array classes. - Reset or release metadata state at the correct root-operation boundary. +Java scoped meta-share TypeInfo occurrences are root-local. The current table size +is the protocol visibility boundary, so resetting it to zero makes retained slots +unreachable from later roots. Root cleanup retains the backing array and its slots +for tables of at most 8192 entries to avoid clearing or allocating on the normal +path. After a root exceeds 8192 entries, cleanup replaces the backing array with +an eight-slot array so an unusual metadata high-water mark is not retained. + +JavaScript applies the same 8192-entry boundary to its root-local read-side +MetaString and TypeMeta occurrence arrays. Bounded roots clear only the logical +length; larger roots replace the arrays so ordinary compatible reads do not +allocate during cleanup. + A class-resolution cache reachable from untrusted deserialization may publish an entry only from explicit trusted configuration or after the active class policy has accepted the resolved class. A cache hit therefore represents an @@ -657,6 +669,11 @@ class-keyed state, or `Class.getName()`. A custom-name registration does not by itself publish the Java class name as an additional alias; ID registration does publish the Java class name. +Read-side warnings selected by remote class or type names must use fixed +one-time-log keys. They must not include remote names or other +untrusted-cardinality values in the message or arguments, because those keys are +retained for the logger lifetime. + Remote metadata that can create persistent read state must be bounded before that state is retained. The check is resource control only: it must not change wire compatibility, type registration, dynamic class loading, unknown-type @@ -764,6 +781,11 @@ Nested `try`/`finally` or equivalent cleanup should be added only when the outer root-operation cleanup cannot cover the state or resource owned by the nested path. +A failure object must not copy or retain the root reference table or the +materialized object graph for diagnostics. Root cleanup owns releasing that +operation-local graph, and error reporting must stay bounded independently of +the graph size. + ## Performance Requirements Security validation must preserve Fory hot-path performance. Do not add From 37531ffea6d4aaee896d2f8ee1e4ac9c0b9da3ce Mon Sep 17 00:00:00 2001 From: chaokunyang Date: Fri, 28 Aug 2026 01:27:50 +0800 Subject: [PATCH 010/168] test(javascript): expect failed root write cleanup --- javascript/test/decimal.test.ts | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/javascript/test/decimal.test.ts b/javascript/test/decimal.test.ts index b6a169cc8e..ffeb8f1b4e 100644 --- a/javascript/test/decimal.test.ts +++ b/javascript/test/decimal.test.ts @@ -176,8 +176,15 @@ describe("decimal", () => { const roundTrip = fory.deserialize(fory.serialize(value)) as Decimal; expect(roundTrip.equals(value)).toBe(true); } else { + const writer = (fory as any).writeContext.writer; + const bodyBefore = Array.from( + writer.getPlatformBuffer().subarray(bodyOffset, bodyOffset + 5), + ); expect(() => fory.serialize(value)).toThrow(/Decimal scale/); - expect((fory as any).writeContext.writer.writeGetCursor()).toBe(bodyOffset); + expect(writer.writeGetCursor()).toBe(0); + expect(Array.from(writer.getPlatformBuffer().subarray(bodyOffset, bodyOffset + 5))).toEqual( + bodyBefore, + ); } const payload = decimalPayload(scale); @@ -210,7 +217,7 @@ describe("decimal", () => { writer.getPlatformBuffer().subarray(bodyOffset, bodyOffset + 5), ); expect(() => fory.serialize(value)).toThrow(/Decimal magnitude/); - expect(writer.writeGetCursor()).toBe(bodyOffset); + expect(writer.writeGetCursor()).toBe(0); expect(Array.from(writer.getPlatformBuffer().subarray(bodyOffset, bodyOffset + 5))).toEqual( bodyBefore, ); From 8df5f66a41e549d4c489a539913ed24193fdaa0f Mon Sep 17 00:00:00 2001 From: chaokunyang Date: Fri, 28 Aug 2026 02:59:29 +0800 Subject: [PATCH 011/168] fix(javascript): restore root state before reuse --- .agents/languages/javascript.md | 7 +- AGENTS.md | 8 +- docs/security/deserialization.md | 9 +- .../xlang_implementation_guide.md | 20 +++-- javascript/packages/core/lib/context.ts | 56 ++---------- javascript/packages/core/lib/fory.ts | 41 ++++----- javascript/test/decimal.test.ts | 2 - javascript/test/depthLimit.test.ts | 4 +- javascript/test/map.test.ts | 1 - javascript/test/rootReadCleanup.test.ts | 87 ++++--------------- 10 files changed, 66 insertions(+), 169 deletions(-) diff --git a/.agents/languages/javascript.md b/.agents/languages/javascript.md index db27e82f0c..18585da68c 100644 --- a/.agents/languages/javascript.md +++ b/.agents/languages/javascript.md @@ -11,10 +11,9 @@ Load this file when changing `javascript/`. - Preserve generated serializer hot paths that bind writer, reader, ref, resolver, and metadata locals in outer closures; do not replace them with per-call context lookups without a measured reason. - Do not add parallel header-low/header-high slot caches in TypeMeta hot paths to chase benchmark gaps. Header-cache hits must use the concrete checked cache owner directly; if a small hit hint is needed, cache TypeMeta objects themselves and compare `TypeMeta.headerHash`, not separate low/high header fields or benchmark-pattern state. - JavaScript TypeMeta header cache hits should compare the 52-bit TypeMeta header hash directly. The hash is precise in JS `Number` and already includes the low header bits as hash input; do not add extra low-bit fields, sentinel state, nullable accepted headers, or parallel slot arrays around it. -- Root-local read metadata occurrence arrays and write metadata owner arrays use an 8192-entry - retention boundary. Clear the logical length at or below that boundary and replace the array - only above it. Root serializers must release reference and metadata owner state in `finally` - after both successful and failed writes. +- Root entry releases reference and metadata state left by the previous operation, including a + failed operation, before the context is reused. Do not put full cleanup on the root exit path or + copy Java backing-array retention policies onto native JavaScript arrays. - Runtime value carriers such as decimal or reduced-precision numeric types belong under the core `types/` ownership boundary, with imports, exports, and codegen externals updated together. - Keep `TypeInfo` as schema metadata. Compatibility-sensitive decisions belong on `TypeResolver` or explicit operations, not as retained resolver state on metadata objects. - Normalize optional boolean config values at config construction; do not carry `null` through runtime paths when it means `false`. diff --git a/AGENTS.md b/AGENTS.md index fe05399eeb..ce141f6dc2 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -157,11 +157,9 @@ This is the entry point for AI guidance in Apache Fory. Read this file first, th only the size to zero; do not clear retained slots. When the size exceeds 8192, replace the backing array with an eight-slot array. Do not add per-count or benchmark-shape specializations to this path. -- JavaScript root-local read metadata occurrence arrays and write metadata owner arrays use the - same 8192-entry retention boundary. Reset a bounded array by clearing its logical length and - replace it only after the root exceeds 8192 entries; ordinary compatible roots must not allocate - replacement arrays during cleanup. Root serializers clear reference and metadata owner state in - `finally` after success or failure. +- JavaScript root entry releases reference and metadata state left by the previous root, including a + failed root, before the context is reused. Do not add full cleanup to the root exit path or copy + Java backing-array retention policies onto native JavaScript arrays. - Root failure exceptions must not copy or retain the operation reference table or materialized object graph for diagnostics. Root cleanup owns releasing that graph, and failure reporting must remain bounded independently of graph size. diff --git a/docs/security/deserialization.md b/docs/security/deserialization.md index 3d3cf073fd..3e52f8b029 100644 --- a/docs/security/deserialization.md +++ b/docs/security/deserialization.md @@ -647,10 +647,11 @@ for tables of at most 8192 entries to avoid clearing or allocating on the normal path. After a root exceeds 8192 entries, cleanup replaces the backing array with an eight-slot array so an unusual metadata high-water mark is not retained. -JavaScript applies the same 8192-entry boundary to its root-local read-side -MetaString and TypeMeta occurrence arrays. Bounded roots clear only the logical -length; larger roots replace the arrays so ordinary compatible reads do not -allocate during cleanup. +JavaScript root entry releases reference and metadata state left by the +previous operation, including a failed operation, before the context is reused. +Its native arrays keep their O(1) replacement reset instead of copying the Java +backing-array retention policy. Full reference and metadata cleanup does not +run on the root exit path. A class-resolution cache reachable from untrusted deserialization may publish an entry only from explicit trusted configuration or after the active class diff --git a/docs/specification/xlang_implementation_guide.md b/docs/specification/xlang_implementation_guide.md index e4c38caacf..663eb88174 100644 --- a/docs/specification/xlang_implementation_guide.md +++ b/docs/specification/xlang_implementation_guide.md @@ -80,15 +80,16 @@ not the place where nested serializers do their work. - delegating nested value encoding to `WriteContext` - delegating nested value decoding to `ReadContext` - owning registration through `TypeResolver` -- resetting operation-local context state in a top-level `finally` +- resetting operation-local context state at the top-level root boundary Nested serializers must not call back into root `serialize(...)` or `deserialize(...)` entry points. ### `WriteContext` and `ReadContext` hold operation-local state -`WriteContext` and `ReadContext` are prepared by `Fory` for one root operation -and reset by `Fory` in a `finally` block before reuse. +`WriteContext` and `ReadContext` are prepared by `Fory` for one root operation. +`Fory` resets state left by the previous root, including a failed root, before +the context is reused. `prepare(...)` should only bind the active buffer and root-operation inputs. `reset()` should clear operation-local mutable state. @@ -919,7 +920,7 @@ The current root write flow is: 2. `Fory` calls `writeContext.prepare(...)`. 3. `Fory` writes the root bitmap. 4. `Fory` delegates the root object to `WriteContext`. -5. `writeContext.reset()` runs in `finally`. +5. State left by the write resets before the next root reuses the context. For a non-null root value, `WriteContext.writeRootValue(...)` performs: @@ -941,7 +942,7 @@ Important rules: - repeated primitive writes should go directly through the buffer - nested serializer flow should stay straight-line; do not add internal `try/finally` blocks just to clean per-operation state -- top-level `Fory.serialize(...)` owns the operation reset `finally` +- top-level `Fory.serialize(...)` owns the operation reset boundary ## Deserialization Flow @@ -954,7 +955,7 @@ The current root read flow mirrors the write flow: 3. `Fory` validates xlang mode and other root framing requirements. 4. `Fory` calls `readContext.prepare(...)`. 5. `Fory` delegates to `ReadContext`. -6. `readContext.reset()` runs in `finally`. +6. State left by the read resets before the next root reuses the context. ### `ReadContext` owns ref reservation and payload materialization @@ -1228,7 +1229,7 @@ Important rules: it - nested serializer flow should stay straight-line; do not add internal `try/finally` blocks just to restore operation-local state -- top-level `Fory.deserialize(...)` owns the operation reset `finally` +- top-level `Fory.deserialize(...)` owns the operation reset boundary ## Depth Tracking @@ -1237,8 +1238,9 @@ Important rules: Depth should stay explicit on the contexts rather than relying on the native call stack alone. At the same time, depth cleanup should not depend on nested -`try/finally` blocks throughout serializer code. Top-level context reset must be -able to recover operation-local state after failures. +`try/finally` blocks throughout serializer code. Top-level context reset must +recover operation-local state before the context is reused after a root +failure. ## Struct Compatibility diff --git a/javascript/packages/core/lib/context.ts b/javascript/packages/core/lib/context.ts index 6f19837a29..d2c010ddf5 100644 --- a/javascript/packages/core/lib/context.ts +++ b/javascript/packages/core/lib/context.ts @@ -276,8 +276,6 @@ export class RefReader { } export class MetaStringWriter { - private static readonly MAX_RETAINED_META_STRING_OWNERS = 8192; - private disposeMetaStringBytes: MetaStringBytes[] = []; private dynamicNameId = 0; private namespaceEncoder = new MetaStringEncoder(".", "_"); @@ -308,22 +306,14 @@ export class MetaStringWriter { } reset() { - const names = this.disposeMetaStringBytes; - for (let i = 0; i < names.length; i++) { - names[i].dynamicWriteStringId = -1; - } + this.disposeMetaStringBytes.forEach((item) => { + item.dynamicWriteStringId = -1; + }); this.dynamicNameId = 0; - if (names.length > MetaStringWriter.MAX_RETAINED_META_STRING_OWNERS) { - this.disposeMetaStringBytes = []; - } else { - names.length = 0; - } } } export class MetaStringReader { - private static readonly MAX_RETAINED_NAMES = 8192; - private names: string[] = []; private namespaceDecoder = new MetaStringDecoder(".", "_"); private typenameDecoder = new MetaStringDecoder("$", "_"); @@ -361,17 +351,11 @@ export class MetaStringReader { } reset() { - if (this.names.length > MetaStringReader.MAX_RETAINED_NAMES) { - this.names = []; - } else { - this.names.length = 0; - } + this.names = []; } } export class WriteContext { - private static readonly MAX_RETAINED_TYPE_META_OWNERS = 8192; - readonly writer: BinaryWriter; readonly refWriter: RefWriter; readonly metaStringWriter: MetaStringWriter; @@ -392,15 +376,10 @@ export class WriteContext { this.writer.reset(); this.refWriter.reset(); this.metaStringWriter.reset(); - const owners = this.disposeTypeMetaOwners; - for (let i = 0; i < owners.length; i++) { - owners[i].dynamicTypeId = -1; - } - if (owners.length > WriteContext.MAX_RETAINED_TYPE_META_OWNERS) { - this.disposeTypeMetaOwners = []; - } else { - owners.length = 0; - } + this.disposeTypeMetaOwners.forEach((owner) => { + owner.dynamicTypeId = -1; + }); + this.disposeTypeMetaOwners = []; this.dynamicTypeId = 0; } @@ -569,7 +548,6 @@ export class WriteContext { export class ReadContext { private static readonly MIN_REMOTE_TYPE_META_LIMIT = 8192; private static readonly MAX_REMOTE_TYPE_KEYS = 8192; - private static readonly MAX_RETAINED_TYPE_META = 8192; readonly reader: BinaryReader; readonly refReader: RefReader; @@ -606,28 +584,12 @@ export class ReadContext { this.reader.reset(bytes); this.refReader.reset(); this.metaStringReader.reset(); - if (this.typeMeta.length !== 0) { - this.typeMeta.length = 0; - } + this.typeMeta = []; this._depth = 0; this.remainingGraphMemoryBytes = this.maxGraphMemoryBytes; this.remainingUnbackedContainerItems = this.maxUnbackedContainerItems; } - resetRootState() { - // Root reads own failure cleanup; nested readers retain their live state when a child throws. - this.refReader.reset(); - this.metaStringReader.reset(); - if (this.typeMeta.length > ReadContext.MAX_RETAINED_TYPE_META) { - this.typeMeta = []; - } else { - this.typeMeta.length = 0; - } - this._depth = 0; - this.remainingGraphMemoryBytes = 0; - this.remainingUnbackedContainerItems = 0; - } - reserveGraphMemory(bytes: number) { const remaining = this.remainingGraphMemoryBytes - bytes; if (remaining >= 0 && bytes >= 0 && (bytes | 0) === bytes) { diff --git a/javascript/packages/core/lib/fory.ts b/javascript/packages/core/lib/fory.ts index 4314faff00..8d188264f5 100644 --- a/javascript/packages/core/lib/fory.ts +++ b/javascript/packages/core/lib/fory.ts @@ -178,16 +178,12 @@ export default class Fory { deserialize(bytes: Uint8Array, serializer: Serializer = this.anySerializer): T | null { this.typeResolver.freezeRegistration(); this.readContext.reset(bytes); - try { - const reader = this.readContext.reader; - const bitmap = reader.readUint8(); - if (bitmap !== ConfigFlags.isCrossLanguageFlag) { - this.throwInvalidRootHeader(bitmap); - } - return serializer.readRef(); - } finally { - this.readContext.resetRootState(); + const reader = this.readContext.reader; + const bitmap = reader.readUint8(); + if (bitmap !== ConfigFlags.isCrossLanguageFlag) { + this.throwInvalidRootHeader(bitmap); } + return serializer.readRef(); } private throwInvalidRootHeader(bitmap: number): never { @@ -211,15 +207,12 @@ export default class Fory { const rootHeader = ConfigFlags.isCrossLanguageFlag; rootSerializer = (data: any) => { this.typeResolver.freezeRegistration(); - try { - writer.writeUint8(rootHeader); - writer.reserve(serializer.fixedSize); - serializer.writeRef(data); - return writer.dump(); - } finally { - // dump() returns an owned copy, so cleanup cannot invalidate a successful result. - writeContext.reset(); - } + // The entry reset releases state from the previous root before this context is reused. + writeContext.reset(); + writer.writeUint8(rootHeader); + writer.reserve(serializer.fixedSize); + serializer.writeRef(data); + return writer.dump(); }; this.rootSerializers.set(serializer, rootSerializer); return rootSerializer; @@ -239,15 +232,11 @@ export default class Fory { rootDeserializer = (bytes: Uint8Array) => { this.typeResolver.freezeRegistration(); readContext.reset(bytes); - try { - const bitmap = reader.readUint8(); - if (bitmap !== rootHeader) { - this.throwInvalidRootHeader(bitmap); - } - return rootSerializer.readRef(); - } finally { - readContext.resetRootState(); + const bitmap = reader.readUint8(); + if (bitmap !== rootHeader) { + this.throwInvalidRootHeader(bitmap); } + return rootSerializer.readRef(); }; this.rootDeserializers.set(serializer, rootDeserializer); return rootDeserializer; diff --git a/javascript/test/decimal.test.ts b/javascript/test/decimal.test.ts index ffeb8f1b4e..58177997ad 100644 --- a/javascript/test/decimal.test.ts +++ b/javascript/test/decimal.test.ts @@ -181,7 +181,6 @@ describe("decimal", () => { writer.getPlatformBuffer().subarray(bodyOffset, bodyOffset + 5), ); expect(() => fory.serialize(value)).toThrow(/Decimal scale/); - expect(writer.writeGetCursor()).toBe(0); expect(Array.from(writer.getPlatformBuffer().subarray(bodyOffset, bodyOffset + 5))).toEqual( bodyBefore, ); @@ -217,7 +216,6 @@ describe("decimal", () => { writer.getPlatformBuffer().subarray(bodyOffset, bodyOffset + 5), ); expect(() => fory.serialize(value)).toThrow(/Decimal magnitude/); - expect(writer.writeGetCursor()).toBe(0); expect(Array.from(writer.getPlatformBuffer().subarray(bodyOffset, bodyOffset + 5))).toEqual( bodyBefore, ); diff --git a/javascript/test/depthLimit.test.ts b/javascript/test/depthLimit.test.ts index 2b2a40f5c3..8e6689d379 100644 --- a/javascript/test/depthLimit.test.ts +++ b/javascript/test/depthLimit.test.ts @@ -291,7 +291,6 @@ describe("depth-limit", () => { expect(() => reader.deserialize(malformedDepth)).toThrow( "Deserialization depth limit exceeded", ); - expect(readerFory.readContext.depth).toBe(0); expect(shallowReader.deserialize(shallowWriter.serialize({ value: 10 }))).toEqual({ value: 10, @@ -299,7 +298,7 @@ describe("depth-limit", () => { expect(readerFory.readContext.depth).toBe(0); }); - test("should reset depth after each deserialization", () => { + test("should reset depth before each deserialization", () => { const fory = new Fory({ compatible: false, maxDepth: 50 }); const typeInfo = Type.struct( { @@ -432,7 +431,6 @@ describe("depth-limit", () => { for (const readRoot of rootReaders) { expect(() => readRoot(serialized.subarray(0, serialized.length - 1))).toThrow(); - expect(fory.readContext.depth).toBe(0); expect(readRoot(serialized)).toEqual(value); expect(fory.readContext.depth).toBe(0); diff --git a/javascript/test/map.test.ts b/javascript/test/map.test.ts index 55f8cbf864..c65cdc5e47 100644 --- a/javascript/test/map.test.ts +++ b/javascript/test/map.test.ts @@ -209,7 +209,6 @@ describe("map", () => { malformed[chunkSizeOffset] = chunkSize; expect(() => serializer.deserialize(malformed)).toThrow(); - expect(fory.readContext.depth).toBe(0); expect(serializer.deserialize(valid)).toEqual(value); } }); diff --git a/javascript/test/rootReadCleanup.test.ts b/javascript/test/rootReadCleanup.test.ts index 9c64a4adf8..50cfad3e7f 100644 --- a/javascript/test/rootReadCleanup.test.ts +++ b/javascript/test/rootReadCleanup.test.ts @@ -44,7 +44,7 @@ describe.each([ registered.deserialize(bytes), }, ])("$name root cleanup", ({ invoke }) => { - test.each(["success", "failure"] as const)("clears generated root state after %s", (outcome) => { + test.each(["success", "failure"] as const)("restores generated root state for %s", (outcome) => { const writerFory = new Fory({ compatible: true, ref: true }); const readerFory = new Fory({ compatible: true, ref: true }); const writer = writerFory.register( @@ -63,15 +63,17 @@ describe.each([ if (outcome === "failure") { expect(read).toThrow(); + expect(invoke(readerFory, reader, bytes)).toEqual({ value: 7 }); } else { - expect(read()).toEqual({ value: 7 }); + const first = read(); + const second = read(); + expect(first).toEqual({ value: 7 }); + expect(second).toEqual({ value: 7 }); + expect(second).not.toBe(first); } - - const readContext = (readerFory as any).readContext; - expectRootStateCleared(readContext); }); - test.each(["success", "failure"] as const)("clears logical tables after %s", (outcome) => { + test.each(["success", "failure"] as const)("restores logical tables for %s", (outcome) => { const fory = new Fory({ compatible: true, ref: true }); const registered = fory.register(Type.struct(7602, {})); const readContext = (fory as any).readContext; @@ -92,71 +94,21 @@ describe.each([ const read = () => invoke(fory, registered, new Uint8Array([1])); if (outcome === "failure") { expect(read).toThrow(); + registered.serializer.readRef = () => { + expectRootStateCleared(readContext); + return 7; + }; + expect(read()).toBe(7); } else { expect(read()).toBe(7); + expect(read()).toBe(7); } - expectRootStateCleared(readContext); expect(readContext.typeMetaCache.get(headerHash)).toBe(typeMeta); }); }); -test("retains bounded read metadata", () => { - const fory = new Fory({ compatible: true }); - const readContext = (fory as any).readContext; - const typeMeta = TypeMeta.fromTypeInfo(Type.struct(7604, {})); - - readContext.typeMeta.push(...new Array(8192).fill(typeMeta)); - const bounded = readContext.typeMeta; - readContext.metaStringReader.names.push(...new Array(8192).fill("stale")); - const boundedNames = readContext.metaStringReader.names; - readContext.resetRootState(); - expect(readContext.typeMeta).toBe(bounded); - expect(readContext.metaStringReader.names).toBe(boundedNames); - expect(readContext.typeMeta).toHaveLength(0); - expect(readContext.metaStringReader.names).toHaveLength(0); - - readContext.typeMeta.push(...new Array(8193).fill(typeMeta)); - const oversized = readContext.typeMeta; - readContext.metaStringReader.names.push(...new Array(8193).fill("stale")); - const oversizedNames = readContext.metaStringReader.names; - readContext.resetRootState(); - expect(readContext.typeMeta).not.toBe(oversized); - expect(readContext.metaStringReader.names).not.toBe(oversizedNames); - expect(readContext.typeMeta).toHaveLength(0); - expect(readContext.metaStringReader.names).toHaveLength(0); -}); - -test("retains bounded write metadata", () => { - const fory = new Fory({ compatible: true }); - const writeContext = (fory as any).writeContext; - const typeMetaOwners = Array.from({ length: 8192 }, (_, dynamicTypeId) => ({ - dynamicTypeId, - })); - const metaStringOwners = Array.from({ length: 8192 }, (_, dynamicWriteStringId) => ({ - dynamicWriteStringId, - })); - - writeContext.disposeTypeMetaOwners.push(...typeMetaOwners); - const bounded = writeContext.disposeTypeMetaOwners; - writeContext.metaStringWriter.disposeMetaStringBytes.push(...metaStringOwners); - const boundedNames = writeContext.metaStringWriter.disposeMetaStringBytes; - writeContext.reset(); - expect(writeContext.disposeTypeMetaOwners).toBe(bounded); - expect(writeContext.metaStringWriter.disposeMetaStringBytes).toBe(boundedNames); - - writeContext.disposeTypeMetaOwners.push(...typeMetaOwners, { dynamicTypeId: 8192 }); - const oversized = writeContext.disposeTypeMetaOwners; - writeContext.metaStringWriter.disposeMetaStringBytes.push(...metaStringOwners, { - dynamicWriteStringId: 8192, - }); - const oversizedNames = writeContext.metaStringWriter.disposeMetaStringBytes; - writeContext.reset(); - expect(writeContext.disposeTypeMetaOwners).not.toBe(oversized); - expect(writeContext.metaStringWriter.disposeMetaStringBytes).not.toBe(oversizedNames); -}); - -test.each(["success", "failure"] as const)("clears root write state after %s", (outcome) => { +test.each(["success", "failure"] as const)("restores root write state for %s", (outcome) => { const fory = new Fory({ compatible: true, ref: true }); const registered = fory.register(Type.struct(7606, {})); const writeContext = (fory as any).writeContext; @@ -175,19 +127,18 @@ test.each(["success", "failure"] as const)("clears root write state after %s", ( if (outcome === "failure") { expect(() => registered.serialize(value)).toThrow("root write failed"); + expect(fory.serialize(7)).toBeDefined(); } else { expect(registered.serialize(value)).toBeDefined(); + expect(fory.serialize(7)).toBeDefined(); } expect(writeContext.refWriter.writeObjects.size).toBe(0); - expect(writeContext.metaStringWriter.disposeMetaStringBytes).toHaveLength(0); expect(writeContext.disposeTypeMetaOwners).toHaveLength(0); expect(name.dynamicWriteStringId).toBe(-1); expect(typeMeta.dynamicTypeId).toBe(-1); - expect(writeContext.writer.writeGetCursor()).toBe(0); - expect(fory.serialize(7)).toBeDefined(); }); -test("releases a failed root write buffer", () => { +test("releases a failed root write buffer before reuse", () => { const fory = new Fory({ compatible: true }); const registered = fory.register(Type.struct(7608, {})); const writer = (fory as any).writeContext.writer; @@ -198,6 +149,6 @@ test("releases a failed root write buffer", () => { }; expect(() => registered.serialize({})).toThrow("root write failed"); + expect(fory.serialize(7)).toBeDefined(); expect(writer.getPlatformBuffer().byteLength).toBeLessThan(4 * 1024 * 1024); - expect(writer.writeGetCursor()).toBe(0); }); From 41219c31549b2f5c38c85ae74c3862c2255bb008 Mon Sep 17 00:00:00 2001 From: chaokunyang Date: Fri, 28 Aug 2026 03:50:59 +0800 Subject: [PATCH 012/168] fix(jvm): recheck registry after serializer construction --- .../serializer/kotlin/KotlinSerializers.java | 1 + .../kotlin/BuiltinClassSerializerTests.kt | 42 +++++++++++++++++++ .../apache/fory/scala/ForySerializer.scala | 4 +- .../scala/ForySerializerDerivationTest.scala | 20 +++++++++ 4 files changed, 66 insertions(+), 1 deletion(-) diff --git a/kotlin/fory-kotlin/src/main/java/org/apache/fory/serializer/kotlin/KotlinSerializers.java b/kotlin/fory-kotlin/src/main/java/org/apache/fory/serializer/kotlin/KotlinSerializers.java index 696dd3459b..14456dd3d0 100644 --- a/kotlin/fory-kotlin/src/main/java/org/apache/fory/serializer/kotlin/KotlinSerializers.java +++ b/kotlin/fory-kotlin/src/main/java/org/apache/fory/serializer/kotlin/KotlinSerializers.java @@ -256,6 +256,7 @@ private static Serializer newRegistrationSerializer(TypeResolver resolver, Cl private static void registerSerializerAfterType(Fory fory, Class cls) { TypeResolver resolver = fory.getTypeResolver(); Serializer serializer = newGeneratedSerializer(resolver, cls); + checkRegistrationOpen(resolver); resolver.setSerializer(cls, serializer); } diff --git a/kotlin/fory-kotlin/src/test/kotlin/org/apache/fory/serializer/kotlin/BuiltinClassSerializerTests.kt b/kotlin/fory-kotlin/src/test/kotlin/org/apache/fory/serializer/kotlin/BuiltinClassSerializerTests.kt index bdc1e807bc..3ab0d14ec0 100644 --- a/kotlin/fory-kotlin/src/test/kotlin/org/apache/fory/serializer/kotlin/BuiltinClassSerializerTests.kt +++ b/kotlin/fory-kotlin/src/test/kotlin/org/apache/fory/serializer/kotlin/BuiltinClassSerializerTests.kt @@ -34,12 +34,34 @@ import kotlin.time.Duration.Companion.seconds import kotlin.uuid.ExperimentalUuidApi import kotlin.uuid.Uuid import org.apache.fory.Fory +import org.apache.fory.context.ReadContext +import org.apache.fory.context.WriteContext import org.apache.fory.exception.ForyException import org.apache.fory.kotlin.ForyKotlin +import org.apache.fory.resolver.TypeResolver +import org.apache.fory.serializer.Serializer import org.apache.fory.util.DefaultValueUtils import org.testng.Assert import org.testng.Assert.assertThrows +private object ReentrantRegistration { + var fory: Fory? = null +} + +class ReentrantStruct + +@Suppress("UNCHECKED_CAST") +class ReentrantStruct_ForySerializer(resolver: TypeResolver, cls: Class<*>) : + Serializer(resolver.config, cls as Class) { + init { + checkNotNull(ReentrantRegistration.fory).serialize("freeze") + } + + override fun write(writeContext: WriteContext, value: ReentrantStruct) = Unit + + override fun read(readContext: ReadContext): ReentrantStruct = ReentrantStruct() +} + class BuiltinClassSerializerTests { @Test fun testLateBootstrapIsReadOnly() { @@ -51,6 +73,26 @@ class BuiltinClassSerializerTests { Assert.assertSame(DefaultValueUtils.getKotlinDefaultValueSupport(), defaultValueSupport) } + @Test + fun testCombinedFreezeRecheck() { + val fory = + ForyKotlin.builder() + .withXlang(true) + .requireClassRegistration(true) + .withRefTracking(false) + .build() + ReentrantRegistration.fory = fory + + try { + assertThrows(ForyException::class.java) { + KotlinSerializers.register(fory, ReentrantStruct::class.java, "kotlin.ReentrantStruct") + } + Assert.assertTrue(fory.typeResolver.isRegistered(ReentrantStruct::class.java)) + } finally { + ReentrantRegistration.fory = null + } + } + @Test fun testSerializePair() { val fory: Fory = ForyKotlin.builder().withXlang(false).requireClassRegistration(true).build() diff --git a/scala/fory-scala/src/main/scala-3/org/apache/fory/scala/ForySerializer.scala b/scala/fory-scala/src/main/scala-3/org/apache/fory/scala/ForySerializer.scala index 31dfc87d8c..27cab8f79d 100644 --- a/scala/fory-scala/src/main/scala-3/org/apache/fory/scala/ForySerializer.scala +++ b/scala/fory-scala/src/main/scala-3/org/apache/fory/scala/ForySerializer.scala @@ -166,7 +166,9 @@ object ForySerializer { } } else { registerType(fory, cls, typeId, namespace, typeName) - resolver.setSerializer(cls, serializer.createSerializer(resolver)) + val generatedSerializer = serializer.createSerializer(resolver) + checkRegistrationOpen(resolver) + resolver.setSerializer(cls, generatedSerializer) } } diff --git a/scala/fory-scala/src/test/scala-3/org/apache/fory/serializer/scala/ForySerializerDerivationTest.scala b/scala/fory-scala/src/test/scala-3/org/apache/fory/serializer/scala/ForySerializerDerivationTest.scala index 305ce069a9..0c643f8e03 100644 --- a/scala/fory-scala/src/test/scala-3/org/apache/fory/serializer/scala/ForySerializerDerivationTest.scala +++ b/scala/fory-scala/src/test/scala-3/org/apache/fory/serializer/scala/ForySerializerDerivationTest.scala @@ -412,6 +412,26 @@ class ForySerializerDerivationTest extends AnyWordSpec with Matchers { runtime.getSerializer(classOf[Person]) shouldBe theSameInstanceAs(originalSerializer) } + "reject combined registration frozen during creation" in { + val runtime = xlangFory() + val factory = summon[ForySerializer[StoredState]] + val reentrantFactory = new ForySerializer[StoredState] { + override def createSerializer( + typeResolver: org.apache.fory.resolver.TypeResolver, + typeDef: TypeDef): org.apache.fory.serializer.Serializer[StoredState] = { + val generatedSerializer = factory.createSerializer(typeResolver, typeDef) + runtime.serialize("freeze") + generatedSerializer + } + } + + intercept[ForyException] { + ForySerializer.register(runtime, classOf[StoredState], "scala_test.ReentrantStoredState")( + using reentrantFactory) + } + runtime.getTypeResolver.isRegistered(classOf[StoredState]) shouldBe true + } + "serialize derived case classes with Scala collection fields" in { val fory = xlangFory() val box = CollectionBox(List("a", "b"), Set("x", "y"), Map("a" -> 1, "b" -> 2)) From 6584bd7c6887a1a35defea51664e1bf9cc2e109f Mon Sep 17 00:00:00 2001 From: chaokunyang Date: Fri, 28 Aug 2026 03:57:20 +0800 Subject: [PATCH 013/168] fix(javascript): clear root metastring owners --- javascript/packages/core/lib/context.ts | 2 ++ javascript/test/rootReadCleanup.test.ts | 1 + 2 files changed, 3 insertions(+) diff --git a/javascript/packages/core/lib/context.ts b/javascript/packages/core/lib/context.ts index d2c010ddf5..a72e63e1af 100644 --- a/javascript/packages/core/lib/context.ts +++ b/javascript/packages/core/lib/context.ts @@ -309,6 +309,8 @@ export class MetaStringWriter { this.disposeMetaStringBytes.forEach((item) => { item.dynamicWriteStringId = -1; }); + // Reset owners are appended again after their ID is cleared, so retain no prior-root entries. + this.disposeMetaStringBytes.length = 0; this.dynamicNameId = 0; } } diff --git a/javascript/test/rootReadCleanup.test.ts b/javascript/test/rootReadCleanup.test.ts index 50cfad3e7f..7cb2169895 100644 --- a/javascript/test/rootReadCleanup.test.ts +++ b/javascript/test/rootReadCleanup.test.ts @@ -134,6 +134,7 @@ test.each(["success", "failure"] as const)("restores root write state for %s", ( } expect(writeContext.refWriter.writeObjects.size).toBe(0); expect(writeContext.disposeTypeMetaOwners).toHaveLength(0); + expect(writeContext.metaStringWriter.disposeMetaStringBytes).toHaveLength(0); expect(name.dynamicWriteStringId).toBe(-1); expect(typeMeta.dynamicTypeId).toBe(-1); }); From 1ab4a913ef775b8fc3754bcd17159a88b3d48657 Mon Sep 17 00:00:00 2001 From: chaokunyang Date: Fri, 28 Aug 2026 03:57:20 +0800 Subject: [PATCH 014/168] docs(python): fix circular reference example --- python/README.md | 44 ++++++++++++++------------------------------ 1 file changed, 14 insertions(+), 30 deletions(-) diff --git a/python/README.md b/python/README.md index 593aed98a9..5f416ac46c 100644 --- a/python/README.md +++ b/python/README.md @@ -651,26 +651,29 @@ data = f.serialize(MyDataClass(field1="value", field2=42)) Handle shared references and circular dependencies safely. Set `ref=True` to deduplicate objects: ```python +from dataclasses import dataclass +from typing import Optional + import pyfory f = pyfory.Fory(xlang=False, ref=True) # Enable reference tracking -# Handle circular references safely +@dataclass class Node: - def __init__(self, value): - self.value = value - self.children = [] - self.parent = None + value: str + next: Optional["Node"] = pyfory.field(ref=True, nullable=True, default=None) + +f.register_type(Node) root = Node("root") child = Node("child") -child.parent = root # Circular reference -root.children.append(child) +root.next = child +child.next = root # Circular reference # Serializes without infinite recursion data = f.serialize(root) result = f.deserialize(data) -assert result.children[0].parent is result # Reference preserved +assert result.next.next is result # Reference preserved ``` ### Type Registration @@ -862,28 +865,9 @@ object identity or cycles matter: f = pyfory.Fory(ref=True) ``` -For configured Python object graphs with circular references, use native mode and register every -application type before the first root: - -```python -f = pyfory.Fory(xlang=False, ref=True, strict=False) - -# Example with circular reference -class Node: - def __init__(self, value): - self.value = value - self.next = None - -node1 = Node(1) -node2 = Node(2) -node1.next = node2 -node2.next = node1 # Circular reference - -f.register_type(Node) -data = f.dumps(node1) -result = f.loads(data) -assert result.next.next is result # Circular reference preserved -``` +For configured Python object graphs with circular references, use native mode, register every +application type before the first root, and declare reference-tracked recursive fields as shown in +[Reference Tracking & Circular References](#reference-tracking--circular-references). ### Debug Mode From 77bb81f95794e021b865e3b17b97c1006e6cafe1 Mon Sep 17 00:00:00 2001 From: chaokunyang Date: Fri, 28 Aug 2026 03:57:20 +0800 Subject: [PATCH 015/168] test(swift): restore precise failure assertions --- .../ExternalTypeSerializationTests.swift | 34 +++++++++---------- 1 file changed, 17 insertions(+), 17 deletions(-) diff --git a/swift/Tests/ForyTests/ExternalTypeSerializationTests.swift b/swift/Tests/ForyTests/ExternalTypeSerializationTests.swift index 7a40ef2e18..f52401e2af 100644 --- a/swift/Tests/ForyTests/ExternalTypeSerializationTests.swift +++ b/swift/Tests/ForyTests/ExternalTypeSerializationTests.swift @@ -538,7 +538,7 @@ func externalIgnoredFieldBudget() throws { ) ) try limited.register(NodeSerializer.self, id: 142) - #expect(throws: (any Error).self) { + #expect(throws: ForyError.self) { let _: Node = try limited.deserialize( bytes, with: NodeSerializer.self @@ -732,7 +732,7 @@ func customCarrierEnforcesBudget() throws { ) try reader.register(UserSerializer.self, id: 138) try reader.register(UserArrayCustomSerializer.self, id: 139) - #expect(throws: (any Error).self) { + #expect(throws: ForyError.self) { let _: [User] = try reader.deserialize( bytes, with: UserArrayCustomSerializer.self @@ -1089,7 +1089,7 @@ func directExternalDynamicRoot() throws { func dynamicTargetFailures() throws { let unregistered = Fory() let unregisteredValue: Any = User(name: "unregistered", age: 1) - #expect(throws: (any Error).self) { + #expect(throws: ForyError.self) { _ = try unregistered.serialize( unregisteredValue, with: DynamicSerializer.self @@ -1101,13 +1101,13 @@ func dynamicTargetFailures() throws { let value: Any = Key(value: "not-named") let bytes = try fory.serialize(value, with: DynamicSerializer.self) - #expect(throws: (any Error).self) { + #expect(throws: ForyError.self) { let _: any NamedValue = try fory.deserialize( bytes, with: DynamicSerializer.self ) } - #expect(throws: (any Error).self) { + #expect(throws: ForyError.self) { let _: AnyObject = try fory.deserialize( bytes, with: DynamicSerializer.self @@ -1143,33 +1143,33 @@ func dynamicReferenceEnvelopeBytes() throws { @Test func registrationRejectsInvalidOwnership() throws { let carrier = Fory() - #expect(throws: (any Error).self) { + #expect(throws: ForyError.self) { try carrier.register(ArraySerializer.self, id: 126) } let wrapper = Fory() - #expect(throws: (any Error).self) { + #expect(throws: ForyError.self) { try wrapper.register(OptionalSerializer.self, id: 132) } let dynamic = Fory() - #expect(throws: (any Error).self) { + #expect(throws: ForyError.self) { try dynamic.register(DynamicSerializer.self, id: 133) } let duplicateTarget = Fory() try duplicateTarget.register(UserSerializer.self, id: 127) - #expect(throws: (any Error).self) { + #expect(throws: ForyError.self) { try duplicateTarget.register(AlternateUserSerializer.self, id: 128) } let builtinTarget = Fory() - #expect(throws: (any Error).self) { + #expect(throws: ForyError.self) { try builtinTarget.register(StringCustomSerializer.self, id: 129) } let valueDeclaration = Fory() - #expect(throws: (any Error).self) { + #expect(throws: ForyError.self) { try valueDeclaration.register(ValueNodeSerializer.self, id: 130) } } @@ -1180,13 +1180,13 @@ func hiddenCarrierAliasIsRejected() throws { try fory.register(UserSerializer.self, id: 78) try fory.register(HiddenCarrierHolder.self, id: 79) - #expect(throws: (any Error).self) { + #expect(throws: ForyError.self) { _ = try fory.serialize(HiddenCarrierHolder(users: [])) } - #expect(throws: (any Error).self) { + #expect(throws: ForyError.self) { _ = try fory.serialize(Int32(1)) } - #expect(throws: (any Error).self) { + #expect(throws: ForyError.self) { try fory.register(KeySerializer.self, id: 80) } } @@ -1195,7 +1195,7 @@ func hiddenCarrierAliasIsRejected() throws { func superclassIsRejected() throws { let fory = Fory() try fory.register(SuperclassChild.self, id: 133) - #expect(throws: (any Error).self) { + #expect(throws: ForyError.self) { _ = try fory.serialize(SuperclassChild()) } } @@ -1204,7 +1204,7 @@ func superclassIsRejected() throws { func numericIDConflictIsAtomic() throws { let fory = Fory() try fory.register(UserSerializer.self, id: 80) - #expect(throws: (any Error).self) { + #expect(throws: ForyError.self) { try fory.register(KeySerializer.self, id: 80) } try fory.register(KeySerializer.self, id: 81) @@ -1227,7 +1227,7 @@ func numericIDConflictIsAtomic() throws { func registrationFreezesAtFirstRoot() throws { let fory = Fory() _ = try fory.serialize(Int32(1)) - #expect(throws: (any Error).self) { + #expect(throws: ForyError.self) { try fory.register(UserSerializer.self, id: 131) } } From a0fd34b6f79c4b24305b6bed74951b1e7f8a8d0d Mon Sep 17 00:00:00 2001 From: chaokunyang Date: Fri, 28 Aug 2026 04:01:09 +0800 Subject: [PATCH 016/168] docs(javascript): define metadata owner reset --- .agents/languages/javascript.md | 4 +++- AGENTS.md | 4 +++- docs/security/deserialization.md | 8 +++++--- docs/specification/xlang_implementation_guide.md | 5 +++++ 4 files changed, 16 insertions(+), 5 deletions(-) diff --git a/.agents/languages/javascript.md b/.agents/languages/javascript.md index 18585da68c..6982b7c060 100644 --- a/.agents/languages/javascript.md +++ b/.agents/languages/javascript.md @@ -13,7 +13,9 @@ Load this file when changing `javascript/`. - JavaScript TypeMeta header cache hits should compare the 52-bit TypeMeta header hash directly. The hash is precise in JS `Number` and already includes the low header bits as hash input; do not add extra low-bit fields, sentinel state, nullable accepted headers, or parallel slot arrays around it. - Root entry releases reference and metadata state left by the previous operation, including a failed operation, before the context is reused. Do not put full cleanup on the root exit path or - copy Java backing-array retention policies onto native JavaScript arrays. + copy Java backing-array retention policies onto native JavaScript arrays. Read-side occurrence + arrays use native replacement reset. Writer metadata owners first restore their dynamic IDs, + then truncate the active owner list because those owners are appended again in the next root. - Runtime value carriers such as decimal or reduced-precision numeric types belong under the core `types/` ownership boundary, with imports, exports, and codegen externals updated together. - Keep `TypeInfo` as schema metadata. Compatibility-sensitive decisions belong on `TypeResolver` or explicit operations, not as retained resolver state on metadata objects. - Normalize optional boolean config values at config construction; do not carry `null` through runtime paths when it means `false`. diff --git a/AGENTS.md b/AGENTS.md index ce141f6dc2..23b4ad31b9 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -159,7 +159,9 @@ This is the entry point for AI guidance in Apache Fory. Read this file first, th to this path. - JavaScript root entry releases reference and metadata state left by the previous root, including a failed root, before the context is reused. Do not add full cleanup to the root exit path or copy - Java backing-array retention policies onto native JavaScript arrays. + Java backing-array retention policies onto native JavaScript arrays. Read-side occurrence arrays + use native replacement reset. Writer metadata owners first restore their dynamic IDs, then + truncate the active owner list because those owners are appended again in the next root. - Root failure exceptions must not copy or retain the operation reference table or materialized object graph for diagnostics. Root cleanup owns releasing that graph, and failure reporting must remain bounded independently of graph size. diff --git a/docs/security/deserialization.md b/docs/security/deserialization.md index 3e52f8b029..8a31d29a83 100644 --- a/docs/security/deserialization.md +++ b/docs/security/deserialization.md @@ -649,9 +649,11 @@ an eight-slot array so an unusual metadata high-water mark is not retained. JavaScript root entry releases reference and metadata state left by the previous operation, including a failed operation, before the context is reused. -Its native arrays keep their O(1) replacement reset instead of copying the Java -backing-array retention policy. Full reference and metadata cleanup does not -run on the root exit path. +Read-side occurrence arrays use native replacement reset instead of copying the +Java backing-array retention policy. Writer metadata owners restore their +dynamic IDs and then truncate the active owner list so the same owners do not +accumulate across roots. Full reference and metadata cleanup does not run on +the root exit path. A class-resolution cache reachable from untrusted deserialization may publish an entry only from explicit trusted configuration or after the active class diff --git a/docs/specification/xlang_implementation_guide.md b/docs/specification/xlang_implementation_guide.md index 663eb88174..33116bac8d 100644 --- a/docs/specification/xlang_implementation_guide.md +++ b/docs/specification/xlang_implementation_guide.md @@ -94,6 +94,11 @@ the context is reused. `prepare(...)` should only bind the active buffer and root-operation inputs. `reset()` should clear operation-local mutable state. +When writer metadata objects carry root-local dynamic IDs, reset must restore +those IDs and discard the active owner-list entries. An owner may be appended +again on first use in the next root; retaining prior entries creates duplicate +cleanup work across roots. + That operation-local state includes: - the current buffer From a40934de124ab7cba715fa8c60d9f6ad62a771a4 Mon Sep 17 00:00:00 2001 From: chaokunyang Date: Fri, 28 Aug 2026 04:16:30 +0800 Subject: [PATCH 017/168] fix(javascript): reuse bounded metastring owners --- javascript/packages/core/lib/context.ts | 21 ++++++++---- javascript/test/rootReadCleanup.test.ts | 44 ++++++++++++++++++++++++- 2 files changed, 58 insertions(+), 7 deletions(-) diff --git a/javascript/packages/core/lib/context.ts b/javascript/packages/core/lib/context.ts index a72e63e1af..44613d8ef9 100644 --- a/javascript/packages/core/lib/context.ts +++ b/javascript/packages/core/lib/context.ts @@ -276,7 +276,10 @@ export class RefReader { } export class MetaStringWriter { + private static readonly MAX_RETAINED_META_STRING_OWNERS = 8192; + private disposeMetaStringBytes: MetaStringBytes[] = []; + private disposeMetaStringBytesSize = 0; private dynamicNameId = 0; private namespaceEncoder = new MetaStringEncoder(".", "_"); private typenameEncoder = new MetaStringEncoder("$", "_"); @@ -287,7 +290,7 @@ export class MetaStringWriter { } else { bytes.dynamicWriteStringId = this.dynamicNameId; this.dynamicNameId += 1; - this.disposeMetaStringBytes.push(bytes); + this.disposeMetaStringBytes[this.disposeMetaStringBytesSize++] = bytes; const len = bytes.bytes.getBytes().byteLength; writer.writeVarUInt32(len << 1); if (len !== 0) { @@ -306,11 +309,17 @@ export class MetaStringWriter { } reset() { - this.disposeMetaStringBytes.forEach((item) => { - item.dynamicWriteStringId = -1; - }); - // Reset owners are appended again after their ID is cleared, so retain no prior-root entries. - this.disposeMetaStringBytes.length = 0; + const owners = this.disposeMetaStringBytes; + const size = this.disposeMetaStringBytesSize; + for (let i = 0; i < size; i++) { + owners[i].dynamicWriteStringId = -1; + } + // These owners remain serializer-owned. Keep bounded backing without making old entries + // protocol-visible, and release only an unusual root's oversized owner table. + if (size > MetaStringWriter.MAX_RETAINED_META_STRING_OWNERS) { + this.disposeMetaStringBytes = []; + } + this.disposeMetaStringBytesSize = 0; this.dynamicNameId = 0; } } diff --git a/javascript/test/rootReadCleanup.test.ts b/javascript/test/rootReadCleanup.test.ts index 7cb2169895..eefbb8682d 100644 --- a/javascript/test/rootReadCleanup.test.ts +++ b/javascript/test/rootReadCleanup.test.ts @@ -134,11 +134,53 @@ test.each(["success", "failure"] as const)("restores root write state for %s", ( } expect(writeContext.refWriter.writeObjects.size).toBe(0); expect(writeContext.disposeTypeMetaOwners).toHaveLength(0); - expect(writeContext.metaStringWriter.disposeMetaStringBytes).toHaveLength(0); + expect(writeContext.metaStringWriter.disposeMetaStringBytesSize).toBe(0); expect(name.dynamicWriteStringId).toBe(-1); expect(typeMeta.dynamicTypeId).toBe(-1); }); +test("reuses root write metastring owners", () => { + const fory = new Fory({ compatible: true }); + const registered = fory.register(Type.struct(7609, {})); + const writeContext = (fory as any).writeContext; + const name = writeContext.metaStringWriter.encodeTypeName("RootName"); + + registered.serializer.writeRef = () => { + writeContext.metaStringWriter.writeBytes(writeContext.writer, name); + }; + + expect(registered.serialize({})).toBeDefined(); + const owners = writeContext.metaStringWriter.disposeMetaStringBytes; + expect(owners).toHaveLength(1); + expect(writeContext.metaStringWriter.disposeMetaStringBytesSize).toBe(1); + + expect(registered.serialize({})).toBeDefined(); + expect(writeContext.metaStringWriter.disposeMetaStringBytes).toBe(owners); + expect(owners).toHaveLength(1); + expect(writeContext.metaStringWriter.disposeMetaStringBytesSize).toBe(1); +}); + +test.each([8192, 8193])("bounds %s root write metastring owners", (ownerCount) => { + const fory = new Fory({ compatible: true }); + const writeContext = (fory as any).writeContext; + const metaStringWriter = writeContext.metaStringWriter; + + for (let i = 0; i < ownerCount; i++) { + const owner = metaStringWriter.encodeTypeName(`name-${i}`); + metaStringWriter.writeBytes(writeContext.writer, owner); + } + const owners = metaStringWriter.disposeMetaStringBytes; + + writeContext.reset(); + expect(metaStringWriter.disposeMetaStringBytesSize).toBe(0); + if (ownerCount === 8192) { + expect(metaStringWriter.disposeMetaStringBytes).toBe(owners); + } else { + expect(metaStringWriter.disposeMetaStringBytes).not.toBe(owners); + expect(metaStringWriter.disposeMetaStringBytes).toHaveLength(0); + } +}); + test("releases a failed root write buffer before reuse", () => { const fory = new Fory({ compatible: true }); const registered = fory.register(Type.struct(7608, {})); From 5a53780d14a595e1834705ef6b9bc4b69ff70bae Mon Sep 17 00:00:00 2001 From: chaokunyang Date: Fri, 28 Aug 2026 04:16:30 +0800 Subject: [PATCH 018/168] docs(javascript): define bounded owner reset --- .agents/languages/javascript.md | 5 +++-- AGENTS.md | 5 +++-- docs/security/deserialization.md | 7 ++++--- docs/specification/xlang_implementation_guide.md | 7 ++++--- 4 files changed, 14 insertions(+), 10 deletions(-) diff --git a/.agents/languages/javascript.md b/.agents/languages/javascript.md index 6982b7c060..3796728418 100644 --- a/.agents/languages/javascript.md +++ b/.agents/languages/javascript.md @@ -14,8 +14,9 @@ Load this file when changing `javascript/`. - Root entry releases reference and metadata state left by the previous operation, including a failed operation, before the context is reused. Do not put full cleanup on the root exit path or copy Java backing-array retention policies onto native JavaScript arrays. Read-side occurrence - arrays use native replacement reset. Writer metadata owners first restore their dynamic IDs, - then truncate the active owner list because those owners are appended again in the next root. + arrays use native replacement reset. The writer metadata-owner table has a separate logical size: + reset owner IDs and the logical size without clearing bounded backing, and replace backing only + after more than 8192 owners. - Runtime value carriers such as decimal or reduced-precision numeric types belong under the core `types/` ownership boundary, with imports, exports, and codegen externals updated together. - Keep `TypeInfo` as schema metadata. Compatibility-sensitive decisions belong on `TypeResolver` or explicit operations, not as retained resolver state on metadata objects. - Normalize optional boolean config values at config construction; do not carry `null` through runtime paths when it means `false`. diff --git a/AGENTS.md b/AGENTS.md index 23b4ad31b9..3b436df7e6 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -160,8 +160,9 @@ This is the entry point for AI guidance in Apache Fory. Read this file first, th - JavaScript root entry releases reference and metadata state left by the previous root, including a failed root, before the context is reused. Do not add full cleanup to the root exit path or copy Java backing-array retention policies onto native JavaScript arrays. Read-side occurrence arrays - use native replacement reset. Writer metadata owners first restore their dynamic IDs, then - truncate the active owner list because those owners are appended again in the next root. + use native replacement reset. The writer metadata-owner table has a separate logical size: reset + owner IDs and the logical size without clearing bounded backing, and replace backing only after + more than 8192 owners. - Root failure exceptions must not copy or retain the operation reference table or materialized object graph for diagnostics. Root cleanup owns releasing that graph, and failure reporting must remain bounded independently of graph size. diff --git a/docs/security/deserialization.md b/docs/security/deserialization.md index 8a31d29a83..daff355d27 100644 --- a/docs/security/deserialization.md +++ b/docs/security/deserialization.md @@ -651,9 +651,10 @@ JavaScript root entry releases reference and metadata state left by the previous operation, including a failed operation, before the context is reused. Read-side occurrence arrays use native replacement reset instead of copying the Java backing-array retention policy. Writer metadata owners restore their -dynamic IDs and then truncate the active owner list so the same owners do not -accumulate across roots. Full reference and metadata cleanup does not run on -the root exit path. +dynamic IDs and reset a separate logical owner count, so a bounded backing +array is reused without making prior-root entries visible or accumulating +duplicate work. After more than 8192 owners, reset replaces that backing array. +Full reference and metadata cleanup does not run on the root exit path. A class-resolution cache reachable from untrusted deserialization may publish an entry only from explicit trusted configuration or after the active class diff --git a/docs/specification/xlang_implementation_guide.md b/docs/specification/xlang_implementation_guide.md index 33116bac8d..e0e3027738 100644 --- a/docs/specification/xlang_implementation_guide.md +++ b/docs/specification/xlang_implementation_guide.md @@ -95,9 +95,10 @@ the context is reused. `reset()` should clear operation-local mutable state. When writer metadata objects carry root-local dynamic IDs, reset must restore -those IDs and discard the active owner-list entries. An owner may be appended -again on first use in the next root; retaining prior entries creates duplicate -cleanup work across roots. +those IDs and reset the active owner count. A bounded owner table may retain its +backing storage, but only entries below the current logical count participate in +the next reset; otherwise prior owners create duplicate cleanup work across +roots. Implementations should release an unusual high-water backing table. That operation-local state includes: From 178aa899e33a90d9238eb974dfedaa34e0969638 Mon Sep 17 00:00:00 2001 From: chaokunyang Date: Fri, 28 Aug 2026 04:35:26 +0800 Subject: [PATCH 019/168] fix(csharp): recheck lifecycle after registration callbacks --- csharp/src/Fory/Fory.cs | 35 +++- csharp/src/Fory/ThreadSafeFory.cs | 16 +- csharp/src/Fory/TypeResolver.cs | 23 ++- .../tests/Fory.Tests/RuntimeEdgeCaseTests.cs | 176 ++++++++++++++++++ 4 files changed, 237 insertions(+), 13 deletions(-) diff --git a/csharp/src/Fory/Fory.cs b/csharp/src/Fory/Fory.cs index e6dd5c5d17..57832945c5 100644 --- a/csharp/src/Fory/Fory.cs +++ b/csharp/src/Fory/Fory.cs @@ -73,7 +73,8 @@ public static ForyBuilder Builder() public Fory Register(uint typeId) { EnsureRegistrationOpen(); - _typeResolver.Register(typeof(T), typeId); + TypeInfo typeInfo = PrepareRegistration(typeof(T)); + _typeResolver.Register(typeof(T), typeId, typeInfo); return this; } @@ -89,7 +90,8 @@ public Fory Register(string name) { EnsureRegistrationOpen(); (string namespaceName, string typeName) = TypeResolver.SplitTypeName(name); - _typeResolver.Register(typeof(T), namespaceName, typeName); + TypeInfo typeInfo = PrepareRegistration(typeof(T)); + _typeResolver.Register(typeof(T), namespaceName, typeName, typeInfo); return this; } @@ -105,7 +107,9 @@ public Fory Register(string name) public Fory Register(string typeNamespace, string typeName) { EnsureRegistrationOpen(); - _typeResolver.Register(typeof(T), typeNamespace, typeName); + TypeResolver.ValidateSplitTypeName(typeNamespace, typeName); + TypeInfo typeInfo = PrepareRegistration(typeof(T)); + _typeResolver.Register(typeof(T), typeNamespace, typeName, typeInfo); return this; } @@ -122,7 +126,7 @@ public Fory Register(uint typeId) where TSerializer : Serializer, new() { EnsureRegistrationOpen(); - TypeInfo typeInfo = _typeResolver.RegisterSerializer(); + TypeInfo typeInfo = PrepareRegistration(); _typeResolver.Register(typeof(T), typeId, typeInfo); return this; } @@ -141,7 +145,7 @@ public Fory Register(string name) { EnsureRegistrationOpen(); (string namespaceName, string typeName) = TypeResolver.SplitTypeName(name); - TypeInfo typeInfo = _typeResolver.RegisterSerializer(); + TypeInfo typeInfo = PrepareRegistration(); _typeResolver.Register(typeof(T), namespaceName, typeName, typeInfo); return this; } @@ -161,7 +165,7 @@ public Fory Register(string typeNamespace, string typeName) { EnsureRegistrationOpen(); TypeResolver.ValidateSplitTypeName(typeNamespace, typeName); - TypeInfo typeInfo = _typeResolver.RegisterSerializer(); + TypeInfo typeInfo = PrepareRegistration(); _typeResolver.Register(typeof(T), typeNamespace, typeName, typeInfo); return this; } @@ -355,6 +359,25 @@ private void EnsureRegistrationOpen() } } + private TypeInfo PrepareRegistration(Type type) + { + TypeInfo typeInfo = _typeResolver.PrepareRegistration(type); + // Serializer factories can call application code that starts a root. Recheck before + // the resolver publishes any type or serializer state. + EnsureRegistrationOpen(); + return typeInfo; + } + + private TypeInfo PrepareRegistration() + where TSerializer : Serializer, new() + { + TypeInfo typeInfo = _typeResolver.PrepareRegistration(); + // Serializer construction can call application code that starts a root. Recheck before + // the resolver publishes any type or serializer state. + EnsureRegistrationOpen(); + return typeInfo; + } + [MethodImpl(MethodImplOptions.NoInlining)] private static void ThrowRegistryFrozen() => throw new InvalidOperationException( diff --git a/csharp/src/Fory/ThreadSafeFory.cs b/csharp/src/Fory/ThreadSafeFory.cs index 6900acff09..f3a21a99a0 100644 --- a/csharp/src/Fory/ThreadSafeFory.cs +++ b/csharp/src/Fory/ThreadSafeFory.cs @@ -235,10 +235,24 @@ private void ApplyRegistration(Action registration) } catch { - _registrationFory = _registrations.Count == 0 ? null : RebuildRegistrationFory(); + if (!_disposed && _registryFrozen == 0) + { + Fory? rebuilt = _registrations.Count == 0 ? null : RebuildRegistrationFory(); + // Rebuilding replays serializer constructors, which can close this wrapper. + if (!_disposed && _registryFrozen == 0) + { + _registrationFory = rebuilt; + } + } throw; } + ThrowIfDisposed(); + if (_registryFrozen != 0) + { + ThrowRegistryFrozen(); + } + _registrations.Add(registration); } } diff --git a/csharp/src/Fory/TypeResolver.cs b/csharp/src/Fory/TypeResolver.cs index 9eb0cc9da1..945c31dd4e 100644 --- a/csharp/src/Fory/TypeResolver.cs +++ b/csharp/src/Fory/TypeResolver.cs @@ -485,17 +485,28 @@ private TypeInfo GetOrCreateTypeInfo(Type type, TypeInfo? explicitTypeInfo) return typeInfo; } - internal TypeInfo RegisterSerializer() - where TSerializer : Serializer, new() + internal TypeInfo PrepareRegistration(Type type) { - TypeInfo typeInfo = TypeInfo.Create(typeof(T), new TSerializer()); - RegisterSerializer(typeof(T), typeInfo); + if (_typeInfos.TryGetValue(TypeMapKey.Get(type), out TypeInfo? existing)) + { + return existing; + } + + // Registration factories can run application code. Return the binding without publishing + // it so the facade can recheck its lifecycle boundary first. + TypeInfo typeInfo = CreateBindingCore(type); + if (typeInfo.Type != type) + { + throw new InvalidDataException($"serializer type mismatch for {type}, got {typeInfo.Type}"); + } + return typeInfo; } - internal void RegisterSerializer(Type type, TypeInfo typeInfo) + internal TypeInfo PrepareRegistration() + where TSerializer : Serializer, new() { - GetOrCreateTypeInfo(type, typeInfo); + return TypeInfo.Create(typeof(T), new TSerializer()); } internal void Register(Type type, uint id, TypeInfo? explicitTypeInfo = null) diff --git a/csharp/tests/Fory.Tests/RuntimeEdgeCaseTests.cs b/csharp/tests/Fory.Tests/RuntimeEdgeCaseTests.cs index e2ba161195..abe937f0cf 100644 --- a/csharp/tests/Fory.Tests/RuntimeEdgeCaseTests.cs +++ b/csharp/tests/Fory.Tests/RuntimeEdgeCaseTests.cs @@ -87,10 +87,12 @@ public sealed class FrozenPayload public sealed class FrozenPayloadSerializer : Serializer { public static int Constructions; + public static Action? ConstructionAction; public FrozenPayloadSerializer() { Interlocked.Increment(ref Constructions); + ConstructionAction?.Invoke(); } public override FrozenPayload DefaultValue => null!; @@ -107,6 +109,35 @@ public override FrozenPayload ReadData(ReadContext context) } } +public enum GeneratedFrozenValue +{ + Zero, + One, +} + +public sealed class GeneratedFrozenSerializer : Serializer +{ + public static Action? ConstructionAction; + + public GeneratedFrozenSerializer() + { + ConstructionAction?.Invoke(); + } + + public override GeneratedFrozenValue DefaultValue => GeneratedFrozenValue.Zero; + + public override void WriteData(WriteContext context, in GeneratedFrozenValue value, bool hasGenerics) + { + _ = hasGenerics; + context.Writer.WriteVarInt32((int)value); + } + + public override GeneratedFrozenValue ReadData(ReadContext context) + { + return (GeneratedFrozenValue)context.Reader.ReadVarInt32(); + } +} + [ForyStruct] public sealed class FailingWritePayload { @@ -838,6 +869,142 @@ public void FrozenRegistryRejectsBeforeMutation() Assert.Equal(0, FrozenPayloadSerializer.Constructions); } + [Fact] + public void ReentrantRegistrationFreezes() + { + ForyRuntime fory = ForyRuntime.Builder().Build(); + FrozenPayloadSerializer.ConstructionAction = () => _ = fory.Serialize(1); + + try + { + Assert.Throws( + () => fory.Register(721)); + Assert.Throws(() => fory.Register(722)); + } + finally + { + FrozenPayloadSerializer.ConstructionAction = null; + } + } + + [Fact] + public void GeneratedReentryFreezes() + { + TypeResolver.RegisterGenerated(); + ForyRuntime fory = ForyRuntime.Builder().Build(); + GeneratedFrozenSerializer.ConstructionAction = () => _ = fory.Serialize(1); + + try + { + Assert.Throws(() => fory.Register(725)); + Assert.Throws(() => fory.Register(726)); + } + finally + { + GeneratedFrozenSerializer.ConstructionAction = null; + } + + Assert.Throws( + () => fory.Serialize(GeneratedFrozenValue.One)); + } + + [Fact] + public void ThreadSafeReentryFreezes() + { + using ThreadSafeFory fory = ForyRuntime.Builder().BuildThreadSafe(); + FrozenPayloadSerializer.ConstructionAction = () => _ = fory.Serialize(1); + + try + { + Assert.Throws( + () => fory.Register(723)); + Assert.Throws(() => fory.Register(724)); + } + finally + { + FrozenPayloadSerializer.ConstructionAction = null; + } + + Assert.Throws(() => + Task.Run(() => fory.Serialize(new FrozenPayload { Value = 1 })) + .GetAwaiter() + .GetResult()); + } + + [Fact] + public void ThreadSafeDisposeReentryStops() + { + ThreadSafeFory fory = ForyRuntime.Builder().BuildThreadSafe(); + FrozenPayloadSerializer.ConstructionAction = fory.Dispose; + + try + { + Assert.Throws( + () => fory.Register(727)); + } + finally + { + FrozenPayloadSerializer.ConstructionAction = null; + fory.Dispose(); + } + } + + [Fact] + public void ThreadSafeFailedReentryClears() + { + using ThreadSafeFory fory = ForyRuntime.Builder().BuildThreadSafe(); + fory.Register(728); + FrozenPayloadSerializer.ConstructionAction = + () => _ = fory.Deserialize(Array.Empty()); + + try + { + Exception error = Assert.ThrowsAny( + () => fory.Register(729)); + Assert.IsAssignableFrom(error.InnerException ?? error); + Assert.Null(RegistrationForyFor(fory)); + Assert.Throws(() => fory.Register(730)); + } + finally + { + FrozenPayloadSerializer.ConstructionAction = null; + } + } + + [Fact] + public void ThreadSafeRebuildFreezeClears() + { + using ThreadSafeFory fory = ForyRuntime.Builder().BuildThreadSafe(); + fory.Register(731); + bool rootStarted = false; + FrozenPayloadSerializer.ConstructionAction = () => + { + if (rootStarted) + { + return; + } + + rootStarted = true; + _ = fory.Serialize(1); + }; + GeneratedFrozenSerializer.ConstructionAction = + () => throw new InvalidOperationException("registration failure"); + + try + { + Exception error = Assert.ThrowsAny( + () => fory.Register(732)); + Assert.IsType(error.InnerException ?? error); + Assert.Null(RegistrationForyFor(fory)); + Assert.Throws(() => fory.Register(733)); + } + finally + { + FrozenPayloadSerializer.ConstructionAction = null; + GeneratedFrozenSerializer.ConstructionAction = null; + } + } + [Fact] public void FailedRootFreezesRegistry() { @@ -1035,6 +1202,15 @@ private static WriteContext WriteContextFor(ForyRuntime fory) return Assert.IsType(field.GetValue(fory)); } + private static ForyRuntime? RegistrationForyFor(ThreadSafeFory fory) + { + System.Reflection.FieldInfo? field = typeof(ThreadSafeFory).GetField( + "_registrationFory", + System.Reflection.BindingFlags.Instance | System.Reflection.BindingFlags.NonPublic); + Assert.NotNull(field); + return field.GetValue(fory) as ForyRuntime; + } + [Fact] public void DeserializeFromReaderReadsFrames() { From 1cb3d4b3d9c8a62f20e2668c512c43afb29868ab Mon Sep 17 00:00:00 2001 From: chaokunyang Date: Fri, 28 Aug 2026 04:38:35 +0800 Subject: [PATCH 020/168] fix(python): recheck registry after serializer construction --- python/pyfory/registry.py | 3 +++ python/pyfory/tests/test_serializer.py | 28 ++++++++++++++++++++++++++ 2 files changed, 31 insertions(+) diff --git a/python/pyfory/registry.py b/python/pyfory/registry.py index a148d6a4c8..49783c7b24 100644 --- a/python/pyfory/registry.py +++ b/python/pyfory/registry.py @@ -821,6 +821,9 @@ def __register_type( if should_create_serializer: serializer = self._create_serializer(cls) + # Serializer construction can run application code and start a root. Recheck before + # publishing any type, serializer, name, or id state. + self._check_registry_mutable() if serializer is not None and type_id in _NO_REF_NUMERIC_TYPE_IDS: serializer.need_to_write_ref = False diff --git a/python/pyfory/tests/test_serializer.py b/python/pyfory/tests/test_serializer.py index c9639f0cc3..a79676237d 100644 --- a/python/pyfory/tests/test_serializer.py +++ b/python/pyfory/tests/test_serializer.py @@ -1035,6 +1035,34 @@ def test_registry_freezes_at_root(root): assert fory.type_resolver.get_type_info(FrozenRegistration).type_id == TypeId.STRUCT +@pytest.mark.parametrize("registration", ["type", "union"]) +def test_reentrant_freeze(registration): + fory = Fory(xlang=True, compatible=False) + before = registration_state(fory) + constructions = 0 + target = FrozenExt if registration == "type" else FrozenUnion + + def serializer_factory(type_resolver, cls): + nonlocal constructions + constructions += 1 + fory.serialize(None) + if registration == "type": + return FrozenExtSerializer(type_resolver, cls) + return UnionSerializer(type_resolver, cls, {0: str}) + + with pytest.raises(RuntimeError, match="first root operation"): + if registration == "type": + fory.register_type(target, type_id=725, serializer=serializer_factory) + else: + fory.register_union(target, type_id=725, serializer=serializer_factory) + + assert constructions == 1 + assert fory.type_resolver.get_type_info(target, create=False) is None + assert registration_state(fory) == before + with pytest.raises(RuntimeError, match="first root operation"): + fory.register_type(RejectedRegistration, type_id=726) + + def test_frozen_serializer_lookup(monkeypatch): fory = Fory(xlang=False, strict=False, compatible=False) fory.serialize(None) From a44aeeb473019592b59990b1f7b1dcd78e75e610 Mon Sep 17 00:00:00 2001 From: chaokunyang Date: Fri, 28 Aug 2026 04:40:03 +0800 Subject: [PATCH 021/168] docs: define callback-safe registry publication --- .agents/languages/csharp.md | 8 ++++++-- .agents/languages/python.md | 2 ++ AGENTS.md | 3 +++ docs/object-serialization/csharp/type-registration.md | 2 ++ docs/object-serialization/python/type-registration.md | 2 ++ docs/security/deserialization.md | 8 ++++++++ docs/specification/xlang_implementation_guide.md | 6 ++++++ 7 files changed, 29 insertions(+), 2 deletions(-) diff --git a/.agents/languages/csharp.md b/.agents/languages/csharp.md index 8fdbac5105..c9bfe38fad 100644 --- a/.agents/languages/csharp.md +++ b/.agents/languages/csharp.md @@ -12,8 +12,12 @@ Load this file when changing `csharp/` or C# xlang behavior. - A direct C# `Fory` owns permanent registration freeze at first root entry, including a failed root. `ThreadSafeFory` linearizes root entry and registration with its registration lock and frozen state, validates registration on its staging `Fory`, and appends only successful actions - to the replay log. New per-thread runtimes replay that log; do not mutate existing runtimes or - introduce another freeze owner. + to the replay log. Serializer construction and generated factories may reenter a root, so direct + registration must recheck the facade after resolving the serializer and before resolver mutation. + `ThreadSafeFory` must recheck disposal and freeze after staging registration and before replay-log + publication, and a staging failure must not rebuild after either lifecycle boundary has closed. + New per-thread runtimes replay that log; do not mutate existing runtimes or introduce another + freeze owner. - Generated C# gRPC service companions are compiler-owned files that depend on application-provided gRPC packages, not `csharp/src/Fory`. Keep gRPC package references out of the Fory runtime package. - C# generated schema modules are source-file owners. Service companions must use that module's `ThreadSafeFory` and must not introduce namespace-owned aliases or duplicate serializer registration paths. - C# external-type serialization is target-keyed. A local diff --git a/.agents/languages/python.md b/.agents/languages/python.md index 7f55ae65ef..44b35b5b08 100644 --- a/.agents/languages/python.md +++ b/.agents/languages/python.md @@ -14,6 +14,8 @@ Load this file when changing `python/`, Cython serialization, or Python xlang be - Python `TypeResolver` owns registry freeze and finalization state. Its Cython companion may cache completion of the one Python-owner dispatch needed to populate native resolver tables, but the `Fory` facade must not mirror that state. Cython roots call the resolver owner directly. + Serializer construction may reenter a root, so the resolver rechecks its frozen state after + construction and before publishing type, serializer, name, or ID state. - In non-strict native mode, public unqualified `register_type` for a built-in native carrier uses the same reserved type identity as pre-root discovery. Ordinary application classes and dataclasses retain their struct registration identity. Configure both through public registration; diff --git a/AGENTS.md b/AGENTS.md index 3b436df7e6..c9a8a4db9b 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -186,6 +186,9 @@ This is the entry point for AI guidance in Apache Fory. Read this file first, th serializer rebinding, metadata rebuilding, or other late-registration machinery. Registration-order finalization before the first root operation remains registration-owned and must not create a runtime invalidation path. + If serializer construction, factory execution, or another application callback + can reenter a root, recheck the authoritative per-instance freeze owner after + that callback and before the first registry mutation or replay-log publication. - Python `TypeResolver` is the sole registry freeze and finalization owner. Its Cython resolver companion may cache completion of the Python-owner dispatch needed to populate native tables, but the `Fory` facade must not mirror that state; Cython roots call the resolver owner directly. diff --git a/docs/object-serialization/csharp/type-registration.md b/docs/object-serialization/csharp/type-registration.md index 37118c3fab..64b824b693 100644 --- a/docs/object-serialization/csharp/type-registration.md +++ b/docs/object-serialization/csharp/type-registration.md @@ -72,6 +72,8 @@ fory.Register("com.example.MyType"); A `Fory` instance accepts registration only before its first root serialization or deserialization attempt. Starting that operation permanently freezes the instance's registry, even when the operation fails. Every later registration throws `InvalidOperationException`. +If a custom serializer constructor starts a root operation, that root freezes the registry and the +in-progress registration fails without publishing the serializer or type mapping. To configure additional types, build a new `Fory` instance, complete its registrations, and then use that new instance for serialization or deserialization. diff --git a/docs/object-serialization/python/type-registration.md b/docs/object-serialization/python/type-registration.md index 48379553ab..784880b7ed 100644 --- a/docs/object-serialization/python/type-registration.md +++ b/docs/object-serialization/python/type-registration.md @@ -85,6 +85,8 @@ the deserialization policy for registered Python-native carriers; it does not permit late discovery or registration. Register callable, class, method, and reduction carriers together with the application types that can appear in the graph before the first root operation. +If a custom serializer factory starts a root operation, that root freezes the registry and the +in-progress registration fails without publishing the serializer or type mapping. Compatible metadata has one data-only exception: when a remote Struct has no local registration, deserialization returns the fixed framework diff --git a/docs/security/deserialization.md b/docs/security/deserialization.md index daff355d27..d07576f126 100644 --- a/docs/security/deserialization.md +++ b/docs/security/deserialization.md @@ -615,6 +615,14 @@ that case, classify the behavior by concrete impact: - Pure strictness about whether a skipped value used one specific encoding shape is not a security issue. +## Registry Lifecycle + +Registration code that invokes serializer constructors, factories, or application callbacks must +recheck the authoritative per-instance registry freeze after the callback and before publishing +resolver or replay-log state. A callback that starts the first root permanently closes the +in-progress registration; implementations must not publish and then repair or invalidate late +state. + ## Metadata And Type Resolution Metadata parsing is security-sensitive when it affects retained read-side state, diff --git a/docs/specification/xlang_implementation_guide.md b/docs/specification/xlang_implementation_guide.md index e0e3027738..d208d01bc7 100644 --- a/docs/specification/xlang_implementation_guide.md +++ b/docs/specification/xlang_implementation_guide.md @@ -82,6 +82,12 @@ not the place where nested serializers do their work. - owning registration through `TypeResolver` - resetting operation-local context state at the top-level root boundary +Registration preparation may invoke serializer constructors, generated factories, or application +callbacks before any registry mutation. If such a callback starts a root operation, registration +must recheck the authoritative per-instance freeze owner when the callback returns and reject the +in-progress registration before publishing type, serializer, name, ID, or replay state. Do not +publish late state and then repair it through invalidation or rollback. + Nested serializers must not call back into root `serialize(...)` or `deserialize(...)` entry points. From b8fb74f7f2416c5f8f67a8aca493c41c3244a017 Mon Sep 17 00:00:00 2001 From: chaokunyang Date: Fri, 28 Aug 2026 04:55:35 +0800 Subject: [PATCH 022/168] fix(csharp): keep one failed-root reset owner --- csharp/src/Fory/Fory.cs | 16 +++------------- csharp/tests/Fory.Tests/RuntimeEdgeCaseTests.cs | 14 ++------------ 2 files changed, 5 insertions(+), 25 deletions(-) diff --git a/csharp/src/Fory/Fory.cs b/csharp/src/Fory/Fory.cs index 57832945c5..cf48ecfb39 100644 --- a/csharp/src/Fory/Fory.cs +++ b/csharp/src/Fory/Fory.cs @@ -184,19 +184,9 @@ public byte[] Serialize(in T value) Serializer serializer = _typeResolver.GetSerializer(); WriteHead(writer); _writeContext.ResetFor(writer); - try - { - RefMode refMode = Config.TrackRef ? RefMode.Tracking : RefMode.NullOnly; - serializer.Write(_writeContext, value, refMode, true, false); - _writeContext.RefWriter.Reset(); - } - catch - { - // A custom serializer can fail after reference and metadata publication. The root - // facade must release that operation-local state before the runtime is reused. - _writeContext.Reset(); - throw; - } + RefMode refMode = Config.TrackRef ? RefMode.Tracking : RefMode.NullOnly; + serializer.Write(_writeContext, value, refMode, true, false); + _writeContext.RefWriter.Reset(); return writer.ToArray(); } diff --git a/csharp/tests/Fory.Tests/RuntimeEdgeCaseTests.cs b/csharp/tests/Fory.Tests/RuntimeEdgeCaseTests.cs index abe937f0cf..0b74165fe2 100644 --- a/csharp/tests/Fory.Tests/RuntimeEdgeCaseTests.cs +++ b/csharp/tests/Fory.Tests/RuntimeEdgeCaseTests.cs @@ -1015,7 +1015,7 @@ public void FailedRootFreezesRegistry() } [Fact] - public void FailedWriteClearsRootState() + public void FailedWriteRestoresNextRoot() { ForyRuntime fory = ForyRuntime.Builder().TrackRef(true).Build(); fory.Register(718); @@ -1024,8 +1024,7 @@ public void FailedWriteClearsRootState() Assert.Throws(() => fory.Serialize(value)); Assert.Throws(() => fory.Register(719)); - ByteWriter probe = new(); - Assert.False(WriteContextFor(fory).RefWriter.TryWriteRef(probe, value)); + Assert.Throws(() => fory.Serialize(value)); Assert.Equal(7, fory.Deserialize(fory.Serialize(7))); } @@ -1193,15 +1192,6 @@ private static ReadContext ReadContextFor(ForyRuntime fory) return Assert.IsType(field.GetValue(fory)); } - private static WriteContext WriteContextFor(ForyRuntime fory) - { - System.Reflection.FieldInfo? field = typeof(ForyRuntime).GetField( - "_writeContext", - System.Reflection.BindingFlags.Instance | System.Reflection.BindingFlags.NonPublic); - Assert.NotNull(field); - return Assert.IsType(field.GetValue(fory)); - } - private static ForyRuntime? RegistrationForyFor(ThreadSafeFory fory) { System.Reflection.FieldInfo? field = typeof(ThreadSafeFory).GetField( From 30500d7f0e77d5141bdbb020c5bdc9a44d1be2fa Mon Sep 17 00:00:00 2001 From: chaokunyang Date: Fri, 28 Aug 2026 05:00:45 +0800 Subject: [PATCH 023/168] fix(go): validate factories before registry publish --- .agents/languages/go.md | 5 ++- go/fory/fory.go | 4 +- go/fory/registry_freeze_lifecycle_test.go | 14 ++++++ go/fory/threadsafe/fory.go | 43 ++++++++++++++++--- .../registry_freeze_lifecycle_test.go | 25 +++++++++++ 5 files changed, 82 insertions(+), 9 deletions(-) diff --git a/.agents/languages/go.md b/.agents/languages/go.md index 30ef97d383..7d2b2cc72f 100644 --- a/.agents/languages/go.md +++ b/.agents/languages/go.md @@ -11,7 +11,10 @@ Load this file when changing `go/fory/` or Go xlang behavior. failed root. Exported resolver registration entries recheck that facade-owned state before mutation. `threadsafe.Fory` owns the cross-pool boundary with one frozen state, one prepared validation instance, and one log of successful named-struct registrations. Failed registrations - are not logged; pool misses after freeze replay the immutable successful log. + are not logged; pool misses after freeze replay the immutable successful log. Its custom factory + runs without the registration mutex because application code may reenter a root; after the + factory returns, registration rechecks the frozen state before publishing prepared or replay + state. - Go `ReadContext` intentionally defers codec errors to existing `HasError` or `CheckError` boundaries. After an error, work may continue only while it remains panic- and bounds-safe and cannot cause disproportionate work or allocation, publish state that survives root cleanup, or diff --git a/go/fory/fory.go b/go/fory/fory.go index a2be046313..f1bb65ca3a 100644 --- a/go/fory/fory.go +++ b/go/fory/fory.go @@ -839,11 +839,13 @@ func (f *Fory) DeserializeWithCallbackBuffers(buffer *ByteBuffer, v any, buffers origBuffer := f.readCtx.buffer f.readCtx.buffer = buffer defer func() { + // Restore the owned buffer before reset so a cleanup panic cannot retain + // the caller-owned buffer. + f.readCtx.buffer = origBuffer f.readCtx.Reset() if f.metaContext != nil { f.metaContext.Reset() } - f.readCtx.buffer = origBuffer f.readCtx.outOfBandBuffers = nil }() // Set up out-of-band buffers if provided diff --git a/go/fory/registry_freeze_lifecycle_test.go b/go/fory/registry_freeze_lifecycle_test.go index 5253251859..980706e5f4 100644 --- a/go/fory/registry_freeze_lifecycle_test.go +++ b/go/fory/registry_freeze_lifecycle_test.go @@ -189,6 +189,20 @@ func TestBorrowedBufferPanicRestore(t *testing.T) { ErrRegistryFrozen) } +func TestCallbackBufferRestoreOrder(t *testing.T) { + f := New(WithXlang(false), WithCompatible(false)) + owned := f.readCtx.buffer + borrowed := NewByteBuffer(nil) + // Force root cleanup to panic so restoring the owned buffer after Reset + // cannot accidentally satisfy this test. + f.readCtx.refReader = nil + + require.Panics(t, func() { + _ = f.DeserializeWithCallbackBuffers(borrowed, nil, nil) + }) + require.Same(t, owned, f.readCtx.buffer) +} + func TestStreamBufferPanicRestore(t *testing.T) { f := New(WithXlang(false), WithCompatible(false)) owned := f.readCtx.buffer diff --git a/go/fory/threadsafe/fory.go b/go/fory/threadsafe/fory.go index 1178824554..b7f291f3d8 100644 --- a/go/fory/threadsafe/fory.go +++ b/go/fory/threadsafe/fory.go @@ -59,19 +59,30 @@ func NewWithFactory(factory func() *fory.Fory) *Fory { return &Fory{factory: factory} } -func (f *Fory) newInner() (*fory.Fory, error) { +func (f *Fory) createInner() *fory.Fory { inner := f.factory() if inner == nil { panic("threadsafe.NewWithFactory factory returned nil") } - // Before the first root, callers hold registrationMu. After registryFrozen - // is published, registrations are immutable, so pool misses can replay them - // without extending the root hot-path lock. + return inner +} + +func (f *Fory) applyRegistrations(inner *fory.Fory) error { for _, registration := range f.registrations { if err := inner.RegisterStructByName(registration.typ, registration.name); err != nil { - return nil, fmt.Errorf("apply registration %q to new Fory instance: %w", registration.name, err) + return fmt.Errorf("apply registration %q to new Fory instance: %w", registration.name, err) } } + return nil +} + +func (f *Fory) newInner() (*fory.Fory, error) { + inner := f.createInner() + // Registry freeze is published before pool misses reach this path, so the + // registration log is immutable and needs no root hot-path lock. + if err := f.applyRegistrations(inner); err != nil { + return nil, err + } return inner, nil } @@ -135,18 +146,36 @@ func (f *Fory) Deserialize(data []byte, v any) error { // RegisterStructByName registers a struct type by name before the first root operation. func (f *Fory) RegisterStructByName(type_ any, name string) error { + f.registrationMu.Lock() + if f.registryFrozen.Load() { + f.registrationMu.Unlock() + return fory.ErrRegistryFrozen + } + if f.prepared != nil { + defer f.registrationMu.Unlock() + return f.registerPrepared(type_, name) + } + f.registrationMu.Unlock() + + // The factory is application code and may reenter a root. Never hold the + // registration mutex across it, and recheck freeze before publishing its result. + inner := f.createInner() + f.registrationMu.Lock() defer f.registrationMu.Unlock() if f.registryFrozen.Load() { return fory.ErrRegistryFrozen } if f.prepared == nil { - inner, err := f.newInner() - if err != nil { + if err := f.applyRegistrations(inner); err != nil { return err } f.prepared = inner } + return f.registerPrepared(type_, name) +} + +func (f *Fory) registerPrepared(type_ any, name string) error { registration := structRegistration{name: name} if err := f.prepared.RegisterStructByName(type_, name); err != nil { // A failed registration is not part of the facade registry. Rebuild from diff --git a/go/fory/threadsafe/registry_freeze_lifecycle_test.go b/go/fory/threadsafe/registry_freeze_lifecycle_test.go index 7c0e619baf..19b3369b4e 100644 --- a/go/fory/threadsafe/registry_freeze_lifecycle_test.go +++ b/go/fory/threadsafe/registry_freeze_lifecycle_test.go @@ -34,6 +34,31 @@ type registryFreezeRace struct { Value int32 } +func TestFactoryRootReentry(t *testing.T) { + var f *Fory + var factoryEntered atomic.Bool + var factoryUnlocked bool + var rootErr error + f = NewWithFactory(func() *fory.Fory { + if factoryEntered.CompareAndSwap(false, true) { + factoryUnlocked = f.registrationMu.TryLock() + if factoryUnlocked { + f.registrationMu.Unlock() + _, rootErr = f.Serialize(int32(1)) + } + } + return fory.New(fory.WithXlang(false), fory.WithCompatible(false)) + }) + + err := f.RegisterStructByName(registryFreezePooled{}, "test.FactoryRootReentry") + require.True(t, factoryUnlocked) + require.NoError(t, rootErr) + require.ErrorIs(t, err, fory.ErrRegistryFrozen) + require.True(t, f.registryFrozen.Load()) + require.Empty(t, f.registrations) + require.Nil(t, f.prepared) +} + func TestRegistryFreezePropagation(t *testing.T) { var factoryCalls atomic.Int32 f := NewWithFactory(func() *fory.Fory { From 257f24f6c60ad69713a133a05abadf72d78a0e3d Mon Sep 17 00:00:00 2001 From: chaokunyang Date: Fri, 28 Aug 2026 05:03:26 +0800 Subject: [PATCH 024/168] refactor(go): remove duplicate root cleanup --- go/fory/fory.go | 6 ------ go/fory/threadsafe/registry_freeze_lifecycle_test.go | 2 +- 2 files changed, 1 insertion(+), 7 deletions(-) diff --git a/go/fory/fory.go b/go/fory/fory.go index f1bb65ca3a..5797d9712d 100644 --- a/go/fory/fory.go +++ b/go/fory/fory.go @@ -799,11 +799,6 @@ func (f *Fory) SerializeWithCallback(buffer *ByteBuffer, v any, callback func(Bu if f.metaContext != nil { f.metaContext.Reset() } - // Set up buffer callback for out-of-band serialization - if callback != nil { - f.writeCtx.bufferCallback = nil - f.writeCtx.outOfBand = false - } }() if !validateRootDecimal(f.writeCtx.Err(), v) { return f.writeCtx.TakeError() @@ -846,7 +841,6 @@ func (f *Fory) DeserializeWithCallbackBuffers(buffer *ByteBuffer, v any, buffers if f.metaContext != nil { f.metaContext.Reset() } - f.readCtx.outOfBandBuffers = nil }() // Set up out-of-band buffers if provided if buffers != nil { diff --git a/go/fory/threadsafe/registry_freeze_lifecycle_test.go b/go/fory/threadsafe/registry_freeze_lifecycle_test.go index 19b3369b4e..c3c9c45efb 100644 --- a/go/fory/threadsafe/registry_freeze_lifecycle_test.go +++ b/go/fory/threadsafe/registry_freeze_lifecycle_test.go @@ -120,7 +120,7 @@ func TestRegistryFreezeReplayFailure(t *testing.T) { } func TestRegistryFreezeOnFactoryPanic(t *testing.T) { - f := NewWithFactory(func() *fory.Fory { return nil }) + f := NewWithFactory(func() *fory.Fory { panic("factory failure") }) require.Panics(t, func() { _, _ = f.Serialize(int32(1)) }) From 7b391d314e091db5706671334304de489a9460f6 Mon Sep 17 00:00:00 2001 From: chaokunyang Date: Fri, 28 Aug 2026 05:04:00 +0800 Subject: [PATCH 025/168] test(csharp): avoid pinning root error type --- .../tests/Fory.Tests/RuntimeEdgeCaseTests.cs | 19 +++++++++---------- 1 file changed, 9 insertions(+), 10 deletions(-) diff --git a/csharp/tests/Fory.Tests/RuntimeEdgeCaseTests.cs b/csharp/tests/Fory.Tests/RuntimeEdgeCaseTests.cs index 0b74165fe2..ab2528431a 100644 --- a/csharp/tests/Fory.Tests/RuntimeEdgeCaseTests.cs +++ b/csharp/tests/Fory.Tests/RuntimeEdgeCaseTests.cs @@ -959,9 +959,8 @@ public void ThreadSafeFailedReentryClears() try { - Exception error = Assert.ThrowsAny( + Assert.ThrowsAny( () => fory.Register(729)); - Assert.IsAssignableFrom(error.InnerException ?? error); Assert.Null(RegistrationForyFor(fory)); Assert.Throws(() => fory.Register(730)); } @@ -1010,7 +1009,7 @@ public void FailedRootFreezesRegistry() { ForyRuntime fory = ForyRuntime.Builder().Build(); - Assert.ThrowsAny(() => fory.Deserialize(Array.Empty())); + Assert.ThrowsAny(() => fory.Deserialize(Array.Empty())); Assert.Throws(() => fory.Register(713)); } @@ -1033,7 +1032,7 @@ public void FailedReaderRootFreezesRegistry() { ForyRuntime fory = ForyRuntime.Builder().Build(); - Assert.ThrowsAny( + Assert.ThrowsAny( () => fory.DeserializeFromReader(new ByteReader(Array.Empty()))); Assert.Throws(() => fory.Register(714)); } @@ -1043,7 +1042,7 @@ public void ThreadSafeFailedRootFreezesRegistry() { using ThreadSafeFory fory = ForyRuntime.Builder().BuildThreadSafe(); - Assert.ThrowsAny(() => fory.Deserialize(Array.Empty())); + Assert.ThrowsAny(() => fory.Deserialize(Array.Empty())); Assert.Throws(() => fory.Register(715)); FrozenPayloadSerializer.Constructions = 0; Assert.Throws( @@ -1113,8 +1112,8 @@ public void TrailingBytesResetReadState(bool useSpan) byte[] invalidPayload = [.. payload, 0x7F]; _ = useSpan - ? Assert.ThrowsAny(() => DeserializeSpan(reader, invalidPayload)) - : Assert.ThrowsAny(() => reader.Deserialize(invalidPayload)); + ? Assert.ThrowsAny(() => DeserializeSpan(reader, invalidPayload)) + : Assert.ThrowsAny(() => reader.Deserialize(invalidPayload)); ReadContext context = ReadContextFor(reader); Assert.Null(context.GetTypeMetaRef(0)); Assert.Null(context.GetReadMetaString(0)); @@ -1131,7 +1130,7 @@ public void RootHeaderFailureClearsMetaCache() TypeMeta first = ReadAndStoreTypeMeta(context, RemoteStructTypeMeta(901, "first")); ulong firstHash = EncodedTypeMetaHash(first); - Assert.ThrowsAny(() => fory.Deserialize([0])); + Assert.ThrowsAny(() => fory.Deserialize([0])); Assert.False(context.TryGetTypeMetaByHash(firstHash, out _)); TypeMeta second = ReadAndStoreTypeMeta(context, RemoteStructTypeMeta(901, "second")); @@ -1154,11 +1153,11 @@ public void TrailingFailureClearsTypeMetaCache(bool useSpan) if (useSpan) { - Assert.ThrowsAny(() => DeserializeIntSpan(fory, invalidPayload)); + Assert.ThrowsAny(() => DeserializeIntSpan(fory, invalidPayload)); } else { - Assert.ThrowsAny(() => fory.Deserialize(invalidPayload)); + Assert.ThrowsAny(() => fory.Deserialize(invalidPayload)); } Assert.False(context.TryGetTypeMetaByHash(firstHash, out _)); From 30e3e0c3279128dfa3051c16484e97ea629b46ce Mon Sep 17 00:00:00 2001 From: chaokunyang Date: Fri, 28 Aug 2026 05:04:39 +0800 Subject: [PATCH 026/168] refactor(cpp): keep registration checks in one owner --- cpp/fory/serialization/serialization_test.cc | 61 ++++++++++++++++++++ cpp/fory/serialization/type_resolver.cc | 12 +++- cpp/fory/serialization/type_resolver.h | 12 ---- 3 files changed, 70 insertions(+), 15 deletions(-) diff --git a/cpp/fory/serialization/serialization_test.cc b/cpp/fory/serialization/serialization_test.cc index 7c420f9dd0..17a705d99e 100644 --- a/cpp/fory/serialization/serialization_test.cc +++ b/cpp/fory/serialization/serialization_test.cc @@ -29,6 +29,7 @@ #include #include #include +#include #include #include #include @@ -76,6 +77,16 @@ struct NestedStruct { FORY_STRUCT(NestedStruct, point, label); }; +struct UnregisteredNested { + int32_t value; + FORY_STRUCT(UnregisteredNested, value); +}; + +struct MissingNestedHolder { + UnregisteredNested nested; + FORY_STRUCT(MissingNestedHolder, nested); +}; + enum class Color { RED, GREEN, BLUE }; enum class SignedScopedStatus : int32_t { NEG = -3, ZERO = 0, LARGE = 42 }; FORY_ENUM(SignedScopedStatus, NEG, ZERO, LARGE); @@ -985,6 +996,15 @@ TEST(SerializationTest, LastElementErrorSafepoints) { declared_ctx, 2)); EXPECT_TRUE(declared_ctx.has_error()); + std::vector forward_bytes{2}; + Buffer forward_buffer(forward_bytes); + ReadContext forward_ctx(config, std::make_unique()); + forward_ctx.attach(forward_buffer); + std::forward_list forward_values; + EXPECT_FALSE(read_declared_same_type_collection(forward_values, + forward_ctx, 2)); + EXPECT_TRUE(forward_ctx.has_error()); + std::vector type_info_bytes{2}; Buffer type_info_buffer(type_info_bytes); ReadContext type_info_ctx(config, std::make_unique()); @@ -995,6 +1015,16 @@ TEST(SerializationTest, LastElementErrorSafepoints) { EXPECT_FALSE(read_same_type_info_collection( type_info_values, type_info_ctx, 2, type_info)); EXPECT_TRUE(type_info_ctx.has_error()); + + std::vector measured_bytes{2}; + Buffer measured_buffer(measured_bytes); + ReadContext measured_ctx(config, std::make_unique()); + measured_ctx.attach(measured_buffer); + type_info.harness.read_data_always_advances = false; + std::vector measured_values; + EXPECT_FALSE(read_same_type_info_collection( + measured_values, measured_ctx, 2, type_info)); + EXPECT_TRUE(measured_ctx.has_error()); } // ============================================================================ @@ -2552,6 +2582,37 @@ TEST(SerializationTest, SourceResolverFinalizes) { expect_finalized_source(*source_resolver); } +TEST(SerializationTest, FinalizationFailureIsAtomic) { + auto source_resolver = std::make_shared(); + auto fory = Fory::builder() + .xlang(true) + .compatible(false) + .track_ref(false) + .type_resolver(source_resolver) + .build(); + ASSERT_TRUE(fory.register_struct<::SimpleStruct>(1).ok()); + ASSERT_TRUE(fory.register_struct<::MissingNestedHolder>(2).ok()); + + auto simple_info = source_resolver->get_type_info<::SimpleStruct>(); + auto holder_info = source_resolver->get_type_info<::MissingNestedHolder>(); + ASSERT_TRUE(simple_info.ok()); + ASSERT_TRUE(holder_info.ok()); + ASSERT_EQ(simple_info.value()->type_meta, nullptr); + ASSERT_TRUE(simple_info.value()->type_def.empty()); + ASSERT_EQ(holder_info.value()->type_meta, nullptr); + ASSERT_TRUE(holder_info.value()->type_def.empty()); + + auto final_resolver = source_resolver->build_final_type_resolver(); + ASSERT_FALSE(final_resolver.ok()); + + EXPECT_EQ(simple_info.value()->type_meta, nullptr); + EXPECT_TRUE(simple_info.value()->type_def.empty()); + EXPECT_EQ(holder_info.value()->type_meta, nullptr); + EXPECT_TRUE(holder_info.value()->type_def.empty()); + EXPECT_FALSE(fory.register_struct<::UnregisteredNested>(3).ok()); + EXPECT_FALSE(source_resolver->get_type_info<::UnregisteredNested>().ok()); +} + TEST(SerializationTest, DirectFailedRootFreezes) { auto source_resolver = std::make_shared(); auto fory = Fory::builder() diff --git a/cpp/fory/serialization/type_resolver.cc b/cpp/fory/serialization/type_resolver.cc index f61d4fbd6f..f97578489c 100644 --- a/cpp/fory/serialization/type_resolver.cc +++ b/cpp/fory/serialization/type_resolver.cc @@ -1762,9 +1762,15 @@ TypeResolver::get_type_info(const std::type_index &type_index) const { return entry->second; } -FORY_NOINLINE Result TypeResolver::registration_frozen_error() { - return Unexpected(Error::invalid( - "TypeResolver registry is frozen, cannot register more types")); +Result TypeResolver::check_registration() { + if (FORY_PREDICT_FALSE(registry_frozen_)) { + return Unexpected(Error::invalid( + "TypeResolver registry is frozen, cannot register more types")); + } + FORY_CHECK(std::this_thread::get_id() == registration_thread_id_) + << "TypeResolver registration methods must be called from the same " + "thread that created the TypeResolver"; + return Result(); } Result, Error> diff --git a/cpp/fory/serialization/type_resolver.h b/cpp/fory/serialization/type_resolver.h index 00be162cb2..6d90c7165e 100644 --- a/cpp/fory/serialization/type_resolver.h +++ b/cpp/fory/serialization/type_resolver.h @@ -1523,8 +1523,6 @@ class TypeResolver { /// Validate registration state while registration_mutex_ is held. Result check_registration(); - static FORY_NOINLINE Result registration_frozen_error(); - void register_builtin_types(); bool compatible_; @@ -1573,16 +1571,6 @@ inline void TypeResolver::apply_config(const Config &config) { track_ref_ = config.track_ref; } -inline Result TypeResolver::check_registration() { - if (FORY_PREDICT_FALSE(registry_frozen_)) { - return registration_frozen_error(); - } - FORY_CHECK(std::this_thread::get_id() == registration_thread_id_) - << "TypeResolver registration methods must be called from the same " - "thread that created the TypeResolver"; - return Result(); -} - template inline void *TypeResolver::harness_struct_read_as(ReadContext &ctx, const TypeInfo *type_info) { From c2fd1a07eefec0e315fee26a373cd388236bb33e Mon Sep 17 00:00:00 2001 From: chaokunyang Date: Fri, 28 Aug 2026 05:05:26 +0800 Subject: [PATCH 027/168] refactor(python): preserve carrier invariants --- python/pyfory/serializer.py | 4 ++++ python/pyfory/tests/test_function.py | 2 +- python/pyfory/tests/test_policy.py | 28 ++++++++++++------------ python/pyfory/tests/test_ref_tracking.py | 4 ++-- python/pyfory/tests/test_struct.py | 2 +- 5 files changed, 22 insertions(+), 18 deletions(-) diff --git a/python/pyfory/serializer.py b/python/pyfory/serializer.py index d1b8ed8320..f06a3e124c 100644 --- a/python/pyfory/serializer.py +++ b/python/pyfory/serializer.py @@ -563,6 +563,8 @@ def write(self, write_context, value): else: write_context.write_int8(NOT_NULL_VALUE_FLAG) write_context.write_no_ref(step) + # Concrete dtype classes differ across NumPy versions. Keep this wire slot owned by + # RangeIndex and encode NumPy's stable descriptor instead of a version-specific object ref. write_context.write_string(value.dtype.str) write_context.write_ref(value.name) @@ -1806,6 +1808,8 @@ def _deserialize_function(self, read_context): freevars.append(read_context.read_string()) globals_dict = read_context.read_ref() + # The writer defines this as a data-only namespace snapshot. Reject subclasses and other + # mappings so their runtime behavior cannot become part of function reconstruction. if type(globals_dict) is not dict: raise ValueError("function globals must be a dict") diff --git a/python/pyfory/tests/test_function.py b/python/pyfory/tests/test_function.py index 02100a62bf..2cc4d664ae 100644 --- a/python/pyfory/tests/test_function.py +++ b/python/pyfory/tests/test_function.py @@ -121,7 +121,7 @@ def test_lambda_functions_serialization(): assert closure_lambda(test_input) == deserialized(test_input) -def test_regular_functions_serialization(): +def test_regular_function_roundtrip(): """Tests serialization of regular functions.""" fory = pyfory.Fory( xlang=False, diff --git a/python/pyfory/tests/test_policy.py b/python/pyfory/tests/test_policy.py index 8bef3148f1..61873348a9 100644 --- a/python/pyfory/tests/test_policy.py +++ b/python/pyfory/tests/test_policy.py @@ -256,7 +256,7 @@ def intercept_setstate(self, obj, state, **kwargs): return None -def test_block_class_type_deserialization(): +def test_block_class_deserialization(): """Test blocking class type (not instance) deserialization.""" class SafeClass: @@ -383,7 +383,7 @@ def __setstate__(self, state): assert result.password == "***REDACTED***" -def test_stateful_intercepts_falsey_state_before_bool(): +def test_falsey_state_hook_before_bool(): """Test stateful path calls intercept_setstate without evaluating state truthiness.""" class BlockSetStatePolicy(DeserializationPolicy): @@ -560,7 +560,7 @@ def __reduce__(self): fory.deserialize(data) -def test_stateful_authorizes_instantiation(): +def test_stateful_instantiation_policy(): """Test authorize_instantiation policy hook for stateful deserialization.""" class StatefulPayload: @@ -591,7 +591,7 @@ def authorize_instantiation(self, cls, **kwargs): assert policy.authorize_instantiation_calls == 1 -def test_reduce_class_callable_authorizes_instantiation(): +def test_reduce_class_instantiation(): """Test authorize_instantiation policy hook for reduce class callables.""" class ReduceTarget: @@ -849,7 +849,7 @@ def validate_class(self, cls, is_local, **kwargs): assert SafeClass.run() == "safe" -def test_type_deserialization_validates_module(): +def test_type_module_policy(): """Test validate_module policy hook for global class deserialization.""" import subprocess @@ -874,7 +874,7 @@ def validate_module(self, module_name, is_local, **kwargs): assert policy.is_local_values == [False] -def test_native_bound_method_uses_validate_method(): +def test_native_method_policy_dispatch(): """Test bound native methods are checked by method policy, not function policy.""" class BlockMethodPolicy(DeserializationPolicy): @@ -900,7 +900,7 @@ def validate_function(self, func, is_local, **kwargs): assert policy.validate_function_calls == 0 -def test_bound_method_policy_runs_before_getattribute_side_effect(): +def test_bound_method_policy_order(): """Test bound method deserialization validates before dynamic attribute lookup.""" class GuardedMethod: @@ -1656,7 +1656,7 @@ def validate_function(self, func, is_local, **kwargs): policy_global_function.__module__ = original_module -def test_global_function_deserialization_validates_module(): +def test_global_function_module_policy(): """Test validate_module policy hook for global function deserialization.""" class BlockModulePolicy(DeserializationPolicy): @@ -1680,7 +1680,7 @@ def validate_module(self, module_name, is_local, **kwargs): assert policy.is_local_values == [False] -def test_local_function_deserialization_validates_module(): +def test_local_function_module_policy(): """Test local function code does not reclassify its module owner.""" def local_function(): @@ -1732,7 +1732,7 @@ def authorize_instantiation(self, cls, **kwargs): assert policy.instantiation_calls == [] -def test_native_function_deserialization_validates_module(): +def test_native_function_module_policy(): """Test validate_module policy hook for native function deserialization.""" import time @@ -1816,7 +1816,7 @@ def validate_class(self, cls, is_local, **kwargs): assert policy.validate_class_calls == 1 -def test_reduce_global_name_validates_module(): +def test_reduce_global_module_policy(): """Test validate_module policy hook for reduce global-name deserialization.""" class GlobalNamePayload: @@ -1844,7 +1844,7 @@ def validate_module(self, module_name, is_local, **kwargs): assert policy.is_local_values == [False] -def test_reduce_global_name_validates_class(): +def test_reduce_global_class_policy(): """Test validate_class policy hook for reduce global-name deserialization.""" class GlobalNamePayload: @@ -1876,7 +1876,7 @@ def validate_class(self, cls, is_local, **kwargs): assert policy.validate_class_calls == 1 -def test_reduce_global_name_validates_function(): +def test_reduce_global_function_policy(): """Test validate_function policy hook for reduce builtins-name deserialization.""" class GlobalNamePayload: @@ -1908,7 +1908,7 @@ def validate_function(self, func, is_local, **kwargs): assert policy.validate_function_calls == 1 -def test_reduce_global_method_resolution_uses_validate_method(): +def test_reduce_global_method_policy(): """Test reduce global-name method deserialization uses validate_method.""" class GlobalNamePayload: diff --git a/python/pyfory/tests/test_ref_tracking.py b/python/pyfory/tests/test_ref_tracking.py index e3f74c512f..3ac1265dba 100644 --- a/python/pyfory/tests/test_ref_tracking.py +++ b/python/pyfory/tests/test_ref_tracking.py @@ -120,7 +120,7 @@ def test_collection_tuple_shared_reference_python_mode(): assert restored[2][0] is restored[0] -def test_collection_set_element_alias_with_outer_reference_python_mode(): +def test_set_element_outer_alias(): fory = pyfory.Fory(xlang=False, ref=True, strict=False, compatible=False) token = HashKey("shared-key") fory.register_type(HashKey) @@ -153,7 +153,7 @@ def test_map_self_cycle_and_shared_submap_python_mode(): assert restored["self"] is restored -def test_map_key_alias_with_outer_reference_python_mode(): +def test_map_key_outer_alias(): fory = pyfory.Fory(xlang=False, ref=True, strict=False, compatible=False) key = HashKey("k") fory.register_type(HashKey) diff --git a/python/pyfory/tests/test_struct.py b/python/pyfory/tests/test_struct.py index 490da93337..8bcd34a943 100644 --- a/python/pyfory/tests/test_struct.py +++ b/python/pyfory/tests/test_struct.py @@ -918,7 +918,7 @@ def test_data_class_serializer_xlang(): @pytest.mark.parametrize("track_ref", [False, True]) -def test_dataclass_with_typed_tuple_field(track_ref): +def test_dataclass_typed_tuple(track_ref): fory = Fory(xlang=False, ref=track_ref, strict=False, compatible=False) fory.register_type(TupleFieldObject) obj = TupleFieldObject(bar=("a", 1)) From 3340108fbd9860ff90bb2b65c536539804e25d95 Mon Sep 17 00:00:00 2001 From: chaokunyang Date: Fri, 28 Aug 2026 05:11:58 +0800 Subject: [PATCH 028/168] fix(java): close facade registration callbacks --- .../apache/fory/FacadeRegistrationGate.java | 35 ++-- .../src/main/java/org/apache/fory/Fory.java | 8 +- .../java/org/apache/fory/ThreadLocalFory.java | 30 ++- .../org/apache/fory/pool/ThreadPoolFory.java | 12 +- .../apache/fory/resolver/TypeResolver.java | 10 +- .../org/apache/fory/ThreadSafeForyTest.java | 188 +++++++++++++++++- .../apache/fory/serializer/RegisterTest.java | 77 +++++++ 7 files changed, 325 insertions(+), 35 deletions(-) diff --git a/java/fory-core/src/main/java/org/apache/fory/FacadeRegistrationGate.java b/java/fory-core/src/main/java/org/apache/fory/FacadeRegistrationGate.java index f13785413d..87b34ea7ce 100644 --- a/java/fory-core/src/main/java/org/apache/fory/FacadeRegistrationGate.java +++ b/java/fory-core/src/main/java/org/apache/fory/FacadeRegistrationGate.java @@ -19,7 +19,6 @@ package org.apache.fory; -import java.util.function.Function; import java.util.function.Supplier; import org.apache.fory.annotation.Internal; import org.apache.fory.exception.ForyException; @@ -28,17 +27,18 @@ @Internal public final class FacadeRegistrationGate { private final Object lock = new Object(); + private final Runnable finishChildren; private volatile boolean frozen; + public FacadeRegistrationGate(Runnable finishChildren) { + this.finishChildren = finishChildren; + } + public void applyRegistration(Runnable action) { synchronized (lock) { - if (frozen) { - throw new ForyException( - "Cannot register class/serializer after registration has been frozen. Please register " - + "all classes before invoking top-level `serialize/deserialize/copy` methods of " - + "ThreadSafeFory."); - } + checkRegistrationAllowed(); action.run(); + checkRegistrationAllowed(); } } @@ -52,19 +52,22 @@ public Fory initializeChild(Supplier initializer) { public void freeze() { if (!frozen) { synchronized (lock) { - frozen = true; + if (!frozen) { + // Set the permanent facade state first. If child finalization fails, registration must + // remain closed rather than reopening a partially finalized facade. + frozen = true; + finishChildren.run(); + } } } } - /** Freezes facade and child registration before invoking an action that can expose the child. */ - public R execute(Fory fory, Function action) { - synchronized (lock) { - // The callback may return or otherwise retain the raw child. Freeze before exposing it, - // because a later root through that escaped reference is invisible to the facade. - frozen = true; - fory.getTypeResolver().finishRegistration(); + private void checkRegistrationAllowed() { + if (frozen) { + throw new ForyException( + "Cannot register class/serializer after registration has been frozen. Please register " + + "all classes before invoking top-level `serialize/deserialize/copy` methods of " + + "ThreadSafeFory."); } - return action.apply(fory); } } diff --git a/java/fory-core/src/main/java/org/apache/fory/Fory.java b/java/fory-core/src/main/java/org/apache/fory/Fory.java index 5830c2aac0..ea602f0eed 100644 --- a/java/fory-core/src/main/java/org/apache/fory/Fory.java +++ b/java/fory-core/src/main/java/org/apache/fory/Fory.java @@ -223,17 +223,13 @@ public void register(String className, String namespace, String typeName) { @Override public void register(ForyModule module) { Preconditions.checkNotNull(module); + checkRegisterAllowed(); if (installedModules.containsKey(module)) { return; } + module.install(this); checkRegisterAllowed(); installedModules.put(module, Boolean.TRUE); - try { - module.install(this); - } catch (RuntimeException | Error e) { - installedModules.remove(module); - throw e; - } } @Override diff --git a/java/fory-core/src/main/java/org/apache/fory/ThreadLocalFory.java b/java/fory-core/src/main/java/org/apache/fory/ThreadLocalFory.java index 7df396f136..1e6901146c 100644 --- a/java/fory-core/src/main/java/org/apache/fory/ThreadLocalFory.java +++ b/java/fory-core/src/main/java/org/apache/fory/ThreadLocalFory.java @@ -47,13 +47,14 @@ public class ThreadLocalFory extends AbstractThreadSafeFory { private final ThreadLocal foryThreadLocal; private Consumer factoryCallback; private final Map allFory; - private final FacadeRegistrationGate registrationGate = new FacadeRegistrationGate(); + private final FacadeRegistrationGate registrationGate; public ThreadLocalFory(Function factory) { SharedRegistry sharedRegistry = new SharedRegistry(); foryFactory = () -> factory.apply(Fory.builder().withSharedRegistry(sharedRegistry)); factoryCallback = f -> {}; allFory = Collections.synchronizedMap(new WeakHashMap<>()); + registrationGate = new FacadeRegistrationGate(this::finishChildRegistration); foryThreadLocal = ThreadLocal.withInitial(this::newFory); // 1. init and warm for current thread. // Fory creation took about 1~2 ms, but first creation @@ -66,12 +67,29 @@ private Fory newFory() { return registrationGate.initializeChild( () -> { Fory fory = foryFactory.get(); - factoryCallback.accept(fory); - allFory.put(fory, null); - return fory; + // Bind and publish the same provisional child before callbacks. A callback which + // reenters the facade then freezes this child instead of recursively creating another. + foryThreadLocal.set(fory); + try { + allFory.put(fory, null); + factoryCallback.accept(fory); + return fory; + } catch (RuntimeException | Error e) { + foryThreadLocal.remove(); + allFory.remove(fory); + throw e; + } }); } + private void finishChildRegistration() { + synchronized (allFory) { + for (Fory fory : allFory.keySet()) { + fory.getTypeResolver().finishRegistration(); + } + } + } + private Fory currentFory() { registrationGate.freeze(); return foryThreadLocal.get(); @@ -91,8 +109,8 @@ public void registerCallback(Consumer callback) { @Override public R execute(Function action) { - Fory fory = foryThreadLocal.get(); - return registrationGate.execute(fory, action); + registrationGate.freeze(); + return action.apply(foryThreadLocal.get()); } @Override diff --git a/java/fory-core/src/main/java/org/apache/fory/pool/ThreadPoolFory.java b/java/fory-core/src/main/java/org/apache/fory/pool/ThreadPoolFory.java index e8d30f3fb0..f91c61f3d2 100644 --- a/java/fory-core/src/main/java/org/apache/fory/pool/ThreadPoolFory.java +++ b/java/fory-core/src/main/java/org/apache/fory/pool/ThreadPoolFory.java @@ -53,7 +53,7 @@ public class ThreadPoolFory extends AbstractThreadSafeFory { private final Fory[] pooledFory; private final Semaphore waiterSignal = new Semaphore(0); private final AtomicInteger waitingBorrowers = new AtomicInteger(); - private final FacadeRegistrationGate registrationGate = new FacadeRegistrationGate(); + private final FacadeRegistrationGate registrationGate; public ThreadPoolFory(Function foryFactory, int poolSize) { if (poolSize <= 0) { @@ -71,6 +71,13 @@ public ThreadPoolFory(Function foryFactory, int poolSize) { pooledFory[i] = fory; slots.set(i, new PooledEntry(fory, i)); } + registrationGate = new FacadeRegistrationGate(this::finishChildRegistration); + } + + private void finishChildRegistration() { + for (Fory fory : pooledFory) { + fory.getTypeResolver().finishRegistration(); + } } private PooledEntry acquire() { @@ -163,9 +170,10 @@ public void registerCallback(Consumer callback) { @Override public R execute(Function action) { + registrationGate.freeze(); PooledEntry entry = acquireEntry(); try { - return registrationGate.execute(entry.fory, action); + return action.apply(entry.fory); } finally { release(entry); } diff --git a/java/fory-core/src/main/java/org/apache/fory/resolver/TypeResolver.java b/java/fory-core/src/main/java/org/apache/fory/resolver/TypeResolver.java index 46bf8a4481..4cddffd339 100644 --- a/java/fory-core/src/main/java/org/apache/fory/resolver/TypeResolver.java +++ b/java/fory-core/src/main/java/org/apache/fory/resolver/TypeResolver.java @@ -416,10 +416,12 @@ public final void finishRegistration() { */ public void registerSerializerAndType( Class type, Class serializerClass) { - if (!isRegistered(type)) { - register(type); - } - registerSerializer(type, serializerClass); + checkRegisterAllowed(); + Serializer serializer = newSerializer(type, serializerClass); + // Serializer construction may invoke application code which starts a root operation. Keep + // type and serializer publication together after the authoritative lifecycle check. + checkRegisterAllowed(); + registerSerializerAndType(type, serializer); } /** diff --git a/java/fory-core/src/test/java/org/apache/fory/ThreadSafeForyTest.java b/java/fory-core/src/test/java/org/apache/fory/ThreadSafeForyTest.java index 938d3ca942..5a5b25ab90 100644 --- a/java/fory-core/src/test/java/org/apache/fory/ThreadSafeForyTest.java +++ b/java/fory-core/src/test/java/org/apache/fory/ThreadSafeForyTest.java @@ -26,14 +26,18 @@ import static org.testng.Assert.assertTrue; import java.nio.ByteBuffer; +import java.util.Map; import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.CountDownLatch; +import java.util.concurrent.ExecutionException; import java.util.concurrent.ExecutorService; import java.util.concurrent.Executors; import java.util.concurrent.Future; import java.util.concurrent.TimeUnit; import java.util.concurrent.TimeoutException; +import java.util.concurrent.atomic.AtomicInteger; import java.util.concurrent.atomic.AtomicReference; +import java.util.concurrent.atomic.AtomicReferenceArray; import lombok.Data; import org.apache.fory.context.MetaReadContext; import org.apache.fory.context.MetaWriteContext; @@ -648,7 +652,7 @@ public void testExecuteFreezesThreadLocal() throws Exception { @Test public void testRegistrationGateLinearization() throws Exception { - FacadeRegistrationGate gate = new FacadeRegistrationGate(); + FacadeRegistrationGate gate = new FacadeRegistrationGate(() -> {}); CountDownLatch registrationEntered = new CountDownLatch(1); CountDownLatch freezeEntered = new CountDownLatch(1); CountDownLatch releaseRegistration = new CountDownLatch(1); @@ -683,6 +687,188 @@ public void testRegistrationGateLinearization() throws Exception { } } + @Test + public void testReentrantRegistrationFreeze() throws Exception { + ThreadLocalFory threadLocal = + Fory.builder() + .withXlang(false) + .requireClassRegistration(true) + .withCompatible(false) + .buildThreadLocalFory(); + assertReentrantRegistrationRejected(threadLocal, threadLocalChildren(threadLocal)); + + ThreadPoolFory threadPool = + (ThreadPoolFory) + Fory.builder() + .withXlang(false) + .requireClassRegistration(true) + .withCompatible(false) + .buildThreadSafeForyPool(2); + Fory[] pooledFory = TestUtils.getFieldValue(threadPool, "pooledFory"); + assertReentrantRegistrationRejected(threadPool, pooledFory); + } + + private static void assertReentrantRegistrationRejected(ThreadSafeFory facade, Fory[] children) { + AtomicInteger creatorCalls = new AtomicInteger(); + Assert.assertThrows( + ForyException.class, + () -> + facade.registerSerializerAndType( + Foo.class, + resolver -> { + creatorCalls.incrementAndGet(); + facade.serialize("freeze"); + return new FooSerializer(resolver, Foo.class); + })); + Assert.assertEquals(creatorCalls.get(), 1); + for (Fory child : children) { + TypeResolver resolver = child.getTypeResolver(); + assertTrue(resolver.isRegistrationFinished()); + assertNull(((ClassResolver) resolver).getRegisteredClassId(Foo.class)); + } + } + + private static Fory[] threadLocalChildren(ThreadLocalFory facade) throws Exception { + ThreadLocal local = TestUtils.getFieldValue(facade, "foryThreadLocal"); + Fory first = local.get(); + ExecutorService executor = Executors.newSingleThreadExecutor(); + try { + Fory second = executor.submit(local::get).get(10, TimeUnit.SECONDS); + return new Fory[] {first, second}; + } finally { + executor.shutdownNow(); + } + } + + @Test + public void testLateChildReplaysRegistration() throws Exception { + ThreadLocalFory facade = + Fory.builder() + .withXlang(false) + .requireClassRegistration(true) + .withCompatible(false) + .buildThreadLocalFory(); + facade.registerSerializerAndType(Foo.class, FooSerializer.class); + facade.serialize("freeze"); + + ExecutorService executor = Executors.newSingleThreadExecutor(); + try { + Foo value = new Foo(); + value.f1 = 42; + Foo result = + executor + .submit(() -> facade.deserialize(facade.serialize(value), Foo.class)) + .get(10, TimeUnit.SECONDS); + assertEquals(result, value); + Class serializerType = + executor + .submit( + () -> + facade.execute( + child -> child.getTypeResolver().getSerializer(Foo.class).getClass())) + .get(10, TimeUnit.SECONDS); + assertSame(serializerType, FooSerializer.class); + } finally { + executor.shutdownNow(); + } + } + + @Test + public void testReentrantReplayCleanup() throws Exception { + ThreadLocalFory facade = + Fory.builder() + .withXlang(false) + .requireClassRegistration(true) + .withCompatible(false) + .buildThreadLocalFory(); + AtomicInteger callbackCalls = new AtomicInteger(); + facade.registerCallback( + child -> { + if (callbackCalls.incrementAndGet() > 1) { + facade.serialize("nested"); + } + child.register(BeanA.class); + }); + facade.serialize("freeze"); + + Map children = TestUtils.getFieldValue(facade, "allFory"); + ExecutorService executor = Executors.newSingleThreadExecutor(); + try { + for (int i = 0; i < 2; i++) { + ExecutionException failure = + Assert.expectThrows( + ExecutionException.class, + () -> + executor + .submit(() -> facade.execute(child -> child)) + .get(10, TimeUnit.SECONDS)); + assertTrue(failure.getCause() instanceof ForyException); + assertEquals(children.size(), 1); + } + assertEquals(callbackCalls.get(), 3); + } finally { + executor.shutdownNow(); + } + } + + @Test + public void testPoolGatePrecedesBorrow() throws Exception { + ThreadPoolFory facade = + (ThreadPoolFory) + Fory.builder() + .withXlang(false) + .requireClassRegistration(true) + .withCompatible(false) + .buildThreadSafeForyPool(1); + AtomicReferenceArray slots = TestUtils.getFieldValue(facade, "slots"); + CountDownLatch callbackEntered = new CountDownLatch(1); + CountDownLatch allowReentrantRoot = new CountDownLatch(1); + CountDownLatch rootStarted = new CountDownLatch(1); + AtomicReference rootThread = new AtomicReference<>(); + ExecutorService executor = Executors.newFixedThreadPool(2); + try { + Future registration = + executor.submit( + () -> + facade.registerCallback( + child -> { + callbackEntered.countDown(); + awaitUnchecked(allowReentrantRoot); + facade.execute(value -> null); + })); + assertTrue(callbackEntered.await(10, TimeUnit.SECONDS)); + Future root = + executor.submit( + () -> { + rootThread.set(Thread.currentThread()); + rootStarted.countDown(); + return facade.execute(value -> null); + }); + assertTrue(rootStarted.await(10, TimeUnit.SECONDS)); + awaitBlocked(rootThread.get()); + assertNotNull(slots.get(0)); + + allowReentrantRoot.countDown(); + ExecutionException registrationFailure = + Assert.expectThrows( + ExecutionException.class, () -> registration.get(10, TimeUnit.SECONDS)); + assertTrue(registrationFailure.getCause() instanceof ForyException); + root.get(10, TimeUnit.SECONDS); + } finally { + allowReentrantRoot.countDown(); + executor.shutdownNow(); + executor.awaitTermination(10, TimeUnit.SECONDS); + } + } + + private static void awaitBlocked(Thread thread) { + long deadline = System.nanoTime() + TimeUnit.SECONDS.toNanos(10); + while (thread.getState() != Thread.State.BLOCKED && System.nanoTime() < deadline) { + Thread.yield(); + } + Assert.assertEquals(thread.getState(), Thread.State.BLOCKED); + } + @Test public void testExecuteFreezesPool() { ThreadPoolFory fory = diff --git a/java/fory-core/src/test/java/org/apache/fory/serializer/RegisterTest.java b/java/fory-core/src/test/java/org/apache/fory/serializer/RegisterTest.java index be7b718a1e..b26428f800 100644 --- a/java/fory-core/src/test/java/org/apache/fory/serializer/RegisterTest.java +++ b/java/fory-core/src/test/java/org/apache/fory/serializer/RegisterTest.java @@ -19,10 +19,14 @@ package org.apache.fory.serializer; +import java.util.IdentityHashMap; import java.util.concurrent.atomic.AtomicBoolean; +import java.util.concurrent.atomic.AtomicInteger; +import java.util.concurrent.atomic.AtomicReference; import org.apache.fory.Fory; import org.apache.fory.ForyModule; import org.apache.fory.ForyTestBase; +import org.apache.fory.TestUtils; import org.apache.fory.config.ForyBuilder; import org.apache.fory.context.ReadContext; import org.apache.fory.context.WriteContext; @@ -197,6 +201,79 @@ public void testFrozenFacadeRegistration() { Assert.assertFalse(creatorCalled.get()); } + @Test + public void testReentrantModuleFreeze() { + Fory fory = + Fory.builder() + .withXlang(false) + .withCodegen(false) + .requireClassRegistration(false) + .withCompatible(false) + .build(); + AtomicBoolean installReturned = new AtomicBoolean(); + ForyModule module = + runtime -> { + runtime.serialize("freeze"); + installReturned.set(true); + }; + + Assert.assertThrows(ForyException.class, () -> fory.register(module)); + Assert.assertTrue(installReturned.get()); + IdentityHashMap installedModules = + TestUtils.getFieldValue(fory, "installedModules"); + Assert.assertFalse(installedModules.containsKey(module)); + } + + @Test + public void testFrozenModuleDuplicateRejected() { + Fory fory = + Fory.builder() + .withXlang(false) + .withCodegen(false) + .requireClassRegistration(false) + .withCompatible(false) + .build(); + AtomicInteger installs = new AtomicInteger(); + ForyModule module = runtime -> installs.incrementAndGet(); + fory.register(module); + fory.serialize("freeze"); + + Assert.assertThrows(ForyException.class, () -> fory.register(module)); + Assert.assertEquals(installs.get(), 1); + } + + @Test(dataProvider = "xlang") + public void testReentrantCombinedRegistration(boolean xlang) { + Fory fory = + Fory.builder() + .withXlang(xlang) + .withCodegen(false) + .requireClassRegistration(true) + .withCompatible(false) + .build(); + ReentrantSerializer.CONSTRUCTION.set(() -> fory.serialize("freeze")); + try { + Assert.assertThrows( + ForyException.class, + () -> fory.registerSerializerAndType(MyExt.class, ReentrantSerializer.class)); + } finally { + ReentrantSerializer.CONSTRUCTION.set(null); + } + + Assert.assertTrue(fory.getTypeResolver().isRegistrationFinished()); + Assert.assertFalse(fory.getTypeResolver().isRegistered(MyExt.class)); + Assert.assertNull(fory.getTypeResolver().getTypeInfo(MyExt.class, false)); + } + + public static class ReentrantSerializer extends MyExtSerializer { + private static final AtomicReference CONSTRUCTION = new AtomicReference<>(); + + public ReentrantSerializer(TypeResolver typeResolver) { + super(typeResolver); + CONSTRUCTION.get().run(); + } + } + public static class MyExtSerializer extends Serializer { public MyExtSerializer(TypeResolver typeResolver) { super(typeResolver.getConfig(), MyExt.class); From 4d8d28d9d1eb643aa7f955c6b4f62a474ecad07d Mon Sep 17 00:00:00 2001 From: chaokunyang Date: Fri, 28 Aug 2026 05:12:29 +0800 Subject: [PATCH 029/168] refactor(jvm): reuse registration owners --- .../serializer/kotlin/KotlinSerializers.java | 26 +++++-------------- .../serializer/scala/ScalaSerializers.java | 3 --- .../apache/fory/scala/ForySerializer.scala | 1 - 3 files changed, 7 insertions(+), 23 deletions(-) diff --git a/kotlin/fory-kotlin/src/main/java/org/apache/fory/serializer/kotlin/KotlinSerializers.java b/kotlin/fory-kotlin/src/main/java/org/apache/fory/serializer/kotlin/KotlinSerializers.java index 14456dd3d0..d9f93e2662 100644 --- a/kotlin/fory-kotlin/src/main/java/org/apache/fory/serializer/kotlin/KotlinSerializers.java +++ b/kotlin/fory-kotlin/src/main/java/org/apache/fory/serializer/kotlin/KotlinSerializers.java @@ -218,27 +218,29 @@ public static void registerType(Fory fory, Class cls, String namespace, Strin public static void register(Fory fory, Class cls) { fory.register(cls); - registerSerializerAfterType(fory, cls); + registerSerializer(fory, cls); } public static void register(Fory fory, Class cls, long typeId) { registerType(fory, cls, typeId); - registerSerializerAfterType(fory, cls); + registerSerializer(fory, cls); } public static void register(Fory fory, Class cls, String name) { registerType(fory, cls, name); - registerSerializerAfterType(fory, cls); + registerSerializer(fory, cls); } public static void register(Fory fory, Class cls, String namespace, String typeName) { registerType(fory, cls, namespace, typeName); - registerSerializerAfterType(fory, cls); + registerSerializer(fory, cls); } public static void registerSerializer(Fory fory, Class cls) { TypeResolver resolver = fory.getTypeResolver(); - Serializer serializer = newRegistrationSerializer(resolver, cls); + checkRegistrationOpen(resolver); + Serializer serializer = newGeneratedSerializer(resolver, cls); + checkRegistrationOpen(resolver); if (resolver.isRegistered(cls)) { resolver.setSerializer(cls, serializer); } else { @@ -246,20 +248,6 @@ public static void registerSerializer(Fory fory, Class cls) { } } - private static Serializer newRegistrationSerializer(TypeResolver resolver, Class cls) { - checkRegistrationOpen(resolver); - Serializer serializer = newGeneratedSerializer(resolver, cls); - checkRegistrationOpen(resolver); - return serializer; - } - - private static void registerSerializerAfterType(Fory fory, Class cls) { - TypeResolver resolver = fory.getTypeResolver(); - Serializer serializer = newGeneratedSerializer(resolver, cls); - checkRegistrationOpen(resolver); - resolver.setSerializer(cls, serializer); - } - private static void checkRegistrationOpen(TypeResolver resolver) { // Resolver setSerializer remains available for lazy internal resolution, so this facade owns // the public freeze checks around construction and before the final replacement. diff --git a/scala/fory-scala/src/main/java/org/apache/fory/serializer/scala/ScalaSerializers.java b/scala/fory-scala/src/main/java/org/apache/fory/serializer/scala/ScalaSerializers.java index 0799360b2e..fc1aa0c773 100644 --- a/scala/fory-scala/src/main/java/org/apache/fory/serializer/scala/ScalaSerializers.java +++ b/scala/fory-scala/src/main/java/org/apache/fory/serializer/scala/ScalaSerializers.java @@ -197,7 +197,6 @@ public static void registerEnum(Fory fory, Class cls, long typeId) { TypeResolver resolver = fory.getTypeResolver(); checkRegistrationOpen(resolver); ScalaEnumSerializer serializer = new ScalaEnumSerializer(resolver, cls); - checkRegistrationOpen(resolver); Object[] values = serializer.getEnumConstants(); resolver.registerEnum(cls, typeId, serializer); registerEnumRuntimeAliases(fory, cls, values); @@ -230,7 +229,6 @@ public static void registerEnum(Fory fory, Class cls, String name) { String[] parts = splitName(name); checkRegistrationOpen(resolver); ScalaEnumSerializer serializer = new ScalaEnumSerializer(resolver, cls); - checkRegistrationOpen(resolver); Object[] values = serializer.getEnumConstants(); resolver.registerEnum(cls, parts[0], parts[1], serializer); registerEnumRuntimeAliases(fory, cls, values); @@ -241,7 +239,6 @@ public static void registerEnum(Fory fory, Class cls, String namespace, Strin TypeResolver resolver = fory.getTypeResolver(); checkRegistrationOpen(resolver); ScalaEnumSerializer serializer = new ScalaEnumSerializer(resolver, cls); - checkRegistrationOpen(resolver); Object[] values = serializer.getEnumConstants(); resolver.registerEnum(cls, namespace, typeName, serializer); registerEnumRuntimeAliases(fory, cls, values); diff --git a/scala/fory-scala/src/main/scala-3/org/apache/fory/scala/ForySerializer.scala b/scala/fory-scala/src/main/scala-3/org/apache/fory/scala/ForySerializer.scala index 27cab8f79d..bad66c927a 100644 --- a/scala/fory-scala/src/main/scala-3/org/apache/fory/scala/ForySerializer.scala +++ b/scala/fory-scala/src/main/scala-3/org/apache/fory/scala/ForySerializer.scala @@ -148,7 +148,6 @@ object ForySerializer { val generatedSerializer = serializer.createSerializer(resolver) checkRegistrationOpen(resolver) val runtimeClasses = serializer.handledRuntimeClasses(cls) - checkRegistrationOpen(resolver) if typeId != null then { resolver.registerUnion(cls, typeId.longValue(), generatedSerializer) } else { From 958c9715747f85e432f5ad0cba5f4f2af6ae9ec5 Mon Sep 17 00:00:00 2001 From: chaokunyang Date: Fri, 28 Aug 2026 05:14:01 +0800 Subject: [PATCH 030/168] fix(python): publish registrations after callbacks --- .agents/languages/python.md | 11 +- .../python/type-registration.md | 7 +- python/pyfory/_fory.py | 49 ++++++-- python/pyfory/registry.py | 113 ++++++++++-------- python/pyfory/tests/test_serializer.py | 25 +++- python/pyfory/tests/test_thread_safe.py | 77 ++++++++++++ 6 files changed, 214 insertions(+), 68 deletions(-) diff --git a/.agents/languages/python.md b/.agents/languages/python.md index 44b35b5b08..ab50fe6282 100644 --- a/.agents/languages/python.md +++ b/.agents/languages/python.md @@ -15,11 +15,20 @@ Load this file when changing `python/`, Cython serialization, or Python xlang be completion of the one Python-owner dispatch needed to populate native resolver tables, but the `Fory` facade must not mirror that state. Cython roots call the resolver owner directly. Serializer construction may reenter a root, so the resolver rechecks its frozen state after - construction and before publishing type, serializer, name, or ID state. + construction and before publishing type, serializer, name, or ID state. Allocate automatic type + IDs only at that common publication point; do not reserve IDs before callbacks or maintain + rollback state. `ThreadSafeFory` validates registrations before retaining their replay callbacks, + and it must not execute application factories or callbacks while holding its pool lock. - In non-strict native mode, public unqualified `register_type` for a built-in native carrier uses the same reserved type identity as pre-root discovery. Ordinary application classes and dataclasses retain their struct registration identity. Configure both through public registration; do not prewarm private resolver state or enumerate version-specific transitive object shapes. +- Function serialization writes captured globals as a data-only exact `dict`. Keep the reader's + exact-type check before sizing or merging the namespace; a dict subclass or other mapping must not + introduce runtime behavior into function reconstruction. +- Pandas `RangeIndex` owns its dtype wire slot. Encode `dtype.str` and reconstruct it with + `numpy.dtype`; do not serialize the dtype object as a reference because concrete NumPy dtype + classes vary across versions and would make the wire depend on version-specific registration. - Use explicit Cython fields and methods for fixed hot-path shapes. Avoid `__getattr__`, generic `object` fields, public bridge internals, or `Fory` backreferences where ownership can stay explicit. - Keep Python and Cython context/ref-tracking branch conditions and stack mutations semantically aligned unless a documented intentional difference exists. - Root deserialization graph memory budget state belongs to pure-Python and Cython `ReadContext`. diff --git a/docs/object-serialization/python/type-registration.md b/docs/object-serialization/python/type-registration.md index 784880b7ed..5d267da4e2 100644 --- a/docs/object-serialization/python/type-registration.md +++ b/docs/object-serialization/python/type-registration.md @@ -79,14 +79,13 @@ classes. Register application classes before serializing or deserializing payloads, and keep the same registration IDs or names on every peer that shares those payloads. -The first root serialization or deserialization attempt permanently freezes the -instance's registry, including when that attempt fails. `strict=False` relaxes +The first root serialization or deserialization attempt permanently closes +registration, including when that attempt fails. `strict=False` relaxes the deserialization policy for registered Python-native carriers; it does not permit late discovery or registration. Register callable, class, method, and reduction carriers together with the application types that can appear in the graph before the first root operation. -If a custom serializer factory starts a root operation, that root freezes the registry and the -in-progress registration fails without publishing the serializer or type mapping. +Later registration attempts fail. Compatible metadata has one data-only exception: when a remote Struct has no local registration, deserialization returns the fixed framework diff --git a/python/pyfory/_fory.py b/python/pyfory/_fory.py index 9444bf5ffe..81ea72c62d 100644 --- a/python/pyfory/_fory.py +++ b/python/pyfory/_fory.py @@ -692,6 +692,8 @@ def __init__(self, fory_factory=None, **kwargs): self._fory_factory = fory_factory self._callbacks = [] self._lock = threading.Lock() + self._registration_lock = threading.Lock() + self._registration_fory = None self._pool = [] if fory_factory is not None: self._fory_class = None @@ -703,30 +705,53 @@ def __init__(self, fory_factory=None, **kwargs): self._fory_class = Fory self._instances_created = False + def _build_fory(self): + if self._fory_factory is not None: + fory = self._fory_factory() + else: + fory = self._fory_class(**self._config) + for callback in self._callbacks: + callback(fory) + return fory + def _get_fory(self): with self._lock: if self._pool: return self._pool.pop() self._instances_created = True - if self._fory_factory is not None: - fory = self._fory_factory() - else: - fory = self._fory_class(**self._config) - for callback in self._callbacks: - callback(fory) + fory = self._registration_fory + self._registration_fory = None + if fory is not None: + # The validation instance already contains every published registration. return fory + # Factories and registration callbacks are application code. Keep them outside the + # non-reentrant pool lock so a callback can enter the same facade root. + return self._build_fory() def _return_fory(self, fory): with self._lock: self._pool.append(fory) def _register_callback(self, callback): - with self._lock: - if self._instances_created: - raise RuntimeError( - "Cannot register types after Fory instances have been created. Please register all types before calling serialize/deserialize." - ) - self._callbacks.append(callback) + with self._registration_lock: + with self._lock: + self._check_registration_open() + registration_fory = self._registration_fory + # A concurrent root must not reuse this instance while the callback mutates it. + self._registration_fory = None + if registration_fory is None: + registration_fory = self._build_fory() + callback(registration_fory) + with self._lock: + self._check_registration_open() + self._callbacks.append(callback) + self._registration_fory = registration_fory + + def _check_registration_open(self): + if self._instances_created: + raise RuntimeError( + "Cannot register types after Fory instances have been created. Please register all types before calling serialize/deserialize." + ) def register( self, diff --git a/python/pyfory/registry.py b/python/pyfory/registry.py index 49783c7b24..e3f48eb772 100644 --- a/python/pyfory/registry.py +++ b/python/pyfory/registry.py @@ -639,30 +639,28 @@ def register_union( raise TypeError("register_union requires a serializer") if typename is not None and type_id is not None: raise TypeError(f"type name {typename} and id {type_id} should not be set at the same time") - previous_type_id_counter = self._type_id_counter - automatic_type_id = typename is None and type_id is None - if typename is None and type_id is None: - type_id = self._next_type_id() - if type_id not in {0, None}: + if type_id is None: + if typename is None: + user_type_id = None + type_id = TypeId.TYPED_UNION + else: + user_type_id = NO_USER_TYPE_ID + type_id = TypeId.NAMED_UNION + elif type_id != 0: user_type_id = type_id type_id = TypeId.TYPED_UNION else: user_type_id = NO_USER_TYPE_ID type_id = TypeId.NAMED_UNION - try: - return self.__register_type( - cls, - type_id=type_id, - user_type_id=user_type_id, - namespace=namespace, - typename=typename, - serializer=serializer, - internal=False, - ) - except BaseException: - if automatic_type_id and self._type_id_counter == user_type_id: - self._type_id_counter = previous_type_id_counter - raise + return self.__register_type( + cls, + type_id=type_id, + user_type_id=user_type_id, + namespace=namespace, + typename=typename, + serializer=serializer, + internal=False, + ) def _register_type( self, @@ -692,26 +690,17 @@ def _register_type( if typeinfo is not None: return typeinfo n_params = len({typename, type_id, None}) - 1 - previous_type_id_counter = self._type_id_counter - automatic_type_id = n_params == 0 and typename is None - if n_params == 0 and typename is None: - type_id = self._next_type_id() if n_params == 2: raise TypeError(f"type name {typename} and id {type_id} should not be set at the same time") - try: - return self._register_xtype( - cls, - type_id=type_id, - user_type_id=user_type_id, - namespace=namespace, - typename=typename, - serializer=serializer, - internal=internal, - ) - except BaseException: - if automatic_type_id and self._type_id_counter == type_id: - self._type_id_counter = previous_type_id_counter - raise + return self._register_xtype( + cls, + type_id=type_id, + user_type_id=user_type_id, + namespace=namespace, + typename=typename, + serializer=serializer, + internal=internal, + ) def _register_xtype( self, @@ -731,8 +720,12 @@ def _register_xtype( if serializer is None: if issubclass(cls, enum.Enum): if type_id is None: - type_id = TypeId.NAMED_ENUM - user_type_id = NO_USER_TYPE_ID + if typename is None: + type_id = TypeId.ENUM + user_type_id = None + else: + type_id = TypeId.NAMED_ENUM + user_type_id = NO_USER_TYPE_ID else: user_type_id = type_id type_id = TypeId.ENUM @@ -740,22 +733,34 @@ def _register_xtype( serializer = None if self.meta_share and evolving: if type_id is None: - type_id = TypeId.NAMED_COMPATIBLE_STRUCT - user_type_id = NO_USER_TYPE_ID + if typename is None: + type_id = TypeId.COMPATIBLE_STRUCT + user_type_id = None + else: + type_id = TypeId.NAMED_COMPATIBLE_STRUCT + user_type_id = NO_USER_TYPE_ID else: user_type_id = type_id type_id = TypeId.COMPATIBLE_STRUCT else: if type_id is None: - type_id = TypeId.NAMED_STRUCT - user_type_id = NO_USER_TYPE_ID + if typename is None: + type_id = TypeId.STRUCT + user_type_id = None + else: + type_id = TypeId.NAMED_STRUCT + user_type_id = NO_USER_TYPE_ID else: user_type_id = type_id type_id = TypeId.STRUCT elif not internal: if type_id is None: - type_id = TypeId.NAMED_EXT - user_type_id = NO_USER_TYPE_ID + if typename is None: + type_id = TypeId.EXT + user_type_id = None + else: + type_id = TypeId.NAMED_EXT + user_type_id = NO_USER_TYPE_ID else: user_type_id = type_id type_id = TypeId.EXT @@ -781,7 +786,6 @@ def __register_type( serializer: Serializer = None, internal: bool = False, ): - dynamic_type = type_id is not None and type_id < 0 namespace_metastr = None typename_metastr = None if typename is not None: @@ -795,9 +799,13 @@ def __register_type( namespace = namespace or "" if not typename: raise ValueError("type name must not be empty") - if not internal and needs_user_type_id(type_id): - if not isinstance(user_type_id, int) or isinstance(user_type_id, bool) or user_type_id < 0 or user_type_id > 0xFFFFFFFE: - raise ValueError(f"user_type_id must be an integer in range [0, 0xfffffffe], got {user_type_id}") + if ( + not internal + and needs_user_type_id(type_id) + and user_type_id is not None + and (not isinstance(user_type_id, int) or isinstance(user_type_id, bool) or user_type_id < 0 or user_type_id > 0xFFFFFFFE) + ): + raise ValueError(f"user_type_id must be an integer in range [0, 0xfffffffe], got {user_type_id}") self._preflight_registration( cls, type_id=type_id, @@ -824,6 +832,13 @@ def __register_type( # Serializer construction can run application code and start a root. Recheck before # publishing any type, serializer, name, or id state. self._check_registry_mutable() + # Allocate automatic IDs only at the common commit point. Nested registrations therefore + # receive IDs in publication order without reservations or rollback state. + if type_id is None: + type_id = self._next_type_id() + elif not internal and needs_user_type_id(type_id) and user_type_id is None: + user_type_id = self._next_type_id() + dynamic_type = type_id < 0 if serializer is not None and type_id in _NO_REF_NUMERIC_TYPE_IDS: serializer.need_to_write_ref = False diff --git a/python/pyfory/tests/test_serializer.py b/python/pyfory/tests/test_serializer.py index a79676237d..2d0856a1d4 100644 --- a/python/pyfory/tests/test_serializer.py +++ b/python/pyfory/tests/test_serializer.py @@ -1050,7 +1050,7 @@ def serializer_factory(type_resolver, cls): return FrozenExtSerializer(type_resolver, cls) return UnionSerializer(type_resolver, cls, {0: str}) - with pytest.raises(RuntimeError, match="first root operation"): + with pytest.raises(RuntimeError): if registration == "type": fory.register_type(target, type_id=725, serializer=serializer_factory) else: @@ -1059,7 +1059,7 @@ def serializer_factory(type_resolver, cls): assert constructions == 1 assert fory.type_resolver.get_type_info(target, create=False) is None assert registration_state(fory) == before - with pytest.raises(RuntimeError, match="first root operation"): + with pytest.raises(RuntimeError): fory.register_type(RejectedRegistration, type_id=726) @@ -1519,6 +1519,27 @@ def __init__(self, *_args): assert actual.user_type_id == expected.user_type_id +@pytest.mark.parametrize("registration", ["type", "union"]) +def test_nested_registration_ids(registration): + fory = Fory(xlang=True, compatible=False) + nested_info = None + target = FrozenExt if registration == "type" else FrozenUnion + + def serializer_factory(type_resolver, cls): + nonlocal nested_info + nested_info = fory.register_type(FrozenSecondExt) + if registration == "type": + return FrozenExtSerializer(type_resolver, cls) + return UnionSerializer(type_resolver, cls, {0: str}) + + if registration == "type": + type_info = fory.register_type(target, serializer=serializer_factory) + else: + type_info = fory.register_union(target, serializer=serializer_factory) + + assert nested_info.user_type_id + 1 == type_info.user_type_id + + def test_duplicate_type_keeps_id(): class FirstValue: pass diff --git a/python/pyfory/tests/test_thread_safe.py b/python/pyfory/tests/test_thread_safe.py index 3bfb7a0d9b..1ed9571b5f 100644 --- a/python/pyfory/tests/test_thread_safe.py +++ b/python/pyfory/tests/test_thread_safe.py @@ -18,7 +18,9 @@ import threading from dataclasses import dataclass +import pytest +import pyfory from pyfory import ThreadSafeFory @@ -193,3 +195,78 @@ def test_thread_safe_fory_register_after_use(): assert False, "Should raise RuntimeError" except RuntimeError as e: assert "Cannot register types after Fory instances have been created" in str(e) + + +def test_invalid_registration(): + failed_constructions = 0 + valid_constructions = 0 + + class BrokenSerializer: + def __init__(self, *_args): + nonlocal failed_constructions + failed_constructions += 1 + raise ValueError("serializer construction failed") + + class AddressSerializer(pyfory.Serializer): + def write(self, write_context, value): + write_context.write_string(value.city) + write_context.write_string(value.country) + + def read(self, read_context): + return Address(read_context.read_string(), read_context.read_string()) + + def serializer_factory(type_resolver, cls): + nonlocal valid_constructions + valid_constructions += 1 + return AddressSerializer(type_resolver, cls) + + fory = ThreadSafeFory(xlang=False, compatible=False) + with pytest.raises(ValueError): + fory.register_type(Address, serializer=BrokenSerializer) + assert failed_constructions == 1 + + fory.register_type(Address, serializer=serializer_factory) + address = Address(city="Oslo", country="Norway") + assert fory.deserialize(fory.serialize(address)) == address + assert failed_constructions == 1 + assert valid_constructions == 1 + + +def test_reentrant_registration(): + class AddressSerializer(pyfory.Serializer): + def write(self, write_context, value): + write_context.write_string(value.city) + write_context.write_string(value.country) + + def read(self, read_context): + return Address(read_context.read_string(), read_context.read_string()) + + fory = ThreadSafeFory(xlang=False, compatible=False) + constructions = 0 + errors = [] + + def serializer_factory(type_resolver, cls): + nonlocal constructions + constructions += 1 + fory.serialize(None) + return AddressSerializer(type_resolver, cls) + + def register(): + try: + fory.register_type(Address, serializer=serializer_factory) + except RuntimeError as exc: + errors.append(exc) + + thread = threading.Thread(target=register, daemon=True) + thread.start() + thread.join(timeout=5) + + assert not thread.is_alive() + assert constructions == 1 + assert len(errors) == 1 + assert isinstance(errors[0], RuntimeError) + assert not fory._callbacks + assert fory._registration_fory is None + assert fory.deserialize(fory.serialize(None)) is None + with pytest.raises(RuntimeError): + fory.register_type(Person) From 5c9ea48136a087cb8795f5727d45041b33278972 Mon Sep 17 00:00:00 2001 From: chaokunyang Date: Fri, 28 Aug 2026 05:15:40 +0800 Subject: [PATCH 031/168] docs: clarify registration lifecycle --- .../csharp/type-registration.md | 9 +-- python/README.md | 72 ++++++++++--------- 2 files changed, 40 insertions(+), 41 deletions(-) diff --git a/docs/object-serialization/csharp/type-registration.md b/docs/object-serialization/csharp/type-registration.md index 64b824b693..7f275c6e47 100644 --- a/docs/object-serialization/csharp/type-registration.md +++ b/docs/object-serialization/csharp/type-registration.md @@ -70,10 +70,8 @@ fory.Register("com.example.MyType"); ## Registration Lifecycle A `Fory` instance accepts registration only before its first root serialization or deserialization -attempt. Starting that operation permanently freezes the instance's registry, even when the -operation fails. Every later registration throws `InvalidOperationException`. -If a custom serializer constructor starts a root operation, that root freezes the registry and the -in-progress registration fails without publishing the serializer or type mapping. +attempt. Starting that operation permanently closes registration, even when the operation fails. +Every later registration throws `InvalidOperationException`. To configure additional types, build a new `Fory` instance, complete its registrations, and then use that new instance for serialization or deserialization. @@ -82,8 +80,7 @@ use that new instance for serialization or deserialization. `ThreadSafeFory` exposes the same registration APIs. Register every type before the first serialization or deserialization attempt. Starting the first root permanently freezes -registration, even when the root fails. A later registration throws `InvalidOperationException` -before changing any per-thread runtime. +registration, even when the root fails. A later registration throws `InvalidOperationException`. ```csharp using ThreadSafeFory fory = Fory.Builder().BuildThreadSafe(); diff --git a/python/README.md b/python/README.md index 5f416ac46c..ab1dafbd46 100644 --- a/python/README.md +++ b/python/README.md @@ -262,8 +262,8 @@ them to `deserialize` in the same order. For contiguous storage, `BufferObject.g expose a `memoryview` without an additional source-side copy; non-contiguous storage may be copied. This does not promise copy-free transport or decoding. -See [Out-of-Band Buffers](../docs/object-serialization/python/out-of-band.md) for the callback, -transport, and stream APIs. +See [Out-of-Band Buffers](https://fory.apache.org/docs/object-serialization/python/out-of-band) for +the callback, transport, and stream APIs. ## Cross-Language Object Graph Serialization @@ -331,8 +331,9 @@ Person person = (Person) fory.deserialize(binaryData); ## Row Format Row Format provides random and partial access to trusted analytical data without reconstructing the -complete object graph. See the [Python Row Format guide](../docs/row-format/python.md) for supported -types, schema requirements, and APIs. +complete object graph. See the +[Python Row Format guide](https://fory.apache.org/docs/row-format/python) for supported types, schema +requirements, and APIs. ### Basic Row Format Usage @@ -509,7 +510,7 @@ class Fory: ### ThreadSafeFory Class -Thread-safe serialization interface using thread-local storage: +Thread-safe serialization interface for sharing one configured facade across threads: ```python class ThreadSafeFory: @@ -523,7 +524,9 @@ class ThreadSafeFory: ) ``` -`ThreadSafeFory` provides thread-safe serialization by maintaining a pool of `Fory` instances protected by a lock. When a thread needs to serialize/deserialize, it gets an instance from the pool, uses it, and returns it. All type registrations must be done before any serialization to ensure consistency across all instances. +Register all types before the first serialization or deserialization attempt. That first attempt +permanently freezes registration, even when it fails. Every later registration attempt raises an +error. **Thread Safety Example:** @@ -555,8 +558,8 @@ for t in threads: t.join() **Key Features:** -- **Instance Pool**: Maintains a pool of `Fory` instances protected by a lock for thread safety -- **Shared Configuration**: All registrations must be done upfront and are applied to all instances +- **Thread-safe use**: One configured facade can be shared across threads +- **Shared Configuration**: Complete all registrations before the first root attempt - **Same root API**: Provides the same serialization and deserialization methods as `Fory` - **Registration Safety**: The first root attempt permanently freezes registration, even if it fails @@ -577,21 +580,17 @@ for t in threads: t.join() **Key Methods:** ```python -# Serialization (serialize/deserialize are identical to dumps/loads) +# Complete registration before the first root API call. +fory.register(MyClass, type_id=123) +# Alternatively, register by name or provide a custom serializer. +# fory.register(MyClass, name="my.package.MyClass") +# fory.register(MyClass, type_id=123, serializer=custom_serializer) + +# serialize/deserialize are identical to dumps/loads. data: bytes = fory.serialize(obj) obj = fory.deserialize(data) - -# Alternative API (aliases) -data: bytes = fory.dumps(obj) +data = fory.dumps(obj) obj = fory.loads(data) - -# Type registration by id -fory.register(MyClass, type_id=123) -fory.register(MyClass, type_id=123, serializer=custom_serializer) - -# Type registration by name -fory.register(MyClass, name="my.package.MyClass") -fory.register(MyClass, name="my.package.MyClass", serializer=custom_serializer) ``` ### Xlang And Native Mode Comparison @@ -679,22 +678,23 @@ assert result.next.next is result # Reference preserved ### Type Registration Register the complete application type surface before the first root operation. See -[Type Registration](../docs/object-serialization/python/type-registration.md) for registration -identity, strict-mode behavior, and the frozen registry lifecycle. See -[Python Security](../docs/object-serialization/python/security.md) before accepting untrusted input. +[Type Registration](https://fory.apache.org/docs/object-serialization/python/type-registration) for +registration identity, strict-mode behavior, and the frozen registry lifecycle. See +[Python Security](https://fory.apache.org/docs/object-serialization/python/security) before +accepting untrusted input. ### Custom Serializers Custom serializers implement the serializer-owned `write` and `read` operations and are registered before the first root operation. See -[Custom Serializers](../docs/object-serialization/python/custom-serializers.md) for the supported -constructor and context APIs. +[Custom Serializers](https://fory.apache.org/docs/object-serialization/python/custom-serializers) for +the supported constructor and context APIs. ### NumPy & Scientific Computing Python native mode supports NumPy ndarrays, including multidimensional and object-dtype arrays. See -[NumPy Integration](../docs/object-serialization/python/numpy-integration.md) for supported behavior -and out-of-band transport. +[NumPy Integration](https://fory.apache.org/docs/object-serialization/python/numpy-integration) for +supported behavior and out-of-band transport. ## Best Practices @@ -730,7 +730,7 @@ Use these configuration rules before measuring an application workload: Python class schema 4. **Use Row Format for partial reads**: Choose it when applications need random access to trusted analytical row data instead of object reconstruction; see the - [Python Row Format guide](../docs/row-format/python.md) + [Python Row Format guide](https://fory.apache.org/docs/row-format/python) ```python # Good: Reuse instance @@ -747,16 +747,16 @@ for obj in objects: ### Type Registration Patterns Use stable names for shared xlang schemas and numeric IDs for Python-native type identity. See -[Type Registration](../docs/object-serialization/python/type-registration.md) for the supported -patterns, including custom serializers and batch registration. +[Type Registration](https://fory.apache.org/docs/object-serialization/python/type-registration) for +the supported patterns, including custom serializers and batch registration. ### Error Handling A failed root never reopens the registry. Create and fully configure a new instance after a missing or invalid registration failure. A fully configured instance can process another root after a failure while reading input data or serializing a value. See -[Error Handling](../docs/object-serialization/python/troubleshooting.md#error-handling) for a -complete example. +[Error Handling](https://fory.apache.org/docs/object-serialization/python/troubleshooting#error-handling) +for a complete example. ## Security Best Practices @@ -814,8 +814,9 @@ else: When `strict=False` is necessary for trusted native-mode payloads, configure a `DeserializationPolicy` before the first root operation to restrict accepted types and object hooks. -See [Python Security](../docs/object-serialization/python/security.md#deserializationpolicy) for the -supported policy hooks and configuration example. +See +[Python Security](https://fory.apache.org/docs/object-serialization/python/security#deserializationpolicy) +for the supported policy hooks and configuration example. ## Troubleshooting @@ -887,7 +888,8 @@ import pyfory # Now uses pure Python implementation Xlang mode defaults to compatible schema evolution. Configure writer and reader schemas on separate instances because each instance's registry freezes on its first root operation. See -[Schema Evolution](../docs/object-serialization/python/schema-evolution.md) for a complete example. +[Schema Evolution](https://fory.apache.org/docs/object-serialization/python/schema-evolution) for a +complete example. **Q: Type registration errors in strict mode** From 6ece493f211abac414fdb6e5022d0f43364c0dc2 Mon Sep 17 00:00:00 2001 From: chaokunyang Date: Fri, 28 Aug 2026 05:24:05 +0800 Subject: [PATCH 032/168] fix(javascript): publish generated registries atomically --- javascript/packages/core/lib/context.ts | 20 +- javascript/packages/core/lib/fory.ts | 4 - javascript/packages/core/lib/gen/builder.ts | 9 +- javascript/packages/core/lib/gen/index.ts | 191 ++++++++++++++---- javascript/packages/core/lib/gen/map.ts | 2 +- .../packages/core/lib/gen/serializer.ts | 3 +- javascript/packages/core/lib/gen/struct.ts | 17 +- javascript/packages/core/lib/typeResolver.ts | 53 ++++- javascript/test/fory.test.ts | 96 +++++++++ ...eadCleanup.test.ts => rootCleanup.test.ts} | 49 ++++- 10 files changed, 383 insertions(+), 61 deletions(-) rename javascript/test/{rootReadCleanup.test.ts => rootCleanup.test.ts} (80%) diff --git a/javascript/packages/core/lib/context.ts b/javascript/packages/core/lib/context.ts index 44613d8ef9..0e166fb275 100644 --- a/javascript/packages/core/lib/context.ts +++ b/javascript/packages/core/lib/context.ts @@ -367,11 +367,14 @@ export class MetaStringReader { } export class WriteContext { + private static readonly MAX_RETAINED_TYPE_META_OWNERS = 8192; + readonly writer: BinaryWriter; readonly refWriter: RefWriter; readonly metaStringWriter: MetaStringWriter; private disposeTypeMetaOwners: Array<{ dynamicTypeId: number }> = []; + private disposeTypeMetaOwnersSize = 0; private dynamicTypeId = 0; constructor( @@ -387,10 +390,17 @@ export class WriteContext { this.writer.reset(); this.refWriter.reset(); this.metaStringWriter.reset(); - this.disposeTypeMetaOwners.forEach((owner) => { - owner.dynamicTypeId = -1; - }); - this.disposeTypeMetaOwners = []; + const owners = this.disposeTypeMetaOwners; + const size = this.disposeTypeMetaOwnersSize; + for (let i = 0; i < size; i++) { + owners[i].dynamicTypeId = -1; + } + // The logical size is the current root's visibility boundary. Reuse bounded backing and + // release only an unusual root's oversized owner table. + if (size > WriteContext.MAX_RETAINED_TYPE_META_OWNERS) { + this.disposeTypeMetaOwners = []; + } + this.disposeTypeMetaOwnersSize = 0; this.dynamicTypeId = 0; } @@ -434,7 +444,7 @@ export class WriteContext { const index = this.dynamicTypeId; owner.dynamicTypeId = index; this.dynamicTypeId += 1; - this.disposeTypeMetaOwners.push(owner); + this.disposeTypeMetaOwners[this.disposeTypeMetaOwnersSize++] = owner; this.writer.writeVarUInt32(index << 1); this.writer.buffer(bytes); } diff --git a/javascript/packages/core/lib/fory.ts b/javascript/packages/core/lib/fory.ts index 8d188264f5..9e2a4a2a78 100644 --- a/javascript/packages/core/lib/fory.ts +++ b/javascript/packages/core/lib/fory.ts @@ -154,19 +154,15 @@ export default class Fory { if (constructor.prototype?.[ForyTypeInfoSymbol]) { const typeInfo: TypeInfo = (constructor.prototype[ForyTypeInfoSymbol] as WithForyClsInfo) .structTypeInfo; - typeInfo.freeze(); serializer = new Gen(this.typeResolver, { creator: constructor, customSerializer, }).generateSerializer(typeInfo); - this.typeResolver.registerSerializer(typeInfo, serializer); } else { const typeInfo = constructor; - typeInfo.freeze(); serializer = new Gen(this.typeResolver, { customSerializer, }).generateSerializer(typeInfo); - this.typeResolver.registerSerializer(typeInfo, serializer); } return { serializer, diff --git a/javascript/packages/core/lib/gen/builder.ts b/javascript/packages/core/lib/gen/builder.ts index 6fc84efe01..a89110693b 100644 --- a/javascript/packages/core/lib/gen/builder.ts +++ b/javascript/packages/core/lib/gen/builder.ts @@ -20,6 +20,11 @@ import { Scope } from "./scope"; import TypeResolver from "../typeResolver"; +export type SerializerLookup = Pick< + TypeResolver, + "getSerializerByTypeInfo" | "getSerializerById" | "getSerializerByName" +>; + export class BinaryReaderBuilder { constructor(private holder: string) {} @@ -426,19 +431,19 @@ export class CodecBuilder { constructor( scope: Scope, readonly resolver: TypeResolver, + readonly serializerLookup: SerializerLookup = resolver, ) { const writeContext = scope.declareByName("writeContext", "typeResolver.writeContext"); const readContext = scope.declareByName("readContext", "typeResolver.readContext"); const br = scope.declareByName("br", "readContext.reader"); const bw = scope.declareByName("bw", "writeContext.writer"); - const cr = scope.declareByName("cr", "typeResolver"); const rw = scope.declareByName("rw", "writeContext.refWriter"); const rr = scope.declareByName("rr", "readContext.refReader"); const mw = scope.declareByName("mw", "writeContext.metaStringWriter"); scope.declareByName("mr", "readContext.metaStringReader"); this.reader = new BinaryReaderBuilder(br); this.writer = new BinaryWriterBuilder(bw); - this.typeResolver = new TypeResolverBuilder(cr); + this.typeResolver = new TypeResolverBuilder("serializerLookup"); this.referenceResolver = new ReferenceResolverBuilder(rr, rw); this.typeMetaResolver = new TypeMetaContextBuilder(writeContext, readContext); this.metaStringResolver = new MetaStringContextBuilder(writeContext, readContext, mw); diff --git a/javascript/packages/core/lib/gen/index.ts b/javascript/packages/core/lib/gen/index.ts index 9e62446722..647377cd47 100644 --- a/javascript/packages/core/lib/gen/index.ts +++ b/javascript/packages/core/lib/gen/index.ts @@ -20,7 +20,7 @@ import { TypeId, Serializer } from "../type"; import { TypeInfo } from "../typeInfo"; import { CodegenRegistry } from "./router"; -import { CodecBuilder } from "./builder"; +import { CodecBuilder, SerializerLookup } from "./builder"; import { Scope } from "./scope"; import { CompatibleScalarConverter } from "../compatible/scalar"; import "./array"; @@ -50,6 +50,7 @@ CodegenRegistry.registerExternal(CompatibleScalarConverter); type SerializerFactoryBuilder = () => ( typeResolver: TypeResolver, + serializerLookup: SerializerLookup, external: unknown, typeInfo: TypeInfo, options: { [key: string]: unknown }, @@ -59,15 +60,35 @@ type SerializerFactoryBuilder = () => ( checkedTypeMetaWireTypeIdSymbol: symbol, ) => Serializer; +type SerializerCreator = (serializerLookup: SerializerLookup) => Serializer; + +interface GeneratedRegistration { + typeInfo: TypeInfo; + serializer: Serializer; + captureOwner: Serializer; + preparing: boolean; +} + export class Gen { static external = CodegenRegistry.getExternal(); + private generatedRegistrations: GeneratedRegistration[] = []; + private readonly serializerLookup: SerializerLookup; + constructor( private typeResolver: TypeResolver, private regOptions: { [key: string]: any } = {}, - ) {} + ) { + // Generator-time TypeInfo queries see initialized local serializers for codegen decisions. + // Factory-init ID/name queries instead return the stable owner captured by runtime closures. + this.serializerLookup = { + getSerializerByTypeInfo: (typeInfo) => this.getGeneratedSerializer(typeInfo), + getSerializerById: (id, userTypeId) => this.getCapturedSerializerById(id, userTypeId), + getSerializerByName: (name) => this.getCapturedSerializerByName(name), + }; + } - private generate(typeInfo: TypeInfo): Serializer { + private prepare(typeInfo: TypeInfo, serializerLookup: SerializerLookup): SerializerCreator { const InnerGeneratorClass = CodegenRegistry.get(typeInfo.typeId); if (!InnerGeneratorClass) { throw new Error(`${typeInfo.typeId} generator not exists`); @@ -75,7 +96,7 @@ export class Gen { const scope = new Scope(); const generator = new InnerGeneratorClass( typeInfo, - new CodecBuilder(scope, this.typeResolver), + new CodecBuilder(scope, this.typeResolver, serializerLookup), scope, ); @@ -91,20 +112,20 @@ export class Gen { } else { factoryBuilder = new Function(funcString) as SerializerFactoryBuilder; } - return factoryBuilder()( - this.typeResolver, - Gen.external, - typeInfo, - this.regOptions, - generator.getLocalTypeMeta(), - localTypeMetaSymbol, - checkedTypeMetaSerializerSymbol, - checkedTypeMetaWireTypeIdSymbol, - ); - } - - private register(typeInfo: TypeInfo, serializer?: Serializer) { - this.typeResolver.registerSerializer(typeInfo, serializer); + const factory = factoryBuilder(); + const localTypeMeta = generator.getLocalTypeMeta(); + return (factoryLookup) => + factory( + this.typeResolver, + factoryLookup, + Gen.external, + typeInfo, + this.regOptions, + localTypeMeta, + localTypeMetaSymbol, + checkedTypeMetaSerializerSymbol, + checkedTypeMetaWireTypeIdSymbol, + ); } private isRegistered(typeInfo: TypeInfo) { @@ -112,10 +133,99 @@ export class Gen { } private isFullyGenerated(typeInfo: TypeInfo) { - const ser = this.typeResolver.getSerializerByTypeInfo(typeInfo); + const ser = this.getGeneratedSerializer(typeInfo); return ser && ser._initialized; } + private sameRegistration(left: TypeInfo, right: TypeInfo) { + const leftTypeId = this.typeResolver.computeTypeId(left); + const rightTypeId = this.typeResolver.computeTypeId(right); + if (leftTypeId !== rightTypeId) { + return false; + } + if (TypeId.isNamedType(leftTypeId)) { + return left.named === right.named; + } + if (TypeId.needsUserTypeId(leftTypeId)) { + return left.userTypeId === right.userTypeId; + } + return true; + } + + private findRegistration(typeInfo: TypeInfo) { + return this.generatedRegistrations.find((entry) => + this.sameRegistration(entry.typeInfo, typeInfo), + ); + } + + private addRegistration(typeInfo: TypeInfo) { + const owner = this.typeResolver.createSerializerPlaceholder(); + const entry: GeneratedRegistration = { + typeInfo, + serializer: owner, + captureOwner: this.typeResolver.getSerializerByTypeInfo(typeInfo) ?? owner, + preparing: false, + }; + this.generatedRegistrations.push(entry); + return entry; + } + + private getGeneratedSerializer(typeInfo: TypeInfo) { + return ( + this.findRegistration(typeInfo)?.serializer ?? + this.typeResolver.getSerializerByTypeInfo(typeInfo) + ); + } + + private getCapturedSerializerById(id: number, userTypeId?: number) { + const entry = this.generatedRegistrations.find((candidate) => { + const typeId = this.typeResolver.computeTypeId(candidate.typeInfo); + if (typeId !== id || TypeId.isNamedType(typeId)) { + return false; + } + if (TypeId.needsUserTypeId(typeId)) { + if (userTypeId !== undefined && userTypeId !== -1) { + return candidate.typeInfo.userTypeId === userTypeId; + } + return candidate.typeInfo.userTypeId === -1; + } + return true; + }); + return entry?.captureOwner ?? this.typeResolver.getSerializerById(id, userTypeId); + } + + private getCapturedSerializerByName(name: number | string) { + const entry = this.generatedRegistrations.find( + (candidate) => + typeof name === "string" && + TypeId.isNamedType(this.typeResolver.computeTypeId(candidate.typeInfo)) && + candidate.typeInfo.named === name, + ); + return entry?.captureOwner ?? this.typeResolver.getSerializerByName(name); + } + + private prepareRegistration(typeInfo: TypeInfo, children: TypeInfo[]) { + let entry = this.findRegistration(typeInfo); + if (entry?.serializer._initialized || entry?.preparing) { + return; + } + if (entry === undefined) { + entry = this.addRegistration(typeInfo); + } else { + entry.typeInfo = typeInfo; + } + entry.preparing = true; + try { + for (const child of children) { + this.traversalContainer(child); + } + const serializer = this.prepare(typeInfo, this.serializerLookup)(this.serializerLookup); + Object.assign(entry.serializer, serializer); + } finally { + entry.preparing = false; + } + } + private traversalContainer(typeInfo: TypeInfo) { if (TypeId.userDefinedType(typeInfo.typeId)) { if (this.isFullyGenerated(typeInfo)) { @@ -127,26 +237,19 @@ export class Gen { typeInfo.typeId === TypeId.TYPED_UNION || typeInfo.typeId === TypeId.NAMED_UNION; if (unionType && options?.cases && Object.keys(options.cases).length > 0) { - this.register(typeInfo); - Object.values(options.cases).forEach((x) => { - this.traversalContainer(x); - }); - this.register(typeInfo, this.generate(typeInfo)); + this.prepareRegistration(typeInfo, Object.values(options.cases)); return; } else if (options?.props && Object.keys(options.props).length > 0) { - this.register(typeInfo); - Object.values(options.props).forEach((x) => { - this.traversalContainer(x); - }); - this.register(typeInfo, this.generate(typeInfo)); + this.prepareRegistration(typeInfo, Object.values(options.props)); } else if (!this.isRegistered(typeInfo) && TypeId.structType(typeInfo.typeId)) { - // Forward reference to a struct type not yet fully defined — register a - // placeholder so that serializer factories can capture the object - // reference. The placeholder will be filled in via Object.assign - // when the real serializer is generated later. - this.register(typeInfo); + // Keep the recursive owner local until every generated factory has completed. If a prior + // registration published a forward owner, factory captures use that owner without mutating + // it; commit initializes it in place so earlier serializers keep the same identity. + if (this.findRegistration(typeInfo) === undefined) { + this.addRegistration(typeInfo); + } } else if (TypeId.enumType(typeInfo.typeId) && !this.isRegistered(typeInfo)) { - this.register(typeInfo, this.generate(typeInfo)); + this.prepareRegistration(typeInfo, []); } } if (typeInfo.typeId === TypeId.LIST) { @@ -170,15 +273,25 @@ export class Gen { } reGenerateSerializer(typeInfo: TypeInfo) { - return this.generate(typeInfo); + return this.prepare(typeInfo, this.typeResolver)(this.typeResolver); } generateSerializer(typeInfo: TypeInfo) { this.traversalContainer(typeInfo); const serializer = this.typeResolver.getSerializerByTypeInfo(typeInfo); - if (serializer?._initialized) { - return serializer; + if (!serializer?._initialized) { + let registration = this.findRegistration(typeInfo); + if (registration === undefined) { + registration = this.addRegistration(typeInfo); + } + if (!registration.serializer._initialized) { + this.prepareRegistration(typeInfo, []); + } } - return this.reGenerateSerializer(typeInfo); + + // Generated factories may execute application-transformed code, so every factory completes + // against local owners before the resolver performs the only global publication step. + this.typeResolver.commitGeneratedSerializers(typeInfo, this.generatedRegistrations); + return this.typeResolver.getSerializerByTypeInfo(typeInfo)!; } } diff --git a/javascript/packages/core/lib/gen/map.ts b/javascript/packages/core/lib/gen/map.ts index 3e18823541..6df1f99a10 100644 --- a/javascript/packages/core/lib/gen/map.ts +++ b/javascript/packages/core/lib/gen/map.ts @@ -393,7 +393,7 @@ export class MapSerializerGenerator extends BaseSerializerGenerator { private useDeclaredType(typeInfo: TypeInfo) { const readWriteTypeInfo = - this.builder.resolver.getSerializerByTypeInfo(typeInfo)?.getTypeInfo() ?? typeInfo; + this.builder.serializerLookup.getSerializerByTypeInfo(typeInfo)?.getTypeInfo() ?? typeInfo; // Evolving structs need per-chunk TypeInfo so a compatible reader can discard a removed map // field. A fixed-schema serializer deliberately keeps the declared form: evolving=false is its // same-schema size and speed opt-out, even when the field declaration is only a placeholder. diff --git a/javascript/packages/core/lib/gen/serializer.ts b/javascript/packages/core/lib/gen/serializer.ts index c2a05dd542..7da7c3490c 100644 --- a/javascript/packages/core/lib/gen/serializer.ts +++ b/javascript/packages/core/lib/gen/serializer.ts @@ -303,6 +303,7 @@ export abstract class BaseSerializerGenerator implements SerializerGenerator { this.scope.assertNameNotDuplicate("write"); this.scope.assertNameNotDuplicate("writeInner"); this.scope.assertNameNotDuplicate("typeResolver"); + this.scope.assertNameNotDuplicate("serializerLookup"); this.scope.assertNameNotDuplicate("external"); this.scope.assertNameNotDuplicate("options"); this.scope.assertNameNotDuplicate("typeInfo"); @@ -372,7 +373,7 @@ export abstract class BaseSerializerGenerator implements SerializerGenerator { // Append read-only capability metadata so existing writer properties keep // their object-layout order on serialization hot paths. return ` - return function (typeResolver, external, typeInfo, options${localTypeMetaParams}) { + return function (typeResolver, serializerLookup, external, typeInfo, options${localTypeMetaParams}) { ${this.scope.generate()} ${serializerDeclaration} ${declare} diff --git a/javascript/packages/core/lib/gen/struct.ts b/javascript/packages/core/lib/gen/struct.ts index d9ffae0d3f..4b730153ff 100644 --- a/javascript/packages/core/lib/gen/struct.ts +++ b/javascript/packages/core/lib/gen/struct.ts @@ -587,6 +587,12 @@ class StructSerializerGenerator extends BaseSerializerGenerator { return JS_STRUCT_OWNER_BYTES + this.sortedProps.length * REFERENCE_BYTES; } + private serializerCaptureExpr(): string { + return TypeId.isNamedType(this.typeInfo.typeId) + ? this.builder.typeResolver.getSerializerByName(this.typeInfo.named!) + : this.builder.typeResolver.getSerializerById(this.typeInfo.typeId, this.typeInfo.userTypeId); + } + readDataAlwaysAdvances(): boolean { if (!this.builder.resolver.isCompatible()) { return true; @@ -603,7 +609,8 @@ class StructSerializerGenerator extends BaseSerializerGenerator { // recursive placeholder remains unknown and selects the guarded loop; // do not recursively walk the schema graph here. if ( - this.builder.resolver.getSerializerByTypeInfo(typeInfo)?.readDataAlwaysAdvances === true + this.builder.serializerLookup.getSerializerByTypeInfo(typeInfo)?.readDataAlwaysAdvances === + true ) { return true; } @@ -1230,11 +1237,11 @@ class StructSerializerGenerator extends BaseSerializerGenerator { // Hoist the serializer lookup into a scope-level const, evaluated once during // factory init. Self-recursive structs may still point at a placeholder, so // only the fully generated serializer path can hoist derived values below. - const hoisted = this.scope.declare("ser", this.serializerExpr); + const hoisted = this.scope.declare("ser", this.serializerCaptureExpr()); const scope = this.scope; const builder = this.builder; const internalTypeId = this.getInternalTypeId(); - const serializer = builder.resolver.getSerializerByTypeInfo(this.typeInfo); + const serializer = builder.serializerLookup.getSerializerByTypeInfo(this.typeInfo); const canInlineCompatibleTypeInfo = internalTypeId === TypeId.COMPATIBLE_STRUCT || internalTypeId === TypeId.NAMED_COMPATIBLE_STRUCT || @@ -1348,7 +1355,7 @@ class StructSerializerGenerator extends BaseSerializerGenerator { writeEmbed() { // Hoist the serializer lookup — safe because writeEmbed() is used by // the parent struct whose factory runs after child serializers exist. - const hoisted = this.scope.declare("ser", this.serializerExpr); + const hoisted = this.scope.declare("ser", this.serializerCaptureExpr()); const scope = this.scope; return new Proxy( {}, @@ -1445,7 +1452,7 @@ class StructSerializerGenerator extends BaseSerializerGenerator { fixedSize += propGenerator.getFixedSize(); }); } else { - fixedSize += this.builder.resolver.getSerializerByName(typeInfo.named!)!.fixedSize; + fixedSize += this.builder.serializerLookup.getSerializerByTypeInfo(typeInfo)!.fixedSize; } return fixedSize; } diff --git a/javascript/packages/core/lib/typeResolver.ts b/javascript/packages/core/lib/typeResolver.ts index 65c5a98ddf..037f50b7d4 100644 --- a/javascript/packages/core/lib/typeResolver.ts +++ b/javascript/packages/core/lib/typeResolver.ts @@ -193,7 +193,7 @@ export default class TypeResolver { private initInternalSerializer() { const registerSerializer = (typeInfo: TypeInfo) => { - return this.registerSerializer(typeInfo, new Gen(this).generateSerializer(typeInfo)); + return new Gen(this).generateSerializer(typeInfo); }; registerSerializer(Type.string()); registerSerializer(new TypeInfo(TypeId.ENUM)); @@ -281,6 +281,57 @@ export default class TypeResolver { } } + createSerializerPlaceholder(): Serializer { + return { ...uninitSerialize }; + } + + commitGeneratedSerializers( + rootTypeInfo: TypeInfo, + entries: readonly { typeInfo: TypeInfo; serializer: Serializer }[], + ) { + const publications = entries.map((entry) => { + const typeId = this.computeTypeId(entry.typeInfo); + let internalTypeId: number | undefined; + let customTypeKey: number | string | undefined; + if (TypeId.isNamedType(typeId)) { + customTypeKey = entry.typeInfo.named!; + } else if (TypeId.needsUserTypeId(typeId) && entry.typeInfo.userTypeId !== -1) { + customTypeKey = this.makeUserTypeKey(entry.typeInfo.userTypeId); + } else if (typeId <= 0xff) { + internalTypeId = typeId; + } else { + customTypeKey = typeId; + } + const existingSerializer = + internalTypeId === undefined + ? this.customSerializer.get(customTypeKey!) + : this.internalSerializer[internalTypeId]; + return { + entry, + internalTypeId, + customTypeKey, + existingSerializer, + descriptors: + existingSerializer === undefined + ? undefined + : Object.getOwnPropertyDescriptors(entry.serializer), + }; + }); + this.ensureRegistrationOpen(); + rootTypeInfo.freeze(); + for (const publication of publications) { + if (publication.existingSerializer !== undefined) { + // Published forward owners are plain resolver-owned placeholders. Define their prepared + // data properties in place so earlier generated serializers retain the same owner. + Object.defineProperties(publication.existingSerializer, publication.descriptors!); + } else if (publication.internalTypeId !== undefined) { + this.internalSerializer[publication.internalTypeId] = publication.entry.serializer; + } else { + this.customSerializer.set(publication.customTypeKey!, publication.entry.serializer); + } + } + } + registerSerializer(typeInfo: TypeInfo, serializer: Serializer = uninitSerialize) { this.ensureRegistrationOpen(); const typeId = this.computeTypeId(typeInfo); diff --git a/javascript/test/fory.test.ts b/javascript/test/fory.test.ts index 9c38b3ba63..9c97292095 100644 --- a/javascript/test/fory.test.ts +++ b/javascript/test/fory.test.ts @@ -137,6 +137,102 @@ describe("fory", () => { expect(generated).toBe(generatedBefore); }); + test("keeps codegen callbacks from publishing registration", () => { + let reenterRoot = false; + let fory: Fory; + fory = new Fory({ + compatible: false, + hooks: { + afterCodeGenerated(code) { + if (reenterRoot) { + reenterRoot = false; + fory.serialize(1); + } + return code; + }, + }, + }); + const typeResolver = fory.typeResolver as any; + const internalBefore = Array.from(typeResolver.internalSerializer); + const customBefore = Array.from(typeResolver.customSerializer.entries()); + const childType = Type.struct(8109, { + value: Type.int32(), + }); + const rootType = Type.struct(8110, { + child: childType, + }); + + reenterRoot = true; + expect(() => fory.register(rootType)).toThrow(); + + expect(Array.from(typeResolver.internalSerializer)).toEqual(internalBefore); + expect(Array.from(typeResolver.customSerializer.entries())).toEqual(customBefore); + expect(typeResolver.getSerializerById(TypeId.STRUCT, childType.userTypeId)).toBeUndefined(); + expect(typeResolver.getSerializerById(TypeId.STRUCT, rootType.userTypeId)).toBeUndefined(); + rootType.setNullable(true); + expect(rootType.nullable).toBe(true); + }); + + test("keeps failed generated factories local", () => { + let failFactory = false; + const fory = new Fory({ + compatible: false, + hooks: { + afterCodeGenerated(code) { + if (!failFactory) { + return code; + } + return code.replace( + /return function \(typeResolver, serializerLookup, external, typeInfo, options([^)]*)\) \{/, + (signature) => `${signature}\nthrow new Error("factory failure");`, + ); + }, + }, + }); + const typeResolver = fory.typeResolver as any; + const internalBefore = Array.from(typeResolver.internalSerializer); + const customBefore = Array.from(typeResolver.customSerializer.entries()); + const childType = Type.struct(8111, { + value: Type.int32(), + }); + const rootType = Type.struct(8112, { + child: childType, + }); + + failFactory = true; + expect(() => fory.register(rootType)).toThrow(); + + expect(Array.from(typeResolver.internalSerializer)).toEqual(internalBefore); + expect(Array.from(typeResolver.customSerializer.entries())).toEqual(customBefore); + expect(typeResolver.getSerializerById(TypeId.STRUCT, childType.userTypeId)).toBeUndefined(); + expect(typeResolver.getSerializerById(TypeId.STRUCT, rootType.userTypeId)).toBeUndefined(); + rootType.setNullable(true); + expect(rootType.nullable).toBe(true); + }); + + test("initializes a published forward owner in place", () => { + const fory = new Fory({ compatible: false }); + const forwardType = Type.struct(8113); + const parent = fory.register( + Type.struct(8114, { + child: forwardType, + }), + ); + const forwardOwner = fory.typeResolver.getSerializerById(TypeId.STRUCT, forwardType.userTypeId); + + fory.register( + Type.struct(8113, { + value: Type.int32(), + }), + ); + + expect(fory.typeResolver.getSerializerById(TypeId.STRUCT, forwardType.userTypeId)).toBe( + forwardOwner, + ); + const value = { child: { value: 7 } }; + expect(parent.deserialize(parent.serialize(value))).toEqual(value); + }); + test.each(["serialize", "deserialize"] as const)( "freezes registration after registered %s succeeds", (operation) => { diff --git a/javascript/test/rootReadCleanup.test.ts b/javascript/test/rootCleanup.test.ts similarity index 80% rename from javascript/test/rootReadCleanup.test.ts rename to javascript/test/rootCleanup.test.ts index eefbb8682d..f92ab0b33f 100644 --- a/javascript/test/rootReadCleanup.test.ts +++ b/javascript/test/rootCleanup.test.ts @@ -126,14 +126,14 @@ test.each(["success", "failure"] as const)("restores root write state for %s", ( }; if (outcome === "failure") { - expect(() => registered.serialize(value)).toThrow("root write failed"); + expect(() => registered.serialize(value)).toThrow(); expect(fory.serialize(7)).toBeDefined(); } else { expect(registered.serialize(value)).toBeDefined(); expect(fory.serialize(7)).toBeDefined(); } expect(writeContext.refWriter.writeObjects.size).toBe(0); - expect(writeContext.disposeTypeMetaOwners).toHaveLength(0); + expect(writeContext.disposeTypeMetaOwnersSize).toBe(0); expect(writeContext.metaStringWriter.disposeMetaStringBytesSize).toBe(0); expect(name.dynamicWriteStringId).toBe(-1); expect(typeMeta.dynamicTypeId).toBe(-1); @@ -160,6 +160,27 @@ test("reuses root write metastring owners", () => { expect(writeContext.metaStringWriter.disposeMetaStringBytesSize).toBe(1); }); +test("reuses root write type metadata owners", () => { + const fory = new Fory({ compatible: true }); + const registered = fory.register(Type.struct(7610, {})); + const writeContext = (fory as any).writeContext; + const typeMeta = TypeMeta.fromTypeInfo(Type.struct(7611, {})); + + registered.serializer.writeRef = () => { + writeContext.writeTypeMeta(typeMeta, typeMeta.toBytes()); + }; + + expect(registered.serialize({})).toBeDefined(); + const owners = writeContext.disposeTypeMetaOwners; + expect(owners).toHaveLength(1); + expect(writeContext.disposeTypeMetaOwnersSize).toBe(1); + + expect(registered.serialize({})).toBeDefined(); + expect(writeContext.disposeTypeMetaOwners).toBe(owners); + expect(owners).toHaveLength(1); + expect(writeContext.disposeTypeMetaOwnersSize).toBe(1); +}); + test.each([8192, 8193])("bounds %s root write metastring owners", (ownerCount) => { const fory = new Fory({ compatible: true }); const writeContext = (fory as any).writeContext; @@ -181,6 +202,28 @@ test.each([8192, 8193])("bounds %s root write metastring owners", (ownerCount) = } }); +test.each([8192, 8193])("bounds %s root write type metadata owners", (ownerCount) => { + const fory = new Fory({ compatible: true }); + const writeContext = (fory as any).writeContext; + const typeMetaOwners = Array.from({ length: ownerCount }, () => ({ dynamicTypeId: -1 })); + const bytes = new Uint8Array(); + + for (const owner of typeMetaOwners) { + writeContext.writeTypeMeta(owner, bytes); + } + const owners = writeContext.disposeTypeMetaOwners; + + writeContext.reset(); + expect(writeContext.disposeTypeMetaOwnersSize).toBe(0); + expect(typeMetaOwners.every((owner) => owner.dynamicTypeId === -1)).toBe(true); + if (ownerCount === 8192) { + expect(writeContext.disposeTypeMetaOwners).toBe(owners); + } else { + expect(writeContext.disposeTypeMetaOwners).not.toBe(owners); + expect(writeContext.disposeTypeMetaOwners).toHaveLength(0); + } +}); + test("releases a failed root write buffer before reuse", () => { const fory = new Fory({ compatible: true }); const registered = fory.register(Type.struct(7608, {})); @@ -191,7 +234,7 @@ test("releases a failed root write buffer before reuse", () => { throw new Error("root write failed"); }; - expect(() => registered.serialize({})).toThrow("root write failed"); + expect(() => registered.serialize({})).toThrow(); expect(fory.serialize(7)).toBeDefined(); expect(writer.getPlatformBuffer().byteLength).toBeLessThan(4 * 1024 * 1024); }); From b680222aeae2e8a349350ace6d3acf707f3849a2 Mon Sep 17 00:00:00 2001 From: chaokunyang Date: Fri, 28 Aug 2026 05:24:22 +0800 Subject: [PATCH 033/168] docs: define registry publication owners --- .agents/languages/java.md | 12 ++++++++ .agents/languages/javascript.md | 12 ++++++-- AGENTS.md | 27 +++++++++++++---- docs/security/deserialization.md | 26 +++++++++++----- .../xlang_implementation_guide.md | 30 +++++++++++++------ 5 files changed, 82 insertions(+), 25 deletions(-) diff --git a/.agents/languages/java.md b/.agents/languages/java.md index cfd7384e19..b7e3c0f3ac 100644 --- a/.agents/languages/java.md +++ b/.agents/languages/java.md @@ -84,6 +84,18 @@ Load this file when changing anything under `java/` or when Java drives a cross- work, dynamic stream bytes-read accounting, or stale narrower-scope formulas. - Generated serializers must not retain runtime context fields. `Fory` should stay a root-operation facade rather than accumulating serializer or convenience state. - When the serializer class and constructor shape are known at the call site, prefer direct constructor lambdas or direct instantiation over reflective `Serializers.newSerializer(...)`. +- `FacadeRegistrationGate` owns registration linearization for Java thread-local and pooled + facades. Starting a root closes registration before child or pool access, then finishes every + already-created child before exposing that root. A child created after closure must replay every + accepted registration, finish registration, and only then become visible; discard a provisional + child when replay or finalization fails. Keep the lock order gate before pool or child storage. +- Registration callbacks must recheck the authoritative freeze owner after returning and before + publishing the entry they prepared. Combined type-and-serializer registration may construct a + serializer only after the canonical type is registered when construction resolves that + `TypeInfo`; recheck before replacing the serializer and do not add rollback or a second + registration path. `ForyModule.install` may perform complete nested registrations, but the + module-installed marker is published only after installation returns and the lifecycle is + rechecked. - For GraalVM, use `fory codegen` to generate serializers when building native images. Do not add reflection configuration except for JDK `proxy`. - In Java native mode (`xlang=false`), only `Types.BOOL` through `Types.STRING` share type IDs with xlang mode. Other native-mode type IDs differ. - Choose one serializer ownership location per logical Java type family. Add native/xlang serializer variants only when the wire format or constructor contract truly differs. diff --git a/.agents/languages/javascript.md b/.agents/languages/javascript.md index 3796728418..c69d78a3da 100644 --- a/.agents/languages/javascript.md +++ b/.agents/languages/javascript.md @@ -14,9 +14,15 @@ Load this file when changing `javascript/`. - Root entry releases reference and metadata state left by the previous operation, including a failed operation, before the context is reused. Do not put full cleanup on the root exit path or copy Java backing-array retention policies onto native JavaScript arrays. Read-side occurrence - arrays use native replacement reset. The writer metadata-owner table has a separate logical size: - reset owner IDs and the logical size without clearing bounded backing, and replace backing only - after more than 8192 owners. + arrays use native replacement reset. The MetaString and TypeMeta writer owner tables each have + their own logical size: reset active owner IDs and that table's logical size without clearing + bounded backing, and replace either backing only after its root has more than 8192 owners. +- Generated registration must initialize the complete recursive serializer graph against + generation-local owners before one `TypeResolver` batch publication. Factory-init serializer + lookup may resolve those local owners for fixed captures; runtime and dynamic lookup must keep the + real resolver. Initialize an existing published forward owner in place during commit so previous + captures retain identity. Do not publish placeholders, nested serializers, descriptors, or cache + state before every generated factory and application code hook succeeds. - Runtime value carriers such as decimal or reduced-precision numeric types belong under the core `types/` ownership boundary, with imports, exports, and codegen externals updated together. - Keep `TypeInfo` as schema metadata. Compatibility-sensitive decisions belong on `TypeResolver` or explicit operations, not as retained resolver state on metadata objects. - Normalize optional boolean config values at config construction; do not carry `null` through runtime paths when it means `false`. diff --git a/AGENTS.md b/AGENTS.md index c9a8a4db9b..7bb0281bd2 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -160,9 +160,15 @@ This is the entry point for AI guidance in Apache Fory. Read this file first, th - JavaScript root entry releases reference and metadata state left by the previous root, including a failed root, before the context is reused. Do not add full cleanup to the root exit path or copy Java backing-array retention policies onto native JavaScript arrays. Read-side occurrence arrays - use native replacement reset. The writer metadata-owner table has a separate logical size: reset - owner IDs and the logical size without clearing bounded backing, and replace backing only after - more than 8192 owners. + use native replacement reset. The MetaString and TypeMeta writer owner tables each have their own + logical size: reset active owner IDs and that table's logical size without clearing bounded + backing, and replace either backing only after its root has more than 8192 owners. +- JavaScript generated registration must build and initialize the complete recursive serializer + graph against generation-local owners before one `TypeResolver` batch publication. Factory-init + serializer lookup may see those local owners, but runtime and dynamic lookup must retain the real + resolver. Existing published forward owners are initialized in place only during commit so prior + generated captures retain identity. Do not publish placeholders, nested serializers, descriptors, + or cache state before every generated factory and application code hook succeeds. - Root failure exceptions must not copy or retain the operation reference table or materialized object graph for diagnostics. Root cleanup owns releasing that graph, and failure reporting must remain bounded independently of graph size. @@ -187,11 +193,22 @@ This is the entry point for AI guidance in Apache Fory. Read this file first, th machinery. Registration-order finalization before the first root operation remains registration-owned and must not create a runtime invalidation path. If serializer construction, factory execution, or another application callback - can reenter a root, recheck the authoritative per-instance freeze owner after - that callback and before the first registry mutation or replay-log publication. + can reenter a root, complete that callback before publishing the entry it + prepares, then recheck the authoritative per-instance freeze owner immediately + before publication. Kotlin and Scala combined generated-struct registration are + the sole type-first exception: publish the canonical type needed by generated + serializer construction, then recheck after construction and before replacing + its serializer. Do not add rollback, staging, or a parallel registration path + for this exception. A module installation may perform complete nested + registrations; recheck after installation before publishing only the module's + installed marker. - Python `TypeResolver` is the sole registry freeze and finalization owner. Its Cython resolver companion may cache completion of the Python-owner dispatch needed to populate native tables, but the `Fory` facade must not mirror that state; Cython roots call the resolver owner directly. + Allocate automatic type IDs only after callback preparation and the final freeze recheck, at the + common registry publication point; do not reserve IDs early or maintain counter rollback state. + `ThreadSafeFory` validates registrations before publishing replay callbacks and never invokes an + application factory or callback under its non-reentrant pool lock. - Use semantic naming only. Name things after protocol or domain concepts, not history, runtime origin, or workaround style; avoid vague names such as `Internal`, `java_style_*`, `Runtime`, `Session`, `Plan`, `Payload`, or `Binding` when they do not name the real concept. Keep class, method, function, and variable names concise; do not encode the whole scenario or implementation history into one identifier. Never name a class or method with a `Plan` suffix; use the real domain concept instead. For Fory codec/read APIs, do not use generic `payload` naming; name the exact owner and data shape, such as bytes, body, frame, field, string, list, map, compressed bytes, or primitive-array encoding. - Keep one implementation path. Do not keep parallel helpers, serializers, harnesses, wrappers, or registration flows for the same concept; extend the existing owner path instead of inventing another one. - Follow current scope exactly. The latest explicit user instruction overrides earlier plans, and when scope narrows, remove leaked out-of-scope edits immediately. diff --git a/docs/security/deserialization.md b/docs/security/deserialization.md index d07576f126..b4d46ccafa 100644 --- a/docs/security/deserialization.md +++ b/docs/security/deserialization.md @@ -619,9 +619,22 @@ that case, classify the behavior by concrete impact: Registration code that invokes serializer constructors, factories, or application callbacks must recheck the authoritative per-instance registry freeze after the callback and before publishing -resolver or replay-log state. A callback that starts the first root permanently closes the +the entry prepared by that callback. A callback that starts the first root permanently closes the in-progress registration; implementations must not publish and then repair or invalidate late -state. +state. Kotlin and Scala combined generated-struct registration retain their canonical type-first +owner because generated construction resolves that type, but must recheck before the subsequent +serializer replacement. Module installation may perform complete nested registrations and must +recheck before publishing its installed marker. Automatic type IDs are allocated only at their +publication point, after callback preparation and the freeze recheck, so failed or nested +registration needs no reservation or rollback state. Thread-safe facades validate a registration +before retaining its replay callback, and application factories and callbacks execute outside +non-reentrant pool locks. + +JavaScript generated registration initializes its recursive serializer graph against local owners +before one guarded resolver publication. Application code hooks and generated factories complete +before global placeholders, nested serializers, descriptors, or cache state are changed. Runtime +and dynamic serializer lookup continues to use the authoritative resolver; the local lookup exists +only while generated factories capture fixed serializer owners. ## Metadata And Type Resolution @@ -657,12 +670,9 @@ an eight-slot array so an unusual metadata high-water mark is not retained. JavaScript root entry releases reference and metadata state left by the previous operation, including a failed operation, before the context is reused. -Read-side occurrence arrays use native replacement reset instead of copying the -Java backing-array retention policy. Writer metadata owners restore their -dynamic IDs and reset a separate logical owner count, so a bounded backing -array is reused without making prior-root entries visible or accumulating -duplicate work. After more than 8192 owners, reset replaces that backing array. -Full reference and metadata cleanup does not run on the root exit path. +Operation-local reader occurrences and writer metadata owner IDs are reset at +that reuse boundary. Full reference and metadata cleanup does not run on the +root exit path. A class-resolution cache reachable from untrusted deserialization may publish an entry only from explicit trusted configuration or after the active class diff --git a/docs/specification/xlang_implementation_guide.md b/docs/specification/xlang_implementation_guide.md index d208d01bc7..dc51b84d81 100644 --- a/docs/specification/xlang_implementation_guide.md +++ b/docs/specification/xlang_implementation_guide.md @@ -83,10 +83,21 @@ not the place where nested serializers do their work. - resetting operation-local context state at the top-level root boundary Registration preparation may invoke serializer constructors, generated factories, or application -callbacks before any registry mutation. If such a callback starts a root operation, registration -must recheck the authoritative per-instance freeze owner when the callback returns and reject the -in-progress registration before publishing type, serializer, name, ID, or replay state. Do not -publish late state and then repair it through invalidation or rollback. +callbacks. Complete the callback before publishing the registry entry it prepares. If the callback +starts a root operation, registration must recheck the authoritative per-instance freeze owner +when the callback returns and reject that publication. Kotlin and Scala combined generated-struct +registration are the type-first exception: publish the canonical type required by generated +serializer construction, then recheck after construction and before replacing its serializer. +Do not add rollback, staging, or a parallel registration path for that exception. Module +installation may consist of complete nested registrations; publish the module-installed marker +only after installation returns and the lifecycle is rechecked. + +JavaScript generated registration constructs and initializes the complete recursive serializer +graph against generation-local owners. Generated factories may use a construction-only lookup for +fixed serializer captures, while runtime and dynamic dispatch retain the real `TypeResolver`. +After every factory and application code hook succeeds, the resolver performs one guarded batch +publication. An already-published forward owner is initialized in place during that commit so +previous generated captures retain its identity. Nested serializers must not call back into root `serialize(...)` or `deserialize(...)` entry points. @@ -100,11 +111,12 @@ the context is reused. `prepare(...)` should only bind the active buffer and root-operation inputs. `reset()` should clear operation-local mutable state. -When writer metadata objects carry root-local dynamic IDs, reset must restore -those IDs and reset the active owner count. A bounded owner table may retain its -backing storage, but only entries below the current logical count participate in -the next reset; otherwise prior owners create duplicate cleanup work across -roots. Implementations should release an unusual high-water backing table. +When MetaString and TypeMeta writer objects carry root-local dynamic IDs, each +owning table must restore those IDs and reset its own active owner count. A +bounded owner table may retain its backing storage, but only entries below that +table's current logical count participate in the next reset; otherwise prior +owners create duplicate cleanup work across roots. Implementations should +release an unusual high-water backing table. That operation-local state includes: From 8fb91e34d268a391f47496b2352891a231532e90 Mon Sep 17 00:00:00 2001 From: chaokunyang Date: Fri, 28 Aug 2026 05:37:38 +0800 Subject: [PATCH 034/168] fix(go): keep registry diagnostics callback-free --- .agents/languages/go.md | 4 +- .../registry_freeze_lifecycle_test.go | 73 +++++++++++++++++++ go/fory/type_resolver.go | 16 ++-- 3 files changed, 84 insertions(+), 9 deletions(-) diff --git a/.agents/languages/go.md b/.agents/languages/go.md index 7d2b2cc72f..ce0f3071f9 100644 --- a/.agents/languages/go.md +++ b/.agents/languages/go.md @@ -14,7 +14,9 @@ Load this file when changing `go/fory/` or Go xlang behavior. are not logged; pool misses after freeze replay the immutable successful log. Its custom factory runs without the registration mutex because application code may reenter a root; after the factory returns, registration rechecks the frozen state before publishing prepared or replay - state. + state. Resolver duplicate diagnostics identify application serializers by concrete type only; + they must not invoke application string or format methods while registration holds the lifecycle + mutex. - Go `ReadContext` intentionally defers codec errors to existing `HasError` or `CheckError` boundaries. After an error, work may continue only while it remains panic- and bounds-safe and cannot cause disproportionate work or allocation, publish state that survives root cleanup, or diff --git a/go/fory/threadsafe/registry_freeze_lifecycle_test.go b/go/fory/threadsafe/registry_freeze_lifecycle_test.go index c3c9c45efb..530d2ab7a3 100644 --- a/go/fory/threadsafe/registry_freeze_lifecycle_test.go +++ b/go/fory/threadsafe/registry_freeze_lifecycle_test.go @@ -18,9 +18,11 @@ package threadsafe import ( + "reflect" "sync" "sync/atomic" "testing" + "time" "github.com/apache/fory/go/fory" "github.com/stretchr/testify/require" @@ -34,6 +36,77 @@ type registryFreezeRace struct { Value int32 } +type reentrantStringSerializer struct { + f *Fory + called chan struct{} +} + +func (s *reentrantStringSerializer) String() string { + select { + case s.called <- struct{}{}: + default: + } + _, _ = s.f.Serialize(int32(1)) + return "reentrant serializer" +} + +func (*reentrantStringSerializer) Write( + *fory.WriteContext, fory.RefMode, bool, bool, reflect.Value, +) { +} + +func (*reentrantStringSerializer) WriteData(*fory.WriteContext, reflect.Value) {} + +func (*reentrantStringSerializer) Read( + *fory.ReadContext, fory.RefMode, bool, bool, reflect.Value, +) { +} + +func (*reentrantStringSerializer) ReadData(*fory.ReadContext, reflect.Value) {} + +func (*reentrantStringSerializer) ReadWithTypeInfo( + *fory.ReadContext, fory.RefMode, *fory.TypeInfo, reflect.Value, +) { +} + +func TestDuplicateSerializerFormatting(t *testing.T) { + called := make(chan struct{}, 1) + serializer := &reentrantStringSerializer{called: called} + var f *Fory + f = NewWithFactory(func() *fory.Fory { + inner := fory.New(fory.WithXlang(false), fory.WithCompatible(false)) + if err := inner.RegisterUnionByName( + registryFreezePooled{}, "test.DuplicateSerializer", serializer, + ); err != nil { + panic(err) + } + return inner + }) + serializer.f = f + + result := make(chan error, 1) + go func() { + result <- f.RegisterStructByName( + registryFreezePooled{}, "test.DuplicateSerializer") + }() + timer := time.NewTimer(2 * time.Second) + defer timer.Stop() + select { + case err := <-result: + require.Error(t, err) + case <-timer.C: + t.Fatal("duplicate registration deadlocked while formatting the serializer") + } + select { + case <-called: + t.Fatal("duplicate registration formatted the application serializer") + default: + } + require.False(t, f.registryFrozen.Load()) + require.Empty(t, f.registrations) + require.Nil(t, f.prepared) +} + func TestFactoryRootReentry(t *testing.T) { var f *Fory var factoryEntered atomic.Bool diff --git a/go/fory/type_resolver.go b/go/fory/type_resolver.go index 0335845857..56cb370cfc 100644 --- a/go/fory/type_resolver.go +++ b/go/fory/type_resolver.go @@ -455,7 +455,7 @@ func (r *TypeResolver) initialize() { func (r *TypeResolver) registerSerializer(type_ reflect.Type, typeId TypeId, s Serializer) error { if prev, ok := r.typeToSerializers[type_]; ok { - return fmt.Errorf("type %s already has a serializer %s registered", type_, prev) + return fmt.Errorf("type %s already has a serializer of type %T registered", type_, prev) } r.typeToSerializers[type_] = s // Skip type ID registration for namespaced types, collection types, and primitive array types @@ -527,7 +527,7 @@ func (r *TypeResolver) RegisterStruct(type_ reflect.Type, typeID TypeId, userTyp } // For struct types, check if serializer already registered if prev, ok := r.typeToSerializers[type_]; ok { - return fmt.Errorf("type %s already has a serializer %s registered", type_, prev) + return fmt.Errorf("type %s already has a serializer of type %T registered", type_, prev) } // Create struct serializer @@ -582,7 +582,7 @@ func (r *TypeResolver) RegisterUnion(type_ reflect.Type, userTypeID uint32, seri return fmt.Errorf("RegisterUnion only supports struct types; got: %v", type_.Kind()) } if prev, ok := r.typeToSerializers[type_]; ok { - return fmt.Errorf("type %s already has a serializer %s registered", type_, prev) + return fmt.Errorf("type %s already has a serializer of type %T registered", type_, prev) } tag := type_.Name() @@ -651,7 +651,7 @@ func (r *TypeResolver) RegisterEnum(type_ reflect.Type, userTypeID uint32) error func (r *TypeResolver) registerEnumByName(type_ reflect.Type, namespace, typeName string) error { // Check if already registered if prev, ok := r.typeToSerializers[type_]; ok { - return fmt.Errorf("type %s already has a serializer %s registered", type_, prev) + return fmt.Errorf("type %s already has a serializer of type %T registered", type_, prev) } if typeName == "" { return fmt.Errorf("typeName must be non-empty") @@ -691,7 +691,7 @@ func (r *TypeResolver) registerEnumByName(type_ reflect.Type, namespace, typeNam func (r *TypeResolver) registerStructByName(type_ reflect.Type, namespace, typeName string) error { if prev, ok := r.typeToSerializers[type_]; ok { - return fmt.Errorf("type %s already has a serializer %s registered", type_, prev) + return fmt.Errorf("type %s already has a serializer of type %T registered", type_, prev) } if typeName == "" { return fmt.Errorf("typeName must be non-empty") @@ -739,7 +739,7 @@ func (r *TypeResolver) registerUnionByName( return fmt.Errorf("RegisterUnionByName requires a non-nil serializer") } if prev, ok := r.typeToSerializers[type_]; ok { - return fmt.Errorf("type %s already has a serializer %s registered", type_, prev) + return fmt.Errorf("type %s already has a serializer of type %T registered", type_, prev) } if type_.Kind() != reflect.Struct { return fmt.Errorf("RegisterUnionByName only supports struct types; got: %v", type_.Kind()) @@ -781,7 +781,7 @@ func (r *TypeResolver) registerExtensionByName( return fmt.Errorf("serializer cannot be nil for extension type %s", type_) } if prev, ok := r.typeToSerializers[type_]; ok { - return fmt.Errorf("type %s already has a serializer %s registered", type_, prev) + return fmt.Errorf("type %s already has a serializer of type %T registered", type_, prev) } if typeName == "" { return fmt.Errorf("typeName must be non-empty") @@ -832,7 +832,7 @@ func (r *TypeResolver) RegisterExtension( return fmt.Errorf("serializer cannot be nil for extension type %s", type_) } if prev, ok := r.typeToSerializers[type_]; ok { - return fmt.Errorf("type %s already has a serializer %s registered", type_, prev) + return fmt.Errorf("type %s already has a serializer of type %T registered", type_, prev) } // Create adapter wrapping the user's ExtensionSerializer From c2187cc0d33ea9c2cd04f955479a596d4631385e Mon Sep 17 00:00:00 2001 From: chaokunyang Date: Fri, 28 Aug 2026 05:37:52 +0800 Subject: [PATCH 035/168] fix(swift): reject reentrant registry finalization --- .agents/languages/swift.md | 3 ++ swift/Sources/Fory/TypeResolver.swift | 5 +++ swift/Tests/ForyTests/ForySwiftTests.swift | 44 ++++++++++++++++++++++ 3 files changed, 52 insertions(+) diff --git a/.agents/languages/swift.md b/.agents/languages/swift.md index af1755ca16..2d2d31ef90 100644 --- a/.agents/languages/swift.md +++ b/.agents/languages/swift.md @@ -41,6 +41,9 @@ Load this file when changing `swift/` or Swift xlang behavior. finalization because macros cannot inspect inherited storage. SwiftSyntax represents both in one inheritance clause, and Swift provides no public superclass query for arbitrary Swift classes; keep the minimal `_getSuperclass` check finalization-owned and out of root hot paths. +- Swift registration finalization invokes application-owned `StructSerializer.foryFieldsInfo`. + Reject a same-`Fory` root that reenters while finalization is in progress, cache that first + failure, and do not retry partially finalized metadata builders. - Direct `Any` and `AnyObject` root overloads remain disfavored forwarding facades over `DynamicSerializer` and `DynamicSerializer`, including their Data-buffer forms. Arbitrary protocol roots explicitly select `DynamicSerializer`. Do not add an unconstrained diff --git a/swift/Sources/Fory/TypeResolver.swift b/swift/Sources/Fory/TypeResolver.swift index a520d5da05..961e70aa3d 100644 --- a/swift/Sources/Fory/TypeResolver.swift +++ b/swift/Sources/Fory/TypeResolver.swift @@ -848,6 +848,11 @@ final class TypeResolver { if let registrationFailure { throw registrationFailure } + // Application-owned field metadata can invoke another root on this Fory. Once frozen, + // an unfinished registry is already inside finalization and must not rerun its builders. + guard !registryFrozen else { + throw ForyError.invalidData("registration finalization is already in progress") + } registryFrozen = true do { for typeInfo in registeredTypeInfos { diff --git a/swift/Tests/ForyTests/ForySwiftTests.swift b/swift/Tests/ForyTests/ForySwiftTests.swift index 1a82f5a48f..68c1639244 100644 --- a/swift/Tests/ForyTests/ForySwiftTests.swift +++ b/swift/Tests/ForyTests/ForySwiftTests.swift @@ -223,6 +223,26 @@ private struct FailingRegistrationSerializer: StructSerializer { } } +private struct ReentrantRegistrationSerializer: StructSerializer { + typealias Target = Self + + nonisolated(unsafe) static var fieldsCallback: (() throws -> Void)? + static var staticTypeId: TypeId { .structType } + + static func defaultValue(_: ReadContext) throws -> Self { Self() } + static func writeData(_: Self, _: WriteContext) throws {} + static func readData(_: ReadContext) throws -> Self { Self() } + static func readCompatible(_: ReadContext, typeInfo _: TypeInfo) throws -> Self { Self() } + + static func foryFieldsInfo( + trackRef _: Bool, + resolveSerializerTypeId _: (Any.Type) throws -> TypeId + ) throws -> [TypeMeta.FieldInfo] { + try fieldsCallback?() + return [] + } +} + @ForyStruct struct LateMetaHolder: Equatable { var ext: LateMetaExt @@ -1219,6 +1239,30 @@ func finalizationPreservesFailure() throws { } } +@Test +func reentrantFinalizationIsRejected() throws { + let fory = Fory() + var callbackCount = 0 + ReentrantRegistrationSerializer.fieldsCallback = { + callbackCount += 1 + _ = try fory.serialize(Int32(1)) + } + defer { + ReentrantRegistrationSerializer.fieldsCallback = nil + } + try fory.register(ReentrantRegistrationSerializer.self, id: 702) + + for _ in 0..<2 { + #expect(throws: ForyError.self) { + _ = try fory.serialize(ReentrantRegistrationSerializer()) + } + } + #expect(callbackCount == 1) + #expect(throws: ForyError.self) { + try fory.register(Address.self, id: 703) + } +} + @Test func serializeToAppendsRoots() throws { let fory = Fory() From e4285316c38c77812702b2da518a9b1f74a72e04 Mon Sep 17 00:00:00 2001 From: chaokunyang Date: Fri, 28 Aug 2026 05:39:34 +0800 Subject: [PATCH 036/168] fix(python): revalidate nested registrations --- .agents/languages/python.md | 14 +++-- python/pyfory/_fory.py | 75 +++++++++++++++++-------- python/pyfory/registry.py | 13 ++++- python/pyfory/tests/test_serializer.py | 56 ++++++++++++++++++ python/pyfory/tests/test_thread_safe.py | 68 ++++++++++++++++++++-- 5 files changed, 191 insertions(+), 35 deletions(-) diff --git a/.agents/languages/python.md b/.agents/languages/python.md index ab50fe6282..a4e0d863dd 100644 --- a/.agents/languages/python.md +++ b/.agents/languages/python.md @@ -14,11 +14,15 @@ Load this file when changing `python/`, Cython serialization, or Python xlang be - Python `TypeResolver` owns registry freeze and finalization state. Its Cython companion may cache completion of the one Python-owner dispatch needed to populate native resolver tables, but the `Fory` facade must not mirror that state. Cython roots call the resolver owner directly. - Serializer construction may reenter a root, so the resolver rechecks its frozen state after - construction and before publishing type, serializer, name, or ID state. Allocate automatic type - IDs only at that common publication point; do not reserve IDs before callbacks or maintain - rollback state. `ThreadSafeFory` validates registrations before retaining their replay callbacks, - and it must not execute application factories or callbacks while holding its pool lock. + Serializer construction may reenter registration or a root, so the resolver rechecks both + registration conflicts and its frozen state after construction and before publishing type, + serializer, name, or ID state. Allocate automatic type IDs only after those checks at the common + publication point; do not reserve IDs before callbacks or maintain rollback state. + `ThreadSafeFory` validates registrations before retaining their replay callbacks, and it must not + execute application factories or callbacks while holding its pool lock. Its registration + linearization is reentrant so nested facade registrations share the same publication order. A + root started during registration must not reuse the staging instance, and root reentry from a + running user `fory_factory` fails without recursively invoking that factory. - In non-strict native mode, public unqualified `register_type` for a built-in native carrier uses the same reserved type identity as pre-root discovery. Ordinary application classes and dataclasses retain their struct registration identity. Configure both through public registration; diff --git a/python/pyfory/_fory.py b/python/pyfory/_fory.py index 81ea72c62d..ff05e38f80 100644 --- a/python/pyfory/_fory.py +++ b/python/pyfory/_fory.py @@ -692,8 +692,10 @@ def __init__(self, fory_factory=None, **kwargs): self._fory_factory = fory_factory self._callbacks = [] self._lock = threading.Lock() - self._registration_lock = threading.Lock() + self._registration_lock = threading.RLock() + self._registration_depth = 0 self._registration_fory = None + self._fory_factory_running = False self._pool = [] if fory_factory is not None: self._fory_class = None @@ -703,23 +705,37 @@ def __init__(self, fory_factory=None, **kwargs): self._fory_class = CythonFory else: self._fory_class = Fory - self._instances_created = False + self._root_started = False def _build_fory(self): - if self._fory_factory is not None: - fory = self._fory_factory() - else: - fory = self._fory_class(**self._config) - for callback in self._callbacks: - callback(fory) - return fory + with self._registration_lock: + if self._fory_factory is not None: + if self._fory_factory_running: + raise RuntimeError( + "Cannot start a root serialization or deserialization operation while the Fory factory is creating an instance." + ) + self._fory_factory_running = True + try: + fory = self._fory_factory() + finally: + self._fory_factory_running = False + else: + fory = self._fory_class(**self._config) + for callback in self._callbacks: + callback(fory) + return fory def _get_fory(self): with self._lock: if self._pool: return self._pool.pop() - self._instances_created = True - fory = self._registration_fory + self._root_started = True + # Nested registrations share the staging instance, but a root may reuse it only + # after the outermost registration has published its callback. + if self._registration_depth == 0: + fory = self._registration_fory + else: + fory = None self._registration_fory = None if fory is not None: # The validation instance already contains every published registration. @@ -733,25 +749,36 @@ def _return_fory(self, fory): self._pool.append(fory) def _register_callback(self, callback): + # The reentrant lock gives nested facade registrations one publication order while the + # pool lock keeps a concurrently starting root atomic with callback publication. with self._registration_lock: with self._lock: self._check_registration_open() + self._registration_depth += 1 registration_fory = self._registration_fory - # A concurrent root must not reuse this instance while the callback mutates it. - self._registration_fory = None - if registration_fory is None: - registration_fory = self._build_fory() - callback(registration_fory) - with self._lock: - self._check_registration_open() - self._callbacks.append(callback) - self._registration_fory = registration_fory + try: + if registration_fory is None: + registration_fory = self._build_fory() + with self._lock: + self._check_registration_open() + self._registration_fory = registration_fory + callback(registration_fory) + with self._lock: + self._check_registration_open() + self._callbacks.append(callback) + self._registration_fory = registration_fory + except BaseException: + with self._lock: + if self._registration_depth == 1: + self._registration_fory = None + raise + finally: + with self._lock: + self._registration_depth -= 1 def _check_registration_open(self): - if self._instances_created: - raise RuntimeError( - "Cannot register types after Fory instances have been created. Please register all types before calling serialize/deserialize." - ) + if self._root_started: + raise RuntimeError("Cannot register types after the first root serialization or deserialization operation has started.") def register( self, diff --git a/python/pyfory/registry.py b/python/pyfory/registry.py index e3f48eb772..1fb5cfe7de 100644 --- a/python/pyfory/registry.py +++ b/python/pyfory/registry.py @@ -829,9 +829,16 @@ def __register_type( if should_create_serializer: serializer = self._create_serializer(cls) - # Serializer construction can run application code and start a root. Recheck before - # publishing any type, serializer, name, or id state. + # Serializer construction can run application code and mutate or freeze this registry. + # Recheck both invariants before allocating an automatic ID or publishing state. self._check_registry_mutable() + self._preflight_registration( + cls, + type_id=type_id, + user_type_id=user_type_id, + namespace=namespace, + typename=typename, + ) # Allocate automatic IDs only at the common commit point. Nested registrations therefore # receive IDs in publication order without reservations or rollback state. if type_id is None: @@ -871,6 +878,8 @@ def _preflight_registration( namespace, typename, ): + if cls in self._types_info: + raise TypeError(f"{cls} registered already") if typename is not None: existing = self._named_type_to_type_info.get((namespace, typename)) if existing is not None and existing.cls is not cls: diff --git a/python/pyfory/tests/test_serializer.py b/python/pyfory/tests/test_serializer.py index 2d0856a1d4..bd102ff8d5 100644 --- a/python/pyfory/tests/test_serializer.py +++ b/python/pyfory/tests/test_serializer.py @@ -1540,6 +1540,62 @@ def serializer_factory(type_resolver, cls): assert nested_info.user_type_id + 1 == type_info.user_type_id +@pytest.mark.parametrize("conflict", ["class", "id", "name"]) +def test_nested_registration_conflict(conflict): + fory = Fory(xlang=True, compatible=False) + resolver = fory.type_resolver + nested_info = None + nested_cls = FrozenExt if conflict == "class" else FrozenSecondExt + outer_args = {} + nested_args = {} + if conflict == "class": + nested_args["type_id"] = 735 + elif conflict == "id": + outer_args["type_id"] = 735 + nested_args["type_id"] = 735 + else: + outer_args["name"] = "test.NestedConflict" + nested_args["name"] = "test.NestedConflict" + + def serializer_factory(type_resolver, cls): + nonlocal nested_info + serializer = FrozenExtSerializer(type_resolver, nested_cls) + nested_info = fory.register_type( + nested_cls, + serializer=serializer, + **nested_args, + ) + return FrozenExtSerializer(type_resolver, cls) + + with pytest.raises(TypeError): + fory.register_type( + FrozenExt, + serializer=serializer_factory, + **outer_args, + ) + + assert resolver.get_type_info(nested_cls, create=False) is nested_info + assert resolver._types_info[nested_cls] is nested_info + if conflict == "class": + assert resolver._user_type_id_to_type_info[735] is nested_info + + class NextValue: + pass + + actual = fory.register_type(NextValue) + expected_fory = Fory(xlang=True, compatible=False) + expected = expected_fory.register_type(NextValue) + assert actual.user_type_id == expected.user_type_id + else: + assert resolver.get_type_info(FrozenExt, create=False) is None + if conflict == "id": + assert resolver._user_type_id_to_type_info[735] is nested_info + else: + assert resolver.get_type_info_by_name("test", "NestedConflict") is nested_info + key = (nested_info.namespace_bytes, nested_info.typename_bytes) + assert resolver._ns_type_to_type_info[key] is nested_info + + def test_duplicate_type_keeps_id(): class FirstValue: pass diff --git a/python/pyfory/tests/test_thread_safe.py b/python/pyfory/tests/test_thread_safe.py index 1ed9571b5f..e31544ba75 100644 --- a/python/pyfory/tests/test_thread_safe.py +++ b/python/pyfory/tests/test_thread_safe.py @@ -190,11 +190,8 @@ def test_thread_safe_fory_register_after_use(): person = Person(name="Alice", age=30) fory.serialize(person) - try: + with pytest.raises(RuntimeError): fory.register(Address) - assert False, "Should raise RuntimeError" - except RuntimeError as e: - assert "Cannot register types after Fory instances have been created" in str(e) def test_invalid_registration(): @@ -270,3 +267,66 @@ def register(): assert fory.deserialize(fory.serialize(None)) is None with pytest.raises(RuntimeError): fory.register_type(Person) + + +def test_nested_registration(): + class AddressSerializer(pyfory.Serializer): + def write(self, write_context, value): + write_context.write_string(value.city) + write_context.write_string(value.country) + + def read(self, read_context): + return Address(read_context.read_string(), read_context.read_string()) + + fory = ThreadSafeFory(xlang=True, compatible=False) + constructions = 0 + errors = [] + + def serializer_factory(type_resolver, cls): + nonlocal constructions + constructions += 1 + if constructions == 1: + fory.register_type(Person) + return AddressSerializer(type_resolver, cls) + + def register(): + try: + fory.register_type(Address, serializer=serializer_factory) + except (RuntimeError, TypeError) as exc: + errors.append(exc) + + thread = threading.Thread(target=register, daemon=True) + thread.start() + thread.join(timeout=5) + + assert not thread.is_alive() + assert not errors + resolver = fory._registration_fory.type_resolver + person_info = resolver.get_type_info(Person, create=False) + address_info = resolver.get_type_info(Address, create=False) + assert person_info.user_type_id + 1 == address_info.user_type_id + address = Address(city="Oslo", country="Norway") + assert fory.deserialize(fory.serialize(address)) == address + assert constructions == 1 + + +def test_factory_root_reentry(): + fory = None + constructions = 0 + + def fory_factory(): + nonlocal constructions + constructions += 1 + fory.serialize(None) + return pyfory.Fory(xlang=False, compatible=False) + + fory = ThreadSafeFory(fory_factory=fory_factory) + with pytest.raises(Exception): + fory.register_type(Person) + + assert constructions == 1 + assert fory._root_started + assert not fory._callbacks + assert fory._registration_fory is None + with pytest.raises(RuntimeError): + fory.register_type(Address) From 9d7b8bbd3f805b29b23b03e9b2f168034815a5c7 Mon Sep 17 00:00:00 2001 From: chaokunyang Date: Fri, 28 Aug 2026 05:45:21 +0800 Subject: [PATCH 037/168] fix(python): reject root reentry during pool builds --- python/pyfory/_fory.py | 28 ++++++++++----------- python/pyfory/tests/test_thread_safe.py | 33 +++++++++++++++++++++++++ 2 files changed, 46 insertions(+), 15 deletions(-) diff --git a/python/pyfory/_fory.py b/python/pyfory/_fory.py index ff05e38f80..5393f268ad 100644 --- a/python/pyfory/_fory.py +++ b/python/pyfory/_fory.py @@ -695,7 +695,7 @@ def __init__(self, fory_factory=None, **kwargs): self._registration_lock = threading.RLock() self._registration_depth = 0 self._registration_fory = None - self._fory_factory_running = False + self._fory_building = False self._pool = [] if fory_factory is not None: self._fory_class = None @@ -709,21 +709,19 @@ def __init__(self, fory_factory=None, **kwargs): def _build_fory(self): with self._registration_lock: - if self._fory_factory is not None: - if self._fory_factory_running: - raise RuntimeError( - "Cannot start a root serialization or deserialization operation while the Fory factory is creating an instance." - ) - self._fory_factory_running = True - try: + if self._fory_building: + raise RuntimeError("Cannot start a root serialization or deserialization operation while a Fory instance is being built.") + self._fory_building = True + try: + if self._fory_factory is not None: fory = self._fory_factory() - finally: - self._fory_factory_running = False - else: - fory = self._fory_class(**self._config) - for callback in self._callbacks: - callback(fory) - return fory + else: + fory = self._fory_class(**self._config) + for callback in self._callbacks: + callback(fory) + return fory + finally: + self._fory_building = False def _get_fory(self): with self._lock: diff --git a/python/pyfory/tests/test_thread_safe.py b/python/pyfory/tests/test_thread_safe.py index e31544ba75..4f1b7a80fe 100644 --- a/python/pyfory/tests/test_thread_safe.py +++ b/python/pyfory/tests/test_thread_safe.py @@ -330,3 +330,36 @@ def fory_factory(): assert fory._registration_fory is None with pytest.raises(RuntimeError): fory.register_type(Address) + + +def test_callback_root_reentry(): + class AddressSerializer(pyfory.Serializer): + def write(self, write_context, value): + write_context.write_string(value.city) + write_context.write_string(value.country) + + def read(self, read_context): + return Address(read_context.read_string(), read_context.read_string()) + + fory = pyfory.ThreadSafeFory(xlang=False, compatible=False) + constructions = 0 + reenter_root = False + + def serializer_factory(type_resolver, cls): + nonlocal constructions, reenter_root + constructions += 1 + if reenter_root: + fory.serialize(None) + return AddressSerializer(type_resolver, cls) + + fory.register_type(Address, type_id=100, serializer=serializer_factory) + initial_constructions = constructions + with pytest.raises(TypeError): + fory.register_type(Person, type_id=100) + + reenter_root = True + with pytest.raises(RuntimeError): + fory.serialize(None) + + assert constructions == initial_constructions + 1 + assert fory._root_started From 946f3ca6b2404af84e7f951ae94bbb44a1cfe47f Mon Sep 17 00:00:00 2001 From: chaokunyang Date: Fri, 28 Aug 2026 05:59:51 +0800 Subject: [PATCH 038/168] fix(javascript): freeze generated schema publication --- .agents/languages/javascript.md | 11 +- .../javascript/type-registration.md | 15 ++ .../xlang_implementation_guide.md | 16 ++- javascript/packages/core/lib/context.ts | 4 +- javascript/packages/core/lib/gen/index.ts | 21 ++- javascript/packages/core/lib/typeInfo.ts | 73 +++++++--- javascript/packages/core/lib/typeResolver.ts | 16 +-- javascript/test/fory.test.ts | 134 +++++++++++++++++- javascript/test/rootCleanup.test.ts | 24 ++++ javascript/test/typemeta.test.ts | 6 +- 10 files changed, 270 insertions(+), 50 deletions(-) diff --git a/.agents/languages/javascript.md b/.agents/languages/javascript.md index c69d78a3da..2316563618 100644 --- a/.agents/languages/javascript.md +++ b/.agents/languages/javascript.md @@ -18,10 +18,13 @@ Load this file when changing `javascript/`. their own logical size: reset active owner IDs and that table's logical size without clearing bounded backing, and replace either backing only after its root has more than 8192 owners. - Generated registration must initialize the complete recursive serializer graph against - generation-local owners before one `TypeResolver` batch publication. Factory-init serializer - lookup may resolve those local owners for fixed captures; runtime and dynamic lookup must keep the - real resolver. Initialize an existing published forward owner in place during commit so previous - captures retain identity. Do not publish placeholders, nested serializers, descriptors, or cache + generation-local owners before one `TypeResolver` batch publication. Freeze the complete + `TypeInfo` schema graph before code generation; schema fields and occurrence modifiers are + immutable afterward, while `dynamicTypeId` remains operation-local writer state. Factory-init + serializer lookup may resolve local owners for fixed captures; runtime and dynamic lookup must + keep the real resolver. Initialize an uninitialized published forward owner in place during + commit so previous captures retain identity, but never overwrite an initialized owner published + by a nested registration. Do not publish placeholders, nested serializers, descriptors, or cache state before every generated factory and application code hook succeeds. - Runtime value carriers such as decimal or reduced-precision numeric types belong under the core `types/` ownership boundary, with imports, exports, and codegen externals updated together. - Keep `TypeInfo` as schema metadata. Compatibility-sensitive decisions belong on `TypeResolver` or explicit operations, not as retained resolver state on metadata objects. diff --git a/docs/object-serialization/javascript/type-registration.md b/docs/object-serialization/javascript/type-registration.md index e2b4a3a109..998a9045e9 100644 --- a/docs/object-serialization/javascript/type-registration.md +++ b/docs/object-serialization/javascript/type-registration.md @@ -137,6 +137,21 @@ const order = deserialize(bytes); Store and reuse this pair — it is the fast path. +Registration freezes the schema and every nested `TypeInfo`. Set field IDs, nullability, reference +tracking, and other schema options before registration. To use an already registered type as a new +field occurrence with different field options, clone it first: + +```ts +const itemType = Type.struct("example.item", { + value: Type.string(), +}); +fory.register(itemType); + +const wrapperType = Type.struct("example.wrapper", { + item: itemType.clone().setId(1).setNullable(true), +}); +``` + ## Field Metadata Field nullability, reference tracking, dynamic field behavior, numeric widths, and per-struct diff --git a/docs/specification/xlang_implementation_guide.md b/docs/specification/xlang_implementation_guide.md index dc51b84d81..4d91717ac0 100644 --- a/docs/specification/xlang_implementation_guide.md +++ b/docs/specification/xlang_implementation_guide.md @@ -92,12 +92,16 @@ Do not add rollback, staging, or a parallel registration path for that exception installation may consist of complete nested registrations; publish the module-installed marker only after installation returns and the lifecycle is rechecked. -JavaScript generated registration constructs and initializes the complete recursive serializer -graph against generation-local owners. Generated factories may use a construction-only lookup for -fixed serializer captures, while runtime and dynamic dispatch retain the real `TypeResolver`. -After every factory and application code hook succeeds, the resolver performs one guarded batch -publication. An already-published forward owner is initialized in place during that commit so -previous generated captures retain its identity. +JavaScript generated registration freezes the complete `TypeInfo` schema graph before code +generation, including nested schemas and field occurrence modifiers. The writer-owned +`dynamicTypeId` remains mutable because it is reset per root. Code generation then constructs and +initializes the complete recursive serializer graph against generation-local owners. Generated +factories may use a construction-only lookup for fixed serializer captures, while runtime and +dynamic dispatch retain the real `TypeResolver`. After every factory and application code hook +succeeds, the resolver performs one guarded batch publication. An uninitialized, already-published +forward owner is initialized in place during that commit so previous generated captures retain its +identity. An initialized owner published by a nested registration is authoritative and must not be +overwritten by the outer registration. Nested serializers must not call back into root `serialize(...)` or `deserialize(...)` entry points. diff --git a/javascript/packages/core/lib/context.ts b/javascript/packages/core/lib/context.ts index 0e166fb275..33e24af3f7 100644 --- a/javascript/packages/core/lib/context.ts +++ b/javascript/packages/core/lib/context.ts @@ -255,9 +255,7 @@ export class RefReader { constructor(private reader: BinaryReader) {} reset() { - if (this.readObjects.length !== 0) { - this.readObjects.length = 0; - } + this.readObjects = []; } getReadRef(refId: number) { diff --git a/javascript/packages/core/lib/gen/index.ts b/javascript/packages/core/lib/gen/index.ts index 647377cd47..f5d48a6be9 100644 --- a/javascript/packages/core/lib/gen/index.ts +++ b/javascript/packages/core/lib/gen/index.ts @@ -65,7 +65,6 @@ type SerializerCreator = (serializerLookup: SerializerLookup) => Serializer; interface GeneratedRegistration { typeInfo: TypeInfo; serializer: Serializer; - captureOwner: Serializer; preparing: boolean; } @@ -163,7 +162,6 @@ export class Gen { const entry: GeneratedRegistration = { typeInfo, serializer: owner, - captureOwner: this.typeResolver.getSerializerByTypeInfo(typeInfo) ?? owner, preparing: false, }; this.generatedRegistrations.push(entry); @@ -178,6 +176,10 @@ export class Gen { } private getCapturedSerializerById(id: number, userTypeId?: number) { + const published = this.typeResolver.getSerializerById(id, userTypeId); + if (published !== undefined) { + return published; + } const entry = this.generatedRegistrations.find((candidate) => { const typeId = this.typeResolver.computeTypeId(candidate.typeInfo); if (typeId !== id || TypeId.isNamedType(typeId)) { @@ -191,17 +193,21 @@ export class Gen { } return true; }); - return entry?.captureOwner ?? this.typeResolver.getSerializerById(id, userTypeId); + return entry?.serializer as Serializer; } private getCapturedSerializerByName(name: number | string) { + const published = this.typeResolver.getSerializerByName(name); + if (published !== undefined) { + return published; + } const entry = this.generatedRegistrations.find( (candidate) => typeof name === "string" && TypeId.isNamedType(this.typeResolver.computeTypeId(candidate.typeInfo)) && candidate.typeInfo.named === name, ); - return entry?.captureOwner ?? this.typeResolver.getSerializerByName(name); + return entry?.serializer; } private prepareRegistration(typeInfo: TypeInfo, children: TypeInfo[]) { @@ -277,6 +283,11 @@ export class Gen { } generateSerializer(typeInfo: TypeInfo) { + this.typeResolver.ensureRegistrationOpen(); + typeInfo.freeze(); + // TypeInfo freezing may invoke application-owned proxy traps. A root entered there closes the + // resolver before code generation or publication can continue. + this.typeResolver.ensureRegistrationOpen(); this.traversalContainer(typeInfo); const serializer = this.typeResolver.getSerializerByTypeInfo(typeInfo); if (!serializer?._initialized) { @@ -291,7 +302,7 @@ export class Gen { // Generated factories may execute application-transformed code, so every factory completes // against local owners before the resolver performs the only global publication step. - this.typeResolver.commitGeneratedSerializers(typeInfo, this.generatedRegistrations); + this.typeResolver.commitGeneratedSerializers(this.generatedRegistrations); return this.typeResolver.getSerializerByTypeInfo(typeInfo)!; } } diff --git a/javascript/packages/core/lib/typeInfo.ts b/javascript/packages/core/lib/typeInfo.ts index 809d7efcbc..77fb200d3a 100644 --- a/javascript/packages/core/lib/typeInfo.ts +++ b/javascript/packages/core/lib/typeInfo.ts @@ -149,24 +149,63 @@ export class TypeInfo extends ExtensibleFunction { }); } + /** Freezes schema-owned state recursively while leaving root write IDs operation-local. */ public freeze() { - Object.defineProperties(this, { - named: { writable: false, configurable: false }, - namespace: { writable: false, configurable: false }, - typeName: { writable: false, configurable: false }, - userTypeId: { writable: false, configurable: false }, - evolving: { writable: false, configurable: false }, - options: { writable: false, configurable: false }, - _typeId: { writable: false, configurable: false }, - nullable: { writable: false, configurable: false }, - }); - Object.freeze(this.options); - if (this.options?.props) { - Object.freeze(this.options!.props); - } - if (this.options?.enumProps) { - Object.freeze(this.options!.enumProps); - } + const seen = new Set(); + const freezeTypeInfo = (typeInfo: TypeInfo) => { + if (seen.has(typeInfo)) { + return; + } + seen.add(typeInfo); + const options = typeInfo.options; + const children: TypeInfo[] = []; + if (options !== undefined) { + if (options.props !== undefined) { + children.push(...Object.values(options.props)); + Object.freeze(options.props); + } + if (options.cases !== undefined) { + children.push(...Object.values(options.cases)); + Object.freeze(options.cases); + } + if (options.fieldEntries !== undefined) { + for (const entry of options.fieldEntries) { + children.push(entry.typeInfo); + Object.freeze(entry); + } + Object.freeze(options.fieldEntries); + } + if (options.inner !== undefined) { + children.push(options.inner); + } + if (options.key !== undefined) { + children.push(options.key); + } + if (options.value !== undefined) { + children.push(options.value); + } + if (options.enumProps !== undefined) { + Object.freeze(options.enumProps); + } + Object.freeze(options); + } + Object.defineProperties(typeInfo, { + named: { writable: false, configurable: false }, + namespace: { writable: false, configurable: false }, + typeName: { writable: false, configurable: false }, + userTypeId: { writable: false, configurable: false }, + evolving: { writable: false, configurable: false }, + options: { writable: false, configurable: false }, + _typeId: { writable: false, configurable: false }, + nullable: { writable: false, configurable: false }, + trackingRef: { writable: false, configurable: false }, + id: { writable: false, configurable: false }, + dynamic: { writable: false, configurable: false }, + }); + // dynamicTypeId is operation-local writer state and remains mutable across roots. + children.forEach(freezeTypeInfo); + }; + freezeTypeInfo(this); } public constructor(typeId: number, userTypeId = -1) { diff --git a/javascript/packages/core/lib/typeResolver.ts b/javascript/packages/core/lib/typeResolver.ts index 037f50b7d4..0bff6116c6 100644 --- a/javascript/packages/core/lib/typeResolver.ts +++ b/javascript/packages/core/lib/typeResolver.ts @@ -285,10 +285,7 @@ export default class TypeResolver { return { ...uninitSerialize }; } - commitGeneratedSerializers( - rootTypeInfo: TypeInfo, - entries: readonly { typeInfo: TypeInfo; serializer: Serializer }[], - ) { + commitGeneratedSerializers(entries: readonly { typeInfo: TypeInfo; serializer: Serializer }[]) { const publications = entries.map((entry) => { const typeId = this.computeTypeId(entry.typeInfo); let internalTypeId: number | undefined; @@ -312,18 +309,19 @@ export default class TypeResolver { customTypeKey, existingSerializer, descriptors: - existingSerializer === undefined + existingSerializer === undefined || existingSerializer._initialized ? undefined : Object.getOwnPropertyDescriptors(entry.serializer), }; }); this.ensureRegistrationOpen(); - rootTypeInfo.freeze(); for (const publication of publications) { if (publication.existingSerializer !== undefined) { - // Published forward owners are plain resolver-owned placeholders. Define their prepared - // data properties in place so earlier generated serializers retain the same owner. - Object.defineProperties(publication.existingSerializer, publication.descriptors!); + if (!publication.existingSerializer._initialized) { + // Complete only a resolver-owned forward placeholder. An initialized owner published by + // a nested registration is authoritative for this identity. + Object.defineProperties(publication.existingSerializer, publication.descriptors!); + } } else if (publication.internalTypeId !== undefined) { this.internalSerializer[publication.internalTypeId] = publication.entry.serializer; } else { diff --git a/javascript/test/fory.test.ts b/javascript/test/fory.test.ts index 9c97292095..ff1d17a23a 100644 --- a/javascript/test/fory.test.ts +++ b/javascript/test/fory.test.ts @@ -169,8 +169,7 @@ describe("fory", () => { expect(Array.from(typeResolver.customSerializer.entries())).toEqual(customBefore); expect(typeResolver.getSerializerById(TypeId.STRUCT, childType.userTypeId)).toBeUndefined(); expect(typeResolver.getSerializerById(TypeId.STRUCT, rootType.userTypeId)).toBeUndefined(); - rootType.setNullable(true); - expect(rootType.nullable).toBe(true); + expect(() => rootType.setNullable(true)).toThrow(); }); test("keeps failed generated factories local", () => { @@ -206,8 +205,7 @@ describe("fory", () => { expect(Array.from(typeResolver.customSerializer.entries())).toEqual(customBefore); expect(typeResolver.getSerializerById(TypeId.STRUCT, childType.userTypeId)).toBeUndefined(); expect(typeResolver.getSerializerById(TypeId.STRUCT, rootType.userTypeId)).toBeUndefined(); - rootType.setNullable(true); - expect(rootType.nullable).toBe(true); + expect(() => rootType.setNullable(true)).toThrow(); }); test("initializes a published forward owner in place", () => { @@ -233,6 +231,134 @@ describe("fory", () => { expect(parent.deserialize(parent.serialize(value))).toEqual(value); }); + test.each(["userTypeId", "name", "options"] as const)( + "rejects root %s changes during codegen", + (change) => { + let mutateDescriptor: (() => void) | undefined; + const fory = new Fory({ + compatible: false, + hooks: { + afterCodeGenerated(code) { + const mutate = mutateDescriptor; + mutateDescriptor = undefined; + mutate?.(); + return code; + }, + }, + }); + const typeInfo = + change === "name" + ? Type.struct("stable.Root", { value: Type.int32() }) + : Type.struct(8115, { value: Type.int32() }); + const typeResolver = fory.typeResolver as any; + const internalBefore = Array.from(typeResolver.internalSerializer); + const customBefore = Array.from(typeResolver.customSerializer.entries()); + if (change === "userTypeId") { + mutateDescriptor = () => { + typeInfo.userTypeId = 9115; + }; + } else if (change === "name") { + mutateDescriptor = () => { + typeInfo.named = "changed$Root"; + }; + } else { + mutateDescriptor = () => { + typeInfo.options!.props!.extra = Type.string(); + }; + } + + expect(() => fory.register(typeInfo)).toThrow(); + + expect(Array.from(typeResolver.internalSerializer)).toEqual(internalBefore); + expect(Array.from(typeResolver.customSerializer.entries())).toEqual(customBefore); + expect(() => typeInfo.setNullable(true)).toThrow(); + if (change === "options") { + expect(() => { + typeInfo.options!.props!.afterFailure = Type.bool(); + }).toThrow(); + } + }, + ); + + test("rejects nested schema changes during codegen", () => { + let mutateDescriptor: (() => void) | undefined; + const fory = new Fory({ + compatible: false, + hooks: { + afterCodeGenerated(code) { + const mutate = mutateDescriptor; + mutateDescriptor = undefined; + mutate?.(); + return code; + }, + }, + }); + const childType = Type.struct(8116, { value: Type.int32() }); + const rootType = Type.struct(8117, { child: childType }); + const typeResolver = fory.typeResolver as any; + const internalBefore = Array.from(typeResolver.internalSerializer); + const customBefore = Array.from(typeResolver.customSerializer.entries()); + mutateDescriptor = () => { + childType.options!.props!.extra = Type.string(); + }; + + expect(() => fory.register(rootType)).toThrow(); + + expect(Array.from(typeResolver.internalSerializer)).toEqual(internalBefore); + expect(Array.from(typeResolver.customSerializer.entries())).toEqual(customBefore); + expect(typeResolver.getSerializerById(TypeId.STRUCT, childType.userTypeId)).toBeUndefined(); + expect(typeResolver.getSerializerById(TypeId.STRUCT, rootType.userTypeId)).toBeUndefined(); + expect(() => rootType.setNullable(true)).toThrow(); + expect(() => childType.setNullable(true)).toThrow(); + }); + + test("captures a reentrant same-key owner", () => { + let registerSameKey = false; + let reentrant: ReturnType; + let fory: Fory; + fory = new Fory({ + compatible: false, + hooks: { + afterCodeGenerated(code) { + if (registerSameKey) { + registerSameKey = false; + reentrant = fory.register(Type.struct(8118, { innerValue: Type.string() })); + } + return code; + }, + }, + }); + const childType = Type.struct(8118, { outerValue: Type.int32() }); + const parentType = Type.struct(8119, { child: childType }); + + registerSameKey = true; + const parent = fory.register(parentType); + const owner = fory.typeResolver.getSerializerById(TypeId.STRUCT, childType.userTypeId); + expect(reentrant!.serializer).toBe(owner); + expect(owner.getTypeInfo()).toBe(reentrant!.serializer.getTypeInfo()); + const write = owner.write; + let childWrites = 0; + owner.write = (value) => { + childWrites++; + write(value); + }; + + const value = { child: { innerValue: "kept" } }; + expect(parent.deserialize(parent.serialize(value as any))).toEqual(value); + expect(childWrites).toBeGreaterThan(0); + }); + + test("freezes a recursive schema graph", () => { + const left = Type.struct(8120, {}); + const right = Type.struct(8121, { left }); + left.options!.props!.right = right; + + left.freeze(); + + expect(() => left.setNullable(true)).toThrow(); + expect(() => right.setTrackingRef(true)).toThrow(); + }); + test.each(["serialize", "deserialize"] as const)( "freezes registration after registered %s succeeds", (operation) => { diff --git a/javascript/test/rootCleanup.test.ts b/javascript/test/rootCleanup.test.ts index f92ab0b33f..7ff9a56e94 100644 --- a/javascript/test/rootCleanup.test.ts +++ b/javascript/test/rootCleanup.test.ts @@ -238,3 +238,27 @@ test("releases a failed root write buffer before reuse", () => { expect(fory.serialize(7)).toBeDefined(); expect(writer.getPlatformBuffer().byteLength).toBeLessThan(4 * 1024 * 1024); }); + +test("releases failed root reference backing before reuse", () => { + const fory = new Fory({ compatible: false, ref: true }); + const registered = fory.register(Type.struct(7612, {})); + const refReader = (fory as any).readContext.refReader; + let failedBacking: unknown[]; + + registered.serializer.readRef = () => { + for (let i = 0; i < 32768; i++) { + refReader.reference({}); + } + failedBacking = refReader.readObjects; + throw new Error("root read failed"); + }; + expect(() => registered.deserialize(new Uint8Array([1]))).toThrow(); + expect(refReader.readObjects).toBe(failedBacking!); + + registered.serializer.readRef = () => { + expect(refReader.readObjects).not.toBe(failedBacking!); + expect(refReader.readObjects).toHaveLength(0); + return {}; + }; + expect(registered.deserialize(new Uint8Array([1]))).toEqual({}); +}); diff --git a/javascript/test/typemeta.test.ts b/javascript/test/typemeta.test.ts index e0d0d555e4..9e95b94d04 100644 --- a/javascript/test/typemeta.test.ts +++ b/javascript/test/typemeta.test.ts @@ -509,6 +509,7 @@ describe("typemeta", () => { readContext.typeMetaCache.set(localTypeMeta.getHash(), cachedTypeMeta); expect(readerFory.deserialize(bytes, reader.serializer)).toBe(Color.Red); + expect(readContext.typeMeta[0]).toBe(localTypeMeta); expect(readContext.typeMetaCache.get(localTypeMeta.getHash())).toBe(cachedTypeMeta); expect(readContext.totalAcceptedSchemaVersions).toBe(0); }); @@ -809,6 +810,7 @@ describe("typemeta", () => { readContext.typeMetaCache.set(localTypeMeta.getHash(), cachedTypeMeta); expect(reader.deserialize(bytes)).toEqual({}); + expect(readContext.typeMeta[0]).toBe(localTypeMeta); expect(readContext.typeMetaCache.get(localTypeMeta.getHash())).toBe(cachedTypeMeta); expect(readContext.totalAcceptedSchemaVersions).toBe(0); }); @@ -1012,7 +1014,7 @@ describe("typemeta", () => { const writerChild = writerFory.register(remoteChild); const writerRoot = writerFory.register( Type.struct(rootId, { - child: remoteChild.setId(1), + child: remoteChild.clone().setId(1), }), ); const remoteTypeMeta = TypeMeta.fromTypeInfo(remoteChild, (writerFory as any).typeResolver); @@ -1027,7 +1029,7 @@ describe("typemeta", () => { const child = fory.register(localChild); const root = fory.register( Type.struct(rootId, { - child: localChild.setId(1), + child: localChild.clone().setId(1), }), ); return { fory, child, root }; From 1eb5fe6b9923754622dafbe7ca7643a8592a7f17 Mon Sep 17 00:00:00 2001 From: chaokunyang Date: Fri, 28 Aug 2026 06:04:39 +0800 Subject: [PATCH 039/168] fix(jvm): finalize registry publication atomically --- .agents/languages/java.md | 12 +- .agents/languages/kotlin.md | 8 + .agents/languages/scala.md | 8 + .../kotlin/configuration.md | 4 + .../kotlin/static-generated-serializers.md | 5 +- .../scala/configuration.md | 4 + .../xlang_implementation_guide.md | 7 + .../apache/fory/AbstractThreadSafeFory.java | 5 - .../main/java/org/apache/fory/BaseFory.java | 9 - .../apache/fory/FacadeRegistrationGate.java | 57 +++- .../src/main/java/org/apache/fory/Fory.java | 5 +- .../java/org/apache/fory/ThreadLocalFory.java | 6 +- .../java/org/apache/fory/ThreadSafeFory.java | 4 + .../org/apache/fory/config/ForyBuilder.java | 2 + .../apache/fory/resolver/TypeResolver.java | 10 +- .../org/apache/fory/ThreadSafeForyTest.java | 114 ++++++++ .../fory/resolver/ClassResolverTest.java | 87 ++++++ .../apache/fory/serializer/RegisterTest.java | 93 +++++++ .../fory/kotlin/xlang/KotlinXlangPeer.kt | 14 +- .../serializer/kotlin/KotlinSerializers.java | 220 ++++++++------- .../org/apache/fory/kotlin/ForyExtensions.kt | 32 +-- .../kotlin/BuiltinClassSerializerTests.kt | 75 +++++ .../serializer/scala/ScalaSerializers.java | 261 +++++++++--------- .../apache/fory/scala/ForyExtensions.scala | 40 ++- .../apache/fory/scala/ForySerializer.scala | 49 +--- .../fory/serializer/scala/ScalaTest.scala | 169 ++++++++++++ 26 files changed, 931 insertions(+), 369 deletions(-) diff --git a/.agents/languages/java.md b/.agents/languages/java.md index b7e3c0f3ac..e10786ccb3 100644 --- a/.agents/languages/java.md +++ b/.agents/languages/java.md @@ -90,12 +90,14 @@ Load this file when changing anything under `java/` or when Java drives a cross- accepted registration, finish registration, and only then become visible; discard a provisional child when replay or finalization fails. Keep the lock order gate before pool or child storage. - Registration callbacks must recheck the authoritative freeze owner after returning and before - publishing the entry they prepared. Combined type-and-serializer registration may construct a - serializer only after the canonical type is registered when construction resolves that - `TypeInfo`; recheck before replacing the serializer and do not add rollback or a second - registration path. `ForyModule.install` may perform complete nested registrations, but the + publishing the entry they prepared. Java combined type-and-serializer registration constructs + against an unpublished type, rechecks, then publishes both together. Use the nonpublishing + `ObjectSerializer` constructor for that exact class; reject static-generated serializer classes + from the combined class overload because their construction requires prior canonical type + registration. `ForyModule.install` may perform complete nested registrations, but the module-installed marker is published only after installation returns and the lifecycle is - rechecked. + rechecked. Direct `Fory` accepts modules before its first root; thread-safe facades accept modules + only through `ForyBuilder.withModule` before construction. - For GraalVM, use `fory codegen` to generate serializers when building native images. Do not add reflection configuration except for JDK `proxy`. - In Java native mode (`xlang=false`), only `Types.BOOL` through `Types.STRING` share type IDs with xlang mode. Other native-mode type IDs differ. - Choose one serializer ownership location per logical Java type family. Add native/xlang serializer variants only when the wire format or constructor contract truly differs. diff --git a/.agents/languages/kotlin.md b/.agents/languages/kotlin.md index a7f0e4465f..3047d44663 100644 --- a/.agents/languages/kotlin.md +++ b/.agents/languages/kotlin.md @@ -21,6 +21,14 @@ Load this file when changing `kotlin/` or compiler code that generates Kotlin so - Combined generated-struct registration must publish the canonical type before constructing its serializer because generated construction resolves the canonical `TypeInfo`. Do not move that construction before type registration or add rollback, staging, or a parallel registration path. +- Bootstrap installation markers must be published only after every nested registration completes + and an authoritative lifecycle recheck succeeds. A failed or root-reentered installation leaves + the marker absent naturally; do not publish early and add rollback or staging state. Use the + concrete `Fory` as the per-runtime monitor for the complete bootstrap, and reject same-runtime + same-thread reentry instead of recursively starting a second installation. +- Install modules for thread-safe facades through `ForyBuilder.withModule` before building them. + Runtime registration extensions target concrete `Fory` instances and must not recreate a + thread-safe module-registration wrapper. - When adding Kotlin gRPC service companions, emit Kotlin source only. Reuse the generated schema module's `ThreadSafeFory` and KSP-generated schema serializers, and keep grpc-java/grpc-kotlin dependencies application-owned instead of adding them as hard `fory-kotlin` dependencies. diff --git a/.agents/languages/scala.md b/.agents/languages/scala.md index 08ad71d746..fa2d7dead9 100644 --- a/.agents/languages/scala.md +++ b/.agents/languages/scala.md @@ -18,6 +18,14 @@ Load this file when changing `scala/`. construction before type registration or add rollback, staging, or a parallel registration path. Union construction is the exception because it does not require canonical registration: finish its serializer-owned callbacks and recheck the freeze before publishing the union type. +- Bootstrap installation markers must be published only after every nested registration completes + and an authoritative lifecycle recheck succeeds. A failed or root-reentered installation leaves + the marker absent naturally; do not publish early and add rollback or staging state. Use the + concrete `Fory` as the per-runtime monitor for the complete bootstrap, and reject same-runtime + same-thread reentry instead of recursively starting a second installation. +- Install modules for thread-safe facades through `ForyBuilder.withModule` before building them. + Runtime registration extensions target concrete `Fory` instances and must not recreate a + thread-safe module-registration wrapper. ## Commands diff --git a/docs/object-serialization/kotlin/configuration.md b/docs/object-serialization/kotlin/configuration.md index f58b2d83a9..42eb964b81 100644 --- a/docs/object-serialization/kotlin/configuration.md +++ b/docs/object-serialization/kotlin/configuration.md @@ -83,6 +83,10 @@ object ForyHolder { } ``` +Install `ForyModule` instances with `withModule(...)` before calling +`buildThreadSafeFory()`. Runtime module registration and the Kotlin reified registration extension +are available only on a direct `Fory` instance. + ### Using Builder Methods ```kotlin diff --git a/docs/object-serialization/kotlin/static-generated-serializers.md b/docs/object-serialization/kotlin/static-generated-serializers.md index fe42f884b1..bd3a21ccef 100644 --- a/docs/object-serialization/kotlin/static-generated-serializers.md +++ b/docs/object-serialization/kotlin/static-generated-serializers.md @@ -273,7 +273,10 @@ fory.register("example.User") `ForyKotlin.builder()` installs the Kotlin serializer bootstrap for the Fory instance. The `fory.register(...)` extension registers your xlang schema type -name and resolves the generated serializer from the target class. +name and resolves the generated serializer from the target class. This extension +targets a direct `Fory` instance. For a `ThreadSafeFory`, put generated type +registrations in a `ForyModule` and pass it to `withModule(...)` before building +the facade. Do not register or reference generated serializer classes in application code. Fory resolves them from the registered target class. diff --git a/docs/object-serialization/scala/configuration.md b/docs/object-serialization/scala/configuration.md index 2524eaeb0e..f2b7349b6f 100644 --- a/docs/object-serialization/scala/configuration.md +++ b/docs/object-serialization/scala/configuration.md @@ -120,6 +120,10 @@ object ForyHolder { } ``` +Install `ForyModule` instances with `withModule(...)` before calling +`buildThreadSafeFory()`. Runtime module registration and Scala generated-serializer registration +extensions are available only on a direct `Fory` instance. + ## Configuration All configuration options from Fory Java are available. See [Java Configuration](../java/configuration.md) for the complete list. diff --git a/docs/specification/xlang_implementation_guide.md b/docs/specification/xlang_implementation_guide.md index 4d91717ac0..57364faf17 100644 --- a/docs/specification/xlang_implementation_guide.md +++ b/docs/specification/xlang_implementation_guide.md @@ -92,6 +92,13 @@ Do not add rollback, staging, or a parallel registration path for that exception installation may consist of complete nested registrations; publish the module-installed marker only after installation returns and the lifecycle is rechecked. +Java combined type-and-serializer registration constructs against an unpublished type, rechecks +the registry lifecycle, and then publishes the type and serializer together. The exact +`ObjectSerializer` path uses its nonpublishing constructor. Static-generated serializer classes +require an already registered canonical type and are therefore rejected by the combined class +overload. Direct Java `Fory` instances may install a module before their first root operation; +thread-safe facades install modules only through `ForyBuilder.withModule` during construction. + JavaScript generated registration freezes the complete `TypeInfo` schema graph before code generation, including nested schemas and field occurrence modifiers. The writer-owned `dynamicTypeId` remains mutable because it is reset per root. Code generation then constructs and diff --git a/java/fory-core/src/main/java/org/apache/fory/AbstractThreadSafeFory.java b/java/fory-core/src/main/java/org/apache/fory/AbstractThreadSafeFory.java index 0098854119..1fa4aa2daa 100644 --- a/java/fory-core/src/main/java/org/apache/fory/AbstractThreadSafeFory.java +++ b/java/fory-core/src/main/java/org/apache/fory/AbstractThreadSafeFory.java @@ -66,11 +66,6 @@ public void register(String className, String namespace, String typeName) { registerCallback(fory -> fory.register(className, namespace, typeName)); } - @Override - public void register(ForyModule module) { - registerCallback(fory -> fory.register(module)); - } - public void registerUnion( Class cls, int id, org.apache.fory.serializer.Serializer serializer) { registerCallback(fory -> fory.registerUnion(cls, id, serializer)); diff --git a/java/fory-core/src/main/java/org/apache/fory/BaseFory.java b/java/fory-core/src/main/java/org/apache/fory/BaseFory.java index c51ac2488c..e6c7ce3ab9 100644 --- a/java/fory-core/src/main/java/org/apache/fory/BaseFory.java +++ b/java/fory-core/src/main/java/org/apache/fory/BaseFory.java @@ -89,15 +89,6 @@ public interface BaseFory { */ void register(String className, String namespace, String typeName); - /** - * Register a runtime module. Direct {@link Fory} instances install the module immediately; - * thread-safe runtimes install it into every underlying runtime instance. - * - *

For thread-safe runtimes, call this during setup before concurrent serialization, - * deserialization, or copy operations start. - */ - void register(ForyModule module); - void registerUnion(Class cls, int id, Serializer serializer); /** diff --git a/java/fory-core/src/main/java/org/apache/fory/FacadeRegistrationGate.java b/java/fory-core/src/main/java/org/apache/fory/FacadeRegistrationGate.java index 87b34ea7ce..e515a2eb7c 100644 --- a/java/fory-core/src/main/java/org/apache/fory/FacadeRegistrationGate.java +++ b/java/fory-core/src/main/java/org/apache/fory/FacadeRegistrationGate.java @@ -26,9 +26,16 @@ /** Owns the permanent registration freeze before a thread-safe facade's first root or callback. */ @Internal public final class FacadeRegistrationGate { + private enum RegistrationState { + OPEN, + FINALIZING, + FROZEN, + FAILED + } + private final Object lock = new Object(); private final Runnable finishChildren; - private volatile boolean frozen; + private volatile RegistrationState state = RegistrationState.OPEN; public FacadeRegistrationGate(Runnable finishChildren) { this.finishChildren = finishChildren; @@ -42,6 +49,15 @@ public void applyRegistration(Runnable action) { } } + public void applyRegistration(Runnable prepare, Runnable publish) { + synchronized (lock) { + checkRegistrationAllowed(); + prepare.run(); + checkRegistrationAllowed(); + publish.run(); + } + } + /** Initializes a child while registration callbacks cannot change. */ public Fory initializeChild(Supplier initializer) { synchronized (lock) { @@ -49,21 +65,42 @@ public Fory initializeChild(Supplier initializer) { } } + void finishChildIfFrozen(Fory child) { + if (state == RegistrationState.FROZEN) { + child.getTypeResolver().finishRegistration(); + } + } + public void freeze() { - if (!frozen) { - synchronized (lock) { - if (!frozen) { - // Set the permanent facade state first. If child finalization fails, registration must - // remain closed rather than reopening a partially finalized facade. - frozen = true; - finishChildren.run(); - } + RegistrationState current = state; + if (current == RegistrationState.FROZEN) { + return; + } + synchronized (lock) { + current = state; + if (current == RegistrationState.FROZEN) { + return; + } + if (current == RegistrationState.FAILED) { + throw new ForyException("ThreadSafeFory registration finalization previously failed."); + } + if (current == RegistrationState.FINALIZING) { + throw new ForyException("ThreadSafeFory registration finalization is already in progress."); + } + state = RegistrationState.FINALIZING; + try { + finishChildren.run(); + state = RegistrationState.FROZEN; + } catch (RuntimeException | Error e) { + // Registration remains permanently closed after a failed first finalization. + state = RegistrationState.FAILED; + throw e; } } } private void checkRegistrationAllowed() { - if (frozen) { + if (state != RegistrationState.OPEN) { throw new ForyException( "Cannot register class/serializer after registration has been frozen. Please register " + "all classes before invoking top-level `serialize/deserialize/copy` methods of " diff --git a/java/fory-core/src/main/java/org/apache/fory/Fory.java b/java/fory-core/src/main/java/org/apache/fory/Fory.java index ea602f0eed..054b806cde 100644 --- a/java/fory-core/src/main/java/org/apache/fory/Fory.java +++ b/java/fory-core/src/main/java/org/apache/fory/Fory.java @@ -220,7 +220,10 @@ public void register(String className, String namespace, String typeName) { getTypeResolver().register(className, namespace, typeName); } - @Override + /** + * Installs a module into this runtime before its first root operation. Configure modules for a + * thread-safe facade through {@link ForyBuilder#withModule(ForyModule)} before building it. + */ public void register(ForyModule module) { Preconditions.checkNotNull(module); checkRegisterAllowed(); diff --git a/java/fory-core/src/main/java/org/apache/fory/ThreadLocalFory.java b/java/fory-core/src/main/java/org/apache/fory/ThreadLocalFory.java index 1e6901146c..1e02ab5ee5 100644 --- a/java/fory-core/src/main/java/org/apache/fory/ThreadLocalFory.java +++ b/java/fory-core/src/main/java/org/apache/fory/ThreadLocalFory.java @@ -73,6 +73,8 @@ private Fory newFory() { try { allFory.put(fory, null); factoryCallback.accept(fory); + // Keep finalization in this failure scope so an unusable late child is not retained. + registrationGate.finishChildIfFrozen(fory); return fory; } catch (RuntimeException | Error e) { foryThreadLocal.remove(); @@ -103,8 +105,8 @@ public void registerCallback(Consumer callback) { synchronized (allFory) { allFory.keySet().forEach(callback); } - factoryCallback = factoryCallback.andThen(callback); - }); + }, + () -> factoryCallback = factoryCallback.andThen(callback)); } @Override diff --git a/java/fory-core/src/main/java/org/apache/fory/ThreadSafeFory.java b/java/fory-core/src/main/java/org/apache/fory/ThreadSafeFory.java index b92fc4729c..6f436ef40f 100644 --- a/java/fory-core/src/main/java/org/apache/fory/ThreadSafeFory.java +++ b/java/fory-core/src/main/java/org/apache/fory/ThreadSafeFory.java @@ -22,6 +22,7 @@ import java.util.function.Consumer; import java.util.function.Function; import org.apache.fory.annotation.Internal; +import org.apache.fory.config.ForyBuilder; import org.apache.fory.resolver.TypeChecker; /** @@ -30,6 +31,9 @@ * *

The runtime class loader is fixed when the thread-safe serializer is built. If you need a * different class loader, build a different {@link ThreadSafeFory} instance. + * + *

Configure runtime modules through {@link ForyBuilder#withModule(ForyModule)} before building + * the facade. */ public interface ThreadSafeFory extends BaseFory { diff --git a/java/fory-core/src/main/java/org/apache/fory/config/ForyBuilder.java b/java/fory-core/src/main/java/org/apache/fory/config/ForyBuilder.java index b1887ce7aa..a4a2de1d22 100644 --- a/java/fory-core/src/main/java/org/apache/fory/config/ForyBuilder.java +++ b/java/fory-core/src/main/java/org/apache/fory/config/ForyBuilder.java @@ -425,6 +425,8 @@ public ForyBuilder withSerializerFactory(SerializerFactory serializerFactory) { * *

Repeated registration of the same module object is ignored. Dedupe uses identity, not {@link * Object#equals(Object)}, so distinct module instances are installed independently. + * + *

Thread-safe facades accept modules only through this builder configuration. */ public ForyBuilder withModule(ForyModule module) { ForyModule checkedModule = Objects.requireNonNull(module); diff --git a/java/fory-core/src/main/java/org/apache/fory/resolver/TypeResolver.java b/java/fory-core/src/main/java/org/apache/fory/resolver/TypeResolver.java index 4cddffd339..2774fda034 100644 --- a/java/fory-core/src/main/java/org/apache/fory/resolver/TypeResolver.java +++ b/java/fory-core/src/main/java/org/apache/fory/resolver/TypeResolver.java @@ -417,7 +417,15 @@ public final void finishRegistration() { public void registerSerializerAndType( Class type, Class serializerClass) { checkRegisterAllowed(); - Serializer serializer = newSerializer(type, serializerClass); + if (StaticGeneratedStructSerializer.class.isAssignableFrom(serializerClass)) { + throw new ForyException( + "Static generated serializers require registering the type first, then installing a " + + "constructed serializer instance with registerSerializer."); + } + Serializer serializer = + serializerClass == ObjectSerializer.class + ? new ObjectSerializer<>(this, type, false) + : newSerializer(type, serializerClass); // Serializer construction may invoke application code which starts a root operation. Keep // type and serializer publication together after the authoritative lifecycle check. checkRegisterAllowed(); diff --git a/java/fory-core/src/test/java/org/apache/fory/ThreadSafeForyTest.java b/java/fory-core/src/test/java/org/apache/fory/ThreadSafeForyTest.java index 5a5b25ab90..3d4a280840 100644 --- a/java/fory-core/src/test/java/org/apache/fory/ThreadSafeForyTest.java +++ b/java/fory-core/src/test/java/org/apache/fory/ThreadSafeForyTest.java @@ -644,6 +644,8 @@ public void testExecuteFreezesThreadLocal() throws Exception { executor.submit(() -> fory.execute(value -> value)).get(10, TimeUnit.SECONDS); ClassResolver otherResolver = (ClassResolver) otherThreadFory.getTypeResolver(); assertNotNull(otherResolver.getRegisteredClassId(BeanA.class)); + assertTrue(otherResolver.isRegistrationFinished()); + Assert.assertThrows(ForyException.class, () -> otherThreadFory.register(BeanB.class)); assertNull(otherResolver.getRegisteredClassId(BeanB.class)); } finally { executor.shutdownNow(); @@ -687,6 +689,85 @@ public void testRegistrationGateLinearization() throws Exception { } } + @Test + public void testFreezeWaitsForChildren() throws Exception { + CountDownLatch finishEntered = new CountDownLatch(1); + CountDownLatch releaseFinish = new CountDownLatch(1); + FacadeRegistrationGate gate = + new FacadeRegistrationGate( + () -> { + finishEntered.countDown(); + awaitUnchecked(releaseFinish); + }); + ExecutorService executor = Executors.newFixedThreadPool(2); + try { + Future first = executor.submit(gate::freeze); + assertTrue(finishEntered.await(10, TimeUnit.SECONDS)); + CountDownLatch secondStarted = new CountDownLatch(1); + Future second = + executor.submit( + () -> { + secondStarted.countDown(); + gate.freeze(); + }); + assertTrue(secondStarted.await(10, TimeUnit.SECONDS)); + Assert.assertThrows(TimeoutException.class, () -> second.get(100, TimeUnit.MILLISECONDS)); + + releaseFinish.countDown(); + first.get(10, TimeUnit.SECONDS); + second.get(10, TimeUnit.SECONDS); + } finally { + releaseFinish.countDown(); + executor.shutdownNow(); + } + } + + @Test + public void testFailedFreezeStaysClosed() { + AtomicInteger finishCalls = new AtomicInteger(); + FacadeRegistrationGate gate = + new FacadeRegistrationGate( + () -> { + finishCalls.incrementAndGet(); + throw new IllegalStateException("failed"); + }); + + Assert.assertThrows(IllegalStateException.class, gate::freeze); + Assert.assertThrows(ForyException.class, gate::freeze); + Assert.assertThrows(ForyException.class, () -> gate.applyRegistration(() -> {})); + assertEquals(finishCalls.get(), 1); + } + + @Test + public void testRejectedCallbackNotReplayed() throws Exception { + ThreadLocalFory facade = + Fory.builder() + .withXlang(false) + .requireClassRegistration(true) + .withCompatible(false) + .buildThreadLocalFory(); + AtomicInteger callbackCalls = new AtomicInteger(); + Assert.assertThrows( + ForyException.class, + () -> + facade.registerCallback( + child -> { + callbackCalls.incrementAndGet(); + facade.serialize("freeze"); + })); + assertEquals(callbackCalls.get(), 1); + + ExecutorService executor = Executors.newSingleThreadExecutor(); + try { + Fory lateChild = + executor.submit(() -> facade.execute(child -> child)).get(10, TimeUnit.SECONDS); + assertTrue(lateChild.getTypeResolver().isRegistrationFinished()); + assertEquals(callbackCalls.get(), 1); + } finally { + executor.shutdownNow(); + } + } + @Test public void testReentrantRegistrationFreeze() throws Exception { ThreadLocalFory threadLocal = @@ -740,6 +821,39 @@ private static Fory[] threadLocalChildren(ThreadLocalFory facade) throws Excepti } } + @Test + public void testBuilderModuleForLateChild() throws Exception { + AtomicInteger installs = new AtomicInteger(); + ForyModule module = + child -> { + installs.incrementAndGet(); + child.registerSerializerAndType(Foo.class, FooSerializer.class); + }; + ThreadLocalFory facade = + Fory.builder() + .withXlang(false) + .requireClassRegistration(true) + .withModule(module) + .withCompatible(false) + .buildThreadLocalFory(); + assertEquals(installs.get(), 1); + facade.serialize("freeze"); + + ExecutorService executor = Executors.newSingleThreadExecutor(); + try { + Foo value = new Foo(); + value.f1 = 42; + Foo result = + executor + .submit(() -> facade.deserialize(facade.serialize(value), Foo.class)) + .get(10, TimeUnit.SECONDS); + assertEquals(result, value); + assertEquals(installs.get(), 2); + } finally { + executor.shutdownNow(); + } + } + @Test public void testLateChildReplaysRegistration() throws Exception { ThreadLocalFory facade = diff --git a/java/fory-core/src/test/java/org/apache/fory/resolver/ClassResolverTest.java b/java/fory-core/src/test/java/org/apache/fory/resolver/ClassResolverTest.java index d7112d2e4f..c626dbd356 100644 --- a/java/fory-core/src/test/java/org/apache/fory/resolver/ClassResolverTest.java +++ b/java/fory-core/src/test/java/org/apache/fory/resolver/ClassResolverTest.java @@ -28,8 +28,11 @@ import com.google.common.collect.ImmutableMap; import com.google.common.primitives.Primitives; +import java.io.ByteArrayOutputStream; +import java.io.PrintStream; import java.io.Serializable; import java.lang.reflect.Constructor; +import java.lang.reflect.InvocationTargetException; import java.lang.reflect.Method; import java.nio.charset.StandardCharsets; import java.util.ArrayList; @@ -61,11 +64,14 @@ import org.apache.fory.context.WriteContext; import org.apache.fory.exception.ForyException; import org.apache.fory.exception.InsecureException; +import org.apache.fory.logging.LogLevel; import org.apache.fory.logging.Logger; import org.apache.fory.logging.LoggerFactory; import org.apache.fory.memory.MemoryBuffer; import org.apache.fory.memory.MemoryUtils; import org.apache.fory.meta.ClassSpec; +import org.apache.fory.meta.EncodedMetaString; +import org.apache.fory.meta.Encoders; import org.apache.fory.meta.FieldTypes; import org.apache.fory.meta.TypeDef; import org.apache.fory.reflect.TypeRef; @@ -296,6 +302,27 @@ public void testGetSerializerClass() throws ClassNotFoundException { MapSerializers.DefaultJavaMapSerializer.class); } + @Test + public void testSuppressXtypeWarnings() throws Exception { + String suppressed = + captureOutput( + () -> { + Fory fory = newUnknownClassFory(true); + resolveMissingXtype(fory, "FirstType"); + resolveMissingXtype(fory, "SecondType"); + }); + assertEquals(count(suppressed, " WARN XtypeResolver:"), 0); + + String unsuppressed = + captureOutput( + () -> { + Fory fory = newUnknownClassFory(false); + resolveMissingXtype(fory, "ThirdType"); + resolveMissingXtype(fory, "FourthType"); + }); + assertEquals(count(unsuppressed, " WARN XtypeResolver:"), 1); + } + @Test public void testSharedRegistrySharesTypeDefCachesAcrossForyInstances() { ForyBuilder builder = @@ -1908,4 +1935,64 @@ public int hashCode() { return Objects.hash(codes); } } + + private static Fory newUnknownClassFory(boolean suppressWarnings) { + return Fory.builder() + .withXlang(true) + .withMetaShare(true) + .withDeserializeUnknownClass(true) + .suppressClassRegistrationWarnings(suppressWarnings) + .build(); + } + + private static String captureOutput(Runnable action) throws Exception { + int previousLogLevel = LoggerFactory.getLogLevel(); + PrintStream previousOut = System.out; + ByteArrayOutputStream out = new ByteArrayOutputStream(); + try { + LoggerFactory.setLogLevel(LogLevel.WARN_LEVEL); + System.setOut(new PrintStream(out, true, StandardCharsets.UTF_8.name())); + action.run(); + } finally { + System.setOut(previousOut); + LoggerFactory.setLogLevel(previousLogLevel); + } + return out.toString(StandardCharsets.UTF_8.name()); + } + + private static void resolveMissingXtype(Fory fory, String typeName) { + try { + Method method = + XtypeResolver.class.getDeclaredMethod( + "loadBytesToTypeInfoWithTypeId", + int.class, + EncodedMetaString.class, + EncodedMetaString.class); + method.setAccessible(true); + method.invoke( + fory.getTypeResolver(), + Types.NAMED_STRUCT, + Encoders.PACKAGE_ENCODER.encodeBinary("missing.pkg"), + Encoders.TYPE_NAME_ENCODER.encodeBinary(typeName)); + } catch (InvocationTargetException e) { + if (!(e.getCause() instanceof IllegalStateException)) { + throw new AssertionError(e.getCause()); + } + } catch (ReflectiveOperationException e) { + throw new AssertionError(e); + } + } + + private static int count(String text, String pattern) { + int count = 0; + int from = 0; + while (true) { + int index = text.indexOf(pattern, from); + if (index < 0) { + return count; + } + count++; + from = index + pattern.length(); + } + } } diff --git a/java/fory-core/src/test/java/org/apache/fory/serializer/RegisterTest.java b/java/fory-core/src/test/java/org/apache/fory/serializer/RegisterTest.java index b26428f800..ec270faa66 100644 --- a/java/fory-core/src/test/java/org/apache/fory/serializer/RegisterTest.java +++ b/java/fory-core/src/test/java/org/apache/fory/serializer/RegisterTest.java @@ -19,7 +19,9 @@ package org.apache.fory.serializer; +import java.util.Collections; import java.util.IdentityHashMap; +import java.util.List; import java.util.concurrent.atomic.AtomicBoolean; import java.util.concurrent.atomic.AtomicInteger; import java.util.concurrent.atomic.AtomicReference; @@ -27,11 +29,14 @@ import org.apache.fory.ForyModule; import org.apache.fory.ForyTestBase; import org.apache.fory.TestUtils; +import org.apache.fory.builder.Generated; import org.apache.fory.config.ForyBuilder; import org.apache.fory.context.ReadContext; import org.apache.fory.context.WriteContext; import org.apache.fory.exception.ForyException; +import org.apache.fory.meta.TypeDef; import org.apache.fory.resolver.TypeResolver; +import org.apache.fory.type.Descriptor; import org.testng.Assert; import org.testng.annotations.Test; @@ -265,6 +270,54 @@ public void testReentrantCombinedRegistration(boolean xlang) { Assert.assertNull(fory.getTypeResolver().getTypeInfo(MyExt.class, false)); } + @Test + public void testReentrantObjectRegistration() { + Fory fory = + Fory.builder() + .withXlang(false) + .withCodegen(false) + .requireClassRegistration(false) + .withCompatible(false) + .build(); + AtomicInteger factoryCalls = new AtomicInteger(); + fory.registerSerializerFactory( + (resolver, type) -> { + if (type != ObjectField.class) { + return null; + } + factoryCalls.incrementAndGet(); + fory.serialize("freeze"); + return new ObjectFieldSerializer(resolver); + }); + + Assert.assertThrows( + ForyException.class, + () -> fory.registerSerializerAndType(ObjectHolder.class, ObjectSerializer.class)); + Assert.assertEquals(factoryCalls.get(), 1); + Assert.assertTrue(fory.getTypeResolver().isRegistrationFinished()); + Assert.assertFalse(fory.getTypeResolver().isRegistered(ObjectHolder.class)); + Assert.assertNull(fory.getTypeResolver().getTypeInfo(ObjectHolder.class, false)); + } + + @Test(dataProvider = "xlang") + public void testStaticGeneratedClassRejected(boolean xlang) { + Fory fory = + Fory.builder() + .withXlang(xlang) + .withCodegen(false) + .requireClassRegistration(false) + .withCompatible(false) + .build(); + RejectedStaticSerializer.CONSTRUCTIONS.set(0); + + Assert.assertThrows( + ForyException.class, + () -> fory.registerSerializerAndType(ObjectHolder.class, RejectedStaticSerializer.class)); + Assert.assertEquals(RejectedStaticSerializer.CONSTRUCTIONS.get(), 0); + Assert.assertFalse(fory.getTypeResolver().isRegistered(ObjectHolder.class)); + Assert.assertNull(fory.getTypeResolver().getTypeInfo(ObjectHolder.class, false)); + } + public static class ReentrantSerializer extends MyExtSerializer { private static final AtomicReference CONSTRUCTION = new AtomicReference<>(); @@ -274,6 +327,46 @@ public ReentrantSerializer(TypeResolver typeResolver) { } } + public static class ObjectHolder { + public ObjectField field; + } + + public static final class ObjectField {} + + public static class ObjectFieldSerializer extends Serializer { + public ObjectFieldSerializer(TypeResolver typeResolver) { + super(typeResolver.getConfig(), ObjectField.class); + } + + @Override + public void write(WriteContext writeContext, ObjectField value) {} + + @Override + public ObjectField read(ReadContext readContext) { + return new ObjectField(); + } + } + + public static final class RejectedStaticSerializer + extends Generated.GeneratedStaticCompatibleSerializer { + private static final AtomicInteger CONSTRUCTIONS = new AtomicInteger(); + + public RejectedStaticSerializer(TypeResolver resolver, Class type, TypeDef typeDef) { + super(resolver, type, typeDef, Collections.emptyList()); + CONSTRUCTIONS.incrementAndGet(); + } + + @Override + public List getGeneratedDescriptors() { + return Collections.emptyList(); + } + + @Override + public Object readCompatible(ReadContext readContext) { + throw new UnsupportedOperationException(); + } + } + public static class MyExtSerializer extends Serializer { public MyExtSerializer(TypeResolver typeResolver) { super(typeResolver.getConfig(), MyExt.class); diff --git a/kotlin/fory-kotlin-tests/src/main/kotlin/org/apache/fory/kotlin/xlang/KotlinXlangPeer.kt b/kotlin/fory-kotlin-tests/src/main/kotlin/org/apache/fory/kotlin/xlang/KotlinXlangPeer.kt index cd9e820043..6a6d8f17c3 100644 --- a/kotlin/fory-kotlin-tests/src/main/kotlin/org/apache/fory/kotlin/xlang/KotlinXlangPeer.kt +++ b/kotlin/fory-kotlin-tests/src/main/kotlin/org/apache/fory/kotlin/xlang/KotlinXlangPeer.kt @@ -32,6 +32,7 @@ import kotlin.time.Duration import kotlin.time.Duration.Companion.milliseconds import org.apache.fory.BaseFory import org.apache.fory.Fory +import org.apache.fory.ForyModule import org.apache.fory.annotation.ArrayType import org.apache.fory.annotation.ForyCase import org.apache.fory.annotation.ForyField @@ -946,15 +947,18 @@ private fun compatibleDefaultRoundTrip() { private fun checkNoArgRegisterReceivers() { checkNoArgRegister(newFory()) - checkNoArgRegister( + val module = ForyModule { it.register() } + checkNoArgRegistered( ForyKotlin.builder() + .withModule(module) .withXlang(true) .requireClassRegistration(true) .withRefTracking(false) .buildThreadLocalFory() ) - checkNoArgRegister( + checkNoArgRegistered( ForyKotlin.builder() + .withModule(module) .withXlang(true) .requireClassRegistration(true) .withRefTracking(false) @@ -962,8 +966,12 @@ private fun checkNoArgRegisterReceivers() { ) } -private fun checkNoArgRegister(fory: BaseFory) { +private fun checkNoArgRegister(fory: Fory) { fory.register() + checkNoArgRegistered(fory) +} + +private fun checkNoArgRegistered(fory: BaseFory) { val value = KotlinInternalUser(id = 7u, name = "receiver") check(fory.deserialize(fory.serialize(value), KotlinInternalUser::class.java) == value) } diff --git a/kotlin/fory-kotlin/src/main/java/org/apache/fory/serializer/kotlin/KotlinSerializers.java b/kotlin/fory-kotlin/src/main/java/org/apache/fory/serializer/kotlin/KotlinSerializers.java index d9f93e2662..69e7ff3294 100644 --- a/kotlin/fory-kotlin/src/main/java/org/apache/fory/serializer/kotlin/KotlinSerializers.java +++ b/kotlin/fory-kotlin/src/main/java/org/apache/fory/serializer/kotlin/KotlinSerializers.java @@ -34,7 +34,6 @@ import kotlin.time.TimedValue; import kotlin.uuid.Uuid; import org.apache.fory.Fory; -import org.apache.fory.ThreadSafeFory; import org.apache.fory.codegen.GeneratedClassNames; import org.apache.fory.config.Config; import org.apache.fory.exception.ForyException; @@ -52,126 +51,125 @@ public class KotlinSerializers { private static final Map INSTALLED_FORY = Collections.synchronizedMap(new WeakHashMap<>()); - public static void registerSerializers(ThreadSafeFory fory) { - fory.register(KotlinSerializers::registerSerializers); - } - public static void registerSerializers(Fory fory) { TypeResolver resolver = fory.getTypeResolver(); checkRegistrationOpen(resolver); - synchronized (INSTALLED_FORY) { + // The runtime is the bootstrap's natural owner, so its monitor linearizes only this install. + // Since Java monitors are reentrant, reject a recursive install before entering it again. + if (Thread.holdsLock(fory)) { if (INSTALLED_FORY.containsKey(fory)) { return; } - INSTALLED_FORY.put(fory, Boolean.TRUE); + throw new ForyException("Reentrant Kotlin serializer bootstrap is not supported."); } - try { - DefaultValueUtils.setKotlinDefaultValueSupport(new KotlinDefaultValueSupport()); - if (resolver.isCrossLanguage()) { + synchronized (fory) { + checkRegistrationOpen(resolver); + if (INSTALLED_FORY.containsKey(fory)) { return; } - Config config = resolver.getConfig(); - - // UByte - Class ubyteClass = KotlinToJavaClass.INSTANCE.getUByteClass(); - registerIfAbsent(resolver, ubyteClass); - resolver.registerSerializer(ubyteClass, new UByteSerializer(config)); - - // UShort - Class ushortClass = KotlinToJavaClass.INSTANCE.getUShortClass(); - registerIfAbsent(resolver, ushortClass); - resolver.registerSerializer(ushortClass, new UShortSerializer(config)); - - // UInt - Class uintClass = KotlinToJavaClass.INSTANCE.getUIntClass(); - registerIfAbsent(resolver, uintClass); - resolver.registerSerializer(uintClass, new UIntSerializer(config)); - - // ULong - Class ulongClass = KotlinToJavaClass.INSTANCE.getULongClass(); - registerIfAbsent(resolver, ulongClass); - resolver.registerSerializer(ulongClass, new ULongSerializer(config)); - - // EmptyList - Class emptyListClass = KotlinToJavaClass.INSTANCE.getEmptyListClass(); - registerIfAbsent(resolver, emptyListClass); - resolver.registerSerializer( - emptyListClass, new CollectionSerializers.EmptyListSerializer(resolver, emptyListClass)); - - // EmptySet - Class emptySetClass = KotlinToJavaClass.INSTANCE.getEmptySetClass(); - registerIfAbsent(resolver, emptySetClass); - resolver.registerSerializer( - emptySetClass, new CollectionSerializers.EmptySetSerializer(resolver, emptySetClass)); - - // EmptyMap - Class emptyMapClass = KotlinToJavaClass.INSTANCE.getEmptyMapClass(); - registerIfAbsent(resolver, emptyMapClass); - resolver.registerSerializer( - emptyMapClass, new MapSerializers.EmptyMapSerializer(resolver, emptyMapClass)); - - // Non-Java collection implementation in kotlin stdlib. - Class arrayDequeClass = KotlinToJavaClass.INSTANCE.getArrayDequeClass(); - registerIfAbsent(resolver, arrayDequeClass); - resolver.registerSerializer( - arrayDequeClass, new KotlinArrayDequeSerializer(resolver, arrayDequeClass)); - - // Unsigned array classes: UByteArray, UShortArray, UIntArray, ULongArray. - registerIfAbsent(resolver, UByteArray.class); - resolver.registerSerializer(UByteArray.class, new UByteArraySerializer(resolver)); - registerIfAbsent(resolver, UShortArray.class); - resolver.registerSerializer(UShortArray.class, new UShortArraySerializer(resolver)); - registerIfAbsent(resolver, UIntArray.class); - resolver.registerSerializer(UIntArray.class, new UIntArraySerializer(resolver)); - registerIfAbsent(resolver, ULongArray.class); - resolver.registerSerializer(ULongArray.class, new ULongArraySerializer(resolver)); - - // Ranges and Progressions. - registerIfAbsent(resolver, kotlin.ranges.CharRange.class); - registerIfAbsent(resolver, kotlin.ranges.CharProgression.class); - registerIfAbsent(resolver, kotlin.ranges.IntRange.class); - registerIfAbsent(resolver, kotlin.ranges.IntProgression.class); - registerIfAbsent(resolver, kotlin.ranges.LongRange.class); - registerIfAbsent(resolver, kotlin.ranges.LongProgression.class); - registerIfAbsent(resolver, kotlin.ranges.UIntRange.class); - registerIfAbsent(resolver, kotlin.ranges.UIntProgression.class); - registerIfAbsent(resolver, kotlin.ranges.ULongRange.class); - registerIfAbsent(resolver, kotlin.ranges.ULongProgression.class); - - // Built-in classes. - registerIfAbsent(resolver, kotlin.Pair.class); - registerIfAbsent(resolver, kotlin.Triple.class); - registerIfAbsent(resolver, kotlin.Result.class); - registerIfAbsent(resolver, Result.Failure.class); - - // kotlin.random - registerIfAbsent(resolver, KotlinToJavaClass.INSTANCE.getRandomDefaultClass()); - registerIfAbsent(resolver, KotlinToJavaClass.INSTANCE.getRandomInternalClass()); - registerIfAbsent(resolver, KotlinToJavaClass.INSTANCE.getRandomSerializedClass()); - - // kotlin.text - registerIfAbsent(resolver, Regex.class); - registerIfAbsent(resolver, KotlinToJavaClass.INSTANCE.getRegexSerializedClass()); - registerIfAbsent(resolver, RegexOption.class); - registerIfAbsent(resolver, CharCategory.class); - registerIfAbsent(resolver, CharDirectionality.class); - registerIfAbsent(resolver, HexFormat.class); - registerIfAbsent(resolver, MatchGroup.class); - - // kotlin.time - registerIfAbsent(resolver, DurationUnit.class); - registerIfAbsent(resolver, Duration.class); - resolver.registerSerializer(Duration.class, new DurationSerializer(config)); - registerIfAbsent(resolver, TimedValue.class); - - // kotlin.uuid - registerIfAbsent(resolver, Uuid.class); - resolver.registerSerializer(Uuid.class, new UuidSerializer(config)); - } catch (RuntimeException | Error e) { - synchronized (INSTALLED_FORY) { - INSTALLED_FORY.remove(fory); + DefaultValueUtils.setKotlinDefaultValueSupport(new KotlinDefaultValueSupport()); + if (!resolver.isCrossLanguage()) { + Config config = resolver.getConfig(); + + // UByte + Class ubyteClass = KotlinToJavaClass.INSTANCE.getUByteClass(); + registerIfAbsent(resolver, ubyteClass); + resolver.registerSerializer(ubyteClass, new UByteSerializer(config)); + + // UShort + Class ushortClass = KotlinToJavaClass.INSTANCE.getUShortClass(); + registerIfAbsent(resolver, ushortClass); + resolver.registerSerializer(ushortClass, new UShortSerializer(config)); + + // UInt + Class uintClass = KotlinToJavaClass.INSTANCE.getUIntClass(); + registerIfAbsent(resolver, uintClass); + resolver.registerSerializer(uintClass, new UIntSerializer(config)); + + // ULong + Class ulongClass = KotlinToJavaClass.INSTANCE.getULongClass(); + registerIfAbsent(resolver, ulongClass); + resolver.registerSerializer(ulongClass, new ULongSerializer(config)); + + // EmptyList + Class emptyListClass = KotlinToJavaClass.INSTANCE.getEmptyListClass(); + registerIfAbsent(resolver, emptyListClass); + resolver.registerSerializer( + emptyListClass, + new CollectionSerializers.EmptyListSerializer(resolver, emptyListClass)); + + // EmptySet + Class emptySetClass = KotlinToJavaClass.INSTANCE.getEmptySetClass(); + registerIfAbsent(resolver, emptySetClass); + resolver.registerSerializer( + emptySetClass, new CollectionSerializers.EmptySetSerializer(resolver, emptySetClass)); + + // EmptyMap + Class emptyMapClass = KotlinToJavaClass.INSTANCE.getEmptyMapClass(); + registerIfAbsent(resolver, emptyMapClass); + resolver.registerSerializer( + emptyMapClass, new MapSerializers.EmptyMapSerializer(resolver, emptyMapClass)); + + // Non-Java collection implementation in kotlin stdlib. + Class arrayDequeClass = KotlinToJavaClass.INSTANCE.getArrayDequeClass(); + registerIfAbsent(resolver, arrayDequeClass); + resolver.registerSerializer( + arrayDequeClass, new KotlinArrayDequeSerializer(resolver, arrayDequeClass)); + + // Unsigned array classes: UByteArray, UShortArray, UIntArray, ULongArray. + registerIfAbsent(resolver, UByteArray.class); + resolver.registerSerializer(UByteArray.class, new UByteArraySerializer(resolver)); + registerIfAbsent(resolver, UShortArray.class); + resolver.registerSerializer(UShortArray.class, new UShortArraySerializer(resolver)); + registerIfAbsent(resolver, UIntArray.class); + resolver.registerSerializer(UIntArray.class, new UIntArraySerializer(resolver)); + registerIfAbsent(resolver, ULongArray.class); + resolver.registerSerializer(ULongArray.class, new ULongArraySerializer(resolver)); + + // Ranges and Progressions. + registerIfAbsent(resolver, kotlin.ranges.CharRange.class); + registerIfAbsent(resolver, kotlin.ranges.CharProgression.class); + registerIfAbsent(resolver, kotlin.ranges.IntRange.class); + registerIfAbsent(resolver, kotlin.ranges.IntProgression.class); + registerIfAbsent(resolver, kotlin.ranges.LongRange.class); + registerIfAbsent(resolver, kotlin.ranges.LongProgression.class); + registerIfAbsent(resolver, kotlin.ranges.UIntRange.class); + registerIfAbsent(resolver, kotlin.ranges.UIntProgression.class); + registerIfAbsent(resolver, kotlin.ranges.ULongRange.class); + registerIfAbsent(resolver, kotlin.ranges.ULongProgression.class); + + // Built-in classes. + registerIfAbsent(resolver, kotlin.Pair.class); + registerIfAbsent(resolver, kotlin.Triple.class); + registerIfAbsent(resolver, kotlin.Result.class); + registerIfAbsent(resolver, Result.Failure.class); + + // kotlin.random + registerIfAbsent(resolver, KotlinToJavaClass.INSTANCE.getRandomDefaultClass()); + registerIfAbsent(resolver, KotlinToJavaClass.INSTANCE.getRandomInternalClass()); + registerIfAbsent(resolver, KotlinToJavaClass.INSTANCE.getRandomSerializedClass()); + + // kotlin.text + registerIfAbsent(resolver, Regex.class); + registerIfAbsent(resolver, KotlinToJavaClass.INSTANCE.getRegexSerializedClass()); + registerIfAbsent(resolver, RegexOption.class); + registerIfAbsent(resolver, CharCategory.class); + registerIfAbsent(resolver, CharDirectionality.class); + registerIfAbsent(resolver, HexFormat.class); + registerIfAbsent(resolver, MatchGroup.class); + + // kotlin.time + registerIfAbsent(resolver, DurationUnit.class); + registerIfAbsent(resolver, Duration.class); + resolver.registerSerializer(Duration.class, new DurationSerializer(config)); + registerIfAbsent(resolver, TimedValue.class); + + // kotlin.uuid + registerIfAbsent(resolver, Uuid.class); + resolver.registerSerializer(Uuid.class, new UuidSerializer(config)); } - throw e; + checkRegistrationOpen(resolver); + INSTALLED_FORY.put(fory, Boolean.TRUE); } } diff --git a/kotlin/fory-kotlin/src/main/kotlin/org/apache/fory/kotlin/ForyExtensions.kt b/kotlin/fory-kotlin/src/main/kotlin/org/apache/fory/kotlin/ForyExtensions.kt index 84cb13ead1..463a2fa5f7 100644 --- a/kotlin/fory-kotlin/src/main/kotlin/org/apache/fory/kotlin/ForyExtensions.kt +++ b/kotlin/fory-kotlin/src/main/kotlin/org/apache/fory/kotlin/ForyExtensions.kt @@ -19,46 +19,40 @@ package org.apache.fory.kotlin -import org.apache.fory.BaseFory import org.apache.fory.Fory -import org.apache.fory.ForyModule import org.apache.fory.serializer.kotlin.KotlinSerializers -public inline fun BaseFory.register() { +public inline fun Fory.register() { registerKotlin(this, T::class.java, null, null, null, null) } -public inline fun BaseFory.register(typeId: Long) { +public inline fun Fory.register(typeId: Long) { registerKotlin(this, T::class.java, typeId, null, null, null) } -public inline fun BaseFory.register(name: String) { +public inline fun Fory.register(name: String) { registerKotlin(this, T::class.java, null, name, null, null) } -public inline fun BaseFory.register(namespace: String, typeName: String) { +public inline fun Fory.register(namespace: String, typeName: String) { registerKotlin(this, T::class.java, null, null, namespace, typeName) } @PublishedApi internal fun registerKotlin( - fory: BaseFory, + fory: Fory, cls: Class<*>, typeId: Long?, name: String?, namespace: String?, typeName: String?, ) { - fory.register( - ForyModule { runtime: Fory -> - runtime.register(ForyKotlin) - when { - typeId != null -> KotlinSerializers.register(runtime, cls, typeId) - name != null -> KotlinSerializers.register(runtime, cls, name) - namespace != null && typeName != null -> - KotlinSerializers.register(runtime, cls, namespace, typeName) - else -> KotlinSerializers.register(runtime, cls) - } - }, - ) + fory.register(ForyKotlin) + when { + typeId != null -> KotlinSerializers.register(fory, cls, typeId) + name != null -> KotlinSerializers.register(fory, cls, name) + namespace != null && typeName != null -> + KotlinSerializers.register(fory, cls, namespace, typeName) + else -> KotlinSerializers.register(fory, cls) + } } diff --git a/kotlin/fory-kotlin/src/test/kotlin/org/apache/fory/serializer/kotlin/BuiltinClassSerializerTests.kt b/kotlin/fory-kotlin/src/test/kotlin/org/apache/fory/serializer/kotlin/BuiltinClassSerializerTests.kt index 3ab0d14ec0..9470d6f1bc 100644 --- a/kotlin/fory-kotlin/src/test/kotlin/org/apache/fory/serializer/kotlin/BuiltinClassSerializerTests.kt +++ b/kotlin/fory-kotlin/src/test/kotlin/org/apache/fory/serializer/kotlin/BuiltinClassSerializerTests.kt @@ -21,6 +21,11 @@ package org.apache.fory.serializer.kotlin import java.math.BigDecimal import java.math.BigInteger +import java.util.concurrent.CountDownLatch +import java.util.concurrent.Executors +import java.util.concurrent.Future +import java.util.concurrent.TimeUnit +import java.util.concurrent.atomic.AtomicReference import kotlin.random.Random import kotlin.test.Test import kotlin.time.Duration @@ -73,6 +78,76 @@ class BuiltinClassSerializerTests { Assert.assertSame(DefaultValueUtils.getKotlinDefaultValueSupport(), defaultValueSupport) } + @Test + fun testFreezeWhileBootstrapWaits() { + val fory = + Fory.builder().withXlang(true).withCodegen(false).requireClassRegistration(true).build() + val executor = Executors.newSingleThreadExecutor() + val worker = AtomicReference() + val started = CountDownLatch(1) + lateinit var result: Future + + try { + synchronized(DefaultValueUtils::class.java) { + result = + executor.submit { + worker.set(Thread.currentThread()) + started.countDown() + try { + KotlinSerializers.registerSerializers(fory) + null + } catch (t: Throwable) { + t + } + } + Assert.assertTrue(started.await(10, TimeUnit.SECONDS)) + val deadline = System.nanoTime() + TimeUnit.SECONDS.toNanos(10) + while (worker.get().state != Thread.State.BLOCKED && System.nanoTime() < deadline) { + Thread.yield() + } + Assert.assertEquals(worker.get().state, Thread.State.BLOCKED) + fory.serialize(1) + } + + Assert.assertTrue(result.get(10, TimeUnit.SECONDS) is ForyException) + assertThrows(ForyException::class.java) { KotlinSerializers.registerSerializers(fory) } + } finally { + executor.shutdownNow() + } + } + + @Test + fun testConcurrentBootstrap() { + val fory = + Fory.builder().withXlang(false).withCodegen(false).requireClassRegistration(true).build() + val executor = Executors.newFixedThreadPool(2) + val ready = CountDownLatch(2) + val start = CountDownLatch(1) + + try { + val results = + List(2) { + executor.submit { + ready.countDown() + start.await(10, TimeUnit.SECONDS) + KotlinSerializers.registerSerializers(fory) + } + } + Assert.assertTrue(ready.await(10, TimeUnit.SECONDS)) + start.countDown() + results.forEach { it.get(10, TimeUnit.SECONDS) } + + Assert.assertTrue(fory.typeResolver.isRegistered(Duration::class.java)) + val serializer = fory.typeResolver.getSerializer(Duration::class.java) + val defaultValueSupport = DefaultValueUtils.getKotlinDefaultValueSupport() + KotlinSerializers.registerSerializers(fory) + Assert.assertSame(fory.typeResolver.getSerializer(Duration::class.java), serializer) + Assert.assertSame(DefaultValueUtils.getKotlinDefaultValueSupport(), defaultValueSupport) + } finally { + executor.shutdownNow() + } + } + @Test fun testCombinedFreezeRecheck() { val fory = diff --git a/scala/fory-scala/src/main/java/org/apache/fory/serializer/scala/ScalaSerializers.java b/scala/fory-scala/src/main/java/org/apache/fory/serializer/scala/ScalaSerializers.java index fc1aa0c773..75146a7783 100644 --- a/scala/fory-scala/src/main/java/org/apache/fory/serializer/scala/ScalaSerializers.java +++ b/scala/fory-scala/src/main/java/org/apache/fory/serializer/scala/ScalaSerializers.java @@ -27,7 +27,6 @@ import java.util.Objects; import java.util.WeakHashMap; import org.apache.fory.Fory; -import org.apache.fory.ThreadSafeFory; import org.apache.fory.annotation.Internal; import org.apache.fory.config.Config; import org.apache.fory.exception.ForyException; @@ -39,157 +38,155 @@ public class ScalaSerializers { private static final Map INSTALLED_FORY = Collections.synchronizedMap(new WeakHashMap<>()); - public static void registerSerializers(ThreadSafeFory fory) { - fory.register(ScalaSerializers::registerSerializers); - } - public static void registerSerializers(Fory fory) { TypeResolver resolver = fory.getTypeResolver(); checkRegistrationOpen(resolver); - synchronized (INSTALLED_FORY) { + // The runtime is the bootstrap's natural owner, so its monitor linearizes only this install. + // Since Java monitors are reentrant, reject a recursive install before entering it again. + if (Thread.holdsLock(fory)) { if (INSTALLED_FORY.containsKey(fory)) { return; } - INSTALLED_FORY.put(fory, Boolean.TRUE); + throw new ForyException("Reentrant Scala serializer bootstrap is not supported."); } - try { - fory.registerSerializerFactory(new ScalaSerializerFactory()); - if (resolver.isCrossLanguage()) { + synchronized (fory) { + checkRegistrationOpen(resolver); + if (INSTALLED_FORY.containsKey(fory)) { return; } - Config config = resolver.getConfig(); + fory.registerSerializerFactory(new ScalaSerializerFactory()); + if (!resolver.isCrossLanguage()) { + Config config = resolver.getConfig(); - resolver.registerSerializer( - IterableToFactoryClass, new ToFactorySerializers.IterableToFactorySerializer(config)); - resolver.registerSerializer( - MapToFactoryClass, new ToFactorySerializers.MapToFactorySerializer(config)); + resolver.registerSerializer( + IterableToFactoryClass, new ToFactorySerializers.IterableToFactorySerializer(config)); + resolver.registerSerializer( + MapToFactoryClass, new ToFactorySerializers.MapToFactorySerializer(config)); - // Seq - resolver.register(scala.collection.immutable.Seq.class); - resolver.register(scala.collection.immutable.Nil$.class); - resolver.register(scala.collection.immutable.List$.class); - resolver.register(scala.collection.immutable.$colon$colon.class); - // StrictOptimizedSeqFactory -> ... extends -> IterableFactory - resolver.register(scala.collection.immutable.Vector$.class); - resolver.register("scala.collection.immutable.VectorImpl"); - resolver.register("scala.collection.immutable.Vector0"); - resolver.register("scala.collection.immutable.Vector1"); - resolver.register("scala.collection.immutable.Vector2"); - resolver.register("scala.collection.immutable.Vector3"); - resolver.register("scala.collection.immutable.Vector4"); - resolver.register("scala.collection.immutable.Vector5"); - resolver.register("scala.collection.immutable.Vector6"); - resolver.register(scala.collection.immutable.Queue.class); - resolver.register(scala.collection.immutable.Queue$.class); - resolver.register(scala.collection.immutable.LazyList.class); - resolver.register(scala.collection.immutable.LazyList$.class); - resolver.register(scala.collection.immutable.ArraySeq.class); - resolver.register(scala.collection.immutable.ArraySeq$.class); + // Seq + resolver.register(scala.collection.immutable.Seq.class); + resolver.register(scala.collection.immutable.Nil$.class); + resolver.register(scala.collection.immutable.List$.class); + resolver.register(scala.collection.immutable.$colon$colon.class); + // StrictOptimizedSeqFactory -> ... extends -> IterableFactory + resolver.register(scala.collection.immutable.Vector$.class); + resolver.register("scala.collection.immutable.VectorImpl"); + resolver.register("scala.collection.immutable.Vector0"); + resolver.register("scala.collection.immutable.Vector1"); + resolver.register("scala.collection.immutable.Vector2"); + resolver.register("scala.collection.immutable.Vector3"); + resolver.register("scala.collection.immutable.Vector4"); + resolver.register("scala.collection.immutable.Vector5"); + resolver.register("scala.collection.immutable.Vector6"); + resolver.register(scala.collection.immutable.Queue.class); + resolver.register(scala.collection.immutable.Queue$.class); + resolver.register(scala.collection.immutable.LazyList.class); + resolver.register(scala.collection.immutable.LazyList$.class); + resolver.register(scala.collection.immutable.ArraySeq.class); + resolver.register(scala.collection.immutable.ArraySeq$.class); - // Set - resolver.register(scala.collection.immutable.Set.class); - // IterableFactory - resolver.register(scala.collection.immutable.Set$.class); - resolver.register(scala.collection.immutable.Set.Set1.class); - resolver.register(scala.collection.immutable.Set.Set2.class); - resolver.register(scala.collection.immutable.Set.Set3.class); - resolver.register(scala.collection.immutable.Set.Set4.class); - resolver.register(scala.collection.immutable.HashSet.class); - resolver.register(scala.collection.immutable.TreeSet.class); - // SortedIterableFactory - resolver.register(scala.collection.immutable.TreeSet$.class); - // IterableFactory - resolver.register(scala.collection.immutable.HashSet$.class); - resolver.register(scala.collection.immutable.ListSet.class); - resolver.register(scala.collection.immutable.ListSet$.class); - resolver.register("scala.collection.immutable.Set$EmptySet$"); - resolver.register("scala.collection.immutable.SetBuilderImpl"); - resolver.register("scala.collection.immutable.SortedMapOps$ImmutableKeySortedSet"); + // Set + resolver.register(scala.collection.immutable.Set.class); + // IterableFactory + resolver.register(scala.collection.immutable.Set$.class); + resolver.register(scala.collection.immutable.Set.Set1.class); + resolver.register(scala.collection.immutable.Set.Set2.class); + resolver.register(scala.collection.immutable.Set.Set3.class); + resolver.register(scala.collection.immutable.Set.Set4.class); + resolver.register(scala.collection.immutable.HashSet.class); + resolver.register(scala.collection.immutable.TreeSet.class); + // SortedIterableFactory + resolver.register(scala.collection.immutable.TreeSet$.class); + // IterableFactory + resolver.register(scala.collection.immutable.HashSet$.class); + resolver.register(scala.collection.immutable.ListSet.class); + resolver.register(scala.collection.immutable.ListSet$.class); + resolver.register("scala.collection.immutable.Set$EmptySet$"); + resolver.register("scala.collection.immutable.SetBuilderImpl"); + resolver.register("scala.collection.immutable.SortedMapOps$ImmutableKeySortedSet"); - // Map - resolver.register(scala.collection.immutable.Map.class); - resolver.register(scala.collection.immutable.Map$.class); - resolver.register(scala.collection.immutable.Map.Map1.class); - resolver.register(scala.collection.immutable.Map.Map2.class); - resolver.register(scala.collection.immutable.Map.Map3.class); - resolver.register(scala.collection.immutable.Map.Map4.class); - resolver.register(scala.collection.immutable.Map.WithDefault.class); - resolver.register("scala.collection.immutable.MapBuilderImpl"); - resolver.register("scala.collection.immutable.Map$EmptyMap$"); - resolver.register("scala.collection.immutable.SeqMap$EmptySeqMap$"); - resolver.register(scala.collection.immutable.HashMap.class); - resolver.register(scala.collection.immutable.HashMap$.class); - resolver.register(scala.collection.immutable.TreeMap.class); - resolver.register(scala.collection.immutable.TreeMap$.class); - resolver.register(scala.collection.immutable.SortedMap$.class); - resolver.register(scala.collection.immutable.TreeSeqMap.class); - resolver.register(scala.collection.immutable.TreeSeqMap$.class); - resolver.register(scala.collection.immutable.ListMap.class); - resolver.register(scala.collection.immutable.ListMap$.class); - resolver.register(scala.collection.immutable.IntMap.class); - resolver.register(scala.collection.immutable.IntMap$.class); - resolver.register(scala.collection.immutable.LongMap.class); - resolver.register(scala.collection.immutable.LongMap$.class); + // Map + resolver.register(scala.collection.immutable.Map.class); + resolver.register(scala.collection.immutable.Map$.class); + resolver.register(scala.collection.immutable.Map.Map1.class); + resolver.register(scala.collection.immutable.Map.Map2.class); + resolver.register(scala.collection.immutable.Map.Map3.class); + resolver.register(scala.collection.immutable.Map.Map4.class); + resolver.register(scala.collection.immutable.Map.WithDefault.class); + resolver.register("scala.collection.immutable.MapBuilderImpl"); + resolver.register("scala.collection.immutable.Map$EmptyMap$"); + resolver.register("scala.collection.immutable.SeqMap$EmptySeqMap$"); + resolver.register(scala.collection.immutable.HashMap.class); + resolver.register(scala.collection.immutable.HashMap$.class); + resolver.register(scala.collection.immutable.TreeMap.class); + resolver.register(scala.collection.immutable.TreeMap$.class); + resolver.register(scala.collection.immutable.SortedMap$.class); + resolver.register(scala.collection.immutable.TreeSeqMap.class); + resolver.register(scala.collection.immutable.TreeSeqMap$.class); + resolver.register(scala.collection.immutable.ListMap.class); + resolver.register(scala.collection.immutable.ListMap$.class); + resolver.register(scala.collection.immutable.IntMap.class); + resolver.register(scala.collection.immutable.IntMap$.class); + resolver.register(scala.collection.immutable.LongMap.class); + resolver.register(scala.collection.immutable.LongMap$.class); - // Range - resolver.register("scala.math.Numeric$IntIsIntegral$"); - resolver.register("scala.math.Numeric$LongIsIntegral$"); - resolver.registerSerializerAndType( - Range.Inclusive.class, new RangeSerializer(resolver, Range.Inclusive.class)); - resolver.registerSerializerAndType( - Range.Exclusive.class, new RangeSerializer(resolver, Range.Exclusive.class)); - resolver.registerSerializerAndType( - NumericRange.class, new NumericRangeSerializer<>(resolver, NumericRange.class)); - resolver.registerSerializerAndType( - NumericRange.Exclusive.class, - new NumericRangeSerializer<>(resolver, NumericRange.Exclusive.class)); - resolver.registerSerializerAndType( - NumericRange.Inclusive.class, - new NumericRangeSerializer<>(resolver, NumericRange.Inclusive.class)); + // Range + resolver.register("scala.math.Numeric$IntIsIntegral$"); + resolver.register("scala.math.Numeric$LongIsIntegral$"); + resolver.registerSerializerAndType( + Range.Inclusive.class, new RangeSerializer(resolver, Range.Inclusive.class)); + resolver.registerSerializerAndType( + Range.Exclusive.class, new RangeSerializer(resolver, Range.Exclusive.class)); + resolver.registerSerializerAndType( + NumericRange.class, new NumericRangeSerializer<>(resolver, NumericRange.class)); + resolver.registerSerializerAndType( + NumericRange.Exclusive.class, + new NumericRangeSerializer<>(resolver, NumericRange.Exclusive.class)); + resolver.registerSerializerAndType( + NumericRange.Inclusive.class, + new NumericRangeSerializer<>(resolver, NumericRange.Inclusive.class)); - resolver.register(scala.collection.generic.SerializeEnd$.class); - resolver.register(scala.collection.generic.DefaultSerializationProxy.class); - resolver.register(scala.runtime.ModuleSerializationProxy.class); + resolver.register(scala.collection.generic.SerializeEnd$.class); + resolver.register(scala.collection.generic.DefaultSerializationProxy.class); + resolver.register(scala.runtime.ModuleSerializationProxy.class); - // mutable collection types - resolver.register(scala.collection.mutable.StringBuilder.class); - resolver.register(scala.collection.mutable.ArrayBuffer.class); - resolver.register(scala.collection.mutable.ArrayBuffer$.class); - resolver.register(scala.collection.mutable.ArraySeq.class); - resolver.register(scala.collection.mutable.ArraySeq$.class); - resolver.register(scala.collection.mutable.ListBuffer.class); - resolver.register(scala.collection.mutable.ListBuffer$.class); - resolver.register(scala.collection.mutable.Buffer$.class); - resolver.register(scala.collection.mutable.ArrayDeque.class); - resolver.register(scala.collection.mutable.ArrayDeque$.class); + // mutable collection types + resolver.register(scala.collection.mutable.StringBuilder.class); + resolver.register(scala.collection.mutable.ArrayBuffer.class); + resolver.register(scala.collection.mutable.ArrayBuffer$.class); + resolver.register(scala.collection.mutable.ArraySeq.class); + resolver.register(scala.collection.mutable.ArraySeq$.class); + resolver.register(scala.collection.mutable.ListBuffer.class); + resolver.register(scala.collection.mutable.ListBuffer$.class); + resolver.register(scala.collection.mutable.Buffer$.class); + resolver.register(scala.collection.mutable.ArrayDeque.class); + resolver.register(scala.collection.mutable.ArrayDeque$.class); - resolver.register(scala.collection.mutable.HashSet.class); - resolver.register(scala.collection.mutable.HashSet$.class); - resolver.register(scala.collection.mutable.TreeSet.class); - resolver.register(scala.collection.mutable.TreeSet$.class); + resolver.register(scala.collection.mutable.HashSet.class); + resolver.register(scala.collection.mutable.HashSet$.class); + resolver.register(scala.collection.mutable.TreeSet.class); + resolver.register(scala.collection.mutable.TreeSet$.class); - resolver.register(scala.collection.mutable.HashMap.class); - resolver.register(scala.collection.mutable.HashMap$.class); - resolver.register(scala.collection.mutable.TreeMap.class); - resolver.register(scala.collection.mutable.TreeMap$.class); - resolver.register(scala.collection.mutable.LinkedHashMap.class); - resolver.register(scala.collection.mutable.LinkedHashMap$.class); - resolver.register(scala.collection.mutable.LinkedHashSet.class); - resolver.register(scala.collection.mutable.LinkedHashSet$.class); - resolver.register(scala.collection.mutable.LongMap.class); - resolver.register(scala.collection.mutable.LongMap$.class); + resolver.register(scala.collection.mutable.HashMap.class); + resolver.register(scala.collection.mutable.HashMap$.class); + resolver.register(scala.collection.mutable.TreeMap.class); + resolver.register(scala.collection.mutable.TreeMap$.class); + resolver.register(scala.collection.mutable.LinkedHashMap.class); + resolver.register(scala.collection.mutable.LinkedHashMap$.class); + resolver.register(scala.collection.mutable.LinkedHashSet.class); + resolver.register(scala.collection.mutable.LinkedHashSet$.class); + resolver.register(scala.collection.mutable.LongMap.class); + resolver.register(scala.collection.mutable.LongMap$.class); - resolver.register(scala.collection.mutable.Queue.class); - resolver.register(scala.collection.mutable.Queue$.class); - resolver.register(scala.collection.mutable.Stack.class); - resolver.register(scala.collection.mutable.Stack$.class); - resolver.register(scala.collection.mutable.BitSet.class); - resolver.register(scala.collection.mutable.BitSet$.class); - } catch (RuntimeException | Error e) { - synchronized (INSTALLED_FORY) { - INSTALLED_FORY.remove(fory); + resolver.register(scala.collection.mutable.Queue.class); + resolver.register(scala.collection.mutable.Queue$.class); + resolver.register(scala.collection.mutable.Stack.class); + resolver.register(scala.collection.mutable.Stack$.class); + resolver.register(scala.collection.mutable.BitSet.class); + resolver.register(scala.collection.mutable.BitSet$.class); } - throw e; + checkRegistrationOpen(resolver); + INSTALLED_FORY.put(fory, Boolean.TRUE); } } diff --git a/scala/fory-scala/src/main/scala-3/org/apache/fory/scala/ForyExtensions.scala b/scala/fory-scala/src/main/scala-3/org/apache/fory/scala/ForyExtensions.scala index 8a3d23076e..8b587301e6 100644 --- a/scala/fory-scala/src/main/scala-3/org/apache/fory/scala/ForyExtensions.scala +++ b/scala/fory-scala/src/main/scala-3/org/apache/fory/scala/ForyExtensions.scala @@ -19,37 +19,33 @@ package org.apache.fory.scala -import org.apache.fory.BaseFory +import org.apache.fory.Fory import scala.reflect.ClassTag -extension (fory: BaseFory) - def register[T](using serializer: ForySerializer[T], tag: ClassTag[T]): Unit = - ForySerializer.registerModule(fory, tag.runtimeClass.asInstanceOf[Class[T]], null, null, null) +extension (fory: Fory) + def register[T](using serializer: ForySerializer[T], tag: ClassTag[T]): Unit = { + fory.register(ForyScala) + ForySerializer.register(fory, tag.runtimeClass.asInstanceOf[Class[T]]) + } - def register[T](typeId: Long)(using serializer: ForySerializer[T], tag: ClassTag[T]): Unit = - ForySerializer.registerModule( - fory, - tag.runtimeClass.asInstanceOf[Class[T]], - java.lang.Long.valueOf(typeId), - null, - null) + def register[T](typeId: Long)(using serializer: ForySerializer[T], tag: ClassTag[T]): Unit = { + fory.register(ForyScala) + ForySerializer.register(fory, tag.runtimeClass.asInstanceOf[Class[T]], typeId) + } - def register[T](name: String)(using serializer: ForySerializer[T], tag: ClassTag[T]): Unit = - val (namespace, typeName) = ForySerializer.splitName(name) - ForySerializer.registerModule( - fory, - tag.runtimeClass.asInstanceOf[Class[T]], - null, - namespace, - typeName) + def register[T](name: String)(using serializer: ForySerializer[T], tag: ClassTag[T]): Unit = { + fory.register(ForyScala) + ForySerializer.register(fory, tag.runtimeClass.asInstanceOf[Class[T]], name) + } def register[T](namespace: String, typeName: String)(using serializer: ForySerializer[T], - tag: ClassTag[T]): Unit = - ForySerializer.registerModule( + tag: ClassTag[T]): Unit = { + fory.register(ForyScala) + ForySerializer.register( fory, tag.runtimeClass.asInstanceOf[Class[T]], - null, namespace, typeName) + } diff --git a/scala/fory-scala/src/main/scala-3/org/apache/fory/scala/ForySerializer.scala b/scala/fory-scala/src/main/scala-3/org/apache/fory/scala/ForySerializer.scala index bad66c927a..08e06703aa 100644 --- a/scala/fory-scala/src/main/scala-3/org/apache/fory/scala/ForySerializer.scala +++ b/scala/fory-scala/src/main/scala-3/org/apache/fory/scala/ForySerializer.scala @@ -19,7 +19,7 @@ package org.apache.fory.scala -import org.apache.fory.{BaseFory, Fory, ForyModule, ThreadSafeFory} +import org.apache.fory.Fory import org.apache.fory.annotation.Internal import org.apache.fory.exception.ForyException import org.apache.fory.meta.TypeDef @@ -171,53 +171,6 @@ object ForySerializer { } } - def register[T]( - fory: ThreadSafeFory, - cls: Class[T])(using serializer: ForySerializer[T]): Unit = { - registerModule(fory, cls, null, null, null) - } - - def register[T]( - fory: ThreadSafeFory, - cls: Class[T], - typeId: Long)(using serializer: ForySerializer[T]): Unit = { - registerModule(fory, cls, java.lang.Long.valueOf(typeId), null, null) - } - - def register[T]( - fory: ThreadSafeFory, - cls: Class[T], - name: String)(using serializer: ForySerializer[T]): Unit = { - val (namespace, typeName) = splitName(name) - registerModule(fory, cls, null, namespace, typeName) - } - - def register[T]( - fory: ThreadSafeFory, - cls: Class[T], - namespace: String, - typeName: String)(using serializer: ForySerializer[T]): Unit = { - checkTypeName(typeName) - registerModule(fory, cls, null, namespace, typeName) - } - - private[scala] def registerModule[T]( - fory: BaseFory, - cls: Class[T], - typeId: java.lang.Long, - namespace: String, - typeName: String)(using serializer: ForySerializer[T]): Unit = { - if typeName != null then { - checkTypeName(typeName) - } - fory.register(new ForyModule { - override def install(runtime: Fory): Unit = { - runtime.register(ForyScala) - register(runtime, cls, typeId, namespace, typeName)(using serializer) - } - }) - } - private def registerType[T]( fory: Fory, cls: Class[T], diff --git a/scala/fory-scala/src/test/scala/org/apache/fory/serializer/scala/ScalaTest.scala b/scala/fory-scala/src/test/scala/org/apache/fory/serializer/scala/ScalaTest.scala index fccc21e8e1..15eb4b1df2 100644 --- a/scala/fory-scala/src/test/scala/org/apache/fory/serializer/scala/ScalaTest.scala +++ b/scala/fory-scala/src/test/scala/org/apache/fory/serializer/scala/ScalaTest.scala @@ -20,7 +20,10 @@ package org.apache.fory.serializer.scala import java.math.{BigDecimal => JBigDecimal, BigInteger} +import java.util.concurrent.{Callable, CountDownLatch, Executors, TimeUnit} +import java.util.concurrent.atomic.{AtomicInteger, AtomicReference} import org.apache.fory.Fory +import org.apache.fory.exception.ForyException import org.apache.fory.scala.ForyScala import org.scalatest.matchers.should.Matchers import org.scalatest.wordspec.AnyWordSpec @@ -66,6 +69,126 @@ class ScalaTest extends AnyWordSpec with Matchers { } } } + "reject a root reentered during bootstrap" in { + val loader = new BootstrapClassLoader(getClass.getClassLoader, reenterRoot = true) + val runtime = Fory.builder() + .withClassLoader(loader) + .withXlang(false) + .withCodegen(false) + .requireClassRegistration(false) + .build() + loader.runtime = runtime + + intercept[ForyException] { + ScalaSerializers.registerSerializers(runtime) + } + intercept[ForyException] { + ScalaSerializers.registerSerializers(runtime) + } + } + "retry bootstrap after installation failure" in { + val loader = new BootstrapClassLoader(getClass.getClassLoader, failOnce = true) + val runtime = Fory.builder() + .withClassLoader(loader) + .withXlang(false) + .withCodegen(false) + .requireClassRegistration(false) + .build() + + intercept[IllegalStateException] { + ScalaSerializers.registerSerializers(runtime) + } + + ScalaSerializers.registerSerializers(runtime) + runtime.getTypeResolver.isRegistered( + Class.forName("scala.collection.immutable.Vector6", false, loader) + ) shouldBe true + } + "reject same-thread bootstrap reentry" in { + val loader = new BootstrapClassLoader(getClass.getClassLoader, reenterBootstrap = true) + val runtime = Fory.builder() + .withClassLoader(loader) + .withXlang(false) + .withCodegen(false) + .requireClassRegistration(false) + .build() + loader.runtime = runtime + + intercept[ForyException] { + ScalaSerializers.registerSerializers(runtime) + } + ScalaSerializers.registerSerializers(runtime) + runtime.getTypeResolver.isRegistered( + Class.forName("scala.collection.immutable.Vector6", false, loader) + ) shouldBe true + } + "allow nested bootstrap for another runtime" in { + val loader = new BootstrapClassLoader(getClass.getClassLoader, reenterBootstrap = true) + val nested = Fory.builder() + .withXlang(false) + .withCodegen(false) + .requireClassRegistration(false) + .build() + val runtime = Fory.builder() + .withClassLoader(loader) + .withXlang(false) + .withCodegen(false) + .requireClassRegistration(false) + .build() + loader.runtime = nested + + ScalaSerializers.registerSerializers(runtime) + nested.getTypeResolver.isRegistered( + Class.forName("scala.collection.immutable.Vector6") + ) shouldBe true + runtime.getTypeResolver.isRegistered( + Class.forName("scala.collection.immutable.Vector6", false, loader) + ) shouldBe true + } + "install bootstrap exactly once concurrently" in { + val loader = new BlockingBootstrapLoader(getClass.getClassLoader) + val runtime = Fory.builder() + .withClassLoader(loader) + .withXlang(false) + .withCodegen(false) + .requireClassRegistration(false) + .build() + val executor = Executors.newFixedThreadPool(2) + val secondThread = new AtomicReference[Thread]() + val secondStarted = new CountDownLatch(1) + + try { + val first = executor.submit(new Callable[Unit] { + override def call(): Unit = ScalaSerializers.registerSerializers(runtime) + }) + loader.targetEntered.await(10, TimeUnit.SECONDS) shouldBe true + val second = executor.submit(new Callable[Unit] { + override def call(): Unit = { + secondThread.set(Thread.currentThread()) + secondStarted.countDown() + ScalaSerializers.registerSerializers(runtime) + } + }) + secondStarted.await(10, TimeUnit.SECONDS) shouldBe true + val deadline = System.nanoTime() + TimeUnit.SECONDS.toNanos(10) + while (secondThread.get().getState != Thread.State.BLOCKED && + System.nanoTime() < deadline) { + Thread.`yield`() + } + secondThread.get().getState shouldBe Thread.State.BLOCKED + loader.releaseTarget.countDown() + first.get(10, TimeUnit.SECONDS) + second.get(10, TimeUnit.SECONDS) + + loader.targetLoads.get() shouldBe 1 + runtime.getTypeResolver.isRegistered( + Class.forName("scala.collection.immutable.Vector6", false, loader) + ) shouldBe true + } finally { + loader.releaseTarget.countDown() + executor.shutdownNow() + } + } } "serialize/deserialize package object in app" in { // If we move code in main here, we can't reproduce https://github.com/apache/fory/issues/1165. @@ -74,6 +197,52 @@ class ScalaTest extends AnyWordSpec with Matchers { } } +private final class BootstrapClassLoader( + parent: ClassLoader, + reenterRoot: Boolean = false, + failOnce: Boolean = false, + reenterBootstrap: Boolean = false +) extends ClassLoader(parent) { + @volatile var runtime: Fory = _ + private var shouldReenter = reenterRoot + private var shouldFail = failOnce + private var shouldReenterBootstrap = reenterBootstrap + + override protected def loadClass(name: String, resolve: Boolean): Class[_] = { + if (name == "scala.collection.immutable.VectorImpl") { + if (shouldReenter) { + shouldReenter = false + runtime.serialize("freeze") + } + if (shouldReenterBootstrap) { + shouldReenterBootstrap = false + ScalaSerializers.registerSerializers(runtime) + } + if (shouldFail) { + shouldFail = false + throw new IllegalStateException("bootstrap class loading failed") + } + } + super.loadClass(name, resolve) + } +} + +private final class BlockingBootstrapLoader(parent: ClassLoader) extends ClassLoader(parent) { + val targetEntered = new CountDownLatch(1) + val releaseTarget = new CountDownLatch(1) + val targetLoads = new AtomicInteger() + + override protected def loadClass(name: String, resolve: Boolean): Class[_] = { + if (name == "scala.collection.immutable.VectorImpl" && targetLoads.incrementAndGet() == 1) { + targetEntered.countDown() + if (!releaseTarget.await(10, TimeUnit.SECONDS)) { + throw new IllegalStateException("timed out waiting to finish bootstrap class loading") + } + } + super.loadClass(name, resolve) + } +} + package object PkgObject { case class Id(value: Int) From 980b6400394dd93d0492355f99054ded6a7f7eec Mon Sep 17 00:00:00 2001 From: chaokunyang Date: Fri, 28 Aug 2026 06:07:43 +0800 Subject: [PATCH 040/168] docs: align lifecycle ownership guidance --- .agents/languages/cpp.md | 4 ++ .agents/languages/python.md | 8 +++- .../python/configuration.md | 38 +++++++++-------- .../python/functions-classes-methods.md | 8 ++-- docs/object-serialization/python/index.md | 5 ++- docs/object-serialization/python/native.md | 8 ++-- .../python/numpy-integration.md | 2 +- .../python/out-of-band.md | 2 +- docs/object-serialization/python/security.md | 8 ++-- .../python/serialization-hooks.md | 2 +- .../python/type-registration.md | 10 ++--- docs/security/deserialization.md | 41 +++++-------------- .../xlang_implementation_guide.md | 8 ++++ python/pyfory/_fory.py | 8 ++-- 14 files changed, 81 insertions(+), 71 deletions(-) diff --git a/.agents/languages/cpp.md b/.agents/languages/cpp.md index 8c23dba28d..f2d658e94f 100644 --- a/.agents/languages/cpp.md +++ b/.agents/languages/cpp.md @@ -23,6 +23,10 @@ Load this file when changing `cpp/`, Cython build plumbing, or C++ xlang behavio acquiring pooled instances. `BaseFory` and the source `TypeResolver` keep separate owner-local freeze gates so direct resolver registration cannot bypass the facade gate. Both reject before mutation; do not collapse these gates or describe permanent registry freeze as finalization. +- Keep `TypeResolver::check_registration()` as the single out-of-line owner of the frozen and + registration-thread checks. Do not add a layout-preserving helper, padding, or call-shape + workaround. Resolver finalization prepares metadata completely and publishes it only after the + completed resolver clone succeeds; failed finalization must not expose partial metadata. - Put private methods last in class definitions, immediately before private fields. - Do not redesign alias-based or low-level public type shapes to add convenience methods unless the user explicitly asks for that API change. - For cross-language feature ports, match protocol behavior but use idiomatic C++ ownership and layering instead of mirroring Java structure literally. diff --git a/.agents/languages/python.md b/.agents/languages/python.md index a4e0d863dd..638b93c3b3 100644 --- a/.agents/languages/python.md +++ b/.agents/languages/python.md @@ -22,7 +22,13 @@ Load this file when changing `python/`, Cython serialization, or Python xlang be execute application factories or callbacks while holding its pool lock. Its registration linearization is reentrant so nested facade registrations share the same publication order. A root started during registration must not reuse the staging instance, and root reentry from a - running user `fory_factory` fails without recursively invoking that factory. + running user `fory_factory` or retained registration callback fails without recursively building + another instance. The non-reentrant pool lock owns only pool state; the separate instance-build + boundary covers the factory and complete callback replay. +- Registry freeze prohibits type and serializer publication after the first root; it does not + prohibit policy-authorized resolution of module-global classes or callables during a non-strict + native read when that resolution does not mutate registry state. Do not describe these two + operations as one kind of late discovery. - In non-strict native mode, public unqualified `register_type` for a built-in native carrier uses the same reserved type identity as pre-root discovery. Ordinary application classes and dataclasses retain their struct registration identity. Configure both through public registration; diff --git a/docs/object-serialization/python/configuration.md b/docs/object-serialization/python/configuration.md index 5bc7f579ab..fa2e9baa15 100644 --- a/docs/object-serialization/python/configuration.md +++ b/docs/object-serialization/python/configuration.md @@ -61,23 +61,23 @@ class ThreadSafeFory: ## Parameters -| Parameter | Type | Default | Description | -| -------------------------------------- | ------------------------------- | ----------- | -------------------------------------------------------------------------------------------------------------------------------------------------------- | -| `xlang` | `bool` | `True` | Use xlang mode. Set `False` for Python native mode. | -| `ref` | `bool` | `False` | Enable reference tracking for shared/circular references. Disable for better performance if your data has no shared references. | -| `strict` | `bool` | `True` | Require registration before loading application classes. Compatible unknown Structs use `UnknownStruct`. `False` does not permit late discovery. | -| `compatible` | `bool \| None` | `None` | Schema evolution mode. `None` enables compatible mode in both xlang and native mode. Set `False` only when every reader and writer uses the same schema. | -| `max_depth` | `int` | `50` | Maximum deserialization depth for security, preventing stack overflow attacks. | -| `max_type_fields` | `int` | `512` | Maximum fields accepted in one received remote struct metadata body. | -| `max_type_meta_bytes` | `int` | `4096` | Maximum encoded body bytes accepted for one received TypeDef body, excluding the 8-byte header and any extended-size varint. | -| `max_schema_versions_per_type` | `int` | `10` | Maximum accepted remote metadata versions for one logical type. | -| `max_average_schema_versions_per_type` | `int` | `3` | Average accepted remote metadata versions across accepted remote types. The effective global floor is `8192` schemas. | -| `max_graph_memory_bytes` | `int` | `134217728` | Approximate graph-memory gate for one root deserialization. Explicit non-positive values are rejected. | -| `max_unbacked_container_items` | `int` | `8192` | Maximum collection elements and map entries whose repeated reads are not backed by input progress. Zero is strict. | -| `policy` | `DeserializationPolicy \| None` | `None` | Deserialization policy used for security checks. Strongly recommended when `strict=False`. | -| `field_nullable` | `bool` | `False` | Treat dataclass fields as nullable by default. | -| `meta_compressor` | `Any` | `None` | Optional metadata compressor used for compatible-mode metadata encoding. | -| `fory_factory` | `Callable \| None` | `None` | `ThreadSafeFory` factory hook. When set, `ThreadSafeFory` creates instances via this callback; otherwise it forwards `**kwargs` to `Fory` construction. | +| Parameter | Type | Default | Description | +| -------------------------------------- | ------------------------------- | ----------- | --------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| `xlang` | `bool` | `True` | Use xlang mode. Set `False` for Python native mode. | +| `ref` | `bool` | `False` | Enable reference tracking for shared/circular references. Disable for better performance if your data has no shared references. | +| `strict` | `bool` | `True` | Require registration before loading application classes. Compatible unknown Structs use `UnknownStruct`. `False` permits policy-authorized native module-global resolution. | +| `compatible` | `bool \| None` | `None` | Schema evolution mode. `None` enables compatible mode in both xlang and native mode. Set `False` only when every reader and writer uses the same schema. | +| `max_depth` | `int` | `50` | Maximum deserialization depth for security, preventing stack overflow attacks. | +| `max_type_fields` | `int` | `512` | Maximum fields accepted in one received remote struct metadata body. | +| `max_type_meta_bytes` | `int` | `4096` | Maximum encoded body bytes accepted for one received TypeDef body, excluding the 8-byte header and any extended-size varint. | +| `max_schema_versions_per_type` | `int` | `10` | Maximum accepted remote metadata versions for one logical type. | +| `max_average_schema_versions_per_type` | `int` | `3` | Average accepted remote metadata versions across accepted remote types. The effective global floor is `8192` schemas. | +| `max_graph_memory_bytes` | `int` | `134217728` | Approximate graph-memory gate for one root deserialization. Explicit non-positive values are rejected. | +| `max_unbacked_container_items` | `int` | `8192` | Maximum collection elements and map entries whose repeated reads are not backed by input progress. Zero is strict. | +| `policy` | `DeserializationPolicy \| None` | `None` | Deserialization policy used for security checks. Strongly recommended when `strict=False`. | +| `field_nullable` | `bool` | `False` | Treat dataclass fields as nullable by default. | +| `meta_compressor` | `Any` | `None` | Optional metadata compressor used for compatible-mode metadata encoding. | +| `fory_factory` | `Callable \| None` | `None` | `ThreadSafeFory` factory hook. When set, `ThreadSafeFory` creates instances via this callback; otherwise it forwards `**kwargs` to `Fory` construction. | ## Key Methods @@ -191,7 +191,9 @@ fory.register_type(types.FunctionType) ``` Use `strict=False` only for trusted data, preferably with a `policy=` deserialization policy. -Register every application and Python-native carrier type before the first root attempt. +Register every application and Python-native carrier type whose serializer must be installed before +the first root attempt. Policy-authorized module-global classes and callables may still be resolved +while reading a native payload; that lookup does not reopen or mutate the frozen registry. ## Security diff --git a/docs/object-serialization/python/functions-classes-methods.md b/docs/object-serialization/python/functions-classes-methods.md index e95f39554a..13502e2c0b 100644 --- a/docs/object-serialization/python/functions-classes-methods.md +++ b/docs/object-serialization/python/functions-classes-methods.md @@ -23,9 +23,11 @@ Python native mode serializes Python-specific callable and type values that are type system. Use `strict=False` only for trusted payloads and apply a deserialization policy when the accepted dynamic surface must be restricted. -Register every callable carrier and application type before the first root operation. The first -serialization or deserialization attempt permanently freezes the registry, even when it fails; -`strict=False` does not enable late discovery or registration. +Register every callable carrier and application type whose serializer must be installed before the +first root operation. The first serialization or deserialization attempt permanently freezes the +registry, even when it fails. With `strict=False`, the configured policy may still resolve a +module-global function or class while reading a trusted native payload; that resolution does not +install a type or serializer. ## Serialize Global Functions diff --git a/docs/object-serialization/python/index.md b/docs/object-serialization/python/index.md index 6c7bad945c..1f8eb91f62 100644 --- a/docs/object-serialization/python/index.md +++ b/docs/object-serialization/python/index.md @@ -150,8 +150,9 @@ print(result) # Person(name='Alice', age=30) Register every application type before the first root serialization or deserialization attempt. In native mode, also register callable, class, method, state, and reduction carrier types that can appear in the object graph. The first root attempt permanently freezes the instance's registry, -including when that attempt fails. Setting `strict=False` does not permit discovery or registration -after that point. +including when that attempt fails. Setting `strict=False` may authorize module-global resolution +during a trusted native read, but it does not permit type or serializer registration after that +point. If the first operation exposes an incomplete or invalid registration, create a new instance and register the complete type surface before retrying. A fully configured instance can process a later diff --git a/docs/object-serialization/python/native.md b/docs/object-serialization/python/native.md index d781c9e176..a6062a9e5e 100644 --- a/docs/object-serialization/python/native.md +++ b/docs/object-serialization/python/native.md @@ -52,9 +52,11 @@ fory = pyfory.Fory(xlang=False, ref=False, strict=True) Keep `strict=True` for registered, trusted type surfaces. Use `strict=False` only when the configured surface includes native carriers such as functions, local classes, or objects reconstructed by -reduction hooks. In either mode, register every application type and native carrier before the first -root serialization or deserialization attempt. The first attempt permanently freezes the instance's -registry even when it fails; `strict=False` does not enable late discovery or registration. +reduction hooks. In either mode, register every application type and native carrier whose serializer +must be installed before the first root serialization or deserialization attempt. The first attempt +permanently freezes the instance's registry even when it fails. With `strict=False`, the configured +policy may still authorize module-global classes and callables resolved while reading a native +payload; that lookup does not add a type or serializer to the frozen registry. ## Common Usage diff --git a/docs/object-serialization/python/numpy-integration.md b/docs/object-serialization/python/numpy-integration.md index f0656fd584..966cffdeea 100644 --- a/docs/object-serialization/python/numpy-integration.md +++ b/docs/object-serialization/python/numpy-integration.md @@ -47,7 +47,7 @@ assert np.array_equal(arrays["matrix"], result["matrix"]) The ndarray carrier itself is available when the instance is created. Register application types that contain ndarrays, plus every custom type that can appear inside an object-dtype ndarray, before the first root attempt. The first root attempt permanently freezes registration, including when it -fails. `strict=False` does not permit late discovery or registration. +fails. `strict=False` does not permit late type or serializer registration. ## Out-of-Band Buffers diff --git a/docs/object-serialization/python/out-of-band.md b/docs/object-serialization/python/out-of-band.md index ab7034a3ba..263989b0fa 100644 --- a/docs/object-serialization/python/out-of-band.md +++ b/docs/object-serialization/python/out-of-band.md @@ -35,7 +35,7 @@ Out-of-band serialization separates the Fory root bytes from selected buffers: `numpy.ndarray` and `pickle.PickleBuffer` are built-in native types. If an application wrapper or an object-dtype ndarray can contain custom values, register every application and Python-native carrier type before the first root attempt. That first attempt permanently freezes registration, -including when it fails; `strict=False` does not permit late discovery. +including when it fails; `strict=False` does not permit late type or serializer registration. ## Basic Out-of-Band Serialization diff --git a/docs/object-serialization/python/security.md b/docs/object-serialization/python/security.md index 0942c0196d..9ee181c76c 100644 --- a/docs/object-serialization/python/security.md +++ b/docs/object-serialization/python/security.md @@ -83,9 +83,11 @@ fory.register_type(types.FunctionType) ``` The first root attempt permanently freezes registration, including when that attempt fails. -`strict=False` does not allow late discovery or registration. If that first operation exposes an -incomplete or invalid registration, create and configure a new instance before retrying. A fully -configured reader can process a later root after a malformed-data failure. +`strict=False` does not permit type or serializer registration after that boundary, but its policy +may authorize module-global classes and callables resolved while reading a trusted native payload. +That resolution does not mutate the registry. If the first operation exposes an incomplete or +invalid registration, create and configure a new instance before retrying. A fully configured +reader can process a later root after a malformed-data failure. Received remote metadata is also limited: diff --git a/docs/object-serialization/python/serialization-hooks.md b/docs/object-serialization/python/serialization-hooks.md index 4eeda78200..022476d44f 100644 --- a/docs/object-serialization/python/serialization-hooks.md +++ b/docs/object-serialization/python/serialization-hooks.md @@ -28,7 +28,7 @@ or state restoration. Native mode supports a configured Python-only type surface that may include Python functions, local classes, closures, and reduction hooks. Register every application type and Python-native carrier before the first root attempt. The first attempt permanently freezes registration, even if it -fails, and `strict=False` does not permit late discovery. +fails, and `strict=False` does not permit late type or serializer registration. Use xlang mode instead when the payload crosses language boundaries or the data model should be a portable schema shared with other Fory implementations. diff --git a/docs/object-serialization/python/type-registration.md b/docs/object-serialization/python/type-registration.md index 5d267da4e2..08e57851e6 100644 --- a/docs/object-serialization/python/type-registration.md +++ b/docs/object-serialization/python/type-registration.md @@ -80,11 +80,11 @@ payloads, and keep the same registration IDs or names on every peer that shares those payloads. The first root serialization or deserialization attempt permanently closes -registration, including when that attempt fails. `strict=False` relaxes -the deserialization policy for registered Python-native carriers; it does not -permit late discovery or registration. Register callable, class, method, and -reduction carriers together with the application types that can appear in the -graph before the first root operation. +registration, including when that attempt fails. `strict=False` permits its +configured policy to resolve module-global classes and callables while reading +trusted native payloads, but that resolution does not reopen the registry or +install a new serializer. Register native carrier types and application types +whose serializers must be installed before the first root operation. Later registration attempts fail. Compatible metadata has one data-only exception: when a remote Struct has no diff --git a/docs/security/deserialization.md b/docs/security/deserialization.md index b4d46ccafa..96534f5f3d 100644 --- a/docs/security/deserialization.md +++ b/docs/security/deserialization.md @@ -617,24 +617,12 @@ that case, classify the behavior by concrete impact: ## Registry Lifecycle -Registration code that invokes serializer constructors, factories, or application callbacks must -recheck the authoritative per-instance registry freeze after the callback and before publishing -the entry prepared by that callback. A callback that starts the first root permanently closes the -in-progress registration; implementations must not publish and then repair or invalidate late -state. Kotlin and Scala combined generated-struct registration retain their canonical type-first -owner because generated construction resolves that type, but must recheck before the subsequent -serializer replacement. Module installation may perform complete nested registrations and must -recheck before publishing its installed marker. Automatic type IDs are allocated only at their -publication point, after callback preparation and the freeze recheck, so failed or nested -registration needs no reservation or rollback state. Thread-safe facades validate a registration -before retaining its replay callback, and application factories and callbacks execute outside -non-reentrant pool locks. - -JavaScript generated registration initializes its recursive serializer graph against local owners -before one guarded resolver publication. Application code hooks and generated factories complete -before global placeholders, nested serializers, descriptors, or cache state are changed. Runtime -and dynamic serializer lookup continues to use the authoritative resolver; the local lookup exists -only while generated factories capture fixed serializer owners. +The first root operation permanently closes type and serializer registration, including when that +operation or registry finalization fails. Registration that invokes application code must recheck +the authoritative lifecycle before publishing callback-derived state. Thread-safe facades retain +only registrations that completed before the freeze. These rules prevent a failed or reentrant +registration from changing the accepted type surface after deserialization has begun. Runtime- +specific publication ownership belongs in the implementation guide and language guidance. ## Metadata And Type Resolution @@ -661,18 +649,11 @@ Metadata readers should: entry so input cannot make the JVM derive an unbounded family of array classes. - Reset or release metadata state at the correct root-operation boundary. -Java scoped meta-share TypeInfo occurrences are root-local. The current table size -is the protocol visibility boundary, so resetting it to zero makes retained slots -unreachable from later roots. Root cleanup retains the backing array and its slots -for tables of at most 8192 entries to avoid clearing or allocating on the normal -path. After a root exceeds 8192 entries, cleanup replaces the backing array with -an eight-slot array so an unusual metadata high-water mark is not retained. - -JavaScript root entry releases reference and metadata state left by the -previous operation, including a failed operation, before the context is reused. -Operation-local reader occurrences and writer metadata owner IDs are reset at -that reuse boundary. Full reference and metadata cleanup does not run on the -root exit path. +Operation-local metadata occurrences and writer IDs must be reset before the context is reused, +including after a failed root. The reset must make prior-root entries unreachable and release +unusual high-water backing without adding allocation or slot-clearing work to normal roots. +Runtime-specific retention thresholds and reset ownership belong in the implementation guide and +language guidance. A class-resolution cache reachable from untrusted deserialization may publish an entry only from explicit trusted configuration or after the active class diff --git a/docs/specification/xlang_implementation_guide.md b/docs/specification/xlang_implementation_guide.md index 57364faf17..8120395b5f 100644 --- a/docs/specification/xlang_implementation_guide.md +++ b/docs/specification/xlang_implementation_guide.md @@ -129,6 +129,14 @@ table's current logical count participate in the next reset; otherwise prior owners create duplicate cleanup work across roots. Implementations should release an unusual high-water backing table. +Java scoped meta-share `TypeInfo` occurrence tables use their logical size as the protocol +visibility boundary. A table with at most 8192 active entries resets only that size and retains its +slots; a larger table replaces its backing with eight slots. This uniform owner rule keeps normal +cleanup allocation-free and must not be specialized for particular entry counts or benchmark +shapes. JavaScript read-side occurrence arrays use native replacement reset instead. Its MetaString +and TypeMeta writer owner tables retain bounded backing through 8192 active owners, reset only +their own logical size after restoring active owner IDs, and release backing above that boundary. + That operation-local state includes: - the current buffer diff --git a/python/pyfory/_fory.py b/python/pyfory/_fory.py index 5393f268ad..d33dfb9e59 100644 --- a/python/pyfory/_fory.py +++ b/python/pyfory/_fory.py @@ -167,9 +167,11 @@ def __init__( classes (default: True). Compatible metadata for an unregistered remote Struct uses the fixed data-only UnknownStruct carrier instead of loading or generating the sender-named class. Disabling strict mode authorizes - configured native carriers but does not permit discovery or registration - after the first root. Dynamic application types can be insecure if - malicious code exists in __new__/__init__/__eq__/__hash__ methods. + configured native carriers. Policy-authorized module globals may be + resolved while reading trusted native payloads, but the first root still + freezes type and serializer registration. Dynamic application types can + be insecure if malicious code exists in __new__/__init__/__eq__/__hash__ + methods. **WARNING**: Only disable in trusted environments. When disabling strict mode, you should provide a custom `policy` parameter to control which types are allowed. We are not responsible for security risks when this option From 1bec57b01001d55e59dd74385f112612a813981d Mon Sep 17 00:00:00 2001 From: chaokunyang Date: Fri, 28 Aug 2026 06:10:10 +0800 Subject: [PATCH 041/168] docs(python): distinguish dynamic resolution from registration --- python/README.md | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/python/README.md b/python/README.md index ab1dafbd46..343af9683a 100644 --- a/python/README.md +++ b/python/README.md @@ -140,9 +140,9 @@ print(result) # Person(name='Bob', age=25, ...) Register every application type and native carrier type before the first root serialization or deserialization call. The first root operation permanently freezes that `Fory` instance's registry, -even when the operation fails. `strict=False` relaxes deserialization policy; it does not permit late -type discovery or registration. If the first operation exposes incomplete or invalid registration, -configure a new instance before retrying. +even when the operation fails. `strict=False` may authorize module-global resolution while reading +a trusted native payload, but it does not permit late type or serializer registration. If the first +operation exposes incomplete or invalid registration, configure a new instance before retrying. **Security Warning**: Configured native carriers can import modules and construct Python objects when `strict=False`. Use this mode only with trusted payloads, and provide a From 50e7df075c9d56787013aa22ffae79d12abebff6 Mon Sep 17 00:00:00 2001 From: chaokunyang Date: Fri, 28 Aug 2026 07:20:10 +0800 Subject: [PATCH 042/168] fix(csharp): publish registrations atomically --- .agents/languages/csharp.md | 3 +- csharp/src/Fory/Fory.cs | 4 +- csharp/src/Fory/ThreadSafeFory.cs | 29 +--- csharp/src/Fory/TypeResolver.cs | 40 ++++- .../tests/Fory.Tests/RuntimeEdgeCaseTests.cs | 140 ++++++++++++++++-- 5 files changed, 165 insertions(+), 51 deletions(-) diff --git a/.agents/languages/csharp.md b/.agents/languages/csharp.md index c9bfe38fad..6c68f1fc32 100644 --- a/.agents/languages/csharp.md +++ b/.agents/languages/csharp.md @@ -15,7 +15,8 @@ Load this file when changing `csharp/` or C# xlang behavior. to the replay log. Serializer construction and generated factories may reenter a root, so direct registration must recheck the facade after resolving the serializer and before resolver mutation. `ThreadSafeFory` must recheck disposal and freeze after staging registration and before replay-log - publication, and a staging failure must not rebuild after either lifecycle boundary has closed. + publication. Resolver registration must prepare all serializer and MetaString state before its + single map commit, so failed validation needs no staging rebuild or identity workaround. New per-thread runtimes replay that log; do not mutate existing runtimes or introduce another freeze owner. - Generated C# gRPC service companions are compiler-owned files that depend on application-provided gRPC packages, not `csharp/src/Fory`. Keep gRPC package references out of the Fory runtime package. diff --git a/csharp/src/Fory/Fory.cs b/csharp/src/Fory/Fory.cs index cf48ecfb39..2e86dae908 100644 --- a/csharp/src/Fory/Fory.cs +++ b/csharp/src/Fory/Fory.cs @@ -181,9 +181,11 @@ public byte[] Serialize(in T value) FreezeRegistry(); ByteWriter writer = _writeContext.Writer; writer.Reset(); + // A previous failed root may leave references behind. Reset before serializer lookup, + // because generated or application serializer factories can fail during that lookup. + _writeContext.ResetFor(writer); Serializer serializer = _typeResolver.GetSerializer(); WriteHead(writer); - _writeContext.ResetFor(writer); RefMode refMode = Config.TrackRef ? RefMode.Tracking : RefMode.NullOnly; serializer.Write(_writeContext, value, refMode, true, false); _writeContext.RefWriter.Reset(); diff --git a/csharp/src/Fory/ThreadSafeFory.cs b/csharp/src/Fory/ThreadSafeFory.cs index f3a21a99a0..6818695ad0 100644 --- a/csharp/src/Fory/ThreadSafeFory.cs +++ b/csharp/src/Fory/ThreadSafeFory.cs @@ -229,23 +229,7 @@ private void ApplyRegistration(Action registration) ThrowRegistryFrozen(); } - try - { - registration(_registrationFory ??= new Fory(_config)); - } - catch - { - if (!_disposed && _registryFrozen == 0) - { - Fory? rebuilt = _registrations.Count == 0 ? null : RebuildRegistrationFory(); - // Rebuilding replays serializer constructors, which can close this wrapper. - if (!_disposed && _registryFrozen == 0) - { - _registrationFory = rebuilt; - } - } - throw; - } + registration(_registrationFory ??= new Fory(_config)); ThrowIfDisposed(); if (_registryFrozen != 0) @@ -282,17 +266,6 @@ private void FreezeRegistry() } } - private Fory RebuildRegistrationFory() - { - Fory fory = new(_config); - foreach (Action registration in _registrations) - { - registration(fory); - } - - return fory; - } - [MethodImpl(MethodImplOptions.NoInlining)] private static void ThrowRegistryFrozen() => throw new InvalidOperationException( diff --git a/csharp/src/Fory/TypeResolver.cs b/csharp/src/Fory/TypeResolver.cs index 945c31dd4e..8aac5bae1b 100644 --- a/csharp/src/Fory/TypeResolver.cs +++ b/csharp/src/Fory/TypeResolver.cs @@ -509,9 +509,14 @@ internal TypeInfo PrepareRegistration() return TypeInfo.Create(typeof(T), new TSerializer()); } - internal void Register(Type type, uint id, TypeInfo? explicitTypeInfo = null) + internal void Register(Type type, uint id) { - TypeInfo typeInfo = GetOrCreateTypeInfo(type, explicitTypeInfo).WithTypeIdRegistration(id); + Register(type, id, PrepareRegistration(type)); + } + + internal void Register(Type type, uint id, TypeInfo typeInfo) + { + typeInfo = PrepareRegistration(type, typeInfo).WithTypeIdRegistration(id); _typeInfos.Set(TypeMapKey.Get(type), typeInfo); _byUserTypeId[id] = typeInfo; InvalidateFinalizedVersion(); @@ -551,18 +556,43 @@ internal static void ValidateSplitTypeName(string namespaceName, string typeName } } - internal void Register(Type type, string namespaceName, string typeName, TypeInfo? explicitTypeInfo = null) + internal void Register(Type type, string namespaceName, string typeName) + { + ValidateSplitTypeName(namespaceName, typeName); + Register(type, namespaceName, typeName, PrepareRegistration(type)); + } + + internal void Register(Type type, string namespaceName, string typeName, TypeInfo typeInfo) { ValidateSplitTypeName(namespaceName, typeName); - TypeInfo typeInfo = GetOrCreateTypeInfo(type, explicitTypeInfo); MetaString namespaceMeta = MetaStringEncoder.Namespace.Encode(namespaceName, TypeMetaEncodings.NamespaceMetaStringEncodings); MetaString typeNameMeta = MetaStringEncoder.TypeName.Encode(typeName, TypeMetaEncodings.TypeNameMetaStringEncodings); - typeInfo = typeInfo.WithTypeNameRegistration(namespaceMeta, typeNameMeta); + typeInfo = PrepareRegistration(type, typeInfo).WithTypeNameRegistration(namespaceMeta, typeNameMeta); _typeInfos.Set(TypeMapKey.Get(type), typeInfo); _byTypeName[(namespaceName, typeName)] = typeInfo; InvalidateFinalizedVersion(); } + private TypeInfo PrepareRegistration(Type type, TypeInfo typeInfo) + { + if (typeInfo.Type != type) + { + throw new InvalidDataException($"serializer type mismatch for {type}, got {typeInfo.Type}"); + } + + if (!_typeInfos.TryGetValue(TypeMapKey.Get(type), out TypeInfo? existing) || ReferenceEquals(existing, typeInfo)) + { + return typeInfo; + } + + if (existing.IsRegistered) + { + throw new InvalidDataException($"cannot override serializer for registered type {type}"); + } + + return typeInfo.WithRegistrationFrom(existing); + } + ///

/// Returns a finalized semantic resolver version used by generated/static caches. /// The version is computed lazily and changes whenever bindings/registrations change. diff --git a/csharp/tests/Fory.Tests/RuntimeEdgeCaseTests.cs b/csharp/tests/Fory.Tests/RuntimeEdgeCaseTests.cs index ab2528431a..1b5e6d69c4 100644 --- a/csharp/tests/Fory.Tests/RuntimeEdgeCaseTests.cs +++ b/csharp/tests/Fory.Tests/RuntimeEdgeCaseTests.cs @@ -138,6 +138,65 @@ public override GeneratedFrozenValue ReadData(ReadContext context) } } +public enum LookupFailureValue +{ + Zero, +} + +public sealed class LookupFailureSerializer : Serializer +{ + public static Action? ConstructionAction; + + public LookupFailureSerializer() + { + ConstructionAction?.Invoke(); + } + + public override LookupFailureValue DefaultValue => LookupFailureValue.Zero; + + public override void WriteData(WriteContext context, in LookupFailureValue value, bool hasGenerics) + { + _ = context; + _ = value; + _ = hasGenerics; + } + + public override LookupFailureValue ReadData(ReadContext context) + { + _ = context; + return LookupFailureValue.Zero; + } +} + +public enum AtomicRegistrationValue +{ + Zero, + One, +} + +public sealed class AtomicRegistrationSerializer : Serializer +{ + public static Action? ConstructionAction; + + public AtomicRegistrationSerializer() + { + ConstructionAction?.Invoke(); + } + + public override AtomicRegistrationValue DefaultValue => AtomicRegistrationValue.Zero; + + public override void WriteData(WriteContext context, in AtomicRegistrationValue value, bool hasGenerics) + { + _ = hasGenerics; + context.Writer.WriteVarInt32((int)value); + } + + public override AtomicRegistrationValue ReadData(ReadContext context) + { + return (AtomicRegistrationValue)context.Reader.ReadVarInt32(); + } +} + [ForyStruct] public sealed class FailingWritePayload { @@ -971,37 +1030,49 @@ public void ThreadSafeFailedReentryClears() } [Fact] - public void ThreadSafeRebuildFreezeClears() + public void ThreadSafeFailureDoesNotPublish() { + TypeResolver.RegisterGenerated(); using ThreadSafeFory fory = ForyRuntime.Builder().BuildThreadSafe(); - fory.Register(731); - bool rootStarted = false; + fory.Register(734); + int constructions = 0; + bool nestedFailed = false; + bool stageRetained = false; + AtomicRegistrationSerializer.ConstructionAction = () => constructions++; FrozenPayloadSerializer.ConstructionAction = () => { - if (rootStarted) + ForyRuntime? stage = RegistrationForyFor(fory); + try { - return; + fory.Register(new string('a', 32_767)); } - - rootStarted = true; - _ = fory.Serialize(1); + catch + { + nestedFailed = true; + } + stageRetained = ReferenceEquals(stage, RegistrationForyFor(fory)); }; - GeneratedFrozenSerializer.ConstructionAction = - () => throw new InvalidOperationException("registration failure"); try { - Exception error = Assert.ThrowsAny( - () => fory.Register(732)); - Assert.IsType(error.InnerException ?? error); - Assert.Null(RegistrationForyFor(fory)); - Assert.Throws(() => fory.Register(733)); + fory.Register(735); + Assert.True(nestedFailed); + Assert.True(stageRetained); + Assert.Equal(1, constructions); + fory.Register(737); + Assert.Equal(2, constructions); } finally { FrozenPayloadSerializer.ConstructionAction = null; - GeneratedFrozenSerializer.ConstructionAction = null; + AtomicRegistrationSerializer.ConstructionAction = null; } + + FrozenPayload value = new() { Value = 1 }; + Assert.Equal(value.Value, fory.Deserialize(fory.Serialize(value)).Value); + Assert.Equal( + AtomicRegistrationValue.One, + fory.Deserialize(fory.Serialize(AtomicRegistrationValue.One))); } [Fact] @@ -1027,6 +1098,34 @@ public void FailedWriteRestoresNextRoot() Assert.Equal(7, fory.Deserialize(fory.Serialize(7))); } + [Fact] + public void FailedLookupRestoresWriteState() + { + TypeResolver.RegisterGenerated(); + ForyRuntime fory = ForyRuntime.Builder().TrackRef(true).Build(); + fory.Register(736); + + Assert.Throws( + () => fory.Serialize(new FailingWritePayload { Value = 1 })); + bool lookupStarted = false; + LookupFailureSerializer.ConstructionAction = () => + { + lookupStarted = true; + throw new InvalidOperationException("serializer lookup failed"); + }; + try + { + Assert.ThrowsAny(() => fory.Serialize(LookupFailureValue.Zero)); + Assert.True(lookupStarted); + } + finally + { + LookupFailureSerializer.ConstructionAction = null; + } + + Assert.Equal(0u, WriteContextFor(fory).RefWriter.ReserveRefId()); + } + [Fact] public void FailedReaderRootFreezesRegistry() { @@ -1191,6 +1290,15 @@ private static ReadContext ReadContextFor(ForyRuntime fory) return Assert.IsType(field.GetValue(fory)); } + private static WriteContext WriteContextFor(ForyRuntime fory) + { + System.Reflection.FieldInfo? field = typeof(ForyRuntime).GetField( + "_writeContext", + System.Reflection.BindingFlags.Instance | System.Reflection.BindingFlags.NonPublic); + Assert.NotNull(field); + return Assert.IsType(field.GetValue(fory)); + } + private static ForyRuntime? RegistrationForyFor(ThreadSafeFory fory) { System.Reflection.FieldInfo? field = typeof(ThreadSafeFory).GetField( From d3e41ebb29ef5baf054e7ec667f3582110a9db22 Mon Sep 17 00:00:00 2001 From: chaokunyang Date: Fri, 28 Aug 2026 07:21:57 +0800 Subject: [PATCH 043/168] fix(python): keep pooled serializers instance-owned --- .agents/languages/python.md | 8 +- .../python/basic-serialization.md | 11 +- .../python/configuration.md | 23 ++-- .../python/custom-serializers.md | 24 ++++ .../python/functions-classes-methods.md | 12 +- .../python/type-registration.md | 4 + python/README.md | 44 +++---- python/pyfory/_fory.py | 55 ++++++--- python/pyfory/tests/test_thread_safe.py | 108 ++++++++++++++++++ 9 files changed, 217 insertions(+), 72 deletions(-) diff --git a/.agents/languages/python.md b/.agents/languages/python.md index 638b93c3b3..15e6fdfa20 100644 --- a/.agents/languages/python.md +++ b/.agents/languages/python.md @@ -23,8 +23,12 @@ Load this file when changing `python/`, Cython serialization, or Python xlang be linearization is reentrant so nested facade registrations share the same publication order. A root started during registration must not reuse the staging instance, and root reentry from a running user `fory_factory` or retained registration callback fails without recursively building - another instance. The non-reentrant pool lock owns only pool state; the separate instance-build - boundary covers the factory and complete callback replay. + another instance. The build thread must be rejected before pool acquisition even when another + instance becomes available during that build. The non-reentrant pool lock owns pool publication, + root-started state, registration depth, and the staging instance; the separate instance-build + boundary covers the factory and complete callback replay. Retained callbacks may capture a + serializer class or factory, but never a resolver-bound serializer instance. Instance-specific + serializer configuration belongs in `fory_factory`, which creates and configures each child. - Registry freeze prohibits type and serializer publication after the first root; it does not prohibit policy-authorized resolution of module-global classes or callables during a non-strict native read when that resolution does not mutate registry state. Do not describe these two diff --git a/docs/object-serialization/python/basic-serialization.md b/docs/object-serialization/python/basic-serialization.md index 1c39ea5da8..77b326a7bf 100644 --- a/docs/object-serialization/python/basic-serialization.md +++ b/docs/object-serialization/python/basic-serialization.md @@ -43,11 +43,12 @@ print(obj) # {'name': 'Alice', 'age': 30, 'scores': [95, 87, 92]} ## Registration Lifecycle -Register every application type before the first root serialization or deserialization attempt. In -Python native mode, also register any callable, class, method, state, or reduction carrier that can -appear in the graph. The first root serialization or deserialization attempt permanently freezes -the instance's registry, even when the operation fails. `strict=False` does not enable late type -discovery or registration. +Complete every explicit type and serializer registration before the first root serialization or +deserialization attempt. In Python native mode, register the application and native carrier types +whose serializers must be installed before that first root. The first root serialization or +deserialization attempt permanently freezes the instance's registry, even when the operation +fails. `strict=False` does not enable late type or serializer registration; its policy may still +authorize module-global resolution during a trusted native read without mutating the registry. If the first operation fails because registration is incomplete or invalid, create a new instance, register the complete type surface, and retry with that instance. A fully configured instance can diff --git a/docs/object-serialization/python/configuration.md b/docs/object-serialization/python/configuration.md index fa2e9baa15..dc8673acaa 100644 --- a/docs/object-serialization/python/configuration.md +++ b/docs/object-serialization/python/configuration.md @@ -54,9 +54,7 @@ Thread-safe serialization interface using a pooled wrapper: ```python class ThreadSafeFory: - def __init__( - self, fory_factory=None, **kwargs - ) + def __init__(self, fory_factory=None, **kwargs) ``` ## Parameters @@ -82,6 +80,17 @@ class ThreadSafeFory: ## Key Methods ```python +# Complete registration before serialization or deserialization. This form is valid in native +# and xlang modes when no explicit stable identity is required. +fory.register(MyClass) + +# Alternatively, use one of these forms when the xlang schema needs an explicit stable identity. +# fory.register(MyClass, type_id=123) +# fory.register(MyClass, name="my.package.MyClass") + +# Direct Fory accepts an instance; ThreadSafeFory requires a serializer class or factory. +# fory.register(MyClass, serializer=MySerializer) + # Serialization (serialize/deserialize are identical to dumps/loads) data: bytes = fory.serialize(obj) obj = fory.deserialize(data) @@ -89,14 +98,6 @@ obj = fory.deserialize(data) # Alternative API (aliases) data: bytes = fory.dumps(obj) obj = fory.loads(data) - -# Type registration by id -fory.register(MyClass, type_id=123) -fory.register(MyClass, type_id=123, serializer=custom_serializer) - -# Type registration by name -fory.register(MyClass, name="my.package.MyClass") -fory.register(MyClass, name="my.package.MyClass", serializer=custom_serializer) ``` Complete registration before the first root serialization or deserialization attempt. That first diff --git a/docs/object-serialization/python/custom-serializers.md b/docs/object-serialization/python/custom-serializers.md index da907d4302..615a6b129f 100644 --- a/docs/object-serialization/python/custom-serializers.md +++ b/docs/object-serialization/python/custom-serializers.md @@ -131,6 +131,30 @@ fory.register(MyClass, type_id=100, serializer=MySerializer(fory.type_resolver, fory.register(MyClass, name="com.example.MyClass", serializer=MySerializer(fory.type_resolver, MyClass)) ``` +### Thread-safe registration + +`ThreadSafeFory` accepts a serializer class or factory rather than an already constructed +serializer. Each pooled `Fory` then owns a serializer bound to its own resolver: + +```python +thread_safe = pyfory.ThreadSafeFory(xlang=False) +thread_safe.register(Foo, type_id=100, serializer=FooSerializer) +``` + +When construction needs instance-specific settings, configure each child through the existing +`fory_factory`: + +```python +def create_fory(): + child = pyfory.Fory(xlang=False) + serializer = FooSerializer(child.type_resolver, Foo) + serializer.application_option = "configured" + child.register(Foo, type_id=100, serializer=serializer) + return child + +thread_safe = pyfory.ThreadSafeFory(fory_factory=create_fory) +``` + ## Related Topics - [Type Registration](type-registration.md) - Registration patterns diff --git a/docs/object-serialization/python/functions-classes-methods.md b/docs/object-serialization/python/functions-classes-methods.md index 13502e2c0b..108aae1d9d 100644 --- a/docs/object-serialization/python/functions-classes-methods.md +++ b/docs/object-serialization/python/functions-classes-methods.md @@ -77,7 +77,8 @@ print(fory.loads(data)(10)) # 100 ## Serialize Methods -Register the method carriers and receiver class before serializing bound, class, or static methods: +Register the method carriers and receiver class before serializing bound instance methods. A +static method is serialized as its underlying function: ```python import pyfory @@ -89,23 +90,16 @@ class Calculator: def scale(self, x): return 3 * x - @classmethod - def ten(cls, x): - return 10 * x - @staticmethod def double(x): return 2 * x -for carrier in (type, types.FunctionType, types.MethodType, staticmethod, classmethod, Calculator): +for carrier in (types.FunctionType, types.MethodType, Calculator): fory.register_type(carrier) # Serialize instance method print(fory.loads(fory.dumps(Calculator().scale))(10)) # 30 -# Serialize class method -print(fory.loads(fory.dumps(Calculator.ten))(10)) # 100 - # Serialize static method print(fory.loads(fory.dumps(Calculator.double))(10)) # 20 ``` diff --git a/docs/object-serialization/python/type-registration.md b/docs/object-serialization/python/type-registration.md index 08e57851e6..cb79c944fd 100644 --- a/docs/object-serialization/python/type-registration.md +++ b/docs/object-serialization/python/type-registration.md @@ -72,6 +72,10 @@ for model_class in [User, Order, Product, Invoice]: type_id += 1 ``` +A direct `Fory` may receive a serializer instance. `ThreadSafeFory` accepts a serializer class or +factory so every pooled child constructs a serializer against its own resolver. Use +`fory_factory` for serializer instances that need per-child configuration. + ## Strict Mode Relationship With `strict=True`, Fory loads and instantiates only registered application diff --git a/python/README.md b/python/README.md index 343af9683a..50c0457edc 100644 --- a/python/README.md +++ b/python/README.md @@ -38,7 +38,7 @@ object serialization together with row-format APIs for analytical data. ### **Security & Safety** -- **Strict mode** prevents deserialization of untrusted types by type registration and checks. +- **Strict mode** limits application class loading to the configured registration surface. - **Reference tracking** for handling circular references safely ## Installation @@ -226,7 +226,8 @@ print(fory.loads(data)(10)) # 100 #### Serialize Methods -Register method carriers and receiver classes before serializing bound, class, or static methods: +Register method carriers and receiver classes before serializing bound instance methods. A static +method is serialized as its underlying function: ```python import pyfory @@ -238,19 +239,14 @@ class Calculator: def scale(self, x): return 3 * x - @classmethod - def ten(cls, x): - return 10 * x - @staticmethod def double(x): return 2 * x -for carrier in (type, types.FunctionType, types.MethodType, staticmethod, classmethod, Calculator): +for carrier in (types.FunctionType, types.MethodType, Calculator): fory.register_type(carrier) print(fory.loads(fory.dumps(Calculator().scale))(10)) # 30 -print(fory.loads(fory.dumps(Calculator.ten))(10)) # 100 print(fory.loads(fory.dumps(Calculator.double))(10)) # 20 ``` @@ -514,16 +510,12 @@ Thread-safe serialization interface for sharing one configured facade across thr ```python class ThreadSafeFory: - def __init__( - self, - xlang: bool = True, - ref: bool = False, - strict: bool = True, - compatible: bool | None = None, - max_depth: int = 50 - ) + def __init__(self, fory_factory=None, **kwargs) ``` +Without `fory_factory`, keyword arguments are forwarded to each pooled `Fory`. Use +`fory_factory` when each pooled instance needs custom instance-level configuration. + Register all types before the first serialization or deserialization attempt. That first attempt permanently freezes registration, even when it fails. Every later registration attempt raises an error. @@ -584,7 +576,7 @@ for t in threads: t.join() fory.register(MyClass, type_id=123) # Alternatively, register by name or provide a custom serializer. # fory.register(MyClass, name="my.package.MyClass") -# fory.register(MyClass, type_id=123, serializer=custom_serializer) +# fory.register(MyClass, type_id=123, serializer=MySerializer) # serialize/deserialize are identical to dumps/loads. data: bytes = fory.serialize(obj) @@ -595,15 +587,15 @@ obj = fory.loads(data) ### Xlang And Native Mode Comparison -| Feature | Native mode (`xlang=False`) | Xlang mode (default) | -| ------------------- | --------------------------------------------- | ------------------------------------- | -| Use case | Pure Python applications | Multi-language systems | -| Compatibility | Python only | Java, Go, Rust, C++, JavaScript, etc. | -| Supported types | Configured Python type surface | Cross-language compatible types | -| Functions/lambdas | Supported with registered native carriers | Not allowed | -| Methods | Supported with registered carriers and owners | Not allowed | -| Stateful/reduce | Supported with registered application types | Not allowed | -| Schema mode default | Compatible | Compatible | +| Feature | Native mode (`xlang=False`) | Xlang mode (default) | +| ------------------- | ----------------------------------------- | ------------------------------------- | +| Use case | Pure Python applications | Multi-language systems | +| Compatibility | Python only | Java, Go, Rust, C++, JavaScript, etc. | +| Supported types | Configured Python type surface | Cross-language compatible types | +| Functions/lambdas | Registered and policy-authorized carriers | Not allowed | +| Instance methods | Registered and policy-authorized carriers | Not allowed | +| Stateful/reduce | Registered and policy-authorized types | Not allowed | +| Schema mode default | Compatible | Compatible | #### Native Mode (`xlang=False`) diff --git a/python/pyfory/_fory.py b/python/pyfory/_fory.py index d33dfb9e59..5fe6b4068a 100644 --- a/python/pyfory/_fory.py +++ b/python/pyfory/_fory.py @@ -16,6 +16,7 @@ # under the License. import os +import threading from abc import ABC, abstractmethod from typing import Iterable, Optional, Union @@ -402,12 +403,6 @@ def register_serializer(self, cls: type, serializer): """ self.type_resolver.register_serializer(cls, serializer) - def _freeze_registry(self): - # The resolver remains authoritative because callers may register through - # it directly. Avoid entering its finalization method after the first root. - if not self.type_resolver._registry_frozen: - self.type_resolver._freeze_registry() - def dumps( self, obj, @@ -435,7 +430,8 @@ def dump(self, obj, stream): the passed object (or a view of it) is unsupported. If your sink needs retention, copy bytes inside ``write``. """ - self._freeze_registry() + if not self.type_resolver._registry_frozen: + self.type_resolver._freeze_registry() try: self.buffer.set_writer_index(0) output_stream = Buffer.wrap_output_stream(stream) @@ -491,7 +487,8 @@ def serialize( >>> print(type(data)) """ - self._freeze_registry() + if not self.type_resolver._registry_frozen: + self.type_resolver._freeze_registry() try: write_buffer = self._serialize( obj, @@ -569,7 +566,8 @@ def deserialize( >>> print(obj) {'key': 'value'} """ - self._freeze_registry() + if not self.type_resolver._registry_frozen: + self.type_resolver._freeze_registry() try: return self._deserialize(buffer, buffers, unsupported_objects) finally: @@ -645,9 +643,13 @@ class ThreadSafeFory: All type registrations must be performed before the first root serialization or deserialization attempt to ensure consistency across all pooled instances. Registration - remains closed even when that first operation fails. + remains closed even when that first operation fails. Custom serializer registrations accept + a serializer class or factory so every pooled instance owns a serializer bound to its own + resolver. Use ``fory_factory`` for configured serializer instances. Args: + fory_factory (Callable): Optional factory that creates and configures each pooled Fory. + When omitted, remaining keyword arguments are forwarded to Fory. xlang (bool): Whether to enable xlang mode. Defaults to True. ref (bool): Whether to enable reference tracking. Defaults to False. strict (bool): Whether to require type registration. Defaults to True. @@ -688,8 +690,6 @@ class ThreadSafeFory: """ def __init__(self, fory_factory=None, **kwargs): - import threading - self._config = kwargs self._fory_factory = fory_factory self._callbacks = [] @@ -697,7 +697,7 @@ def __init__(self, fory_factory=None, **kwargs): self._registration_lock = threading.RLock() self._registration_depth = 0 self._registration_fory = None - self._fory_building = False + self._building_thread = None self._pool = [] if fory_factory is not None: self._fory_class = None @@ -710,10 +710,11 @@ def __init__(self, fory_factory=None, **kwargs): self._root_started = False def _build_fory(self): + thread_id = threading.get_ident() with self._registration_lock: - if self._fory_building: + if self._building_thread == thread_id: raise RuntimeError("Cannot start a root serialization or deserialization operation while a Fory instance is being built.") - self._fory_building = True + self._building_thread = thread_id try: if self._fory_factory is not None: fory = self._fory_factory() @@ -723,9 +724,16 @@ def _build_fory(self): callback(fory) return fory finally: - self._fory_building = False + self._building_thread = None def _get_fory(self): + # Check the active builder before the pool: factory callbacks must not evade root-reentry + # rejection by returning an instance to the facade that is still building it. + building_thread = self._building_thread + if building_thread is not None and building_thread == threading.get_ident(): + with self._lock: + self._root_started = True + raise RuntimeError("Cannot start a root serialization or deserialization operation while a Fory instance is being built.") with self._lock: if self._pool: return self._pool.pop() @@ -780,6 +788,15 @@ def _check_registration_open(self): if self._root_started: raise RuntimeError("Cannot register types after the first root serialization or deserialization operation has started.") + @staticmethod + def _check_serializer_factory(serializer): + if serializer is None: + return + from pyfory.serializer import Serializer + + if isinstance(serializer, Serializer): + raise TypeError("ThreadSafeFory requires a serializer class or factory; use fory_factory to install serializer instances per Fory") + def register( self, cls, @@ -788,6 +805,7 @@ def register( name: str = None, serializer=None, ): + self._check_serializer_factory(serializer) self._register_callback(lambda f: f.register(cls, type_id=type_id, name=name, serializer=serializer)) def register_type( @@ -798,6 +816,7 @@ def register_type( name: str = None, serializer=None, ): + self._check_serializer_factory(serializer) self._register_callback(lambda f: f.register_type(cls, type_id=type_id, name=name, serializer=serializer)) def register_union( @@ -808,11 +827,9 @@ def register_union( name: str = None, serializer=None, ): + self._check_serializer_factory(serializer) self._register_callback(lambda f: f.register_union(cls, type_id=type_id, name=name, serializer=serializer)) - def register_serializer(self, cls: type, serializer): - self._register_callback(lambda f: f.register_serializer(cls, serializer)) - def serialize( self, obj, diff --git a/python/pyfory/tests/test_thread_safe.py b/python/pyfory/tests/test_thread_safe.py index 4f1b7a80fe..3f092c5796 100644 --- a/python/pyfory/tests/test_thread_safe.py +++ b/python/pyfory/tests/test_thread_safe.py @@ -229,6 +229,97 @@ def serializer_factory(type_resolver, cls): assert valid_constructions == 1 +@pytest.mark.parametrize("method", ["register", "register_type", "register_union"]) +def test_serializer_instance_rejected(method): + class AddressSerializer(pyfory.Serializer): + def write(self, write_context, value): + write_context.write_string(value.city) + write_context.write_string(value.country) + + def read(self, read_context): + return Address(read_context.read_string(), read_context.read_string()) + + owner = pyfory.Fory(xlang=False, compatible=False) + serializer = AddressSerializer(owner.type_resolver, Address) + builds = 0 + + def fory_factory(): + nonlocal builds + builds += 1 + return pyfory.Fory(xlang=False, compatible=False) + + fory = ThreadSafeFory(fory_factory=fory_factory) + + with pytest.raises(TypeError): + getattr(fory, method)(Address, serializer=serializer) + + assert fory._callbacks == [] + assert fory._registration_fory is None + assert not fory._root_started + assert builds == 0 + + +def test_serializer_factory_per_child(): + class AddressSerializer(pyfory.Serializer): + def write(self, write_context, value): + write_context.write_string(value.city) + write_context.write_string(value.country) + + def read(self, read_context): + return Address(read_context.read_string(), read_context.read_string()) + + serializers = [] + + def serializer_factory(type_resolver, cls): + serializer = AddressSerializer(type_resolver, cls) + serializers.append(serializer) + return serializer + + fory = ThreadSafeFory(xlang=False, compatible=False) + fory.register_type(Address, serializer=serializer_factory) + first = fory._registration_fory + second = fory._build_fory() + + assert len(serializers) == 2 + assert serializers[0] is not serializers[1] + assert serializers[0].type_resolver is first.type_resolver + assert serializers[1].type_resolver is second.type_resolver + address = Address(city="Oslo", country="Norway") + assert second.deserialize(second.serialize(address)) == address + assert fory.deserialize(fory.serialize(address)) == address + + +def test_factory_serializer_owner(): + class AddressSerializer(pyfory.Serializer): + def write(self, write_context, value): + write_context.write_string(value.city) + write_context.write_string(value.country) + + def read(self, read_context): + return Address(read_context.read_string(), read_context.read_string()) + + serializers = [] + + def fory_factory(): + fory = pyfory.Fory(xlang=False, compatible=False) + serializer = AddressSerializer(fory.type_resolver, Address) + fory.register_type(Address, serializer=serializer) + serializers.append(serializer) + return fory + + fory = ThreadSafeFory(fory_factory=fory_factory) + first = fory._build_fory() + second = fory._build_fory() + + assert len(serializers) == 2 + assert serializers[0] is not serializers[1] + assert serializers[0].type_resolver is first.type_resolver + assert serializers[1].type_resolver is second.type_resolver + address = Address(city="Oslo", country="Norway") + assert first.deserialize(first.serialize(address)) == address + assert second.deserialize(second.serialize(address)) == address + + def test_reentrant_registration(): class AddressSerializer(pyfory.Serializer): def write(self, write_context, value): @@ -332,6 +423,23 @@ def fory_factory(): fory.register_type(Address) +def test_build_owner_precedes_pool(): + fory = None + pooled = pyfory.Fory(xlang=False, compatible=False) + + def fory_factory(): + fory._return_fory(pooled) + fory.serialize(None) + return pyfory.Fory(xlang=False, compatible=False) + + fory = ThreadSafeFory(fory_factory=fory_factory) + + with pytest.raises(RuntimeError): + fory.serialize(None) + + assert fory._pool == [pooled] + + def test_callback_root_reentry(): class AddressSerializer(pyfory.Serializer): def write(self, write_context, value): From 20b66fe42404de6471c57e2b5368c2eedc82093f Mon Sep 17 00:00:00 2001 From: chaokunyang Date: Fri, 28 Aug 2026 07:22:07 +0800 Subject: [PATCH 044/168] fix(javascript): publish generated serializers once --- .agents/languages/javascript.md | 12 +- .../javascript/type-registration.md | 4 + javascript/packages/core/lib/context.ts | 5 +- javascript/packages/core/lib/gen/index.ts | 99 ++++++-- javascript/packages/core/lib/typeResolver.ts | 227 +++++------------- javascript/test/fory.test.ts | 87 ++++--- javascript/test/rootCleanup.test.ts | 10 +- javascript/test/typemeta.test.ts | 24 -- 8 files changed, 199 insertions(+), 269 deletions(-) diff --git a/.agents/languages/javascript.md b/.agents/languages/javascript.md index 2316563618..789f3a3955 100644 --- a/.agents/languages/javascript.md +++ b/.agents/languages/javascript.md @@ -13,8 +13,8 @@ Load this file when changing `javascript/`. - JavaScript TypeMeta header cache hits should compare the 52-bit TypeMeta header hash directly. The hash is precise in JS `Number` and already includes the low header bits as hash input; do not add extra low-bit fields, sentinel state, nullable accepted headers, or parallel slot arrays around it. - Root entry releases reference and metadata state left by the previous operation, including a failed operation, before the context is reused. Do not put full cleanup on the root exit path or - copy Java backing-array retention policies onto native JavaScript arrays. Read-side occurrence - arrays use native replacement reset. The MetaString and TypeMeta writer owner tables each have + copy Java backing-array retention policies onto native JavaScript arrays. Read-side metadata + occurrence arrays use native replacement reset. The MetaString and TypeMeta writer owner tables each have their own logical size: reset active owner IDs and that table's logical size without clearing bounded backing, and replace either backing only after its root has more than 8192 owners. - Generated registration must initialize the complete recursive serializer graph against @@ -22,10 +22,10 @@ Load this file when changing `javascript/`. `TypeInfo` schema graph before code generation; schema fields and occurrence modifiers are immutable afterward, while `dynamicTypeId` remains operation-local writer state. Factory-init serializer lookup may resolve local owners for fixed captures; runtime and dynamic lookup must - keep the real resolver. Initialize an uninitialized published forward owner in place during - commit so previous captures retain identity, but never overwrite an initialized owner published - by a nested registration. Do not publish placeholders, nested serializers, descriptors, or cache - state before every generated factory and application code hook succeeds. + keep the real resolver. A nested identity-only Struct must already be registered or resolve to an + owner in the current complete recursive schema graph. Reject any unresolved nested identity + before resolver publication. Never publish placeholders, nested serializers, descriptors, or + cache state before every generated factory and application code hook succeeds. - Runtime value carriers such as decimal or reduced-precision numeric types belong under the core `types/` ownership boundary, with imports, exports, and codegen externals updated together. - Keep `TypeInfo` as schema metadata. Compatibility-sensitive decisions belong on `TypeResolver` or explicit operations, not as retained resolver state on metadata objects. - Normalize optional boolean config values at config construction; do not carry `null` through runtime paths when it means `false`. diff --git a/docs/object-serialization/javascript/type-registration.md b/docs/object-serialization/javascript/type-registration.md index 998a9045e9..a18e3d6bc2 100644 --- a/docs/object-serialization/javascript/type-registration.md +++ b/docs/object-serialization/javascript/type-registration.md @@ -152,6 +152,10 @@ const wrapperType = Type.struct("example.wrapper", { }); ``` +A nested Struct that supplies only an ID or name must already be registered. Alternatively, define +the complete nested schema in the same recursive `TypeInfo` graph. Registration rejects an +unresolved nested identity without publishing either the parent or a placeholder. + ## Field Metadata Field nullability, reference tracking, dynamic field behavior, numeric widths, and per-struct diff --git a/javascript/packages/core/lib/context.ts b/javascript/packages/core/lib/context.ts index 33e24af3f7..023bb6beb3 100644 --- a/javascript/packages/core/lib/context.ts +++ b/javascript/packages/core/lib/context.ts @@ -46,7 +46,6 @@ type TypeResolverLike = { getSerializerByData(value: any): Serializer | null | undefined; isCompatible(): boolean; generateReadSerializer(typeInfo: TypeInfo): Serializer; - regenerateReadSerializer(typeInfo: TypeInfo): Serializer; getUnknownStructSerializer(typeMeta?: TypeMeta, wireTypeId?: number): Serializer; }; @@ -255,7 +254,9 @@ export class RefReader { constructor(private reader: BinaryReader) {} reset() { - this.readObjects = []; + if (this.readObjects.length !== 0) { + this.readObjects.length = 0; + } } getReadRef(refId: number) { diff --git a/javascript/packages/core/lib/gen/index.ts b/javascript/packages/core/lib/gen/index.ts index f5d48a6be9..10324ec186 100644 --- a/javascript/packages/core/lib/gen/index.ts +++ b/javascript/packages/core/lib/gen/index.ts @@ -60,7 +60,63 @@ type SerializerFactoryBuilder = () => ( checkedTypeMetaWireTypeIdSymbol: symbol, ) => Serializer; -type SerializerCreator = (serializerLookup: SerializerLookup) => Serializer; +const uninitializedSerializer: Serializer = { + _initialized: false, + fixedSize: 0, + getTypeInfo: () => { + throw new Error("serializer is not initialized"); + }, + getTypeId: () => { + throw new Error("serializer is not initialized"); + }, + getUserTypeId: () => { + throw new Error("serializer is not initialized"); + }, + needToWriteRef: () => { + throw new Error("serializer is not initialized"); + }, + getHash: () => { + throw new Error("serializer is not initialized"); + }, + write: (value: any) => { + void value; + throw new Error("serializer is not initialized"); + }, + writeRef: (value: any) => { + void value; + throw new Error("serializer is not initialized"); + }, + writeNoRef: (value: any) => { + void value; + throw new Error("serializer is not initialized"); + }, + writeRefOrNull: (value: any) => { + void value; + throw new Error("serializer is not initialized"); + }, + writeTypeInfo: (value: any) => { + void value; + throw new Error("serializer is not initialized"); + }, + read: (fromRef: boolean) => { + void fromRef; + throw new Error("serializer is not initialized"); + }, + readRef: () => { + throw new Error("serializer is not initialized"); + }, + readRefWithoutTypeInfo: () => { + throw new Error("serializer is not initialized"); + }, + readNoRef: (fromRef: boolean) => { + void fromRef; + throw new Error("serializer is not initialized"); + }, + readTypeInfo: () => { + throw new Error("serializer is not initialized"); + }, + readDataAlwaysAdvances: false, +}; interface GeneratedRegistration { typeInfo: TypeInfo; @@ -87,7 +143,7 @@ export class Gen { }; } - private prepare(typeInfo: TypeInfo, serializerLookup: SerializerLookup): SerializerCreator { + private prepare(typeInfo: TypeInfo, serializerLookup: SerializerLookup): Serializer { const InnerGeneratorClass = CodegenRegistry.get(typeInfo.typeId); if (!InnerGeneratorClass) { throw new Error(`${typeInfo.typeId} generator not exists`); @@ -113,18 +169,17 @@ export class Gen { } const factory = factoryBuilder(); const localTypeMeta = generator.getLocalTypeMeta(); - return (factoryLookup) => - factory( - this.typeResolver, - factoryLookup, - Gen.external, - typeInfo, - this.regOptions, - localTypeMeta, - localTypeMetaSymbol, - checkedTypeMetaSerializerSymbol, - checkedTypeMetaWireTypeIdSymbol, - ); + return factory( + this.typeResolver, + serializerLookup, + Gen.external, + typeInfo, + this.regOptions, + localTypeMeta, + localTypeMetaSymbol, + checkedTypeMetaSerializerSymbol, + checkedTypeMetaWireTypeIdSymbol, + ); } private isRegistered(typeInfo: TypeInfo) { @@ -158,7 +213,7 @@ export class Gen { } private addRegistration(typeInfo: TypeInfo) { - const owner = this.typeResolver.createSerializerPlaceholder(); + const owner = { ...uninitializedSerializer }; const entry: GeneratedRegistration = { typeInfo, serializer: owner, @@ -225,7 +280,7 @@ export class Gen { for (const child of children) { this.traversalContainer(child); } - const serializer = this.prepare(typeInfo, this.serializerLookup)(this.serializerLookup); + const serializer = this.prepare(typeInfo, this.serializerLookup); Object.assign(entry.serializer, serializer); } finally { entry.preparing = false; @@ -248,11 +303,8 @@ export class Gen { } else if (options?.props && Object.keys(options.props).length > 0) { this.prepareRegistration(typeInfo, Object.values(options.props)); } else if (!this.isRegistered(typeInfo) && TypeId.structType(typeInfo.typeId)) { - // Keep the recursive owner local until every generated factory has completed. If a prior - // registration published a forward owner, factory captures use that owner without mutating - // it; commit initializes it in place so earlier serializers keep the same identity. if (this.findRegistration(typeInfo) === undefined) { - this.addRegistration(typeInfo); + throw new Error("nested struct schema must be registered or defined before use"); } } else if (TypeId.enumType(typeInfo.typeId) && !this.isRegistered(typeInfo)) { this.prepareRegistration(typeInfo, []); @@ -279,7 +331,7 @@ export class Gen { } reGenerateSerializer(typeInfo: TypeInfo) { - return this.prepare(typeInfo, this.typeResolver)(this.typeResolver); + return this.prepare(typeInfo, this.typeResolver); } generateSerializer(typeInfo: TypeInfo) { @@ -288,6 +340,11 @@ export class Gen { // TypeInfo freezing may invoke application-owned proxy traps. A root entered there closes the // resolver before code generation or publication can continue. this.typeResolver.ensureRegistrationOpen(); + if (!this.typeResolver.getSerializerByTypeInfo(typeInfo)?._initialized) { + // Seed the root owner before traversal so empty roots and self-recursive fields share the + // same transaction-local serializer without publishing an incomplete resolver entry. + this.addRegistration(typeInfo); + } this.traversalContainer(typeInfo); const serializer = this.typeResolver.getSerializerByTypeInfo(typeInfo); if (!serializer?._initialized) { diff --git a/javascript/packages/core/lib/typeResolver.ts b/javascript/packages/core/lib/typeResolver.ts index 0bff6116c6..33d819f075 100644 --- a/javascript/packages/core/lib/typeResolver.ts +++ b/javascript/packages/core/lib/typeResolver.ts @@ -35,64 +35,6 @@ import { BoolArray } from "./types/boolArray"; import { isFloat16Array } from "./types/float16"; import { getUnknownTypeMeta, UnknownStructSerializer } from "./unknownStruct"; -const uninitSerialize = { - _initialized: false, - fixedSize: 0, - getTypeInfo: () => { - throw new Error("uninitSerialize"); - }, - getTypeId: () => { - throw new Error("uninitSerialize"); - }, - getUserTypeId: () => { - throw new Error("uninitSerialize"); - }, - needToWriteRef: () => { - throw new Error("uninitSerialize"); - }, - getHash: () => { - throw new Error("uninitSerialize"); - }, - write: (v: any) => { - void v; - throw new Error("uninitSerialize"); - }, - writeRef: (v: any) => { - void v; - throw new Error("uninitSerialize"); - }, - writeNoRef: (v: any) => { - void v; - throw new Error("uninitSerialize"); - }, - writeRefOrNull: (v: any) => { - void v; - throw new Error("uninitSerialize"); - }, - writeTypeInfo: (v: any) => { - void v; - throw new Error("uninitSerialize"); - }, - read: (fromRef: boolean) => { - void fromRef; - throw new Error("uninitSerialize"); - }, - readRef: () => { - throw new Error("uninitSerialize"); - }, - readRefWithoutTypeInfo: () => { - throw new Error("uninitSerialize"); - }, - readNoRef: (fromRef: boolean) => { - void fromRef; - throw new Error("uninitSerialize"); - }, - readTypeInfo: () => { - throw new Error("uninitSerialize"); - }, - readDataAlwaysAdvances: false, -}; - export default class TypeResolver { readonly trackingRef: boolean; private internalSerializer: Serializer[] = new Array(300); @@ -192,53 +134,53 @@ export default class TypeResolver { } private initInternalSerializer() { - const registerSerializer = (typeInfo: TypeInfo) => { + const generateInternalSerializer = (typeInfo: TypeInfo) => { return new Gen(this).generateSerializer(typeInfo); }; - registerSerializer(Type.string()); - registerSerializer(new TypeInfo(TypeId.ENUM)); - registerSerializer(new TypeInfo(TypeId.NAMED_ENUM)); - registerSerializer(Type.any()); - registerSerializer(Type.list(Type.any())); - registerSerializer(Type.map(Type.any(), Type.any())); - registerSerializer(Type.bool()); - registerSerializer(Type.int8()); - registerSerializer(Type.int16()); - registerSerializer(Type.int32({ encoding: "fixed" })); - registerSerializer(Type.int32()); - registerSerializer(Type.uint32({ encoding: "fixed" })); - registerSerializer(Type.uint64({ encoding: "fixed" })); - registerSerializer(Type.int64({ encoding: "fixed" })); - registerSerializer(Type.int64()); - registerSerializer(Type.uint8()); - registerSerializer(Type.uint16()); - registerSerializer(Type.uint32()); - registerSerializer(Type.uint64()); - registerSerializer(Type.uint64({ encoding: "tagged" })); - registerSerializer(Type.int64({ encoding: "tagged" })); - registerSerializer(Type.float16()); - registerSerializer(Type.bfloat16()); - registerSerializer(Type.float32()); - registerSerializer(Type.float64()); - registerSerializer(Type.timestamp()); - registerSerializer(Type.duration()); - registerSerializer(Type.date()); - registerSerializer(Type.decimal()); - registerSerializer(Type.set(Type.any())); - registerSerializer(Type.binary()); - registerSerializer(Type.boolArray()); - registerSerializer(Type.uint8Array()); - registerSerializer(Type.int8Array()); - registerSerializer(Type.uint16Array()); - registerSerializer(Type.int16Array()); - registerSerializer(Type.uint32Array()); - registerSerializer(Type.int32Array()); - registerSerializer(Type.uint64Array()); - registerSerializer(Type.int64Array()); - registerSerializer(Type.float16Array()); - registerSerializer(Type.bfloat16Array()); - registerSerializer(Type.float32Array()); - registerSerializer(Type.float64Array()); + generateInternalSerializer(Type.string()); + generateInternalSerializer(new TypeInfo(TypeId.ENUM)); + generateInternalSerializer(new TypeInfo(TypeId.NAMED_ENUM)); + generateInternalSerializer(Type.any()); + generateInternalSerializer(Type.list(Type.any())); + generateInternalSerializer(Type.map(Type.any(), Type.any())); + generateInternalSerializer(Type.bool()); + generateInternalSerializer(Type.int8()); + generateInternalSerializer(Type.int16()); + generateInternalSerializer(Type.int32({ encoding: "fixed" })); + generateInternalSerializer(Type.int32()); + generateInternalSerializer(Type.uint32({ encoding: "fixed" })); + generateInternalSerializer(Type.uint64({ encoding: "fixed" })); + generateInternalSerializer(Type.int64({ encoding: "fixed" })); + generateInternalSerializer(Type.int64()); + generateInternalSerializer(Type.uint8()); + generateInternalSerializer(Type.uint16()); + generateInternalSerializer(Type.uint32()); + generateInternalSerializer(Type.uint64()); + generateInternalSerializer(Type.uint64({ encoding: "tagged" })); + generateInternalSerializer(Type.int64({ encoding: "tagged" })); + generateInternalSerializer(Type.float16()); + generateInternalSerializer(Type.bfloat16()); + generateInternalSerializer(Type.float32()); + generateInternalSerializer(Type.float64()); + generateInternalSerializer(Type.timestamp()); + generateInternalSerializer(Type.duration()); + generateInternalSerializer(Type.date()); + generateInternalSerializer(Type.decimal()); + generateInternalSerializer(Type.set(Type.any())); + generateInternalSerializer(Type.binary()); + generateInternalSerializer(Type.boolArray()); + generateInternalSerializer(Type.uint8Array()); + generateInternalSerializer(Type.int8Array()); + generateInternalSerializer(Type.uint16Array()); + generateInternalSerializer(Type.int16Array()); + generateInternalSerializer(Type.uint32Array()); + generateInternalSerializer(Type.int32Array()); + generateInternalSerializer(Type.uint64Array()); + generateInternalSerializer(Type.int64Array()); + generateInternalSerializer(Type.float16Array()); + generateInternalSerializer(Type.bfloat16Array()); + generateInternalSerializer(Type.float32Array()); + generateInternalSerializer(Type.float64Array()); this.float64Serializer = this.getSerializerById(TypeId.FLOAT64); this.float32Serializer = this.getSerializerById(TypeId.FLOAT32); @@ -272,7 +214,9 @@ export default class TypeResolver { } freezeRegistration() { - this.registrationFrozen = true; + if (!this.registrationFrozen) { + this.registrationFrozen = true; + } } ensureRegistrationOpen() { @@ -281,12 +225,12 @@ export default class TypeResolver { } } - createSerializerPlaceholder(): Serializer { - return { ...uninitSerialize }; - } - commitGeneratedSerializers(entries: readonly { typeInfo: TypeInfo; serializer: Serializer }[]) { + this.ensureRegistrationOpen(); const publications = entries.map((entry) => { + if (!entry.serializer._initialized) { + throw new Error("generated serializer graph is incomplete"); + } const typeId = this.computeTypeId(entry.typeInfo); let internalTypeId: number | undefined; let customTypeKey: number | string | undefined; @@ -308,21 +252,13 @@ export default class TypeResolver { internalTypeId, customTypeKey, existingSerializer, - descriptors: - existingSerializer === undefined || existingSerializer._initialized - ? undefined - : Object.getOwnPropertyDescriptors(entry.serializer), }; }); - this.ensureRegistrationOpen(); for (const publication of publications) { if (publication.existingSerializer !== undefined) { - if (!publication.existingSerializer._initialized) { - // Complete only a resolver-owned forward placeholder. An initialized owner published by - // a nested registration is authoritative for this identity. - Object.defineProperties(publication.existingSerializer, publication.descriptors!); - } - } else if (publication.internalTypeId !== undefined) { + continue; + } + if (publication.internalTypeId !== undefined) { this.internalSerializer[publication.internalTypeId] = publication.entry.serializer; } else { this.customSerializer.set(publication.customTypeKey!, publication.entry.serializer); @@ -330,63 +266,10 @@ export default class TypeResolver { } } - registerSerializer(typeInfo: TypeInfo, serializer: Serializer = uninitSerialize) { - this.ensureRegistrationOpen(); - const typeId = this.computeTypeId(typeInfo); - if (!TypeId.isNamedType(typeId)) { - if (TypeId.needsUserTypeId(typeId) && typeInfo.userTypeId !== -1) { - const key = this.makeUserTypeKey(typeInfo.userTypeId); - if (this.customSerializer.has(key)) { - Object.assign(this.customSerializer.get(key)!, serializer); - } else { - this.customSerializer.set(key, { ...serializer }); - } - return this.customSerializer.get(key); - } - if (typeId <= 0xff) { - if (this.internalSerializer[typeId]) { - Object.assign(this.internalSerializer[typeId], serializer); - } else { - this.internalSerializer[typeId] = { ...serializer }; - } - return this.internalSerializer[typeId]; - } - if (this.customSerializer.has(typeId)) { - Object.assign(this.customSerializer.get(typeId)!, serializer); - } else { - this.customSerializer.set(typeId, { ...serializer }); - } - return this.customSerializer.get(typeId); - } - - const name = typeInfo.named!; - if (this.customSerializer.has(name)) { - Object.assign(this.customSerializer.get(name)!, serializer); - } else { - this.customSerializer.set(name, { ...serializer }); - } - return this.customSerializer.get(name); - } - generateReadSerializer(typeInfo: TypeInfo) { return new Gen(this, { creator: typeInfo.options?.creator }).reGenerateSerializer(typeInfo); } - regenerateReadSerializer(typeInfo: TypeInfo) { - this.ensureRegistrationOpen(); - const serializer = this.generateReadSerializer(typeInfo); - return this.registerSerializer(typeInfo, { - readDataAlwaysAdvances: serializer.readDataAlwaysAdvances, - getHash: serializer.getHash, - getTypeInfo: serializer.getTypeInfo, - read: serializer.read, - readNoRef: serializer.readNoRef, - readRef: serializer.readRef, - readTypeInfo: serializer.readTypeInfo, - readRefWithoutTypeInfo: serializer.readRefWithoutTypeInfo, - } as any)!; - } - getSerializerByTypeInfo(typeInfo: TypeInfo) { const typeId = this.computeTypeId(typeInfo); if (TypeId.isNamedType(typeId)) { diff --git a/javascript/test/fory.test.ts b/javascript/test/fory.test.ts index ff1d17a23a..ee93e5f73f 100644 --- a/javascript/test/fory.test.ts +++ b/javascript/test/fory.test.ts @@ -101,14 +101,6 @@ describe("fory", () => { }, ); - test("freezes direct resolver registration", () => { - const fory = new Fory({ compatible: false }); - fory.serialize(1); - - expect(() => fory.typeResolver.registerSerializer(Type.struct(8105, {}))).toThrow(); - expect(fory.typeResolver.getSerializerById(TypeId.STRUCT, 8105)).toBeUndefined(); - }); - test("keeps rejected descriptor mutable", () => { const fory = new Fory({ compatible: false }); const typeInfo = Type.struct(8106, {}); @@ -119,24 +111,6 @@ describe("fory", () => { expect(typeInfo.nullable).toBe(true); }); - test("rejects regeneration before codegen", () => { - let generated = 0; - const fory = new Fory({ - compatible: false, - hooks: { - afterCodeGenerated(code) { - generated++; - return code; - }, - }, - }); - fory.serialize(1); - const generatedBefore = generated; - - expect(() => fory.typeResolver.regenerateReadSerializer(Type.struct(8107, {}))).toThrow(); - expect(generated).toBe(generatedBefore); - }); - test("keeps codegen callbacks from publishing registration", () => { let reenterRoot = false; let fory: Fory; @@ -208,29 +182,70 @@ describe("fory", () => { expect(() => rootType.setNullable(true)).toThrow(); }); - test("initializes a published forward owner in place", () => { + test("rejects an unresolved nested schema", () => { const fory = new Fory({ compatible: false }); + const typeResolver = fory.typeResolver as any; + const internalBefore = Array.from(typeResolver.internalSerializer); + const customBefore = Array.from(typeResolver.customSerializer.entries()); const forwardType = Type.struct(8113); - const parent = fory.register( - Type.struct(8114, { - child: forwardType, - }), - ); - const forwardOwner = fory.typeResolver.getSerializerById(TypeId.STRUCT, forwardType.userTypeId); + const parentType = Type.struct(8114, { child: forwardType }); + + expect(() => fory.register(parentType)).toThrow(); + + expect(Array.from(typeResolver.internalSerializer)).toEqual(internalBefore); + expect(Array.from(typeResolver.customSerializer.entries())).toEqual(customBefore); + expect(typeResolver.getSerializerById(TypeId.STRUCT, forwardType.userTypeId)).toBeUndefined(); + expect(typeResolver.getSerializerById(TypeId.STRUCT, parentType.userTypeId)).toBeUndefined(); fory.register( Type.struct(8113, { value: Type.int32(), }), ); - - expect(fory.typeResolver.getSerializerById(TypeId.STRUCT, forwardType.userTypeId)).toBe( - forwardOwner, + const parent = fory.register( + Type.struct(8114, { + child: Type.struct(8113), + }), ); const value = { child: { value: 7 } }; expect(parent.deserialize(parent.serialize(value))).toEqual(value); }); + test("registers an empty root schema", () => { + const registered = new Fory({ compatible: false }).register(Type.struct(8122, {})); + + expect(registered.deserialize(registered.serialize({}))).toEqual({}); + }); + + test("registers a self-recursive schema", () => { + const nodeType = Type.struct(8123, { + value: Type.int32(), + next: Type.struct(8123).setNullable(true).setTrackingRef(true), + }); + const registered = new Fory({ compatible: false, ref: true }).register(nodeType); + const value: any = { value: 7 }; + value.next = value; + + const result: any = registered.deserialize(registered.serialize(value)); + expect(result.value).toBe(7); + expect(result.next).toBe(result); + }); + + test("registers a mutually recursive schema", () => { + const rightType = Type.struct(8125, { + value: Type.string(), + left: Type.struct(8124).setNullable(true), + }); + const leftType = Type.struct(8124, { + value: Type.int32(), + right: rightType, + }); + const registered = new Fory({ compatible: false }).register(leftType); + const value = { value: 7, right: { value: "right", left: null } }; + + expect(registered.deserialize(registered.serialize(value))).toEqual(value); + }); + test.each(["userTypeId", "name", "options"] as const)( "rejects root %s changes during codegen", (change) => { diff --git a/javascript/test/rootCleanup.test.ts b/javascript/test/rootCleanup.test.ts index 7ff9a56e94..70fa6c02af 100644 --- a/javascript/test/rootCleanup.test.ts +++ b/javascript/test/rootCleanup.test.ts @@ -239,24 +239,18 @@ test("releases a failed root write buffer before reuse", () => { expect(writer.getPlatformBuffer().byteLength).toBeLessThan(4 * 1024 * 1024); }); -test("releases failed root reference backing before reuse", () => { +test("clears failed root references before reuse", () => { const fory = new Fory({ compatible: false, ref: true }); const registered = fory.register(Type.struct(7612, {})); const refReader = (fory as any).readContext.refReader; - let failedBacking: unknown[]; registered.serializer.readRef = () => { - for (let i = 0; i < 32768; i++) { - refReader.reference({}); - } - failedBacking = refReader.readObjects; + refReader.reference({}); throw new Error("root read failed"); }; expect(() => registered.deserialize(new Uint8Array([1]))).toThrow(); - expect(refReader.readObjects).toBe(failedBacking!); registered.serializer.readRef = () => { - expect(refReader.readObjects).not.toBe(failedBacking!); expect(refReader.readObjects).toHaveLength(0); return {}; }; diff --git a/javascript/test/typemeta.test.ts b/javascript/test/typemeta.test.ts index 9e95b94d04..2038f28094 100644 --- a/javascript/test/typemeta.test.ts +++ b/javascript/test/typemeta.test.ts @@ -550,9 +550,6 @@ describe("typemeta", () => { generateReadSerializer: () => { throw new Error("unused"); }, - regenerateReadSerializer: () => { - throw new Error("unused"); - }, } as any, config, ); @@ -761,9 +758,6 @@ describe("typemeta", () => { generateReadSerializer: () => { throw new Error("unused"); }, - regenerateReadSerializer: () => { - throw new Error("unused"); - }, } as any, config, ); @@ -1460,21 +1454,6 @@ describe("typemeta", () => { expect(map.get("second").second).toBe("two"); }); - test("regenerated read serializers keep getTypeInfo", () => { - const fory = new Fory({ compatible: true }); - const serializer = (fory as any).typeResolver.regenerateReadSerializer( - Type.struct( - { namespace: "example", typeName: "repro_struct" }, - { - value: Type.int32(), - }, - ), - ); - - expect(typeof serializer.getTypeInfo).toBe("function"); - expect(serializer.getTypeInfo().named).toBe("example$repro_struct"); - }); - test("caches compatible readers for alternating nested schemas", () => { const stringWriterFory = new Fory({ compatible: true }); const boolWriterFory = new Fory({ compatible: true }); @@ -1558,9 +1537,6 @@ describe("typemeta", () => { generateReadSerializer: () => { throw new Error("unused"); }, - regenerateReadSerializer: () => { - throw new Error("unused"); - }, } as any, config, ); From 3e1a7297d587d9ae5177ce66a6565bd6b3084a12 Mon Sep 17 00:00:00 2001 From: chaokunyang Date: Fri, 28 Aug 2026 07:25:38 +0800 Subject: [PATCH 045/168] fix(jvm): keep module lifecycle in Fory --- .agents/languages/kotlin.md | 9 +- .agents/languages/scala.md | 9 +- .../fory/idl_tests/KotlinIdlRoundTripPeer.kt | 13 +- .../idl_tests/ScalaIdlRoundTripTest.scala | 8 +- .../serializer/kotlin/KotlinSerializers.java | 228 +++++++------- .../org/apache/fory/kotlin/ForyKotlin.kt | 2 +- .../kotlin/BuiltinClassSerializerTests.kt | 75 ----- .../serializer/scala/ScalaSerializers.java | 291 +++++++++--------- .../org/apache/fory/scala/ForyScala.scala | 2 +- .../fory/serializer/scala/ScalaTest.scala | 170 ---------- 10 files changed, 266 insertions(+), 541 deletions(-) diff --git a/.agents/languages/kotlin.md b/.agents/languages/kotlin.md index 3047d44663..66ec103a30 100644 --- a/.agents/languages/kotlin.md +++ b/.agents/languages/kotlin.md @@ -21,11 +21,10 @@ Load this file when changing `kotlin/` or compiler code that generates Kotlin so - Combined generated-struct registration must publish the canonical type before constructing its serializer because generated construction resolves the canonical `TypeInfo`. Do not move that construction before type registration or add rollback, staging, or a parallel registration path. -- Bootstrap installation markers must be published only after every nested registration completes - and an authoritative lifecycle recheck succeeds. A failed or root-reentered installation leaves - the marker absent naturally; do not publish early and add rollback or staging state. Use the - concrete `Fory` as the per-runtime monitor for the complete bootstrap, and reject same-runtime - same-thread reentry instead of recursively starting a second installation. +- `Fory.register(ForyModule)` is the only owner of bootstrap identity, cycle breaking, and + idempotence. Kotlin bootstrap code must not add a marker, monitor, or separate reentry policy. + Keep the install body replay-safe until its final non-repeatable publication; publish global + Kotlin default-value support only after all per-runtime registrations succeed. - Install modules for thread-safe facades through `ForyBuilder.withModule` before building them. Runtime registration extensions target concrete `Fory` instances and must not recreate a thread-safe module-registration wrapper. diff --git a/.agents/languages/scala.md b/.agents/languages/scala.md index fa2d7dead9..0e11d7ecfb 100644 --- a/.agents/languages/scala.md +++ b/.agents/languages/scala.md @@ -18,11 +18,10 @@ Load this file when changing `scala/`. construction before type registration or add rollback, staging, or a parallel registration path. Union construction is the exception because it does not require canonical registration: finish its serializer-owned callbacks and recheck the freeze before publishing the union type. -- Bootstrap installation markers must be published only after every nested registration completes - and an authoritative lifecycle recheck succeeds. A failed or root-reentered installation leaves - the marker absent naturally; do not publish early and add rollback or staging state. Use the - concrete `Fory` as the per-runtime monitor for the complete bootstrap, and reject same-runtime - same-thread reentry instead of recursively starting a second installation. +- `Fory.register(ForyModule)` is the only owner of bootstrap identity, cycle breaking, and + idempotence. Scala bootstrap code must not add a marker, monitor, or separate reentry policy. + Keep the install body replay-safe and append its serializer factory only after all repeatable + per-runtime registrations succeed. - Install modules for thread-safe facades through `ForyBuilder.withModule` before building them. Runtime registration extensions target concrete `Fory` instances and must not recreate a thread-safe module-registration wrapper. diff --git a/integration_tests/idl_tests/kotlin/src/main/kotlin/org/apache/fory/idl_tests/KotlinIdlRoundTripPeer.kt b/integration_tests/idl_tests/kotlin/src/main/kotlin/org/apache/fory/idl_tests/KotlinIdlRoundTripPeer.kt index f6d4bddcee..4b29634f8c 100644 --- a/integration_tests/idl_tests/kotlin/src/main/kotlin/org/apache/fory/idl_tests/KotlinIdlRoundTripPeer.kt +++ b/integration_tests/idl_tests/kotlin/src/main/kotlin/org/apache/fory/idl_tests/KotlinIdlRoundTripPeer.kt @@ -23,6 +23,7 @@ package org.apache.fory.idl_tests import addressbook.AddressBook import addressbook.AddressbookForyModule +import basic.BasicForyModule import basic.Money import example.ExampleForyModule import example.ExampleMessage @@ -109,10 +110,10 @@ private fun runGeneratedSurfaceChecks() { assertRoundTrip(fory, ExampleMessageUnion.Uint32Array(uintArrayOf(1u, UInt.MAX_VALUE))) assertRoundTrip(fory, ExampleMessageUnion.Float16Array(Float16Array.of(1.0f, -2.0f))) assertRoundTrip(fory, ExampleMessageUnion.Bfloat16Array(BFloat16Array.of(3.0f, -4.0f))) - assertBaseForyExtensionReceivers() + assertRegistrationOwners() } -private fun assertBaseForyExtensionReceivers() { +private fun assertRegistrationOwners() { val expected = Money(BigDecimal("12.34"), "USD") val direct = ForyKotlin.builder().withXlang(true).build() @@ -120,13 +121,13 @@ private fun assertBaseForyExtensionReceivers() { val directBytes = direct.serialize(expected) require(direct.deserialize(directBytes, Money::class.java) == expected) - val threadLocal = ForyKotlin.builder().withXlang(true).buildThreadLocalFory() - threadLocal.register(130L) + val threadLocal = + ForyKotlin.builder().withXlang(true).withModule(BasicForyModule).buildThreadLocalFory() val threadLocalBytes = threadLocal.serialize(expected) require(threadLocal.deserialize(threadLocalBytes, Money::class.java) == expected) - val pooled = ForyKotlin.builder().withXlang(true).buildThreadSafeForyPool(1) - pooled.register(130L) + val pooled = + ForyKotlin.builder().withXlang(true).withModule(BasicForyModule).buildThreadSafeForyPool(1) val pooledBytes = pooled.serialize(expected) require(pooled.deserialize(pooledBytes, Money::class.java) == expected) } diff --git a/integration_tests/idl_tests/scala/src/test/scala/org/apache/fory/idl_tests/ScalaIdlRoundTripTest.scala b/integration_tests/idl_tests/scala/src/test/scala/org/apache/fory/idl_tests/ScalaIdlRoundTripTest.scala index 1a74795397..0908fd3065 100644 --- a/integration_tests/idl_tests/scala/src/test/scala/org/apache/fory/idl_tests/ScalaIdlRoundTripTest.scala +++ b/integration_tests/idl_tests/scala/src/test/scala/org/apache/fory/idl_tests/ScalaIdlRoundTripTest.scala @@ -62,19 +62,17 @@ final class ScalaIdlRoundTripTest extends AnyWordSpec with Matchers { fory.deserialize(fory.serialize(envelope)) shouldEqual envelope } - "register generated serializers through BaseFory extension receivers" in { + "register generated serializers through their lifecycle owners" in { val expected = Money(new BigDecimal("12.34"), "USD") val direct = ForyScala.builder().withXlang(true).build() direct.register[Money](130L) direct.deserialize(direct.serialize(expected)).asInstanceOf[Money] shouldEqual expected - val threadLocal = ForyScala.builder().withXlang(true).buildThreadLocalFory() - threadLocal.register[Money](130L) + val threadLocal = ForyScala.builder().withXlang(true).withModule(BasicForyModule).buildThreadLocalFory() threadLocal.deserialize(threadLocal.serialize(expected)).asInstanceOf[Money] shouldEqual expected - val pooled = ForyScala.builder().withXlang(true).buildThreadSafeForyPool(1) - pooled.register[Money](130L) + val pooled = ForyScala.builder().withXlang(true).withModule(BasicForyModule).buildThreadSafeForyPool(1) pooled.deserialize(pooled.serialize(expected)).asInstanceOf[Money] shouldEqual expected } diff --git a/kotlin/fory-kotlin/src/main/java/org/apache/fory/serializer/kotlin/KotlinSerializers.java b/kotlin/fory-kotlin/src/main/java/org/apache/fory/serializer/kotlin/KotlinSerializers.java index 69e7ff3294..c0e40e2ff7 100644 --- a/kotlin/fory-kotlin/src/main/java/org/apache/fory/serializer/kotlin/KotlinSerializers.java +++ b/kotlin/fory-kotlin/src/main/java/org/apache/fory/serializer/kotlin/KotlinSerializers.java @@ -19,10 +19,7 @@ package org.apache.fory.serializer.kotlin; -import java.util.Collections; -import java.util.Map; import java.util.Objects; -import java.util.WeakHashMap; import kotlin.*; import kotlin.UByteArray; import kotlin.UIntArray; @@ -34,9 +31,11 @@ import kotlin.time.TimedValue; import kotlin.uuid.Uuid; import org.apache.fory.Fory; +import org.apache.fory.annotation.Internal; import org.apache.fory.codegen.GeneratedClassNames; import org.apache.fory.config.Config; import org.apache.fory.exception.ForyException; +import org.apache.fory.kotlin.ForyKotlin; import org.apache.fory.resolver.TypeResolver; import org.apache.fory.serializer.EnumSerializer; import org.apache.fory.serializer.Serializer; @@ -48,129 +47,116 @@ @SuppressWarnings({"rawtypes", "unchecked"}) public class KotlinSerializers { private static final String XLANG_GENERATED_SERIALIZER_SUFFIX = "_ForySerializer"; - private static final Map INSTALLED_FORY = - Collections.synchronizedMap(new WeakHashMap<>()); public static void registerSerializers(Fory fory) { + fory.register(ForyKotlin.INSTANCE); + } + + @Internal + public static void installSerializers(Fory fory) { TypeResolver resolver = fory.getTypeResolver(); checkRegistrationOpen(resolver); - // The runtime is the bootstrap's natural owner, so its monitor linearizes only this install. - // Since Java monitors are reentrant, reject a recursive install before entering it again. - if (Thread.holdsLock(fory)) { - if (INSTALLED_FORY.containsKey(fory)) { - return; - } - throw new ForyException("Reentrant Kotlin serializer bootstrap is not supported."); - } - synchronized (fory) { - checkRegistrationOpen(resolver); - if (INSTALLED_FORY.containsKey(fory)) { - return; - } - DefaultValueUtils.setKotlinDefaultValueSupport(new KotlinDefaultValueSupport()); - if (!resolver.isCrossLanguage()) { - Config config = resolver.getConfig(); - - // UByte - Class ubyteClass = KotlinToJavaClass.INSTANCE.getUByteClass(); - registerIfAbsent(resolver, ubyteClass); - resolver.registerSerializer(ubyteClass, new UByteSerializer(config)); - - // UShort - Class ushortClass = KotlinToJavaClass.INSTANCE.getUShortClass(); - registerIfAbsent(resolver, ushortClass); - resolver.registerSerializer(ushortClass, new UShortSerializer(config)); - - // UInt - Class uintClass = KotlinToJavaClass.INSTANCE.getUIntClass(); - registerIfAbsent(resolver, uintClass); - resolver.registerSerializer(uintClass, new UIntSerializer(config)); - - // ULong - Class ulongClass = KotlinToJavaClass.INSTANCE.getULongClass(); - registerIfAbsent(resolver, ulongClass); - resolver.registerSerializer(ulongClass, new ULongSerializer(config)); - - // EmptyList - Class emptyListClass = KotlinToJavaClass.INSTANCE.getEmptyListClass(); - registerIfAbsent(resolver, emptyListClass); - resolver.registerSerializer( - emptyListClass, - new CollectionSerializers.EmptyListSerializer(resolver, emptyListClass)); - - // EmptySet - Class emptySetClass = KotlinToJavaClass.INSTANCE.getEmptySetClass(); - registerIfAbsent(resolver, emptySetClass); - resolver.registerSerializer( - emptySetClass, new CollectionSerializers.EmptySetSerializer(resolver, emptySetClass)); - - // EmptyMap - Class emptyMapClass = KotlinToJavaClass.INSTANCE.getEmptyMapClass(); - registerIfAbsent(resolver, emptyMapClass); - resolver.registerSerializer( - emptyMapClass, new MapSerializers.EmptyMapSerializer(resolver, emptyMapClass)); - - // Non-Java collection implementation in kotlin stdlib. - Class arrayDequeClass = KotlinToJavaClass.INSTANCE.getArrayDequeClass(); - registerIfAbsent(resolver, arrayDequeClass); - resolver.registerSerializer( - arrayDequeClass, new KotlinArrayDequeSerializer(resolver, arrayDequeClass)); - - // Unsigned array classes: UByteArray, UShortArray, UIntArray, ULongArray. - registerIfAbsent(resolver, UByteArray.class); - resolver.registerSerializer(UByteArray.class, new UByteArraySerializer(resolver)); - registerIfAbsent(resolver, UShortArray.class); - resolver.registerSerializer(UShortArray.class, new UShortArraySerializer(resolver)); - registerIfAbsent(resolver, UIntArray.class); - resolver.registerSerializer(UIntArray.class, new UIntArraySerializer(resolver)); - registerIfAbsent(resolver, ULongArray.class); - resolver.registerSerializer(ULongArray.class, new ULongArraySerializer(resolver)); - - // Ranges and Progressions. - registerIfAbsent(resolver, kotlin.ranges.CharRange.class); - registerIfAbsent(resolver, kotlin.ranges.CharProgression.class); - registerIfAbsent(resolver, kotlin.ranges.IntRange.class); - registerIfAbsent(resolver, kotlin.ranges.IntProgression.class); - registerIfAbsent(resolver, kotlin.ranges.LongRange.class); - registerIfAbsent(resolver, kotlin.ranges.LongProgression.class); - registerIfAbsent(resolver, kotlin.ranges.UIntRange.class); - registerIfAbsent(resolver, kotlin.ranges.UIntProgression.class); - registerIfAbsent(resolver, kotlin.ranges.ULongRange.class); - registerIfAbsent(resolver, kotlin.ranges.ULongProgression.class); - - // Built-in classes. - registerIfAbsent(resolver, kotlin.Pair.class); - registerIfAbsent(resolver, kotlin.Triple.class); - registerIfAbsent(resolver, kotlin.Result.class); - registerIfAbsent(resolver, Result.Failure.class); - - // kotlin.random - registerIfAbsent(resolver, KotlinToJavaClass.INSTANCE.getRandomDefaultClass()); - registerIfAbsent(resolver, KotlinToJavaClass.INSTANCE.getRandomInternalClass()); - registerIfAbsent(resolver, KotlinToJavaClass.INSTANCE.getRandomSerializedClass()); - - // kotlin.text - registerIfAbsent(resolver, Regex.class); - registerIfAbsent(resolver, KotlinToJavaClass.INSTANCE.getRegexSerializedClass()); - registerIfAbsent(resolver, RegexOption.class); - registerIfAbsent(resolver, CharCategory.class); - registerIfAbsent(resolver, CharDirectionality.class); - registerIfAbsent(resolver, HexFormat.class); - registerIfAbsent(resolver, MatchGroup.class); - - // kotlin.time - registerIfAbsent(resolver, DurationUnit.class); - registerIfAbsent(resolver, Duration.class); - resolver.registerSerializer(Duration.class, new DurationSerializer(config)); - registerIfAbsent(resolver, TimedValue.class); - - // kotlin.uuid - registerIfAbsent(resolver, Uuid.class); - resolver.registerSerializer(Uuid.class, new UuidSerializer(config)); - } - checkRegistrationOpen(resolver); - INSTALLED_FORY.put(fory, Boolean.TRUE); + if (!resolver.isCrossLanguage()) { + Config config = resolver.getConfig(); + + // UByte + Class ubyteClass = KotlinToJavaClass.INSTANCE.getUByteClass(); + registerIfAbsent(resolver, ubyteClass); + resolver.registerSerializer(ubyteClass, new UByteSerializer(config)); + + // UShort + Class ushortClass = KotlinToJavaClass.INSTANCE.getUShortClass(); + registerIfAbsent(resolver, ushortClass); + resolver.registerSerializer(ushortClass, new UShortSerializer(config)); + + // UInt + Class uintClass = KotlinToJavaClass.INSTANCE.getUIntClass(); + registerIfAbsent(resolver, uintClass); + resolver.registerSerializer(uintClass, new UIntSerializer(config)); + + // ULong + Class ulongClass = KotlinToJavaClass.INSTANCE.getULongClass(); + registerIfAbsent(resolver, ulongClass); + resolver.registerSerializer(ulongClass, new ULongSerializer(config)); + + // EmptyList + Class emptyListClass = KotlinToJavaClass.INSTANCE.getEmptyListClass(); + registerIfAbsent(resolver, emptyListClass); + resolver.registerSerializer( + emptyListClass, new CollectionSerializers.EmptyListSerializer(resolver, emptyListClass)); + + // EmptySet + Class emptySetClass = KotlinToJavaClass.INSTANCE.getEmptySetClass(); + registerIfAbsent(resolver, emptySetClass); + resolver.registerSerializer( + emptySetClass, new CollectionSerializers.EmptySetSerializer(resolver, emptySetClass)); + + // EmptyMap + Class emptyMapClass = KotlinToJavaClass.INSTANCE.getEmptyMapClass(); + registerIfAbsent(resolver, emptyMapClass); + resolver.registerSerializer( + emptyMapClass, new MapSerializers.EmptyMapSerializer(resolver, emptyMapClass)); + + // Non-Java collection implementation in kotlin stdlib. + Class arrayDequeClass = KotlinToJavaClass.INSTANCE.getArrayDequeClass(); + registerIfAbsent(resolver, arrayDequeClass); + resolver.registerSerializer( + arrayDequeClass, new KotlinArrayDequeSerializer(resolver, arrayDequeClass)); + + // Unsigned array classes: UByteArray, UShortArray, UIntArray, ULongArray. + registerIfAbsent(resolver, UByteArray.class); + resolver.registerSerializer(UByteArray.class, new UByteArraySerializer(resolver)); + registerIfAbsent(resolver, UShortArray.class); + resolver.registerSerializer(UShortArray.class, new UShortArraySerializer(resolver)); + registerIfAbsent(resolver, UIntArray.class); + resolver.registerSerializer(UIntArray.class, new UIntArraySerializer(resolver)); + registerIfAbsent(resolver, ULongArray.class); + resolver.registerSerializer(ULongArray.class, new ULongArraySerializer(resolver)); + + // Ranges and Progressions. + registerIfAbsent(resolver, kotlin.ranges.CharRange.class); + registerIfAbsent(resolver, kotlin.ranges.CharProgression.class); + registerIfAbsent(resolver, kotlin.ranges.IntRange.class); + registerIfAbsent(resolver, kotlin.ranges.IntProgression.class); + registerIfAbsent(resolver, kotlin.ranges.LongRange.class); + registerIfAbsent(resolver, kotlin.ranges.LongProgression.class); + registerIfAbsent(resolver, kotlin.ranges.UIntRange.class); + registerIfAbsent(resolver, kotlin.ranges.UIntProgression.class); + registerIfAbsent(resolver, kotlin.ranges.ULongRange.class); + registerIfAbsent(resolver, kotlin.ranges.ULongProgression.class); + + // Built-in classes. + registerIfAbsent(resolver, kotlin.Pair.class); + registerIfAbsent(resolver, kotlin.Triple.class); + registerIfAbsent(resolver, kotlin.Result.class); + registerIfAbsent(resolver, Result.Failure.class); + + // kotlin.random + registerIfAbsent(resolver, KotlinToJavaClass.INSTANCE.getRandomDefaultClass()); + registerIfAbsent(resolver, KotlinToJavaClass.INSTANCE.getRandomInternalClass()); + registerIfAbsent(resolver, KotlinToJavaClass.INSTANCE.getRandomSerializedClass()); + + // kotlin.text + registerIfAbsent(resolver, Regex.class); + registerIfAbsent(resolver, KotlinToJavaClass.INSTANCE.getRegexSerializedClass()); + registerIfAbsent(resolver, RegexOption.class); + registerIfAbsent(resolver, CharCategory.class); + registerIfAbsent(resolver, CharDirectionality.class); + registerIfAbsent(resolver, HexFormat.class); + registerIfAbsent(resolver, MatchGroup.class); + + // kotlin.time + registerIfAbsent(resolver, DurationUnit.class); + registerIfAbsent(resolver, Duration.class); + resolver.registerSerializer(Duration.class, new DurationSerializer(config)); + registerIfAbsent(resolver, TimedValue.class); + + // kotlin.uuid + registerIfAbsent(resolver, Uuid.class); + resolver.registerSerializer(Uuid.class, new UuidSerializer(config)); } + checkRegistrationOpen(resolver); + DefaultValueUtils.setKotlinDefaultValueSupport(new KotlinDefaultValueSupport()); } private static void registerIfAbsent(TypeResolver resolver, Class cls) { diff --git a/kotlin/fory-kotlin/src/main/kotlin/org/apache/fory/kotlin/ForyKotlin.kt b/kotlin/fory-kotlin/src/main/kotlin/org/apache/fory/kotlin/ForyKotlin.kt index 9d26b250b3..41061483da 100644 --- a/kotlin/fory-kotlin/src/main/kotlin/org/apache/fory/kotlin/ForyKotlin.kt +++ b/kotlin/fory-kotlin/src/main/kotlin/org/apache/fory/kotlin/ForyKotlin.kt @@ -28,6 +28,6 @@ public object ForyKotlin : ForyModule { @JvmStatic public fun builder(): ForyBuilder = Fory.builder().withModule(this) override fun install(fory: Fory) { - KotlinSerializers.registerSerializers(fory) + KotlinSerializers.installSerializers(fory) } } diff --git a/kotlin/fory-kotlin/src/test/kotlin/org/apache/fory/serializer/kotlin/BuiltinClassSerializerTests.kt b/kotlin/fory-kotlin/src/test/kotlin/org/apache/fory/serializer/kotlin/BuiltinClassSerializerTests.kt index 9470d6f1bc..3ab0d14ec0 100644 --- a/kotlin/fory-kotlin/src/test/kotlin/org/apache/fory/serializer/kotlin/BuiltinClassSerializerTests.kt +++ b/kotlin/fory-kotlin/src/test/kotlin/org/apache/fory/serializer/kotlin/BuiltinClassSerializerTests.kt @@ -21,11 +21,6 @@ package org.apache.fory.serializer.kotlin import java.math.BigDecimal import java.math.BigInteger -import java.util.concurrent.CountDownLatch -import java.util.concurrent.Executors -import java.util.concurrent.Future -import java.util.concurrent.TimeUnit -import java.util.concurrent.atomic.AtomicReference import kotlin.random.Random import kotlin.test.Test import kotlin.time.Duration @@ -78,76 +73,6 @@ class BuiltinClassSerializerTests { Assert.assertSame(DefaultValueUtils.getKotlinDefaultValueSupport(), defaultValueSupport) } - @Test - fun testFreezeWhileBootstrapWaits() { - val fory = - Fory.builder().withXlang(true).withCodegen(false).requireClassRegistration(true).build() - val executor = Executors.newSingleThreadExecutor() - val worker = AtomicReference() - val started = CountDownLatch(1) - lateinit var result: Future - - try { - synchronized(DefaultValueUtils::class.java) { - result = - executor.submit { - worker.set(Thread.currentThread()) - started.countDown() - try { - KotlinSerializers.registerSerializers(fory) - null - } catch (t: Throwable) { - t - } - } - Assert.assertTrue(started.await(10, TimeUnit.SECONDS)) - val deadline = System.nanoTime() + TimeUnit.SECONDS.toNanos(10) - while (worker.get().state != Thread.State.BLOCKED && System.nanoTime() < deadline) { - Thread.yield() - } - Assert.assertEquals(worker.get().state, Thread.State.BLOCKED) - fory.serialize(1) - } - - Assert.assertTrue(result.get(10, TimeUnit.SECONDS) is ForyException) - assertThrows(ForyException::class.java) { KotlinSerializers.registerSerializers(fory) } - } finally { - executor.shutdownNow() - } - } - - @Test - fun testConcurrentBootstrap() { - val fory = - Fory.builder().withXlang(false).withCodegen(false).requireClassRegistration(true).build() - val executor = Executors.newFixedThreadPool(2) - val ready = CountDownLatch(2) - val start = CountDownLatch(1) - - try { - val results = - List(2) { - executor.submit { - ready.countDown() - start.await(10, TimeUnit.SECONDS) - KotlinSerializers.registerSerializers(fory) - } - } - Assert.assertTrue(ready.await(10, TimeUnit.SECONDS)) - start.countDown() - results.forEach { it.get(10, TimeUnit.SECONDS) } - - Assert.assertTrue(fory.typeResolver.isRegistered(Duration::class.java)) - val serializer = fory.typeResolver.getSerializer(Duration::class.java) - val defaultValueSupport = DefaultValueUtils.getKotlinDefaultValueSupport() - KotlinSerializers.registerSerializers(fory) - Assert.assertSame(fory.typeResolver.getSerializer(Duration::class.java), serializer) - Assert.assertSame(DefaultValueUtils.getKotlinDefaultValueSupport(), defaultValueSupport) - } finally { - executor.shutdownNow() - } - } - @Test fun testCombinedFreezeRecheck() { val fory = diff --git a/scala/fory-scala/src/main/java/org/apache/fory/serializer/scala/ScalaSerializers.java b/scala/fory-scala/src/main/java/org/apache/fory/serializer/scala/ScalaSerializers.java index 75146a7783..77cb7ca3ad 100644 --- a/scala/fory-scala/src/main/java/org/apache/fory/serializer/scala/ScalaSerializers.java +++ b/scala/fory-scala/src/main/java/org/apache/fory/serializer/scala/ScalaSerializers.java @@ -22,172 +22,159 @@ import static org.apache.fory.serializer.scala.ToFactorySerializers.IterableToFactoryClass; import static org.apache.fory.serializer.scala.ToFactorySerializers.MapToFactoryClass; -import java.util.Collections; -import java.util.Map; import java.util.Objects; -import java.util.WeakHashMap; import org.apache.fory.Fory; import org.apache.fory.annotation.Internal; import org.apache.fory.config.Config; import org.apache.fory.exception.ForyException; import org.apache.fory.resolver.TypeResolver; +import org.apache.fory.scala.ForyScala$; import scala.collection.immutable.NumericRange; import scala.collection.immutable.Range; public class ScalaSerializers { - private static final Map INSTALLED_FORY = - Collections.synchronizedMap(new WeakHashMap<>()); - public static void registerSerializers(Fory fory) { + fory.register(ForyScala$.MODULE$); + } + + @Internal + public static void installSerializers(Fory fory) { TypeResolver resolver = fory.getTypeResolver(); checkRegistrationOpen(resolver); - // The runtime is the bootstrap's natural owner, so its monitor linearizes only this install. - // Since Java monitors are reentrant, reject a recursive install before entering it again. - if (Thread.holdsLock(fory)) { - if (INSTALLED_FORY.containsKey(fory)) { - return; - } - throw new ForyException("Reentrant Scala serializer bootstrap is not supported."); - } - synchronized (fory) { - checkRegistrationOpen(resolver); - if (INSTALLED_FORY.containsKey(fory)) { - return; - } - fory.registerSerializerFactory(new ScalaSerializerFactory()); - if (!resolver.isCrossLanguage()) { - Config config = resolver.getConfig(); - - resolver.registerSerializer( - IterableToFactoryClass, new ToFactorySerializers.IterableToFactorySerializer(config)); - resolver.registerSerializer( - MapToFactoryClass, new ToFactorySerializers.MapToFactorySerializer(config)); - - // Seq - resolver.register(scala.collection.immutable.Seq.class); - resolver.register(scala.collection.immutable.Nil$.class); - resolver.register(scala.collection.immutable.List$.class); - resolver.register(scala.collection.immutable.$colon$colon.class); - // StrictOptimizedSeqFactory -> ... extends -> IterableFactory - resolver.register(scala.collection.immutable.Vector$.class); - resolver.register("scala.collection.immutable.VectorImpl"); - resolver.register("scala.collection.immutable.Vector0"); - resolver.register("scala.collection.immutable.Vector1"); - resolver.register("scala.collection.immutable.Vector2"); - resolver.register("scala.collection.immutable.Vector3"); - resolver.register("scala.collection.immutable.Vector4"); - resolver.register("scala.collection.immutable.Vector5"); - resolver.register("scala.collection.immutable.Vector6"); - resolver.register(scala.collection.immutable.Queue.class); - resolver.register(scala.collection.immutable.Queue$.class); - resolver.register(scala.collection.immutable.LazyList.class); - resolver.register(scala.collection.immutable.LazyList$.class); - resolver.register(scala.collection.immutable.ArraySeq.class); - resolver.register(scala.collection.immutable.ArraySeq$.class); - - // Set - resolver.register(scala.collection.immutable.Set.class); - // IterableFactory - resolver.register(scala.collection.immutable.Set$.class); - resolver.register(scala.collection.immutable.Set.Set1.class); - resolver.register(scala.collection.immutable.Set.Set2.class); - resolver.register(scala.collection.immutable.Set.Set3.class); - resolver.register(scala.collection.immutable.Set.Set4.class); - resolver.register(scala.collection.immutable.HashSet.class); - resolver.register(scala.collection.immutable.TreeSet.class); - // SortedIterableFactory - resolver.register(scala.collection.immutable.TreeSet$.class); - // IterableFactory - resolver.register(scala.collection.immutable.HashSet$.class); - resolver.register(scala.collection.immutable.ListSet.class); - resolver.register(scala.collection.immutable.ListSet$.class); - resolver.register("scala.collection.immutable.Set$EmptySet$"); - resolver.register("scala.collection.immutable.SetBuilderImpl"); - resolver.register("scala.collection.immutable.SortedMapOps$ImmutableKeySortedSet"); - - // Map - resolver.register(scala.collection.immutable.Map.class); - resolver.register(scala.collection.immutable.Map$.class); - resolver.register(scala.collection.immutable.Map.Map1.class); - resolver.register(scala.collection.immutable.Map.Map2.class); - resolver.register(scala.collection.immutable.Map.Map3.class); - resolver.register(scala.collection.immutable.Map.Map4.class); - resolver.register(scala.collection.immutable.Map.WithDefault.class); - resolver.register("scala.collection.immutable.MapBuilderImpl"); - resolver.register("scala.collection.immutable.Map$EmptyMap$"); - resolver.register("scala.collection.immutable.SeqMap$EmptySeqMap$"); - resolver.register(scala.collection.immutable.HashMap.class); - resolver.register(scala.collection.immutable.HashMap$.class); - resolver.register(scala.collection.immutable.TreeMap.class); - resolver.register(scala.collection.immutable.TreeMap$.class); - resolver.register(scala.collection.immutable.SortedMap$.class); - resolver.register(scala.collection.immutable.TreeSeqMap.class); - resolver.register(scala.collection.immutable.TreeSeqMap$.class); - resolver.register(scala.collection.immutable.ListMap.class); - resolver.register(scala.collection.immutable.ListMap$.class); - resolver.register(scala.collection.immutable.IntMap.class); - resolver.register(scala.collection.immutable.IntMap$.class); - resolver.register(scala.collection.immutable.LongMap.class); - resolver.register(scala.collection.immutable.LongMap$.class); - - // Range - resolver.register("scala.math.Numeric$IntIsIntegral$"); - resolver.register("scala.math.Numeric$LongIsIntegral$"); - resolver.registerSerializerAndType( - Range.Inclusive.class, new RangeSerializer(resolver, Range.Inclusive.class)); - resolver.registerSerializerAndType( - Range.Exclusive.class, new RangeSerializer(resolver, Range.Exclusive.class)); - resolver.registerSerializerAndType( - NumericRange.class, new NumericRangeSerializer<>(resolver, NumericRange.class)); - resolver.registerSerializerAndType( - NumericRange.Exclusive.class, - new NumericRangeSerializer<>(resolver, NumericRange.Exclusive.class)); - resolver.registerSerializerAndType( - NumericRange.Inclusive.class, - new NumericRangeSerializer<>(resolver, NumericRange.Inclusive.class)); - - resolver.register(scala.collection.generic.SerializeEnd$.class); - resolver.register(scala.collection.generic.DefaultSerializationProxy.class); - resolver.register(scala.runtime.ModuleSerializationProxy.class); - - // mutable collection types - resolver.register(scala.collection.mutable.StringBuilder.class); - resolver.register(scala.collection.mutable.ArrayBuffer.class); - resolver.register(scala.collection.mutable.ArrayBuffer$.class); - resolver.register(scala.collection.mutable.ArraySeq.class); - resolver.register(scala.collection.mutable.ArraySeq$.class); - resolver.register(scala.collection.mutable.ListBuffer.class); - resolver.register(scala.collection.mutable.ListBuffer$.class); - resolver.register(scala.collection.mutable.Buffer$.class); - resolver.register(scala.collection.mutable.ArrayDeque.class); - resolver.register(scala.collection.mutable.ArrayDeque$.class); - - resolver.register(scala.collection.mutable.HashSet.class); - resolver.register(scala.collection.mutable.HashSet$.class); - resolver.register(scala.collection.mutable.TreeSet.class); - resolver.register(scala.collection.mutable.TreeSet$.class); - - resolver.register(scala.collection.mutable.HashMap.class); - resolver.register(scala.collection.mutable.HashMap$.class); - resolver.register(scala.collection.mutable.TreeMap.class); - resolver.register(scala.collection.mutable.TreeMap$.class); - resolver.register(scala.collection.mutable.LinkedHashMap.class); - resolver.register(scala.collection.mutable.LinkedHashMap$.class); - resolver.register(scala.collection.mutable.LinkedHashSet.class); - resolver.register(scala.collection.mutable.LinkedHashSet$.class); - resolver.register(scala.collection.mutable.LongMap.class); - resolver.register(scala.collection.mutable.LongMap$.class); - - resolver.register(scala.collection.mutable.Queue.class); - resolver.register(scala.collection.mutable.Queue$.class); - resolver.register(scala.collection.mutable.Stack.class); - resolver.register(scala.collection.mutable.Stack$.class); - resolver.register(scala.collection.mutable.BitSet.class); - resolver.register(scala.collection.mutable.BitSet$.class); - } - checkRegistrationOpen(resolver); - INSTALLED_FORY.put(fory, Boolean.TRUE); + if (!resolver.isCrossLanguage()) { + Config config = resolver.getConfig(); + + resolver.registerSerializer( + IterableToFactoryClass, new ToFactorySerializers.IterableToFactorySerializer(config)); + resolver.registerSerializer( + MapToFactoryClass, new ToFactorySerializers.MapToFactorySerializer(config)); + + // Seq + resolver.register(scala.collection.immutable.Seq.class); + resolver.register(scala.collection.immutable.Nil$.class); + resolver.register(scala.collection.immutable.List$.class); + resolver.register(scala.collection.immutable.$colon$colon.class); + // StrictOptimizedSeqFactory -> ... extends -> IterableFactory + resolver.register(scala.collection.immutable.Vector$.class); + resolver.register("scala.collection.immutable.VectorImpl"); + resolver.register("scala.collection.immutable.Vector0"); + resolver.register("scala.collection.immutable.Vector1"); + resolver.register("scala.collection.immutable.Vector2"); + resolver.register("scala.collection.immutable.Vector3"); + resolver.register("scala.collection.immutable.Vector4"); + resolver.register("scala.collection.immutable.Vector5"); + resolver.register("scala.collection.immutable.Vector6"); + resolver.register(scala.collection.immutable.Queue.class); + resolver.register(scala.collection.immutable.Queue$.class); + resolver.register(scala.collection.immutable.LazyList.class); + resolver.register(scala.collection.immutable.LazyList$.class); + resolver.register(scala.collection.immutable.ArraySeq.class); + resolver.register(scala.collection.immutable.ArraySeq$.class); + + // Set + resolver.register(scala.collection.immutable.Set.class); + // IterableFactory + resolver.register(scala.collection.immutable.Set$.class); + resolver.register(scala.collection.immutable.Set.Set1.class); + resolver.register(scala.collection.immutable.Set.Set2.class); + resolver.register(scala.collection.immutable.Set.Set3.class); + resolver.register(scala.collection.immutable.Set.Set4.class); + resolver.register(scala.collection.immutable.HashSet.class); + resolver.register(scala.collection.immutable.TreeSet.class); + // SortedIterableFactory + resolver.register(scala.collection.immutable.TreeSet$.class); + // IterableFactory + resolver.register(scala.collection.immutable.HashSet$.class); + resolver.register(scala.collection.immutable.ListSet.class); + resolver.register(scala.collection.immutable.ListSet$.class); + resolver.register("scala.collection.immutable.Set$EmptySet$"); + resolver.register("scala.collection.immutable.SetBuilderImpl"); + resolver.register("scala.collection.immutable.SortedMapOps$ImmutableKeySortedSet"); + + // Map + resolver.register(scala.collection.immutable.Map.class); + resolver.register(scala.collection.immutable.Map$.class); + resolver.register(scala.collection.immutable.Map.Map1.class); + resolver.register(scala.collection.immutable.Map.Map2.class); + resolver.register(scala.collection.immutable.Map.Map3.class); + resolver.register(scala.collection.immutable.Map.Map4.class); + resolver.register(scala.collection.immutable.Map.WithDefault.class); + resolver.register("scala.collection.immutable.MapBuilderImpl"); + resolver.register("scala.collection.immutable.Map$EmptyMap$"); + resolver.register("scala.collection.immutable.SeqMap$EmptySeqMap$"); + resolver.register(scala.collection.immutable.HashMap.class); + resolver.register(scala.collection.immutable.HashMap$.class); + resolver.register(scala.collection.immutable.TreeMap.class); + resolver.register(scala.collection.immutable.TreeMap$.class); + resolver.register(scala.collection.immutable.SortedMap$.class); + resolver.register(scala.collection.immutable.TreeSeqMap.class); + resolver.register(scala.collection.immutable.TreeSeqMap$.class); + resolver.register(scala.collection.immutable.ListMap.class); + resolver.register(scala.collection.immutable.ListMap$.class); + resolver.register(scala.collection.immutable.IntMap.class); + resolver.register(scala.collection.immutable.IntMap$.class); + resolver.register(scala.collection.immutable.LongMap.class); + resolver.register(scala.collection.immutable.LongMap$.class); + + // Range + resolver.register("scala.math.Numeric$IntIsIntegral$"); + resolver.register("scala.math.Numeric$LongIsIntegral$"); + resolver.registerSerializerAndType( + Range.Inclusive.class, new RangeSerializer(resolver, Range.Inclusive.class)); + resolver.registerSerializerAndType( + Range.Exclusive.class, new RangeSerializer(resolver, Range.Exclusive.class)); + resolver.registerSerializerAndType( + NumericRange.class, new NumericRangeSerializer<>(resolver, NumericRange.class)); + resolver.registerSerializerAndType( + NumericRange.Exclusive.class, + new NumericRangeSerializer<>(resolver, NumericRange.Exclusive.class)); + resolver.registerSerializerAndType( + NumericRange.Inclusive.class, + new NumericRangeSerializer<>(resolver, NumericRange.Inclusive.class)); + + resolver.register(scala.collection.generic.SerializeEnd$.class); + resolver.register(scala.collection.generic.DefaultSerializationProxy.class); + resolver.register(scala.runtime.ModuleSerializationProxy.class); + + // mutable collection types + resolver.register(scala.collection.mutable.StringBuilder.class); + resolver.register(scala.collection.mutable.ArrayBuffer.class); + resolver.register(scala.collection.mutable.ArrayBuffer$.class); + resolver.register(scala.collection.mutable.ArraySeq.class); + resolver.register(scala.collection.mutable.ArraySeq$.class); + resolver.register(scala.collection.mutable.ListBuffer.class); + resolver.register(scala.collection.mutable.ListBuffer$.class); + resolver.register(scala.collection.mutable.Buffer$.class); + resolver.register(scala.collection.mutable.ArrayDeque.class); + resolver.register(scala.collection.mutable.ArrayDeque$.class); + + resolver.register(scala.collection.mutable.HashSet.class); + resolver.register(scala.collection.mutable.HashSet$.class); + resolver.register(scala.collection.mutable.TreeSet.class); + resolver.register(scala.collection.mutable.TreeSet$.class); + + resolver.register(scala.collection.mutable.HashMap.class); + resolver.register(scala.collection.mutable.HashMap$.class); + resolver.register(scala.collection.mutable.TreeMap.class); + resolver.register(scala.collection.mutable.TreeMap$.class); + resolver.register(scala.collection.mutable.LinkedHashMap.class); + resolver.register(scala.collection.mutable.LinkedHashMap$.class); + resolver.register(scala.collection.mutable.LinkedHashSet.class); + resolver.register(scala.collection.mutable.LinkedHashSet$.class); + resolver.register(scala.collection.mutable.LongMap.class); + resolver.register(scala.collection.mutable.LongMap$.class); + + resolver.register(scala.collection.mutable.Queue.class); + resolver.register(scala.collection.mutable.Queue$.class); + resolver.register(scala.collection.mutable.Stack.class); + resolver.register(scala.collection.mutable.Stack$.class); + resolver.register(scala.collection.mutable.BitSet.class); + resolver.register(scala.collection.mutable.BitSet$.class); } + checkRegistrationOpen(resolver); + // Install the factory only after the repeatable registrations above have completed, so a + // failed module installation cannot append the same factory again on retry. + fory.registerSerializerFactory(new ScalaSerializerFactory()); } public static void registerEnum(Fory fory, Class cls, long typeId) { diff --git a/scala/fory-scala/src/main/scala/org/apache/fory/scala/ForyScala.scala b/scala/fory-scala/src/main/scala/org/apache/fory/scala/ForyScala.scala index f891272bcb..c71a966a77 100644 --- a/scala/fory-scala/src/main/scala/org/apache/fory/scala/ForyScala.scala +++ b/scala/fory-scala/src/main/scala/org/apache/fory/scala/ForyScala.scala @@ -27,5 +27,5 @@ import org.apache.fory.serializer.scala.ScalaSerializers object ForyScala extends ForyModule { def builder(): ForyBuilder = Fory.builder().withModule(this) - override def install(fory: Fory): Unit = ScalaSerializers.registerSerializers(fory) + override def install(fory: Fory): Unit = ScalaSerializers.installSerializers(fory) } diff --git a/scala/fory-scala/src/test/scala/org/apache/fory/serializer/scala/ScalaTest.scala b/scala/fory-scala/src/test/scala/org/apache/fory/serializer/scala/ScalaTest.scala index 15eb4b1df2..ab03794c02 100644 --- a/scala/fory-scala/src/test/scala/org/apache/fory/serializer/scala/ScalaTest.scala +++ b/scala/fory-scala/src/test/scala/org/apache/fory/serializer/scala/ScalaTest.scala @@ -20,10 +20,7 @@ package org.apache.fory.serializer.scala import java.math.{BigDecimal => JBigDecimal, BigInteger} -import java.util.concurrent.{Callable, CountDownLatch, Executors, TimeUnit} -import java.util.concurrent.atomic.{AtomicInteger, AtomicReference} import org.apache.fory.Fory -import org.apache.fory.exception.ForyException import org.apache.fory.scala.ForyScala import org.scalatest.matchers.should.Matchers import org.scalatest.wordspec.AnyWordSpec @@ -69,126 +66,6 @@ class ScalaTest extends AnyWordSpec with Matchers { } } } - "reject a root reentered during bootstrap" in { - val loader = new BootstrapClassLoader(getClass.getClassLoader, reenterRoot = true) - val runtime = Fory.builder() - .withClassLoader(loader) - .withXlang(false) - .withCodegen(false) - .requireClassRegistration(false) - .build() - loader.runtime = runtime - - intercept[ForyException] { - ScalaSerializers.registerSerializers(runtime) - } - intercept[ForyException] { - ScalaSerializers.registerSerializers(runtime) - } - } - "retry bootstrap after installation failure" in { - val loader = new BootstrapClassLoader(getClass.getClassLoader, failOnce = true) - val runtime = Fory.builder() - .withClassLoader(loader) - .withXlang(false) - .withCodegen(false) - .requireClassRegistration(false) - .build() - - intercept[IllegalStateException] { - ScalaSerializers.registerSerializers(runtime) - } - - ScalaSerializers.registerSerializers(runtime) - runtime.getTypeResolver.isRegistered( - Class.forName("scala.collection.immutable.Vector6", false, loader) - ) shouldBe true - } - "reject same-thread bootstrap reentry" in { - val loader = new BootstrapClassLoader(getClass.getClassLoader, reenterBootstrap = true) - val runtime = Fory.builder() - .withClassLoader(loader) - .withXlang(false) - .withCodegen(false) - .requireClassRegistration(false) - .build() - loader.runtime = runtime - - intercept[ForyException] { - ScalaSerializers.registerSerializers(runtime) - } - ScalaSerializers.registerSerializers(runtime) - runtime.getTypeResolver.isRegistered( - Class.forName("scala.collection.immutable.Vector6", false, loader) - ) shouldBe true - } - "allow nested bootstrap for another runtime" in { - val loader = new BootstrapClassLoader(getClass.getClassLoader, reenterBootstrap = true) - val nested = Fory.builder() - .withXlang(false) - .withCodegen(false) - .requireClassRegistration(false) - .build() - val runtime = Fory.builder() - .withClassLoader(loader) - .withXlang(false) - .withCodegen(false) - .requireClassRegistration(false) - .build() - loader.runtime = nested - - ScalaSerializers.registerSerializers(runtime) - nested.getTypeResolver.isRegistered( - Class.forName("scala.collection.immutable.Vector6") - ) shouldBe true - runtime.getTypeResolver.isRegistered( - Class.forName("scala.collection.immutable.Vector6", false, loader) - ) shouldBe true - } - "install bootstrap exactly once concurrently" in { - val loader = new BlockingBootstrapLoader(getClass.getClassLoader) - val runtime = Fory.builder() - .withClassLoader(loader) - .withXlang(false) - .withCodegen(false) - .requireClassRegistration(false) - .build() - val executor = Executors.newFixedThreadPool(2) - val secondThread = new AtomicReference[Thread]() - val secondStarted = new CountDownLatch(1) - - try { - val first = executor.submit(new Callable[Unit] { - override def call(): Unit = ScalaSerializers.registerSerializers(runtime) - }) - loader.targetEntered.await(10, TimeUnit.SECONDS) shouldBe true - val second = executor.submit(new Callable[Unit] { - override def call(): Unit = { - secondThread.set(Thread.currentThread()) - secondStarted.countDown() - ScalaSerializers.registerSerializers(runtime) - } - }) - secondStarted.await(10, TimeUnit.SECONDS) shouldBe true - val deadline = System.nanoTime() + TimeUnit.SECONDS.toNanos(10) - while (secondThread.get().getState != Thread.State.BLOCKED && - System.nanoTime() < deadline) { - Thread.`yield`() - } - secondThread.get().getState shouldBe Thread.State.BLOCKED - loader.releaseTarget.countDown() - first.get(10, TimeUnit.SECONDS) - second.get(10, TimeUnit.SECONDS) - - loader.targetLoads.get() shouldBe 1 - runtime.getTypeResolver.isRegistered( - Class.forName("scala.collection.immutable.Vector6", false, loader) - ) shouldBe true - } finally { - loader.releaseTarget.countDown() - executor.shutdownNow() - } - } } "serialize/deserialize package object in app" in { // If we move code in main here, we can't reproduce https://github.com/apache/fory/issues/1165. @@ -197,53 +74,6 @@ class ScalaTest extends AnyWordSpec with Matchers { } } -private final class BootstrapClassLoader( - parent: ClassLoader, - reenterRoot: Boolean = false, - failOnce: Boolean = false, - reenterBootstrap: Boolean = false -) extends ClassLoader(parent) { - @volatile var runtime: Fory = _ - private var shouldReenter = reenterRoot - private var shouldFail = failOnce - private var shouldReenterBootstrap = reenterBootstrap - - override protected def loadClass(name: String, resolve: Boolean): Class[_] = { - if (name == "scala.collection.immutable.VectorImpl") { - if (shouldReenter) { - shouldReenter = false - runtime.serialize("freeze") - } - if (shouldReenterBootstrap) { - shouldReenterBootstrap = false - ScalaSerializers.registerSerializers(runtime) - } - if (shouldFail) { - shouldFail = false - throw new IllegalStateException("bootstrap class loading failed") - } - } - super.loadClass(name, resolve) - } -} - -private final class BlockingBootstrapLoader(parent: ClassLoader) extends ClassLoader(parent) { - val targetEntered = new CountDownLatch(1) - val releaseTarget = new CountDownLatch(1) - val targetLoads = new AtomicInteger() - - override protected def loadClass(name: String, resolve: Boolean): Class[_] = { - if (name == "scala.collection.immutable.VectorImpl" && targetLoads.incrementAndGet() == 1) { - targetEntered.countDown() - if (!releaseTarget.await(10, TimeUnit.SECONDS)) { - throw new IllegalStateException("timed out waiting to finish bootstrap class loading") - } - } - super.loadClass(name, resolve) - } -} - - package object PkgObject { case class Id(value: Int) case class IdAnyVal(value: Int) extends AnyVal From 35a072817e7b82d29888e9185bacea3d64b7b969 Mon Sep 17 00:00:00 2001 From: chaokunyang Date: Fri, 28 Aug 2026 08:05:44 +0800 Subject: [PATCH 046/168] fix(java): publish serializer construction atomically --- .agents/languages/java.md | 24 +- AGENTS.md | 30 +- docs/security/deserialization.md | 8 +- .../xlang_implementation_guide.md | 77 ++-- .../apache/fory/FacadeRegistrationGate.java | 90 ++++- .../src/main/java/org/apache/fory/Fory.java | 25 +- .../java/org/apache/fory/ThreadLocalFory.java | 31 +- .../builder/StaticCompatibleCodecBuilder.java | 14 +- .../org/apache/fory/pool/ThreadPoolFory.java | 4 + .../fory/resolver/AllowListChecker.java | 4 +- .../apache/fory/resolver/ClassResolver.java | 193 ++++++---- .../apache/fory/resolver/TypeResolver.java | 198 +++++++++- .../apache/fory/resolver/XtypeResolver.java | 179 +++++++-- .../apache/fory/serializer/FieldGroups.java | 4 +- .../fory/serializer/ObjectSerializer.java | 5 +- .../serializer/ReplaceResolveSerializer.java | 3 +- .../apache/fory/serializer/Serializers.java | 2 +- .../StaticGeneratedStructSerializer.java | 6 +- .../org/apache/fory/memory/MemoryBuffer.java | 17 +- .../org/apache/fory/ThreadSafeForyTest.java | 82 ++-- .../StaticCompatibleCodecBuilderTest.java | 3 +- .../apache/fory/memory/MemoryBufferTest.java | 16 + .../apache/fory/serializer/RegisterTest.java | 354 +++++++++++++++++- 23 files changed, 1112 insertions(+), 257 deletions(-) diff --git a/.agents/languages/java.md b/.agents/languages/java.md index e10786ccb3..a714f7dfb1 100644 --- a/.agents/languages/java.md +++ b/.agents/languages/java.md @@ -88,16 +88,22 @@ Load this file when changing anything under `java/` or when Java drives a cross- facades. Starting a root closes registration before child or pool access, then finishes every already-created child before exposing that root. A child created after closure must replay every accepted registration, finish registration, and only then become visible; discard a provisional - child when replay or finalization fails. Keep the lock order gate before pool or child storage. + child when replay or finalization fails. A callback registration is one facade transaction across + all children: reject root or registration reentry while it is active and permanently fail the + facade after any callback failure, rather than expose partial child mutation, divergent replay + order, or rollback state. Keep the lock order gate before pool or child storage. - Registration callbacks must recheck the authoritative freeze owner after returning and before - publishing the entry they prepared. Java combined type-and-serializer registration constructs - against an unpublished type, rechecks, then publishes both together. Use the nonpublishing - `ObjectSerializer` constructor for that exact class; reject static-generated serializer classes - from the combined class overload because their construction requires prior canonical type - registration. `ForyModule.install` may perform complete nested registrations, but the - module-installed marker is published only after installation returns and the lifecycle is - rechecked. Direct `Fory` accepts modules before its first root; thread-safe facades accept modules - only through `ForyBuilder.withModule` before construction. + publishing the entry they prepared. `TypeResolver` owns one construction-local `TypeInfo` graph + for Java serializer constructors, including self and mutual recursion. After construction and + the lifecycle recheck succeed, the Class/Xtype resolver's normal commit sink publishes the graph + and retains an existing canonical `TypeInfo` owner when its wire and user IDs match. No + constructor-specific publication path or nonpublishing serializer factory is allowed. Reject + static-generated serializer classes from the combined class overload because their construction + requires prior canonical type registration. `Fory.register(ForyModule)` owns module identity, + cycle breaking, and idempotence in one identity set: add the identity before the callback, remove + it on failure, and retain it on success. Do not add separate installing/completed module states. + Direct `Fory` accepts modules before its first root; thread-safe facades accept modules only + through `ForyBuilder.withModule` before construction. - For GraalVM, use `fory codegen` to generate serializers when building native images. Do not add reflection configuration except for JDK `proxy`. - In Java native mode (`xlang=false`), only `Types.BOOL` through `Types.STRING` share type IDs with xlang mode. Other native-mode type IDs differ. - Choose one serializer ownership location per logical Java type family. Add native/xlang serializer variants only when the wire format or constructor contract truly differs. diff --git a/AGENTS.md b/AGENTS.md index 7bb0281bd2..66cf5360bc 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -159,16 +159,17 @@ This is the entry point for AI guidance in Apache Fory. Read this file first, th to this path. - JavaScript root entry releases reference and metadata state left by the previous root, including a failed root, before the context is reused. Do not add full cleanup to the root exit path or copy - Java backing-array retention policies onto native JavaScript arrays. Read-side occurrence arrays - use native replacement reset. The MetaString and TypeMeta writer owner tables each have their own + Java backing-array retention policies onto native JavaScript arrays. Read-side metadata + occurrence arrays use native replacement reset. The MetaString and TypeMeta writer owner tables each have their own logical size: reset active owner IDs and that table's logical size without clearing bounded backing, and replace either backing only after its root has more than 8192 owners. - JavaScript generated registration must build and initialize the complete recursive serializer graph against generation-local owners before one `TypeResolver` batch publication. Factory-init serializer lookup may see those local owners, but runtime and dynamic lookup must retain the real - resolver. Existing published forward owners are initialized in place only during commit so prior - generated captures retain identity. Do not publish placeholders, nested serializers, descriptors, - or cache state before every generated factory and application code hook succeeds. + resolver. A nested identity-only Struct must already have a fully initialized registered owner or + resolve to an owner in the current complete recursive schema graph; otherwise registration fails + before resolver publication. Do not publish placeholders, nested serializers, descriptors, or + cache state before every generated factory and application code hook succeeds. - Root failure exceptions must not copy or retain the operation reference table or materialized object graph for diagnostics. Root cleanup owns releasing that graph, and failure reporting must remain bounded independently of graph size. @@ -200,15 +201,28 @@ This is the entry point for AI guidance in Apache Fory. Read this file first, th serializer construction, then recheck after construction and before replacing its serializer. Do not add rollback, staging, or a parallel registration path for this exception. A module installation may perform complete nested - registrations; recheck after installation before publishing only the module's - installed marker. + registrations. Java `Fory.register(ForyModule)` owns cycle breaking and + idempotence in one identity set: add the identity before installation, remove + it on failure, and retain it on success; do not add parallel installing and + completed states. Java serializer construction uses one resolver-owned local + `TypeInfo` graph for recursive captures, then the normal Class/Xtype commit + sink publishes it after callback and lifecycle checks. Preserve an existing + canonical `TypeInfo` owner when its wire and user IDs match, and do not add a + constructor-specific publication path. Java thread-safe callback registration is one facade-owned + transaction across every child. Reject a root or facade registration reentered + by that callback and permanently fail the facade after callback failure rather + than exposing partial child state, divergent replay order, or resolver + rollback. Keep a late thread-local child provisional until replay and + finalization both complete. - Python `TypeResolver` is the sole registry freeze and finalization owner. Its Cython resolver companion may cache completion of the Python-owner dispatch needed to populate native tables, but the `Fory` facade must not mirror that state; Cython roots call the resolver owner directly. Allocate automatic type IDs only after callback preparation and the final freeze recheck, at the common registry publication point; do not reserve IDs early or maintain counter rollback state. `ThreadSafeFory` validates registrations before publishing replay callbacks and never invokes an - application factory or callback under its non-reentrant pool lock. + application factory or callback under its non-reentrant pool lock. Reject root reentry from the + active instance-build thread before looking in the pool, including when another thread returned + an instance during the build. - Use semantic naming only. Name things after protocol or domain concepts, not history, runtime origin, or workaround style; avoid vague names such as `Internal`, `java_style_*`, `Runtime`, `Session`, `Plan`, `Payload`, or `Binding` when they do not name the real concept. Keep class, method, function, and variable names concise; do not encode the whole scenario or implementation history into one identifier. Never name a class or method with a `Plan` suffix; use the real domain concept instead. For Fory codec/read APIs, do not use generic `payload` naming; name the exact owner and data shape, such as bytes, body, frame, field, string, list, map, compressed bytes, or primitive-array encoding. - Keep one implementation path. Do not keep parallel helpers, serializers, harnesses, wrappers, or registration flows for the same concept; extend the existing owner path instead of inventing another one. - Follow current scope exactly. The latest explicit user instruction overrides earlier plans, and when scope narrows, remove leaked out-of-scope edits immediately. diff --git a/docs/security/deserialization.md b/docs/security/deserialization.md index 96534f5f3d..7ffb4f9249 100644 --- a/docs/security/deserialization.md +++ b/docs/security/deserialization.md @@ -620,9 +620,11 @@ that case, classify the behavior by concrete impact: The first root operation permanently closes type and serializer registration, including when that operation or registry finalization fails. Registration that invokes application code must recheck the authoritative lifecycle before publishing callback-derived state. Thread-safe facades retain -only registrations that completed before the freeze. These rules prevent a failed or reentrant -registration from changing the accepted type surface after deserialization has begun. Runtime- -specific publication ownership belongs in the implementation guide and language guidance. +only registrations that completed before the freeze. A thread-safe facade that cannot roll back an +opaque registration callback must become permanently unusable when that callback fails, rather +than expose children with partially applied registration. These rules prevent a failed or +reentrant registration from changing the accepted type surface after deserialization has begun. +Runtime-specific publication ownership belongs in the implementation guide and language guidance. ## Metadata And Type Resolution diff --git a/docs/specification/xlang_implementation_guide.md b/docs/specification/xlang_implementation_guide.md index 8120395b5f..18b3b4df78 100644 --- a/docs/specification/xlang_implementation_guide.md +++ b/docs/specification/xlang_implementation_guide.md @@ -89,15 +89,37 @@ when the callback returns and reject that publication. Kotlin and Scala combined registration are the type-first exception: publish the canonical type required by generated serializer construction, then recheck after construction and before replacing its serializer. Do not add rollback, staging, or a parallel registration path for that exception. Module -installation may consist of complete nested registrations; publish the module-installed marker -only after installation returns and the lifecycle is rechecked. - -Java combined type-and-serializer registration constructs against an unpublished type, rechecks -the registry lifecycle, and then publishes the type and serializer together. The exact -`ObjectSerializer` path uses its nonpublishing constructor. Static-generated serializer classes -require an already registered canonical type and are therefore rejected by the combined class -overload. Direct Java `Fory` instances may install a module before their first root operation; -thread-safe facades install modules only through `ForyBuilder.withModule` during construction. +installation may consist of complete nested registrations. `Fory.register(ForyModule)` alone owns +module identity, cycle breaking, and idempotence; language bootstrap helpers must not add markers, +monitors, or separate reentry policies. Keep a retryable install body replay-safe until its final +non-repeatable publication. + +Java thread-safe facades serialize each registration callback across their children. A root or +facade registration reentered by that callback rejects the in-progress registration, and any +callback failure closes the facade permanently. This fail-closed boundary prevents partially +applied child state or divergent callback order from becoming observable without adding resolver +rollback, snapshots, or a second registration path. A late thread-local child remains provisional +until every accepted callback has replayed and its resolver has finalized; facade access cannot +expose that child sooner. + +C# `ThreadSafeFory` validates registration on its staging runtime and publishes only a successful +replay action. The resolver prepares serializer bindings and encoded names before one map commit; +a failed callback does not require rebuilding or replacing the staging runtime. + +Python `ThreadSafeFory` likewise validates callbacks before retaining them. Retained callbacks may +capture a serializer class or factory so every child constructs a serializer against its own +resolver; they must not replay one resolver-bound serializer instance. Use the existing +`fory_factory` when each child needs a separately configured serializer instance. + +Java `TypeResolver` owns one construction-local `TypeInfo` graph for serializer constructors, +including self and mutual recursion. After construction and the registry lifecycle recheck +succeed, the normal Class/Xtype resolver commit path publishes the graph. It retains an existing +canonical `TypeInfo` owner when its wire and user IDs match because generated serializers and field +metadata may already capture that owner. There is no constructor-specific publication path or +nonpublishing serializer factory. Static-generated serializer classes require an already +registered canonical type and are therefore rejected by the combined class overload. Direct Java +`Fory` instances may install a module before their first root operation; thread-safe facades +install modules only through `ForyBuilder.withModule` during construction. JavaScript generated registration freezes the complete `TypeInfo` schema graph before code generation, including nested schemas and field occurrence modifiers. The writer-owned @@ -105,10 +127,11 @@ generation, including nested schemas and field occurrence modifiers. The writer- initializes the complete recursive serializer graph against generation-local owners. Generated factories may use a construction-only lookup for fixed serializer captures, while runtime and dynamic dispatch retain the real `TypeResolver`. After every factory and application code hook -succeeds, the resolver performs one guarded batch publication. An uninitialized, already-published -forward owner is initialized in place during that commit so previous generated captures retain its -identity. An initialized owner published by a nested registration is authoritative and must not be -overwritten by the outer registration. +succeeds, the resolver performs one guarded batch publication. A nested identity-only Struct must +already have a fully initialized registered owner or belong to the current complete recursive +schema graph. An unresolved nested identity fails registration before resolver publication. An +initialized owner published by a nested registration is authoritative and must not be overwritten +by the outer registration. Nested serializers must not call back into root `serialize(...)` or `deserialize(...)` entry points. @@ -133,8 +156,8 @@ Java scoped meta-share `TypeInfo` occurrence tables use their logical size as th visibility boundary. A table with at most 8192 active entries resets only that size and retains its slots; a larger table replaces its backing with eight slots. This uniform owner rule keeps normal cleanup allocation-free and must not be specialized for particular entry counts or benchmark -shapes. JavaScript read-side occurrence arrays use native replacement reset instead. Its MetaString -and TypeMeta writer owner tables retain bounded backing through 8192 active owners, reset only +shapes. JavaScript read-side metadata occurrence arrays use native replacement reset instead. Its +MetaString and TypeMeta writer owner tables retain bounded backing through 8192 active owners, reset only their own logical size after restoring active owner IDs, and release backing above that boundary. That operation-local state includes: @@ -959,11 +982,12 @@ Keep the root bitmap separate from per-object ref markers: The current root write flow is: -1. `Fory.serialize(...)` or `serializeTo(...)` prepares the target buffer. -2. `Fory` calls `writeContext.prepare(...)`. -3. `Fory` writes the root bitmap. -4. `Fory` delegates the root object to `WriteContext`. -5. State left by the write resets before the next root reuses the context. +1. `Fory.serialize(...)` or `serializeTo(...)` permanently freezes registration. +2. `Fory` prepares the target buffer. +3. `Fory` calls `writeContext.prepare(...)`. +4. `Fory` writes the root bitmap. +5. `Fory` delegates the root object to `WriteContext`. +6. State left by the write resets before the next root reuses the context. For a non-null root value, `WriteContext.writeRootValue(...)` performs: @@ -993,12 +1017,13 @@ Important rules: The current root read flow mirrors the write flow: -1. `Fory.deserialize(...)` or `deserializeFrom(...)` reads the root bitmap. -2. null roots return immediately. -3. `Fory` validates xlang mode and other root framing requirements. -4. `Fory` calls `readContext.prepare(...)`. -5. `Fory` delegates to `ReadContext`. -6. State left by the read resets before the next root reuses the context. +1. `Fory.deserialize(...)` or `deserializeFrom(...)` permanently freezes registration. +2. `Fory` reads the root bitmap. +3. null roots return immediately. +4. `Fory` validates xlang mode and other root framing requirements. +5. `Fory` calls `readContext.prepare(...)`. +6. `Fory` delegates to `ReadContext`. +7. State left by the read resets before the next root reuses the context. ### `ReadContext` owns ref reservation and payload materialization diff --git a/java/fory-core/src/main/java/org/apache/fory/FacadeRegistrationGate.java b/java/fory-core/src/main/java/org/apache/fory/FacadeRegistrationGate.java index e515a2eb7c..bf410e2227 100644 --- a/java/fory-core/src/main/java/org/apache/fory/FacadeRegistrationGate.java +++ b/java/fory-core/src/main/java/org/apache/fory/FacadeRegistrationGate.java @@ -22,12 +22,14 @@ import java.util.function.Supplier; import org.apache.fory.annotation.Internal; import org.apache.fory.exception.ForyException; +import org.apache.fory.util.ExceptionUtils; -/** Owns the permanent registration freeze before a thread-safe facade's first root or callback. */ +/** Owns registration/root linearization and the permanent freeze at the first facade root. */ @Internal public final class FacadeRegistrationGate { private enum RegistrationState { OPEN, + REGISTERING, FINALIZING, FROZEN, FAILED @@ -35,6 +37,7 @@ private enum RegistrationState { private final Object lock = new Object(); private final Runnable finishChildren; + private boolean childInitializing; private volatile RegistrationState state = RegistrationState.OPEN; public FacadeRegistrationGate(Runnable finishChildren) { @@ -43,25 +46,50 @@ public FacadeRegistrationGate(Runnable finishChildren) { public void applyRegistration(Runnable action) { synchronized (lock) { - checkRegistrationAllowed(); - action.run(); - checkRegistrationAllowed(); + beginRegistration(); + try { + action.run(); + finishRegistration(); + } catch (Throwable e) { + state = RegistrationState.FAILED; + throw ExceptionUtils.throwException(e); + } } } public void applyRegistration(Runnable prepare, Runnable publish) { synchronized (lock) { - checkRegistrationAllowed(); - prepare.run(); - checkRegistrationAllowed(); - publish.run(); + beginRegistration(); + try { + prepare.run(); + requireRegistrationActive(); + publish.run(); + finishRegistration(); + } catch (Throwable e) { + state = RegistrationState.FAILED; + throw ExceptionUtils.throwException(e); + } } } /** Initializes a child while registration callbacks cannot change. */ public Fory initializeChild(Supplier initializer) { synchronized (lock) { - return initializer.get(); + // The monitor is reentrant, so an active initialization here is necessarily the same + // thread reentering the facade before its provisional child is ready. + if (childInitializing) { + throw new IllegalStateException( + "ThreadSafeFory cannot start a root while a child is being initialized."); + } + childInitializing = true; + try { + return initializer.get(); + } catch (Throwable e) { + state = RegistrationState.FAILED; + throw ExceptionUtils.throwException(e); + } finally { + childInitializing = false; + } } } @@ -84,6 +112,11 @@ public void freeze() { if (current == RegistrationState.FAILED) { throw new ForyException("ThreadSafeFory registration finalization previously failed."); } + if (current == RegistrationState.REGISTERING) { + state = RegistrationState.FAILED; + throw new ForyException( + "Cannot start a root operation while ThreadSafeFory registration is in progress."); + } if (current == RegistrationState.FINALIZING) { throw new ForyException("ThreadSafeFory registration finalization is already in progress."); } @@ -91,20 +124,43 @@ public void freeze() { try { finishChildren.run(); state = RegistrationState.FROZEN; - } catch (RuntimeException | Error e) { + } catch (Throwable e) { // Registration remains permanently closed after a failed first finalization. state = RegistrationState.FAILED; - throw e; + throw ExceptionUtils.throwException(e); } } } - private void checkRegistrationAllowed() { - if (state != RegistrationState.OPEN) { - throw new ForyException( - "Cannot register class/serializer after registration has been frozen. Please register " - + "all classes before invoking top-level `serialize/deserialize/copy` methods of " - + "ThreadSafeFory."); + private void beginRegistration() { + RegistrationState current = state; + if (current == RegistrationState.OPEN) { + state = RegistrationState.REGISTERING; + return; } + // The lock is reentrant, so REGISTERING here can only be same-thread facade reentry. Applying + // its callback now would give existing children and future replay different registration order. + if (current == RegistrationState.REGISTERING) { + state = RegistrationState.FAILED; + } + throw registrationClosed(); + } + + private void finishRegistration() { + requireRegistrationActive(); + state = RegistrationState.OPEN; + } + + private void requireRegistrationActive() { + if (state != RegistrationState.REGISTERING) { + throw registrationClosed(); + } + } + + private ForyException registrationClosed() { + return new ForyException( + "Cannot register class/serializer after registration has been frozen or failed. Please " + + "register all classes before invoking top-level `serialize/deserialize/copy` " + + "methods of ThreadSafeFory."); } } diff --git a/java/fory-core/src/main/java/org/apache/fory/Fory.java b/java/fory-core/src/main/java/org/apache/fory/Fory.java index 054b806cde..d65f4d0874 100644 --- a/java/fory-core/src/main/java/org/apache/fory/Fory.java +++ b/java/fory-core/src/main/java/org/apache/fory/Fory.java @@ -22,7 +22,9 @@ import java.io.IOException; import java.io.OutputStream; import java.nio.ByteBuffer; +import java.util.Collections; import java.util.IdentityHashMap; +import java.util.Set; import java.util.function.Consumer; import java.util.function.Function; import javax.annotation.concurrent.NotThreadSafe; @@ -99,7 +101,8 @@ public final class Fory implements BaseFory { private final WriteContext writeContext; private final ReadContext readContext; private final CopyContext copyContext; - private final IdentityHashMap installedModules = new IdentityHashMap<>(); + private final Set moduleRegistrations = + Collections.newSetFromMap(new IdentityHashMap<>()); private final byte headerBitmap; private MemoryBuffer buffer; @@ -227,12 +230,18 @@ public void register(String className, String namespace, String typeName) { public void register(ForyModule module) { Preconditions.checkNotNull(module); checkRegisterAllowed(); - if (installedModules.containsKey(module)) { + if (!moduleRegistrations.add(module)) { return; } - module.install(this); - checkRegisterAllowed(); - installedModules.put(module, Boolean.TRUE); + try { + // Publishing the identity before the callback breaks self and mutual installation cycles. + // A failure removes it, while a successful installation retains the identity for idempotence. + module.install(this); + checkRegisterAllowed(); + } catch (Throwable e) { + moduleRegistrations.remove(module); + throw ExceptionUtils.throwException(e); + } } @Override @@ -271,7 +280,7 @@ public void registerSerializer(Class type, Serializer serializer) { public void registerSerializer( Class type, Function> serializerCreator) { checkRegisterAllowed(); - getTypeResolver().registerSerializer(type, serializerCreator.apply(typeResolver)); + getTypeResolver().registerSerializer(type, serializerCreator); } @Override @@ -291,7 +300,7 @@ public void registerSerializerAndType(Class type, Serializer serializer) { public void registerSerializerAndType( Class type, Function> serializerCreator) { checkRegisterAllowed(); - getTypeResolver().registerSerializerAndType(type, serializerCreator.apply(typeResolver)); + getTypeResolver().registerSerializerAndType(type, serializerCreator); } @Override @@ -709,7 +718,7 @@ SharedRegistry getSharedRegistry() { } private void checkRegisterAllowed() { - if (typeResolver.isRegistrationFinished()) { + if (typeResolver.isRegistrationFrozen()) { throw new ForyException( "Cannot register class/serializer after registration has been frozen. Please register " + "all classes before invoking top-level `serialize/deserialize/copy` methods of " diff --git a/java/fory-core/src/main/java/org/apache/fory/ThreadLocalFory.java b/java/fory-core/src/main/java/org/apache/fory/ThreadLocalFory.java index 1e02ab5ee5..964aff00d9 100644 --- a/java/fory-core/src/main/java/org/apache/fory/ThreadLocalFory.java +++ b/java/fory-core/src/main/java/org/apache/fory/ThreadLocalFory.java @@ -35,6 +35,7 @@ import org.apache.fory.memory.MemoryBuffer; import org.apache.fory.resolver.SharedRegistry; import org.apache.fory.serializer.BufferCallback; +import org.apache.fory.util.ExceptionUtils; /** * A thread safe serialization entrance for {@link Fory} by binding a {@link Fory} for every thread. @@ -66,20 +67,23 @@ public ThreadLocalFory(Function factory) { private Fory newFory() { return registrationGate.initializeChild( () -> { - Fory fory = foryFactory.get(); - // Bind and publish the same provisional child before callbacks. A callback which - // reenters the facade then freezes this child instead of recursively creating another. - foryThreadLocal.set(fory); + Fory fory = null; try { + fory = foryFactory.get(); allFory.put(fory, null); factoryCallback.accept(fory); + if (fory.getTypeResolver().isRegistrationFrozen()) { + throw new IllegalStateException( + "A ThreadSafeFory child started a root operation during registration replay."); + } // Keep finalization in this failure scope so an unusable late child is not retained. registrationGate.finishChildIfFrozen(fory); return fory; - } catch (RuntimeException | Error e) { - foryThreadLocal.remove(); - allFory.remove(fory); - throw e; + } catch (Throwable e) { + if (fory != null) { + allFory.remove(fory); + } + throw ExceptionUtils.throwException(e); } }); } @@ -103,7 +107,13 @@ public void registerCallback(Consumer callback) { registrationGate.applyRegistration( () -> { synchronized (allFory) { - allFory.keySet().forEach(callback); + for (Fory fory : allFory.keySet()) { + callback.accept(fory); + if (fory.getTypeResolver().isRegistrationFrozen()) { + throw new IllegalStateException( + "A ThreadSafeFory child started a root operation during registration."); + } + } } }, () -> factoryCallback = factoryCallback.andThen(callback)); @@ -111,8 +121,7 @@ public void registerCallback(Consumer callback) { @Override public R execute(Function action) { - registrationGate.freeze(); - return action.apply(foryThreadLocal.get()); + return action.apply(currentFory()); } @Override diff --git a/java/fory-core/src/main/java/org/apache/fory/builder/StaticCompatibleCodecBuilder.java b/java/fory-core/src/main/java/org/apache/fory/builder/StaticCompatibleCodecBuilder.java index 4241f258a2..1cee849580 100644 --- a/java/fory-core/src/main/java/org/apache/fory/builder/StaticCompatibleCodecBuilder.java +++ b/java/fory-core/src/main/java/org/apache/fory/builder/StaticCompatibleCodecBuilder.java @@ -239,19 +239,21 @@ private String genRecordCompatibleRead() { Code.ExprCode newRecord = new Invoke(generatedObjectInstantiator(), "newInstanceWithArguments", OBJECT_TYPE, values) .genCode(ctx); - code.append("try {\n"); + code.append("Object _f_record;\n").append("try {\n"); if (StringUtils.isNotBlank(newRecord.code())) { code.append(indent(newRecord.code(), 2)).append('\n'); } - code.append(" Object _f_record = ") + code.append(" _f_record = ") .append(newRecord.value()) .append(";\n") - .append(" return _f_record;\n") - .append("} finally {\n"); + .append("} catch (Throwable _f_error) {\n") + .append(" java.util.Arrays.fill(_f_recordArgs, null);\n") + .append(" throw org.apache.fory.util.ExceptionUtils.throwException(_f_error);\n") + .append("}\n"); for (int i = 0; i < components.length; i++) { - code.append(" _f_recordArgs[").append(i).append("] = null;\n"); + code.append("_f_recordArgs[").append(i).append("] = null;\n"); } - code.append("}"); + code.append("return _f_record;"); return code.toString(); } diff --git a/java/fory-core/src/main/java/org/apache/fory/pool/ThreadPoolFory.java b/java/fory-core/src/main/java/org/apache/fory/pool/ThreadPoolFory.java index f91c61f3d2..3b866ed10c 100644 --- a/java/fory-core/src/main/java/org/apache/fory/pool/ThreadPoolFory.java +++ b/java/fory-core/src/main/java/org/apache/fory/pool/ThreadPoolFory.java @@ -164,6 +164,10 @@ public void registerCallback(Consumer callback) { () -> { for (Fory fory : pooledFory) { callback.accept(fory); + if (fory.getTypeResolver().isRegistrationFrozen()) { + throw new IllegalStateException( + "A ThreadSafeFory child started a root operation during registration."); + } } }); } diff --git a/java/fory-core/src/main/java/org/apache/fory/resolver/AllowListChecker.java b/java/fory-core/src/main/java/org/apache/fory/resolver/AllowListChecker.java index d1cff26a1c..85b02453e8 100644 --- a/java/fory-core/src/main/java/org/apache/fory/resolver/AllowListChecker.java +++ b/java/fory-core/src/main/java/org/apache/fory/resolver/AllowListChecker.java @@ -250,7 +250,7 @@ void addListener(TypeResolver resolver) { try { lock.writeLock().lock(); if ((!disallowList.isEmpty() || !disallowListPrefix.isEmpty()) - && resolver.isRegistrationFinished()) { + && resolver.isRegistrationFrozen()) { throw new IllegalStateException( "A checker with disallow entries cannot be installed after registration."); } @@ -271,7 +271,7 @@ void removeListener(TypeResolver resolver) { private void checkRegistrationOpen() { for (TypeResolver resolver : listeners.keySet()) { - if (resolver.isRegistrationFinished()) { + if (resolver.isRegistrationFrozen()) { throw new IllegalStateException("Classes cannot be disallowed after registration."); } } diff --git a/java/fory-core/src/main/java/org/apache/fory/resolver/ClassResolver.java b/java/fory-core/src/main/java/org/apache/fory/resolver/ClassResolver.java index b347eb3865..37452c2010 100644 --- a/java/fory-core/src/main/java/org/apache/fory/resolver/ClassResolver.java +++ b/java/fory-core/src/main/java/org/apache/fory/resolver/ClassResolver.java @@ -479,6 +479,7 @@ private void registerDefaultClasses() { */ @Override public void register(Class cls) { + checkRegisterAllowed(); if (!extRegistry.registeredClassIdMap.containsKey(cls)) { while (containsUserTypeId(extRegistry.userIdGenerator)) { extRegistry.userIdGenerator++; @@ -1218,7 +1219,7 @@ public static boolean requireJavaSerialization(Class clz) { public void registerSerializer(Class type, Class serializerClass) { checkRegisterAllowed(); checkSerializerRegistration(type, serializerClass); - registerSerializerImpl(type, Serializers.newSerializer(this, type, serializerClass)); + registerSerializer(type, resolver -> resolver.newSerializer(type, serializerClass)); } @Override @@ -1259,38 +1260,12 @@ public void registerInternalSerializer(Class type, Serializer serializer) private void registerSerializerImpl(Class type, Serializer serializer) { checkRegisterAllowed(); - // Serializer registration trusts the Java name, but must not replace an existing custom name. - if (extRegistry.registeredClasses.inverse().get(type) == null) { - extRegistry.registeredClasses.put(type.getName(), type); - } - TypeInfo existingTypeInfo = classInfoMap.get(type); - boolean localOverride = existingTypeInfo != null && existingTypeInfo.serializer != null; - boolean shareable = serializer instanceof Shareable; - if (shareable && !localOverride) { - serializer = sharedRegistry.cacheRegisteredSerializer(type, serializer); - } - addSerializer(type, serializer); - TypeInfo typeInfo = classInfoMap.get(type); - if (shareable && !localOverride) { - TypeInfo sharedTypeInfo = sharedRegistry.cacheRegisteredTypeInfo(type, typeInfo); - if (sharedTypeInfo != typeInfo) { - typeInfo = sharedTypeInfo; - updateTypeInfo(type, typeInfo); - clearTypeInfoCache(); - } - } - if (typeInfo.namespace != null && typeInfo.typeName != null) { - compositeNameBytes2TypeInfo.put( - new TypeNameBytes(typeInfo.namespace, typeInfo.typeName), typeInfo); - } - // in order to support custom serializer for abstract or interface. - if (!type.isPrimitive() && (ReflectionUtils.isAbstract(type) || type.isInterface())) { - extRegistry.abstractTypeInfo.put(type, typeInfo); - extRegistry.registeredTypeInfos.add(typeInfo); - } + TypeInfo typeInfo = newSerializerTypeInfo(type, serializer, false); + publishSerializerTypeInfo(typeInfo, false, true); } - private void checkSerializerRegistration(Class type, Class serializerClass) { + @Override + protected void checkSerializerRegistration(Class type, Class serializerClass) { boolean replaceResolveSerializer = ReplaceResolveSerializer.class.isAssignableFrom(serializerClass) && useReplaceResolveSerializer(type); @@ -1317,6 +1292,100 @@ private void checkSerializerRegistration(Class type, Class serializerClass } } + @Override + protected TypeInfo newSerializerTypeInfo( + Class type, Serializer serializer, boolean registerType) { + TypeInfo existingInfo = classInfoMap.get(type); + if (registerType && !isRegistered(type)) { + int userId = extRegistry.userIdGenerator; + while (containsUserTypeId(userId)) { + userId++; + } + checkRegistration(type, userId, type.getName(), false); + return new TypeInfo(this, type, serializer, buildUserTypeId(type, serializer), userId); + } + int typeId; + int userTypeId = INVALID_USER_TYPE_ID; + Integer registeredId = extRegistry.registeredClassIdMap.get(type); + if (registeredId != null) { + boolean internal = isInternalRegisteredClassId(type, registeredId); + typeId = internal ? registeredId : buildUserTypeId(type, serializer); + userTypeId = internal ? INVALID_USER_TYPE_ID : registeredId; + } else { + typeId = buildUnregisteredTypeId(type, serializer); + } + TypeInfo typeInfo; + if (existingInfo == null) { + typeInfo = new TypeInfo(this, type, serializer, typeId, userTypeId); + } else { + typeInfo = + new TypeInfo( + type, existingInfo.namespace, existingInfo.typeName, serializer, typeId, userTypeId); + typeInfo.typeDef = existingInfo.typeDef; + typeInfo.setSerializer(this, serializer); + } + return typeInfo; + } + + @Override + protected TypeInfo publishSerializerTypeInfo( + TypeInfo typeInfo, boolean registerType, boolean explicitRegistration) { + Class type = typeInfo.type; + TypeInfo currentInfo = classInfoMap.get(type); + TypeInfo publishedInfo = typeInfo; + boolean localOverride = currentInfo != null && currentInfo.serializer != null; + boolean shareable = explicitRegistration && typeInfo.serializer instanceof Shareable; + if (shareable && !localOverride) { + Serializer serializer = + sharedRegistry.cacheRegisteredSerializer(type, typeInfo.serializer); + typeInfo.setSerializer(this, serializer); + } + if (currentInfo != null + && currentInfo.typeId == typeInfo.typeId + && currentInfo.userTypeId == typeInfo.userTypeId) { + currentInfo.setSerializer(this, typeInfo.serializer); + typeInfo = currentInfo; + publishedInfo = typeInfo; + } + if (shareable && !localOverride) { + TypeInfo sharedInfo = sharedRegistry.cacheRegisteredTypeInfo(type, typeInfo); + if (sharedInfo.typeId == typeInfo.typeId && sharedInfo.userTypeId == typeInfo.userTypeId) { + publishedInfo = sharedInfo; + } + } + if (registerType && typeInfo.userTypeId != INVALID_USER_TYPE_ID) { + extRegistry.registeredClassIdMap.put(type, typeInfo.userTypeId); + } + if (publishedInfo.typeId == REPLACE_STUB_ID) { + classInfoMap.put(type, publishedInfo); + } else { + updateTypeInfo(type, publishedInfo); + } + if (explicitRegistration && extRegistry.registeredClasses.inverse().get(type) == null) { + extRegistry.registeredClasses.put(type.getName(), type); + } + boolean publishName = + explicitRegistration + || (!config.requireClassRegistration() + && (extRegistry.typeChecker == DEFAULT_TYPE_CHECKER + || sharedRegistry.isTypeAccepted(type.getName()))); + if (publishName && publishedInfo.namespace != null && publishedInfo.typeName != null) { + compositeNameBytes2TypeInfo.put( + new TypeNameBytes(publishedInfo.namespace, publishedInfo.typeName), publishedInfo); + } + if (explicitRegistration + && !type.isPrimitive() + && (ReflectionUtils.isAbstract(type) || type.isInterface())) { + extRegistry.abstractTypeInfo.put(type, publishedInfo); + extRegistry.registeredTypeInfos.add(publishedInfo); + } + if (registerType) { + registerGraalvmClass(type); + } + clearTypeInfoCache(); + return publishedInfo; + } + /** * Set the serializer for cls, overwrite serializer if exists. Note if class info is * already related with a class, this method should try to reuse that class info, otherwise jit @@ -1325,6 +1394,10 @@ private void checkSerializerRegistration(Class type, Class serializerClass */ @Override public void setSerializer(Class cls, Serializer serializer) { + if (isConstructingSerializer()) { + bindConstructedSerializer(cls, serializer); + return; + } addSerializer(cls, serializer); } @@ -1336,6 +1409,13 @@ public void setSerializer(Class cls, Serializer serializer) { */ @Override public void setSerializerIfAbsent(Class cls, Serializer serializer) { + if (isConstructingSerializer()) { + TypeInfo typeInfo = getConstructedTypeInfo(cls); + if (typeInfo == null || typeInfo.serializer == null) { + bindConstructedSerializer(cls, serializer); + } + return; + } Serializer s = getSerializer(cls, false); if (s == null) { setSerializer(cls, serializer); @@ -1353,45 +1433,8 @@ public void clearSerializer(Class cls) { /** Add serializer for specified class. */ public void addSerializer(Class type, Serializer serializer) { Preconditions.checkNotNull(serializer); - TypeInfo typeInfo; - Integer classId = extRegistry.registeredClassIdMap.get(type); - boolean registered = classId != null; - if (registered) { - int id = classId; - boolean internal = isInternalRegisteredClassId(type, id); - int typeId = internal ? id : buildUserTypeId(type, serializer); - typeInfo = classInfoMap.get(type); - if (typeInfo == null) { - typeInfo = new TypeInfo(this, type, null, typeId, internal ? INVALID_USER_TYPE_ID : id); - } else { - typeInfo = typeInfo.copy(typeId); - } - updateTypeInfo(type, typeInfo); - } else { - int typeId = buildUnregisteredTypeId(type, serializer); - typeInfo = classInfoMap.get(type); - if (typeInfo == null) { - typeInfo = new TypeInfo(this, type, null, typeId, INVALID_USER_TYPE_ID); - } else { - typeInfo = typeInfo.copy(typeId); - } - if (typeId == REPLACE_STUB_ID) { - classInfoMap.put(type, typeInfo); - } else { - updateTypeInfo(type, typeInfo); - } - // Automatic serializer creation may publish only a name accepted earlier by isSecure. - // Explicit registerSerializer publishes below in registerSerializerImpl as a trust event. - if (!config.requireClassRegistration() - && (extRegistry.typeChecker == DEFAULT_TYPE_CHECKER - || sharedRegistry.isTypeAccepted(type.getName())) - && typeInfo.namespace != null - && typeInfo.typeName != null) { - compositeNameBytes2TypeInfo.put( - new TypeNameBytes(typeInfo.namespace, typeInfo.typeName), typeInfo); - } - } - typeInfo.setSerializer(this, serializer); + TypeInfo typeInfo = newAutomaticTypeInfo(type, serializer); + publishSerializerTypeInfo(typeInfo, false, false); } @SuppressWarnings("unchecked") @@ -1718,7 +1761,9 @@ private TypeInfo getOrUpdateTypeInfo(Class cls, int depth) { if (typeInfo == null || typeInfo.serializer == null) { typeInfo = createTypeInfo(cls); } - typeInfoCache[depth] = typeInfo; + if (!isConstructingSerializer()) { + typeInfoCache[depth] = typeInfo; + } } return typeInfo; } @@ -1730,7 +1775,11 @@ private TypeInfo createTypeInfo(Class cls) { // the declaring enum, so its registered ID or class name must be used instead of `$1`. return getTypeInfo(enumClass); } - addSerializer(cls, createSerializer(cls)); + Serializer serializer = createSerializer(cls); + if (isConstructingSerializer()) { + return bindConstructedSerializer(cls, serializer); + } + addSerializer(cls, serializer); return Objects.requireNonNull(classInfoMap.get(cls)); } diff --git a/java/fory-core/src/main/java/org/apache/fory/resolver/TypeResolver.java b/java/fory-core/src/main/java/org/apache/fory/resolver/TypeResolver.java index 2774fda034..416346d992 100644 --- a/java/fory-core/src/main/java/org/apache/fory/resolver/TypeResolver.java +++ b/java/fory-core/src/main/java/org/apache/fory/resolver/TypeResolver.java @@ -140,6 +140,21 @@ private static final class TransformedTypeInfo { } } + private static final class SerializerConstruction { + // Self and mutually recursive constructors must capture these exact TypeInfo owners. Publishing + // them before the callback and lifecycle recheck succeeds would leave stale recursive captures + // and partially registered types after failure. + final Class registrationType; + final boolean registerType; + final IdentityHashMap, TypeInfo> typeInfos = new IdentityHashMap<>(); + boolean rejected; + + SerializerConstruction(Class registrationType, boolean registerType) { + this.registrationType = registrationType; + this.registerType = registerType; + } + } + final Config config; final boolean metaContextShareEnabled; final SharedRegistry sharedRegistry; @@ -155,7 +170,9 @@ private static final class TransformedTypeInfo { // Caches for readTypeInfo(ReadContext) - persist between calls to avoid reloading // dynamically created classes that can't be found by Class.forName private final TypeInfo[] typeInfoCache; + private boolean registrationFrozen; private boolean registrationFinished; + private SerializerConstruction serializerConstruction; protected TypeResolver( Config config, @@ -197,7 +214,13 @@ public final boolean isRegistrationFinished() { return registrationFinished; } + @Internal + public final boolean isRegistrationFrozen() { + return registrationFrozen; + } + protected final void setRegistrationFinished() { + registrationFrozen = true; registrationFinished = true; } @@ -230,11 +253,11 @@ public final Class getDefaultJDKStreamSerializerType() { } protected final void checkRegisterAllowed() { - if (registrationFinished) { + checkRegistrationOpen(); + if (serializerConstruction != null) { + serializerConstruction.rejected = true; throw new ForyException( - "Cannot register class/serializer after registration has been frozen. Please register " - + "all classes before invoking top-level `serialize/deserialize/copy` methods of " - + "Fory."); + "Cannot start an independent registration while a serializer is being constructed."); } } @@ -378,6 +401,14 @@ public final ObjectInstantiator getObjectInstantiator(Class type) { public abstract void registerSerializer( Class type, Class serializerClass); + /** Registers a serializer produced by {@code serializerCreator}. */ + @Internal + public final void registerSerializer( + Class type, Function> serializerCreator) { + checkRegisterAllowed(); + constructSerializer(type, serializerCreator, false); + } + /** * Registers a serializer for internal types (those with fixed IDs in the type system). This * method is used for built-in types like ArrayList, HashMap, etc. @@ -400,6 +431,13 @@ public final void finishRegistration() { if (registrationFinished) { return; } + registrationFrozen = true; + boolean constructionActive = serializerConstruction != null; + if (constructionActive) { + serializerConstruction.rejected = true; + throw new ForyException( + "Cannot start a root operation while a serializer is being constructed."); + } sharedRegistry.setRegistrationIfAbsent( extRegistry.registeredClassIdMap, extRegistry.registeredClasses); extRegistry.finishRegistration( @@ -417,19 +455,21 @@ public final void finishRegistration() { public void registerSerializerAndType( Class type, Class serializerClass) { checkRegisterAllowed(); + checkSerializerRegistration(type, serializerClass); if (StaticGeneratedStructSerializer.class.isAssignableFrom(serializerClass)) { throw new ForyException( "Static generated serializers require registering the type first, then installing a " + "constructed serializer instance with registerSerializer."); } - Serializer serializer = - serializerClass == ObjectSerializer.class - ? new ObjectSerializer<>(this, type, false) - : newSerializer(type, serializerClass); - // Serializer construction may invoke application code which starts a root operation. Keep - // type and serializer publication together after the authoritative lifecycle check. + constructSerializer(type, resolver -> resolver.newSerializer(type, serializerClass), true); + } + + /** Registers a type and a serializer produced by {@code serializerCreator}. */ + @Internal + public final void registerSerializerAndType( + Class type, Function> serializerCreator) { checkRegisterAllowed(); - registerSerializerAndType(type, serializer); + constructSerializer(type, serializerCreator, true); } /** @@ -439,12 +479,88 @@ public void registerSerializerAndType( * @param serializer the serializer instance to use */ public void registerSerializerAndType(Class type, Serializer serializer) { - if (!isRegistered(type)) { - register(type); + checkRegisterAllowed(); + checkSerializerRegistration(type, serializer.getClass()); + TypeInfo typeInfo = newSerializerTypeInfo(type, serializer, true); + checkRegistrationOpen(); + publishSerializerTypeInfo(typeInfo, true, true); + } + + private void constructSerializer( + Class type, + Function> serializerCreator, + boolean registerType) { + jitContext.lock(); + try { + SerializerConstruction construction = new SerializerConstruction(type, registerType); + serializerConstruction = construction; + try { + Serializer serializer = Preconditions.checkNotNull(serializerCreator.apply(this)); + checkSerializerRegistration(type, serializer.getClass()); + bindConstructedSerializer(type, serializer); + checkRegistrationOpen(); + if (construction.rejected) { + throw new ForyException( + "Serializer construction attempted an independent registration for " + + type.getName()); + } + publishConstruction(construction); + } finally { + serializerConstruction = null; + } + } finally { + jitContext.unlock(); + } + } + + private void publishConstruction(SerializerConstruction construction) { + TypeInfo registrationInfo = construction.typeInfos.get(construction.registrationType); + Preconditions.checkNotNull(registrationInfo); + // The target may still fail shareable-serializer conflict validation. Publish it first so a + // rejected target cannot leave otherwise complete recursive dependencies in canonical maps. + Serializer constructedSerializer = registrationInfo.serializer; + TypeInfo publishedInfo = + publishSerializerTypeInfo(registrationInfo, construction.registerType, true); + // Reusing a shared serializer discards the candidate constructor and every dependency it + // discovered. Only dependencies retained by the published serializer belong in this resolver. + if (publishedInfo.serializer != constructedSerializer) { + return; + } + construction.typeInfos.forEach( + (type, typeInfo) -> { + if (type != construction.registrationType) { + publishSerializerTypeInfo(typeInfo, false, false); + } + }); + } + + private void checkRegistrationOpen() { + if (registrationFrozen) { + throw new ForyException( + "Cannot register class/serializer after registration has been frozen. Please register " + + "all classes before invoking top-level `serialize/deserialize/copy` methods of " + + "Fory."); } - registerSerializer(type, serializer); } + protected abstract void checkSerializerRegistration(Class type, Class serializerClass); + + protected abstract TypeInfo newSerializerTypeInfo( + Class type, Serializer serializer, boolean registerType); + + protected TypeInfo newAutomaticTypeInfo(Class type, Serializer serializer) { + return newSerializerTypeInfo(type, serializer, false); + } + + /** + * Publishes prepared serializer metadata through the resolver's canonical commit path. + * + *

When the wire and user IDs are unchanged, the resolver must update the existing {@link + * TypeInfo} owner because generated serializers and field metadata may already retain it. + */ + protected abstract TypeInfo publishSerializerTypeInfo( + TypeInfo typeInfo, boolean registerType, boolean explicitRegistration); + /** * Whether to track reference for this type. If false, reference tracing of subclasses may be * ignored too. @@ -1610,10 +1726,64 @@ private Serializer getNativeTypedValueSerializer(int typeId, Class rawType public abstract void setSerializerIfAbsent(Class cls, Serializer serializer); + /** Returns construction-local metadata for a declared field when one exists. */ + @Internal + public final TypeInfo getFieldTypeInfo(Class type) { + if (serializerConstruction != null) { + TypeInfo typeInfo = serializerConstruction.typeInfos.get(type); + if (typeInfo != null) { + return typeInfo; + } + } + return getTypeInfo(type); + } + + /** Returns the serializer-construction owner without creating type metadata. */ + @Internal + public final TypeInfo getConstructionTypeInfo(Class type) { + TypeInfo typeInfo = getConstructedTypeInfo(type); + return typeInfo == null ? getTypeInfo(type, false) : typeInfo; + } + + protected final TypeInfo getConstructedTypeInfo(Class type) { + return serializerConstruction == null ? null : serializerConstruction.typeInfos.get(type); + } + + protected final boolean isConstructingSerializer() { + return serializerConstruction != null; + } + + protected final TypeInfo bindConstructedSerializer(Class type, Serializer serializer) { + SerializerConstruction construction = Preconditions.checkNotNull(serializerConstruction); + TypeInfo typeInfo = construction.typeInfos.get(type); + if (typeInfo == null) { + if (type == construction.registrationType) { + typeInfo = newSerializerTypeInfo(type, serializer, construction.registerType); + } else { + typeInfo = newAutomaticTypeInfo(type, serializer); + } + construction.typeInfos.put(type, typeInfo); + } else { + typeInfo.setSerializer(this, serializer); + } + return typeInfo; + } + + protected final TypeInfo stageConstructedTypeInfo(Class type, TypeInfo typeInfo) { + Preconditions.checkArgument(typeInfo.type == type); + Preconditions.checkNotNull(serializerConstruction).typeInfos.put(type, typeInfo); + return typeInfo; + } + /** * Reset serializer if {@code serializer} is not null, otherwise clear serializer for {@code cls}. */ public void resetSerializer(Class cls, Serializer serializer) { + TypeInfo constructedTypeInfo = getConstructedTypeInfo(cls); + if (constructedTypeInfo != null) { + constructedTypeInfo.setSerializer(this, serializer); + return; + } if (serializer == null) { TypeInfo typeInfo = getTypeInfo(cls, false); if (typeInfo != null) { diff --git a/java/fory-core/src/main/java/org/apache/fory/resolver/XtypeResolver.java b/java/fory-core/src/main/java/org/apache/fory/resolver/XtypeResolver.java index e3e39199ae..3fc24a6b15 100644 --- a/java/fory-core/src/main/java/org/apache/fory/resolver/XtypeResolver.java +++ b/java/fory-core/src/main/java/org/apache/fory/resolver/XtypeResolver.java @@ -204,6 +204,7 @@ protected void updateTypeInfo(Class cls, TypeInfo typeInfo) { @Override public void register(Class type) { + checkRegisterAllowed(); while (containsUserTypeId(xtypeIdGenerator)) { xtypeIdGenerator++; } @@ -515,39 +516,20 @@ private TypeInfo newTypeInfo( public void registerSerializer(Class type, Class serializerClass) { checkRegisterAllowed(); - registerSerializer(type, newSerializer(type, serializerClass)); + checkSerializerRegistration(type, serializerClass); + registerSerializer(type, resolver -> resolver.newSerializer(type, serializerClass)); } public void registerSerializer(Class type, Serializer serializer) { checkRegisterAllowed(); - TypeInfo typeInfo = checkClassRegistration(type); + checkClassRegistration(type); checkSerializerRegistration(type, serializer.getClass()); - boolean localOverride = typeInfo.serializer != null; - boolean shouldShare = serializer instanceof Shareable && !localOverride; - if (shouldShare) { - serializer = sharedRegistry.cacheRegisteredSerializer(type, serializer); - } - int oldTypeId = typeInfo.typeId; - int foryId = oldTypeId; - - if (foryId == Types.STRUCT || foryId == Types.COMPATIBLE_STRUCT) { - foryId = Types.EXT; - } else if (foryId == Types.NAMED_STRUCT || foryId == Types.NAMED_COMPATIBLE_STRUCT) { - foryId = Types.NAMED_EXT; - } - typeInfo = typeInfo.copy(foryId); - typeInfo.setSerializer(this, serializer); - if (shouldShare) { - typeInfo = sharedRegistry.cacheRegisteredTypeInfo(type, typeInfo); - } - updateTypeInfo(type, typeInfo); - if (typeInfo.typeName != null) { - TypeNameBytes typeNameBytes = new TypeNameBytes(typeInfo.namespace, typeInfo.typeName); - compositeClassNameBytes2TypeInfo.put(typeNameBytes, typeInfo); - } + TypeInfo typeInfo = newSerializerTypeInfo(type, serializer, false); + publishSerializerTypeInfo(typeInfo, false, true); } - private void checkSerializerRegistration(Class type, Class serializerClass) { + @Override + protected void checkSerializerRegistration(Class type, Class serializerClass) { if (isCollection(type) || Collection.class.isAssignableFrom(type)) { if (!CollectionLikeSerializer.class.isAssignableFrom(serializerClass)) { throw new IllegalArgumentException( @@ -566,6 +548,110 @@ private void checkSerializerRegistration(Class type, Class serializerClass } } + @Override + protected TypeInfo newSerializerTypeInfo( + Class type, Serializer serializer, boolean registerType) { + TypeInfo existingInfo = classInfoMap.get(type); + if (!registerType && existingInfo == null) { + checkClassRegistration(type); + } + if (registerType && existingInfo == null) { + if (type.isArray()) { + return newTypeInfo(type, serializer, determineTypeIdForClass(type)); + } + int userTypeId = xtypeIdGenerator; + while (containsUserTypeId(userTypeId)) { + userTypeId++; + } + int typeId = type.isEnum() ? Types.ENUM : Types.EXT; + return newTypeInfo(type, serializer, typeId, userTypeId); + } + int typeId = existingInfo == null ? determineTypeIdForClass(type) : existingInfo.typeId; + if (typeId == Types.STRUCT || typeId == Types.COMPATIBLE_STRUCT) { + typeId = Types.EXT; + } else if (typeId == Types.NAMED_STRUCT || typeId == Types.NAMED_COMPATIBLE_STRUCT) { + typeId = Types.NAMED_EXT; + } + TypeInfo typeInfo; + if (existingInfo == null) { + typeInfo = newTypeInfo(type, serializer, typeId); + } else { + typeInfo = + new TypeInfo( + type, + existingInfo.namespace, + existingInfo.typeName, + serializer, + typeId, + existingInfo.userTypeId); + typeInfo.typeDef = existingInfo.typeDef; + typeInfo.setSerializer(this, serializer); + } + return typeInfo; + } + + @Override + protected TypeInfo newAutomaticTypeInfo(Class type, Serializer serializer) { + TypeInfo existingInfo = classInfoMap.get(type); + if (existingInfo == null) { + return newTypeInfo(type, serializer, determineTypeIdForClass(type)); + } + TypeInfo typeInfo = + new TypeInfo( + type, + existingInfo.namespace, + existingInfo.typeName, + serializer, + existingInfo.typeId, + existingInfo.userTypeId); + typeInfo.typeDef = existingInfo.typeDef; + typeInfo.setSerializer(this, serializer); + return typeInfo; + } + + @Override + protected TypeInfo publishSerializerTypeInfo( + TypeInfo typeInfo, boolean registerType, boolean explicitRegistration) { + Class type = typeInfo.type; + TypeInfo currentInfo = classInfoMap.get(type); + TypeInfo publishedInfo = typeInfo; + boolean localOverride = currentInfo != null && currentInfo.serializer != null; + boolean shareable = explicitRegistration && typeInfo.serializer instanceof Shareable; + if (shareable && !localOverride) { + Serializer serializer = + sharedRegistry.cacheRegisteredSerializer(type, typeInfo.serializer); + typeInfo.setSerializer(this, serializer); + } + if (currentInfo != null + && currentInfo.typeId == typeInfo.typeId + && currentInfo.userTypeId == typeInfo.userTypeId) { + currentInfo.setSerializer(this, typeInfo.serializer); + typeInfo = currentInfo; + publishedInfo = typeInfo; + } + if (shareable && !localOverride) { + TypeInfo sharedInfo = sharedRegistry.cacheRegisteredTypeInfo(type, typeInfo); + if (sharedInfo.typeId == typeInfo.typeId && sharedInfo.userTypeId == typeInfo.userTypeId) { + publishedInfo = sharedInfo; + } + } + updateTypeInfo(type, publishedInfo); + if (explicitRegistration && typeInfo.typeName != null) { + compositeClassNameBytes2TypeInfo.put( + new TypeNameBytes(publishedInfo.namespace, publishedInfo.typeName), publishedInfo); + } + if (registerType && !(type.isArray() && typeInfo.userTypeId == INVALID_USER_TYPE_ID)) { + if (typeInfo.userTypeId != INVALID_USER_TYPE_ID && typeInfo.userTypeId >= xtypeIdGenerator) { + xtypeIdGenerator = typeInfo.userTypeId + 1; + } + String namespace = publishedInfo.decodeNamespace(); + String typeName = publishedInfo.decodeTypeName(); + extRegistry.registeredClasses.put(qualifiedName(namespace, typeName), type); + registerGraalvmClass(type); + } + return publishedInfo; + } + @Override public void registerInternalSerializer(Class type, Serializer serializer) { checkRegisterAllowed(); @@ -837,6 +923,10 @@ public TypeInfo getUserTypeInfo(int userTypeId) { // buildGenericType methods are inherited from TypeResolver private TypeInfo buildTypeInfo(Class cls) { + TypeInfo constructedTypeInfo = getConstructedTypeInfo(cls); + if (constructedTypeInfo != null && constructedTypeInfo.serializer != null) { + return constructedTypeInfo; + } TypeInfo typeInfo = classInfoMap.get(cls); if (typeInfo != null && typeInfo.serializer != null) { return typeInfo; @@ -844,7 +934,11 @@ private TypeInfo buildTypeInfo(Class cls) { if (typeInfo != null) { Class serializerClass = getSerializerClassFromGraalvmRegistry(cls); if (serializerClass != null) { - typeInfo.setSerializer(this, Serializers.newSerializer(this, cls, serializerClass)); + Serializer serializer = Serializers.newSerializer(this, cls, serializerClass); + if (isConstructingSerializer()) { + return bindConstructedSerializer(cls, serializer); + } + typeInfo.setSerializer(this, serializer); return typeInfo; } } @@ -879,11 +973,7 @@ private TypeInfo buildTypeInfo(Class cls) { typeId = Types.MAP; } else if (UnknownClass.class.isAssignableFrom(cls)) { serializer = UnknownClassSerializers.getSerializer(this, "Unknown", cls); - if (cls.isEnum()) { - typeId = Types.ENUM; - } else { - typeId = shareMeta ? Types.COMPATIBLE_STRUCT : Types.STRUCT; - } + typeId = cls.isEnum() ? Types.ENUM : shareMeta ? Types.COMPATIBLE_STRUCT : Types.STRUCT; } else if (cls == Object.class) { // Object.class is handled as unknown type in xlang return getTypeInfo(cls); @@ -891,6 +981,9 @@ private TypeInfo buildTypeInfo(Class cls) { Class enclosingClass = (Class) cls.getEnclosingClass(); if (enclosingClass != null && enclosingClass.isEnum()) { TypeInfo enumInfo = getTypeInfo(enclosingClass); + if (isConstructingSerializer()) { + return enumInfo; + } classInfoMap.put(cls, enumInfo); return enumInfo; } else { @@ -898,8 +991,15 @@ private TypeInfo buildTypeInfo(Class cls) { } } TypeInfo info = newTypeInfo(cls, serializer, typeId); - classInfoMap.put(cls, info); - return info; + if (isConstructingSerializer()) { + TypeInfo constructedInfo = getConstructedTypeInfo(cls); + if (constructedInfo != null) { + return bindConstructedSerializer(cls, serializer); + } + return stageConstructedTypeInfo(cls, info); + } + publishSerializerTypeInfo(info, false, false); + return classInfoMap.get(cls); } private Serializer getCollectionSerializer(Class cls) { @@ -1240,11 +1340,22 @@ public Serializer getRawSerializer(Class cls) { @Override public void setSerializer(Class cls, Serializer serializer) { + if (isConstructingSerializer()) { + bindConstructedSerializer(cls, serializer); + return; + } getTypeInfo(cls).setSerializer(this, serializer); } @Override public void setSerializerIfAbsent(Class cls, Serializer serializer) { + if (isConstructingSerializer()) { + TypeInfo typeInfo = getConstructedTypeInfo(cls); + if (typeInfo == null || typeInfo.serializer == null) { + bindConstructedSerializer(cls, serializer); + } + return; + } TypeInfo typeInfo = classInfoMap.get(cls); Preconditions.checkNotNull(typeInfo); if (typeInfo.serializer == null) { diff --git a/java/fory-core/src/main/java/org/apache/fory/serializer/FieldGroups.java b/java/fory-core/src/main/java/org/apache/fory/serializer/FieldGroups.java index 92e0ec4f6b..644948dd1b 100644 --- a/java/fory-core/src/main/java/org/apache/fory/serializer/FieldGroups.java +++ b/java/fory-core/src/main/java/org/apache/fory/serializer/FieldGroups.java @@ -210,7 +210,7 @@ public SerializationFieldInfo(TypeResolver resolver, Descriptor d) { && resolver.isCollectionDescriptor(d); // invoke `copy` to avoid ObjectSerializer construct clear serializer by `clearSerializer`. if (resolver.isMonomorphic(descriptor)) { - typeInfo = resolver.getTypeInfo(typeRef.getRawType()); + typeInfo = resolver.getFieldTypeInfo(typeRef.getRawType()); if (!resolver.isShareMeta() && !resolver.isCompatible() && typeInfo.getSerializer() instanceof ReplaceResolveSerializer) { @@ -301,7 +301,7 @@ public SerializationFieldInfo(TypeResolver resolver, Descriptor d) { } else { if (!primitiveListCollection && (resolver.isMap(cls) || resolver.isCollection(cls) || resolver.isSet(cls))) { - containerTypeInfo = resolver.getTypeInfo(cls); + containerTypeInfo = resolver.getFieldTypeInfo(cls); } else { containerTypeInfo = null; } diff --git a/java/fory-core/src/main/java/org/apache/fory/serializer/ObjectSerializer.java b/java/fory-core/src/main/java/org/apache/fory/serializer/ObjectSerializer.java index 718d0e45e0..4d21a4d270 100644 --- a/java/fory-core/src/main/java/org/apache/fory/serializer/ObjectSerializer.java +++ b/java/fory-core/src/main/java/org/apache/fory/serializer/ObjectSerializer.java @@ -84,10 +84,9 @@ public ObjectSerializer( super(typeResolver, cls, objectInstantiator); trackingRef = config.trackingRef(); checkClassVersion = typeResolver.checkClassVersion(); - // avoid recursive building serializers. - // Use `setSerializerIfAbsent` to avoid overwriting existing serializer for class when used - // as data serializer. if (resolveParent) { + // Recursive field construction must see this serializer before its fields are resolved. The + // resolver keeps this binding construction-local during combined registration. typeResolver.setSerializerIfAbsent(cls, this); } Collection descriptors; diff --git a/java/fory-core/src/main/java/org/apache/fory/serializer/ReplaceResolveSerializer.java b/java/fory-core/src/main/java/org/apache/fory/serializer/ReplaceResolveSerializer.java index c07ae31326..88dbc686ec 100644 --- a/java/fory-core/src/main/java/org/apache/fory/serializer/ReplaceResolveSerializer.java +++ b/java/fory-core/src/main/java/org/apache/fory/serializer/ReplaceResolveSerializer.java @@ -293,7 +293,8 @@ private static Class dataSerializerClass( private static Serializer createDataSerializer( TypeResolver typeResolver, Class cls, Class sc) { ClassResolver classResolver = (ClassResolver) typeResolver; - Serializer prev = classResolver.getSerializer(cls, false); + TypeInfo typeInfo = classResolver.getConstructionTypeInfo(cls); + Serializer prev = typeInfo == null ? null : typeInfo.getSerializer(); Serializer serializer = Serializers.newSerializer(typeResolver, cls, sc); classResolver.resetSerializer(cls, prev); return serializer; diff --git a/java/fory-core/src/main/java/org/apache/fory/serializer/Serializers.java b/java/fory-core/src/main/java/org/apache/fory/serializer/Serializers.java index 9612d09754..8efe8a40d3 100644 --- a/java/fory-core/src/main/java/org/apache/fory/serializer/Serializers.java +++ b/java/fory-core/src/main/java/org/apache/fory/serializer/Serializers.java @@ -108,7 +108,7 @@ public static Serializer newSerializer( */ public static Serializer newSerializer( TypeResolver typeResolver, Class type, Class serializerClass) { - TypeInfo typeInfo = typeResolver.getTypeInfo(type, false); + TypeInfo typeInfo = typeResolver.getConstructionTypeInfo(type); Serializer serializer = typeInfo == null ? null : typeInfo.getSerializer(); try { return buildSerializer(typeResolver, type, serializerClass); diff --git a/java/fory-core/src/main/java/org/apache/fory/serializer/StaticGeneratedStructSerializer.java b/java/fory-core/src/main/java/org/apache/fory/serializer/StaticGeneratedStructSerializer.java index 1f599f3951..285b6be92e 100644 --- a/java/fory-core/src/main/java/org/apache/fory/serializer/StaticGeneratedStructSerializer.java +++ b/java/fory-core/src/main/java/org/apache/fory/serializer/StaticGeneratedStructSerializer.java @@ -93,12 +93,12 @@ public StaticGeneratedStructSerializer( } private void setSerializerIfAbsent(TypeResolver typeResolver, Class type) { - TypeInfo typeInfo = typeResolver.getTypeInfo(type, false); + TypeInfo typeInfo = typeResolver.getConstructionTypeInfo(type); if (!typeResolver.isCrossLanguage() || typeInfo != null) { // Field-group construction resolves monomorphic field serializers. A generated serializer can // therefore encounter its own type before the subclass constructor has finished, just like - // ObjectSerializer. Install this instance early so recursive fields reuse it instead of - // constructing another serializer for the same type. + // ObjectSerializer. The resolver routes combined registration to its construction owner so + // recursive fields never observe an incomplete serializer in the runtime registry. if (typeInfo != null && typeInfo.getSerializer() instanceof DeferedLazySerializer) { typeResolver.setSerializer(type, this); } else { diff --git a/java/fory-core/src/main/java25/org/apache/fory/memory/MemoryBuffer.java b/java/fory-core/src/main/java25/org/apache/fory/memory/MemoryBuffer.java index 2e3a834b71..9b36502407 100644 --- a/java/fory-core/src/main/java25/org/apache/fory/memory/MemoryBuffer.java +++ b/java/fory-core/src/main/java25/org/apache/fory/memory/MemoryBuffer.java @@ -205,13 +205,16 @@ private void initOffHeapBuffer(long offHeapAddress, int size, ByteBuffer offHeap checkNotNull(offHeapBuffer, "JDK25 MemoryBuffer requires a ByteBuffer owner for off-heap data"); checkArgument( offHeapBuffer.isDirect(), "Only direct ByteBuffers can back off-heap MemoryBuffer"); - this.offHeapBuffer = offHeapBuffer; - ByteBuffer nativeBuffer = offHeapBuffer.duplicate().order(NATIVE_ORDER); - // Stream readers can expand the owner buffer limit after this duplicate is created. Keep the - // absolute-access view capacity-wide so JDK25 public ByteBuffer checks match the logical buffer - // size tracked by MemoryBuffer. - nativeBuffer.clear(); - this.nativeOffHeapBuffer = nativeBuffer; + if (this.offHeapBuffer != offHeapBuffer || nativeOffHeapBuffer == null) { + this.offHeapBuffer = offHeapBuffer; + ByteBuffer nativeBuffer = offHeapBuffer.duplicate().order(NATIVE_ORDER); + // Stream readers can expand the owner buffer limit after this duplicate is created. Keep the + // absolute-access view capacity-wide so JDK25 public ByteBuffer checks match the logical + // buffer size tracked by MemoryBuffer. Reinitializing the same owner only changes its logical + // span, so retain this view instead of allocating one for every stream-root compaction. + nativeBuffer.clear(); + this.nativeOffHeapBuffer = nativeBuffer; + } this.heapMemory = null; this.address = offHeapAddress; this.addressLimit = this.address + size; diff --git a/java/fory-core/src/test/java/org/apache/fory/ThreadSafeForyTest.java b/java/fory-core/src/test/java/org/apache/fory/ThreadSafeForyTest.java index 3d4a280840..2234caeadf 100644 --- a/java/fory-core/src/test/java/org/apache/fory/ThreadSafeForyTest.java +++ b/java/fory-core/src/test/java/org/apache/fory/ThreadSafeForyTest.java @@ -52,6 +52,7 @@ import org.apache.fory.serializer.Serializer; import org.apache.fory.test.bean.BeanA; import org.apache.fory.test.bean.BeanB; +import org.apache.fory.util.ExceptionUtils; import org.testng.Assert; import org.testng.annotations.Test; @@ -738,6 +739,21 @@ public void testFailedFreezeStaysClosed() { assertEquals(finishCalls.get(), 1); } + @Test + public void testCheckedFailureStaysClosed() { + FacadeRegistrationGate gate = new FacadeRegistrationGate(() -> {}); + + Assert.assertThrows( + Exception.class, + () -> + gate.applyRegistration( + () -> { + throw ExceptionUtils.throwException(new Exception("failed")); + })); + Assert.assertThrows(ForyException.class, gate::freeze); + Assert.assertThrows(ForyException.class, () -> gate.applyRegistration(() -> {})); + } + @Test public void testRejectedCallbackNotReplayed() throws Exception { ThreadLocalFory facade = @@ -756,16 +772,38 @@ public void testRejectedCallbackNotReplayed() throws Exception { facade.serialize("freeze"); })); assertEquals(callbackCalls.get(), 1); + Assert.assertThrows(ForyException.class, () -> facade.serialize("closed")); + assertEquals(callbackCalls.get(), 1); + } - ExecutorService executor = Executors.newSingleThreadExecutor(); - try { - Fory lateChild = - executor.submit(() -> facade.execute(child -> child)).get(10, TimeUnit.SECONDS); - assertTrue(lateChild.getTypeResolver().isRegistrationFinished()); - assertEquals(callbackCalls.get(), 1); - } finally { - executor.shutdownNow(); - } + @Test + public void testNestedRegistrationCloses() throws Exception { + ThreadLocalFory facade = + Fory.builder() + .withXlang(false) + .requireClassRegistration(true) + .withCompatible(false) + .buildThreadLocalFory(); + threadLocalChildren(facade); + AtomicInteger callbackCalls = new AtomicInteger(); + + Assert.assertThrows( + ForyException.class, + () -> + facade.registerCallback( + child -> { + callbackCalls.incrementAndGet(); + try { + facade.register(BeanB.class); + } catch (ForyException ignored) { + // The outer callback must still observe the failed gate before publication. + } + child.register(BeanA.class); + })); + + assertTrue(callbackCalls.get() > 0); + Assert.assertThrows(ForyException.class, () -> facade.serialize("closed")); + Assert.assertThrows(ForyException.class, () -> facade.register(BeanA.class)); } @Test @@ -804,9 +842,9 @@ private static void assertReentrantRegistrationRejected(ThreadSafeFory facade, F Assert.assertEquals(creatorCalls.get(), 1); for (Fory child : children) { TypeResolver resolver = child.getTypeResolver(); - assertTrue(resolver.isRegistrationFinished()); assertNull(((ClassResolver) resolver).getRegisteredClassId(Foo.class)); } + Assert.assertThrows(ForyException.class, () -> facade.serialize("closed")); } private static Fory[] threadLocalChildren(ThreadLocalFory facade) throws Exception { @@ -908,18 +946,14 @@ public void testReentrantReplayCleanup() throws Exception { Map children = TestUtils.getFieldValue(facade, "allFory"); ExecutorService executor = Executors.newSingleThreadExecutor(); try { - for (int i = 0; i < 2; i++) { - ExecutionException failure = - Assert.expectThrows( - ExecutionException.class, - () -> - executor - .submit(() -> facade.execute(child -> child)) - .get(10, TimeUnit.SECONDS)); - assertTrue(failure.getCause() instanceof ForyException); - assertEquals(children.size(), 1); - } - assertEquals(callbackCalls.get(), 3); + Assert.expectThrows( + ExecutionException.class, + () -> executor.submit(() -> facade.execute(child -> child)).get(10, TimeUnit.SECONDS)); + assertEquals(children.size(), 1); + assertEquals(callbackCalls.get(), 2); + + Assert.assertThrows(ForyException.class, () -> facade.serialize("closed")); + assertEquals(callbackCalls.get(), 2); } finally { executor.shutdownNow(); } @@ -967,7 +1001,9 @@ public void testPoolGatePrecedesBorrow() throws Exception { Assert.expectThrows( ExecutionException.class, () -> registration.get(10, TimeUnit.SECONDS)); assertTrue(registrationFailure.getCause() instanceof ForyException); - root.get(10, TimeUnit.SECONDS); + ExecutionException rootFailure = + Assert.expectThrows(ExecutionException.class, () -> root.get(10, TimeUnit.SECONDS)); + assertTrue(rootFailure.getCause() instanceof ForyException); } finally { allowReentrantRoot.countDown(); executor.shutdownNow(); diff --git a/java/fory-core/src/test/java/org/apache/fory/builder/StaticCompatibleCodecBuilderTest.java b/java/fory-core/src/test/java/org/apache/fory/builder/StaticCompatibleCodecBuilderTest.java index 7c9e04fed3..97bfa3c7b0 100644 --- a/java/fory-core/src/test/java/org/apache/fory/builder/StaticCompatibleCodecBuilderTest.java +++ b/java/fory-core/src/test/java/org/apache/fory/builder/StaticCompatibleCodecBuilderTest.java @@ -303,7 +303,8 @@ public void testInaccessibleRecordInstantiator() throws Exception { new StaticCompatibleCodecBuilder(TypeRef.of(readerType), reader, remoteTypeDef).genCode(); Assert.assertTrue(generatedSource.contains("newInstanceWithArguments")); Assert.assertTrue(generatedSource.contains("Object[] _f_recordArgs = this._f_recordArgs")); - Assert.assertTrue(generatedSource.contains("finally")); + Assert.assertTrue(generatedSource.contains("java.util.Arrays.fill(_f_recordArgs, null)")); + Assert.assertFalse(generatedSource.contains("finally")); Assert.assertTrue(generatedSource.contains("_f_recordArgs[0] = null")); Assert.assertFalse( generatedSource.contains("return new org.apache.fory.builder." + simpleName)); diff --git a/java/fory-core/src/test/java/org/apache/fory/memory/MemoryBufferTest.java b/java/fory-core/src/test/java/org/apache/fory/memory/MemoryBufferTest.java index 99fe14686b..281dad004d 100644 --- a/java/fory-core/src/test/java/org/apache/fory/memory/MemoryBufferTest.java +++ b/java/fory-core/src/test/java/org/apache/fory/memory/MemoryBufferTest.java @@ -105,6 +105,22 @@ public void testDirectBufferRejectsHeap() { () -> MemoryBuffer.fromDirectByteBuffer(ByteBuffer.allocate(8), 8, null)); } + @Test + public void testDirectReinitRetainsView() { + if (JdkVersion.MAJOR_VERSION < 25) { + throw new SkipException("The retained direct view is specific to JDK 25+"); + } + ByteBuffer owner = ByteBuffer.allocateDirect(8); + MemoryBuffer buffer = MemoryBuffer.fromDirectByteBuffer(owner, 8, null); + ByteBuffer nativeView = TestUtils.getFieldValue(buffer, "nativeOffHeapBuffer"); + + buffer.initByteBuffer(owner, 4); + Assert.assertSame(TestUtils.getFieldValue(buffer, "nativeOffHeapBuffer"), nativeView); + + buffer.initByteBuffer(ByteBuffer.allocateDirect(8), 4); + Assert.assertNotSame(TestUtils.getFieldValue(buffer, "nativeOffHeapBuffer"), nativeView); + } + @Test public void testBackingRangeChecks() { requireRootMemoryBuffer(); diff --git a/java/fory-core/src/test/java/org/apache/fory/serializer/RegisterTest.java b/java/fory-core/src/test/java/org/apache/fory/serializer/RegisterTest.java index ec270faa66..bdd8c9929d 100644 --- a/java/fory-core/src/test/java/org/apache/fory/serializer/RegisterTest.java +++ b/java/fory-core/src/test/java/org/apache/fory/serializer/RegisterTest.java @@ -19,9 +19,10 @@ package org.apache.fory.serializer; +import java.util.ArrayList; import java.util.Collections; -import java.util.IdentityHashMap; import java.util.List; +import java.util.Set; import java.util.concurrent.atomic.AtomicBoolean; import java.util.concurrent.atomic.AtomicInteger; import java.util.concurrent.atomic.AtomicReference; @@ -35,8 +36,11 @@ import org.apache.fory.context.WriteContext; import org.apache.fory.exception.ForyException; import org.apache.fory.meta.TypeDef; +import org.apache.fory.resolver.SharedRegistry; +import org.apache.fory.resolver.TypeInfo; import org.apache.fory.resolver.TypeResolver; import org.apache.fory.type.Descriptor; +import org.apache.fory.util.ExceptionUtils; import org.testng.Assert; import org.testng.annotations.Test; @@ -176,6 +180,277 @@ public static class MyExt { public String id; } + public static class ParentValue { + public int parent; + } + + public static class ChildValue extends ParentValue { + public int child; + } + + public static class RecursiveValue { + public int value; + public RecursiveValue next; + } + + public static class LeftValue { + public int value; + public RightValue right; + } + + public static class RightValue { + public int value; + public LeftValue left; + } + + public static class CustomList extends ArrayList {} + + @Test + public void testCombinedObjectFields() { + Fory fory = + Fory.builder() + .withXlang(false) + .withCodegen(false) + .requireClassRegistration(true) + .withCompatible(false) + .build(); + fory.registerSerializerAndType(ChildValue.class, ObjectSerializer.class); + ChildValue value = new ChildValue(); + value.parent = 1; + value.child = 2; + + ChildValue result = fory.deserialize(fory.serialize(value), ChildValue.class); + + Assert.assertEquals(result.parent, 1); + Assert.assertEquals(result.child, 2); + } + + @Test + public void testCombinedRecursiveObject() { + Fory fory = newStrictNativeFory(); + fory.registerSerializerAndType(RecursiveValue.class, ObjectSerializer.class); + RecursiveValue value = new RecursiveValue(); + value.value = 1; + value.next = value; + + RecursiveValue result = fory.deserialize(fory.serialize(value), RecursiveValue.class); + + Assert.assertEquals(result.value, 1); + Assert.assertSame(result.next, result); + } + + @Test + public void testCombinedMutualObject() { + Fory fory = newStrictNativeFory(); + fory.register(RightValue.class); + fory.registerSerializerAndType( + LeftValue.class, resolver -> new ObjectSerializer<>(resolver, LeftValue.class)); + LeftValue value = new LeftValue(); + value.value = 1; + value.right = new RightValue(); + value.right.value = 2; + value.right.left = value; + + LeftValue result = fory.deserialize(fory.serialize(value), LeftValue.class); + + Assert.assertEquals(result.value, 1); + Assert.assertEquals(result.right.value, 2); + Assert.assertSame(result.right.left, result); + } + + @Test(dataProvider = "xlang") + public void testCombinedInstanceValidation(boolean xlang) { + Fory fory = + Fory.builder() + .withXlang(xlang) + .withCodegen(false) + .requireClassRegistration(true) + .withCompatible(false) + .build(); + Serializer serializer = new MyExtSerializer(fory.getTypeResolver()); + + Assert.assertThrows( + IllegalArgumentException.class, + () -> fory.registerSerializerAndType(CustomList.class, serializer)); + Assert.assertFalse(fory.getTypeResolver().isRegistered(CustomList.class)); + } + + @Test + public void testSerializerKeepsTypeOwner() { + Fory fory = + Fory.builder() + .withXlang(false) + .withCodegen(false) + .requireClassRegistration(true) + .withCompatible(false) + .build(); + fory.register(MyExt.class); + TypeInfo typeInfo = fory.getTypeResolver().getTypeInfo(MyExt.class, false); + + fory.registerSerializer(MyExt.class, ObjectSerializer.class); + + Assert.assertSame(fory.getTypeResolver().getTypeInfo(MyExt.class, false), typeInfo); + Assert.assertTrue( + fory.getTypeResolver().getRawSerializer(MyExt.class) instanceof ObjectSerializer); + } + + @Test(dataProvider = "xlang") + public void testNestedCombinedRegistration(boolean xlang) { + Fory fory = + Fory.builder() + .withXlang(xlang) + .withCodegen(false) + .requireClassRegistration(true) + .withCompatible(false) + .build(); + + Assert.assertThrows( + ForyException.class, + () -> + fory.registerSerializerAndType( + MyExt.class, + resolver -> { + resolver.register(ObjectField.class); + return new MyExtSerializer(resolver); + })); + Assert.assertFalse(fory.getTypeResolver().isRegistered(MyExt.class)); + Assert.assertFalse(fory.getTypeResolver().isRegistered(ObjectField.class)); + } + + @Test(dataProvider = "xlang") + public void testCombinedConstructorFailure(boolean xlang) { + Fory fory = + Fory.builder() + .withXlang(xlang) + .withCodegen(false) + .requireClassRegistration(true) + .withCompatible(false) + .build(); + + Assert.assertThrows( + IllegalStateException.class, + () -> fory.registerSerializerAndType(MyExt.class, FailingSerializer.class)); + Assert.assertFalse(fory.getTypeResolver().isRegistered(MyExt.class)); + Assert.assertNull(fory.getTypeResolver().getTypeInfo(MyExt.class, false)); + } + + @Test + public void testFailedDependencyCache() { + Fory fory = newStrictNativeFory(); + AtomicReference> stagedSerializer = new AtomicReference<>(); + + Assert.assertThrows( + IllegalStateException.class, + () -> + fory.registerSerializerAndType( + MyExt.class, + resolver -> { + stagedSerializer.set(resolver.getSerializer(ObjectField.class)); + throw new IllegalStateException("failed"); + })); + Assert.assertNull(fory.getTypeResolver().getTypeInfo(ObjectField.class, false)); + Assert.assertNotSame( + fory.getTypeResolver().getSerializer(ObjectField.class), stagedSerializer.get()); + } + + @Test + public void testShareConflictIsAtomic() { + SharedRegistry sharedRegistry = new SharedRegistry(); + ForyBuilder builder = + Fory.builder() + .withSharedRegistry(sharedRegistry) + .withXlang(false) + .withCodegen(false) + .requireClassRegistration(false) + .withCompatible(false); + Fory first = builder.build(); + first.registerSerializer(MyExt.class, new FirstShareableSerializer(first.getTypeResolver())); + Serializer sharedSerializer = + first.getTypeResolver().getTypeInfo(MyExt.class, false).getSerializer(); + Fory registered = builder.build(); + registered.registerSerializerAndType(MyExt.class, FirstShareableSerializer.class); + Assert.assertTrue(registered.getTypeResolver().isRegistered(MyExt.class)); + TypeInfo registeredInfo = registered.getTypeResolver().getTypeInfo(MyExt.class, false); + Assert.assertTrue(registeredInfo.getUserTypeId() >= 0); + Assert.assertSame(registeredInfo.getSerializer(), sharedSerializer); + Assert.assertNull(registered.getTypeResolver().getTypeInfo(ObjectField.class, false)); + Fory second = builder.build(); + + Assert.assertThrows( + IllegalArgumentException.class, + () -> second.registerSerializerAndType(MyExt.class, SecondShareableSerializer.class)); + Assert.assertFalse(second.getTypeResolver().isRegistered(MyExt.class)); + Assert.assertNull(second.getTypeResolver().getTypeInfo(MyExt.class, false)); + Assert.assertNull(second.getTypeResolver().getTypeInfo(ObjectField.class, false)); + } + + @Test(dataProvider = "xlang") + public void testReentrantSerializerCreator(boolean xlang) { + Fory fory = + Fory.builder() + .withXlang(xlang) + .withCodegen(false) + .requireClassRegistration(true) + .withCompatible(false) + .build(); + fory.register(MyExt.class); + TypeInfo typeInfo = fory.getTypeResolver().getTypeInfo(MyExt.class, false); + Serializer serializer = typeInfo.getSerializer(); + + Assert.assertThrows( + ForyException.class, + () -> + fory.registerSerializer( + MyExt.class, + resolver -> { + Assert.assertThrows(ForyException.class, () -> fory.serialize("freeze")); + Assert.assertThrows(ForyException.class, () -> fory.serialize("still frozen")); + Assert.assertThrows(ForyException.class, () -> fory.register(runtime -> {})); + return new MyExtSerializer(resolver); + })); + Assert.assertTrue(fory.getTypeResolver().isRegistrationFrozen()); + Assert.assertFalse(fory.getTypeResolver().isRegistrationFinished()); + Assert.assertSame(fory.getTypeResolver().getTypeInfo(MyExt.class, false), typeInfo); + Assert.assertSame(typeInfo.getSerializer(), serializer); + } + + @Test(dataProvider = "xlang") + public void testReentrantSerializerClass(boolean xlang) { + Fory fory = + Fory.builder() + .withXlang(xlang) + .withCodegen(false) + .requireClassRegistration(true) + .withCompatible(false) + .build(); + fory.register(MyExt.class); + TypeInfo typeInfo = fory.getTypeResolver().getTypeInfo(MyExt.class, false); + Serializer serializer = typeInfo.getSerializer(); + ReentrantSerializer.CONSTRUCTION.set(() -> fory.serialize("freeze")); + try { + Assert.assertThrows( + ForyException.class, + () -> fory.registerSerializer(MyExt.class, ReentrantSerializer.class)); + } finally { + ReentrantSerializer.CONSTRUCTION.set(null); + } + + Assert.assertTrue(fory.getTypeResolver().isRegistrationFrozen()); + Assert.assertFalse(fory.getTypeResolver().isRegistrationFinished()); + Assert.assertSame(fory.getTypeResolver().getTypeInfo(MyExt.class, false), typeInfo); + Assert.assertSame(typeInfo.getSerializer(), serializer); + } + + private static Fory newStrictNativeFory() { + return Fory.builder() + .withXlang(false) + .withCodegen(false) + .withRefTracking(true) + .requireClassRegistration(true) + .withCompatible(false) + .build(); + } + @Test public void testFrozenFacadeRegistration() { Fory fory = @@ -224,9 +499,27 @@ public void testReentrantModuleFreeze() { Assert.assertThrows(ForyException.class, () -> fory.register(module)); Assert.assertTrue(installReturned.get()); - IdentityHashMap installedModules = - TestUtils.getFieldValue(fory, "installedModules"); - Assert.assertFalse(installedModules.containsKey(module)); + Set modules = TestUtils.getFieldValue(fory, "moduleRegistrations"); + Assert.assertFalse(modules.contains(module)); + } + + @Test + public void testCheckedModuleFailure() { + Fory fory = Fory.builder().withXlang(false).requireClassRegistration(false).build(); + AtomicBoolean fail = new AtomicBoolean(true); + ForyModule module = + runtime -> { + if (fail.getAndSet(false)) { + throw ExceptionUtils.throwException(new Exception("failed")); + } + }; + + Assert.assertThrows(Exception.class, () -> fory.register(module)); + Set modules = TestUtils.getFieldValue(fory, "moduleRegistrations"); + Assert.assertFalse(modules.contains(module)); + + fory.register(module); + Assert.assertTrue(modules.contains(module)); } @Test @@ -247,6 +540,30 @@ public void testFrozenModuleDuplicateRejected() { Assert.assertEquals(installs.get(), 1); } + @Test + public void testModuleCycle() { + Fory fory = Fory.builder().withXlang(false).requireClassRegistration(false).build(); + AtomicInteger firstInstalls = new AtomicInteger(); + AtomicInteger secondInstalls = new AtomicInteger(); + ForyModule[] modules = new ForyModule[2]; + modules[0] = + runtime -> { + firstInstalls.incrementAndGet(); + runtime.register(modules[1]); + }; + modules[1] = + runtime -> { + secondInstalls.incrementAndGet(); + runtime.register(modules[0]); + }; + + fory.register(modules[0]); + fory.register(modules[1]); + + Assert.assertEquals(firstInstalls.get(), 1); + Assert.assertEquals(secondInstalls.get(), 1); + } + @Test(dataProvider = "xlang") public void testReentrantCombinedRegistration(boolean xlang) { Fory fory = @@ -265,7 +582,8 @@ public void testReentrantCombinedRegistration(boolean xlang) { ReentrantSerializer.CONSTRUCTION.set(null); } - Assert.assertTrue(fory.getTypeResolver().isRegistrationFinished()); + Assert.assertTrue(fory.getTypeResolver().isRegistrationFrozen()); + Assert.assertFalse(fory.getTypeResolver().isRegistrationFinished()); Assert.assertFalse(fory.getTypeResolver().isRegistered(MyExt.class)); Assert.assertNull(fory.getTypeResolver().getTypeInfo(MyExt.class, false)); } @@ -294,9 +612,11 @@ public void testReentrantObjectRegistration() { ForyException.class, () -> fory.registerSerializerAndType(ObjectHolder.class, ObjectSerializer.class)); Assert.assertEquals(factoryCalls.get(), 1); - Assert.assertTrue(fory.getTypeResolver().isRegistrationFinished()); + Assert.assertTrue(fory.getTypeResolver().isRegistrationFrozen()); + Assert.assertFalse(fory.getTypeResolver().isRegistrationFinished()); Assert.assertFalse(fory.getTypeResolver().isRegistered(ObjectHolder.class)); Assert.assertNull(fory.getTypeResolver().getTypeInfo(ObjectHolder.class, false)); + Assert.assertNull(fory.getTypeResolver().getTypeInfo(ObjectField.class, false)); } @Test(dataProvider = "xlang") @@ -327,6 +647,28 @@ public ReentrantSerializer(TypeResolver typeResolver) { } } + public static class FailingSerializer extends MyExtSerializer { + public FailingSerializer(TypeResolver typeResolver) { + super(typeResolver); + typeResolver.setSerializer(MyExt.class, this); + throw new IllegalStateException("failed"); + } + } + + public static final class FirstShareableSerializer extends MyExtSerializer implements Shareable { + public FirstShareableSerializer(TypeResolver typeResolver) { + super(typeResolver); + typeResolver.getTypeInfo(ObjectField.class); + } + } + + public static final class SecondShareableSerializer extends MyExtSerializer implements Shareable { + public SecondShareableSerializer(TypeResolver typeResolver) { + super(typeResolver); + typeResolver.getTypeInfo(ObjectField.class); + } + } + public static class ObjectHolder { public ObjectField field; } From 148bd13d3f57006f30745220585d77d2583c32fc Mon Sep 17 00:00:00 2001 From: chaokunyang Date: Fri, 28 Aug 2026 08:35:06 +0800 Subject: [PATCH 047/168] fix(java): retain canonical serializer owners --- .agents/languages/java.md | 13 +- AGENTS.md | 12 +- .../xlang_implementation_guide.md | 13 +- .../apache/fory/resolver/ClassResolver.java | 9 +- .../apache/fory/resolver/TypeResolver.java | 81 +++++++++++-- .../apache/fory/resolver/XtypeResolver.java | 19 ++- .../apache/fory/serializer/FieldGroups.java | 16 ++- .../serializer/ReplaceResolveSerializer.java | 3 +- .../apache/fory/serializer/Serializers.java | 4 +- .../StaticGeneratedStructSerializer.java | 3 +- .../apache/fory/serializer/RegisterTest.java | 111 ++++++++++++++++-- 11 files changed, 227 insertions(+), 57 deletions(-) diff --git a/.agents/languages/java.md b/.agents/languages/java.md index a714f7dfb1..bfab8d2d21 100644 --- a/.agents/languages/java.md +++ b/.agents/languages/java.md @@ -93,11 +93,14 @@ Load this file when changing anything under `java/` or when Java drives a cross- facade after any callback failure, rather than expose partial child mutation, divergent replay order, or rollback state. Keep the lock order gate before pool or child storage. - Registration callbacks must recheck the authoritative freeze owner after returning and before - publishing the entry they prepared. `TypeResolver` owns one construction-local `TypeInfo` graph - for Java serializer constructors, including self and mutual recursion. After construction and - the lifecycle recheck succeed, the Class/Xtype resolver's normal commit sink publishes the graph - and retains an existing canonical `TypeInfo` owner when its wire and user IDs match. No - constructor-specific publication path or nonpublishing serializer factory is allowed. Reject + publishing the entry they prepared. `TypeResolver` owns one construction-local graph for Java + serializer constructors, including self and mutual recursion. The graph separates final + `TypeInfo` owners from unpublished serializer candidates: recursive fields capture the final + owner immediately, while construction owners resolving recursive fields or candidate state use + the construction-local serializer. Ordinary resolver lookups retain their runtime semantics. + When wire and user IDs match, the final owner is the existing canonical `TypeInfo`. After + construction and the lifecycle recheck succeed, the Class/Xtype resolver's normal commit sink installs the candidate. + No constructor-specific publication path or nonpublishing serializer factory is allowed. Reject static-generated serializer classes from the combined class overload because their construction requires prior canonical type registration. `Fory.register(ForyModule)` owns module identity, cycle breaking, and idempotence in one identity set: add the identity before the callback, remove diff --git a/AGENTS.md b/AGENTS.md index 66cf5360bc..987887c55f 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -205,10 +205,14 @@ This is the entry point for AI guidance in Apache Fory. Read this file first, th idempotence in one identity set: add the identity before installation, remove it on failure, and retain it on success; do not add parallel installing and completed states. Java serializer construction uses one resolver-owned local - `TypeInfo` graph for recursive captures, then the normal Class/Xtype commit - sink publishes it after callback and lifecycle checks. Preserve an existing - canonical `TypeInfo` owner when its wire and user IDs match, and do not add a - constructor-specific publication path. Java thread-safe callback registration is one facade-owned + graph that separates final `TypeInfo` owners from unpublished serializer + candidates. Recursive fields capture the final owner during construction; when + wire and user IDs match, that owner is the existing canonical `TypeInfo`. + Construction owners that resolve recursive fields or candidate state use the + construction-local serializer; ordinary resolver lookups keep their existing + runtime semantics. The normal Class/Xtype commit sink installs the candidate + only after callback and lifecycle checks. + Do not add a constructor-specific publication path. Java thread-safe callback registration is one facade-owned transaction across every child. Reject a root or facade registration reentered by that callback and permanently fail the facade after callback failure rather than exposing partial child state, divergent replay order, or resolver diff --git a/docs/specification/xlang_implementation_guide.md b/docs/specification/xlang_implementation_guide.md index 18b3b4df78..d111cec231 100644 --- a/docs/specification/xlang_implementation_guide.md +++ b/docs/specification/xlang_implementation_guide.md @@ -111,11 +111,14 @@ capture a serializer class or factory so every child constructs a serializer aga resolver; they must not replay one resolver-bound serializer instance. Use the existing `fory_factory` when each child needs a separately configured serializer instance. -Java `TypeResolver` owns one construction-local `TypeInfo` graph for serializer constructors, -including self and mutual recursion. After construction and the registry lifecycle recheck -succeed, the normal Class/Xtype resolver commit path publishes the graph. It retains an existing -canonical `TypeInfo` owner when its wire and user IDs match because generated serializers and field -metadata may already capture that owner. There is no constructor-specific publication path or +Java `TypeResolver` owns one construction-local graph for serializer constructors, including self +and mutual recursion. The graph separates final `TypeInfo` owners from unpublished serializer +candidates. Recursive fields capture the final owner during construction. Construction owners that +resolve recursive fields or candidate state use the construction-local serializer; ordinary +resolver lookups retain their runtime semantics. When wire and user IDs match, the final owner is +the existing canonical `TypeInfo`, so generated serializers and field metadata never retain +temporary metadata. After construction and the registry lifecycle recheck succeed, the normal +Class/Xtype resolver commit path installs the candidate. There is no constructor-specific publication path or nonpublishing serializer factory. Static-generated serializer classes require an already registered canonical type and are therefore rejected by the combined class overload. Direct Java `Fory` instances may install a module before their first root operation; thread-safe facades diff --git a/java/fory-core/src/main/java/org/apache/fory/resolver/ClassResolver.java b/java/fory-core/src/main/java/org/apache/fory/resolver/ClassResolver.java index 37452c2010..f39bc47c3c 100644 --- a/java/fory-core/src/main/java/org/apache/fory/resolver/ClassResolver.java +++ b/java/fory-core/src/main/java/org/apache/fory/resolver/ClassResolver.java @@ -1333,6 +1333,7 @@ protected TypeInfo publishSerializerTypeInfo( Class type = typeInfo.type; TypeInfo currentInfo = classInfoMap.get(type); TypeInfo publishedInfo = typeInfo; + boolean retainedLocalOwner = false; boolean localOverride = currentInfo != null && currentInfo.serializer != null; boolean shareable = explicitRegistration && typeInfo.serializer instanceof Shareable; if (shareable && !localOverride) { @@ -1346,10 +1347,13 @@ protected TypeInfo publishSerializerTypeInfo( currentInfo.setSerializer(this, typeInfo.serializer); typeInfo = currentInfo; publishedInfo = typeInfo; + retainedLocalOwner = true; } if (shareable && !localOverride) { TypeInfo sharedInfo = sharedRegistry.cacheRegisteredTypeInfo(type, typeInfo); - if (sharedInfo.typeId == typeInfo.typeId && sharedInfo.userTypeId == typeInfo.userTypeId) { + if (!retainedLocalOwner + && sharedInfo.typeId == typeInfo.typeId + && sharedInfo.userTypeId == typeInfo.userTypeId) { publishedInfo = sharedInfo; } } @@ -1410,8 +1414,7 @@ public void setSerializer(Class cls, Serializer serializer) { @Override public void setSerializerIfAbsent(Class cls, Serializer serializer) { if (isConstructingSerializer()) { - TypeInfo typeInfo = getConstructedTypeInfo(cls); - if (typeInfo == null || typeInfo.serializer == null) { + if (!hasConstructedSerializer(cls)) { bindConstructedSerializer(cls, serializer); } return; diff --git a/java/fory-core/src/main/java/org/apache/fory/resolver/TypeResolver.java b/java/fory-core/src/main/java/org/apache/fory/resolver/TypeResolver.java index 416346d992..0bb4281305 100644 --- a/java/fory-core/src/main/java/org/apache/fory/resolver/TypeResolver.java +++ b/java/fory-core/src/main/java/org/apache/fory/resolver/TypeResolver.java @@ -141,12 +141,14 @@ private static final class TransformedTypeInfo { } private static final class SerializerConstruction { - // Self and mutually recursive constructors must capture these exact TypeInfo owners. Publishing - // them before the callback and lifecycle recheck succeeds would leave stale recursive captures - // and partially registered types after failure. + // Recursive fields must capture their final TypeInfo owner during construction, while the + // candidate serializers remain unpublished until the whole construction succeeds. Mixing the + // two makes fields retain temporary metadata or lets a failed constructor mutate a canonical + // owner. final Class registrationType; final boolean registerType; final IdentityHashMap, TypeInfo> typeInfos = new IdentityHashMap<>(); + final IdentityHashMap, Serializer> serializers = new IdentityHashMap<>(); boolean rejected; SerializerConstruction(Class registrationType, boolean registerType) { @@ -516,11 +518,18 @@ private void constructSerializer( private void publishConstruction(SerializerConstruction construction) { TypeInfo registrationInfo = construction.typeInfos.get(construction.registrationType); Preconditions.checkNotNull(registrationInfo); + Serializer constructedSerializer = + Preconditions.checkNotNull(construction.serializers.get(construction.registrationType)); + TypeInfo preparedInfo = + prepareConstructionTypeInfo( + construction.registrationType, + registrationInfo, + constructedSerializer, + construction.registerType); // The target may still fail shareable-serializer conflict validation. Publish it first so a // rejected target cannot leave otherwise complete recursive dependencies in canonical maps. - Serializer constructedSerializer = registrationInfo.serializer; TypeInfo publishedInfo = - publishSerializerTypeInfo(registrationInfo, construction.registerType, true); + publishSerializerTypeInfo(preparedInfo, construction.registerType, true); // Reusing a shared serializer discards the candidate constructor and every dependency it // discovered. Only dependencies retained by the published serializer belong in this resolver. if (publishedInfo.serializer != constructedSerializer) { @@ -529,11 +538,26 @@ private void publishConstruction(SerializerConstruction construction) { construction.typeInfos.forEach( (type, typeInfo) -> { if (type != construction.registrationType) { - publishSerializerTypeInfo(typeInfo, false, false); + publishSerializerTypeInfo( + prepareConstructionTypeInfo( + type, typeInfo, construction.serializers.get(type), false), + false, + false); } }); } + private TypeInfo prepareConstructionTypeInfo( + Class type, TypeInfo typeInfo, Serializer serializer, boolean registerType) { + if (classInfoMap.get(type) == typeInfo) { + return registerType + ? newSerializerTypeInfo(type, serializer, true) + : newAutomaticTypeInfo(type, serializer); + } + typeInfo.setSerializer(this, serializer); + return typeInfo; + } + private void checkRegistrationOpen() { if (registrationFrozen) { throw new ForyException( @@ -1726,7 +1750,7 @@ private Serializer getNativeTypedValueSerializer(int typeId, Class rawType public abstract void setSerializerIfAbsent(Class cls, Serializer serializer); - /** Returns construction-local metadata for a declared field when one exists. */ + /** Returns the final metadata owner for a declared field during serializer construction. */ @Internal public final TypeInfo getFieldTypeInfo(Class type) { if (serializerConstruction != null) { @@ -1745,6 +1769,16 @@ public final TypeInfo getConstructionTypeInfo(Class type) { return typeInfo == null ? getTypeInfo(type, false) : typeInfo; } + /** Returns the serializer visible to the active construction without creating type metadata. */ + @Internal + public final Serializer getConstructionSerializer(Class type) { + if (serializerConstruction != null && serializerConstruction.serializers.containsKey(type)) { + return serializerConstruction.serializers.get(type); + } + TypeInfo typeInfo = getTypeInfo(type, false); + return typeInfo == null ? null : typeInfo.serializer; + } + protected final TypeInfo getConstructedTypeInfo(Class type) { return serializerConstruction == null ? null : serializerConstruction.typeInfos.get(type); } @@ -1753,17 +1787,32 @@ protected final boolean isConstructingSerializer() { return serializerConstruction != null; } + protected final boolean hasConstructedSerializer(Class type) { + return serializerConstruction != null && serializerConstruction.serializers.containsKey(type); + } + protected final TypeInfo bindConstructedSerializer(Class type, Serializer serializer) { SerializerConstruction construction = Preconditions.checkNotNull(serializerConstruction); TypeInfo typeInfo = construction.typeInfos.get(type); if (typeInfo == null) { + TypeInfo preparedInfo; if (type == construction.registrationType) { - typeInfo = newSerializerTypeInfo(type, serializer, construction.registerType); + preparedInfo = newSerializerTypeInfo(type, serializer, construction.registerType); } else { - typeInfo = newAutomaticTypeInfo(type, serializer); + preparedInfo = newAutomaticTypeInfo(type, serializer); + } + TypeInfo currentInfo = classInfoMap.get(type); + if (currentInfo != null + && currentInfo.typeId == preparedInfo.typeId + && currentInfo.userTypeId == preparedInfo.userTypeId) { + typeInfo = currentInfo; + } else { + typeInfo = preparedInfo; } construction.typeInfos.put(type, typeInfo); - } else { + } + construction.serializers.put(type, serializer); + if (classInfoMap.get(type) != typeInfo) { typeInfo.setSerializer(this, serializer); } return typeInfo; @@ -1771,7 +1820,9 @@ protected final TypeInfo bindConstructedSerializer(Class type, Serializer protected final TypeInfo stageConstructedTypeInfo(Class type, TypeInfo typeInfo) { Preconditions.checkArgument(typeInfo.type == type); - Preconditions.checkNotNull(serializerConstruction).typeInfos.put(type, typeInfo); + SerializerConstruction construction = Preconditions.checkNotNull(serializerConstruction); + construction.typeInfos.put(type, typeInfo); + construction.serializers.put(type, typeInfo.serializer); return typeInfo; } @@ -1781,7 +1832,13 @@ protected final TypeInfo stageConstructedTypeInfo(Class type, TypeInfo typeIn public void resetSerializer(Class cls, Serializer serializer) { TypeInfo constructedTypeInfo = getConstructedTypeInfo(cls); if (constructedTypeInfo != null) { - constructedTypeInfo.setSerializer(this, serializer); + serializerConstruction.serializers.put(cls, serializer); + if (classInfoMap.get(cls) != constructedTypeInfo) { + constructedTypeInfo.setSerializer(this, serializer); + } + return; + } + if (serializerConstruction != null) { return; } if (serializer == null) { diff --git a/java/fory-core/src/main/java/org/apache/fory/resolver/XtypeResolver.java b/java/fory-core/src/main/java/org/apache/fory/resolver/XtypeResolver.java index 3fc24a6b15..be16895e4d 100644 --- a/java/fory-core/src/main/java/org/apache/fory/resolver/XtypeResolver.java +++ b/java/fory-core/src/main/java/org/apache/fory/resolver/XtypeResolver.java @@ -615,6 +615,7 @@ protected TypeInfo publishSerializerTypeInfo( Class type = typeInfo.type; TypeInfo currentInfo = classInfoMap.get(type); TypeInfo publishedInfo = typeInfo; + boolean retainedLocalOwner = false; boolean localOverride = currentInfo != null && currentInfo.serializer != null; boolean shareable = explicitRegistration && typeInfo.serializer instanceof Shareable; if (shareable && !localOverride) { @@ -628,10 +629,13 @@ protected TypeInfo publishSerializerTypeInfo( currentInfo.setSerializer(this, typeInfo.serializer); typeInfo = currentInfo; publishedInfo = typeInfo; + retainedLocalOwner = true; } if (shareable && !localOverride) { TypeInfo sharedInfo = sharedRegistry.cacheRegisteredTypeInfo(type, typeInfo); - if (sharedInfo.typeId == typeInfo.typeId && sharedInfo.userTypeId == typeInfo.userTypeId) { + if (!retainedLocalOwner + && sharedInfo.typeId == typeInfo.typeId + && sharedInfo.userTypeId == typeInfo.userTypeId) { publishedInfo = sharedInfo; } } @@ -834,12 +838,16 @@ public boolean isMonomorphic(Class clz) { if (clz == UnknownStruct.class) { return false; } - TypeInfo typeInfo = getTypeInfo(clz, false); + TypeInfo typeInfo = getConstructedTypeInfo(clz); + if (typeInfo == null) { + typeInfo = getTypeInfo(clz, false); + } if (typeInfo != null) { if (Types.isEnumType(typeInfo.typeId) || Types.isUnionType(typeInfo.typeId)) { return true; } - Serializer s = typeInfo.serializer; + Serializer s = + isConstructingSerializer() ? getConstructionSerializer(clz) : typeInfo.serializer; if (s instanceof TimeSerializers.TimeSerializer || s instanceof MapLikeSerializer || s instanceof CollectionLikeSerializer @@ -924,7 +932,7 @@ public TypeInfo getUserTypeInfo(int userTypeId) { private TypeInfo buildTypeInfo(Class cls) { TypeInfo constructedTypeInfo = getConstructedTypeInfo(cls); - if (constructedTypeInfo != null && constructedTypeInfo.serializer != null) { + if (constructedTypeInfo != null && hasConstructedSerializer(cls)) { return constructedTypeInfo; } TypeInfo typeInfo = classInfoMap.get(cls); @@ -1350,8 +1358,7 @@ public void setSerializer(Class cls, Serializer serializer) { @Override public void setSerializerIfAbsent(Class cls, Serializer serializer) { if (isConstructingSerializer()) { - TypeInfo typeInfo = getConstructedTypeInfo(cls); - if (typeInfo == null || typeInfo.serializer == null) { + if (!hasConstructedSerializer(cls)) { bindConstructedSerializer(cls, serializer); } return; diff --git a/java/fory-core/src/main/java/org/apache/fory/serializer/FieldGroups.java b/java/fory-core/src/main/java/org/apache/fory/serializer/FieldGroups.java index 644948dd1b..a4291ded61 100644 --- a/java/fory-core/src/main/java/org/apache/fory/serializer/FieldGroups.java +++ b/java/fory-core/src/main/java/org/apache/fory/serializer/FieldGroups.java @@ -208,26 +208,24 @@ public SerializationFieldInfo(TypeResolver resolver, Descriptor d) { boolean primitiveListCollection = TypeUtils.isPrimitiveListClass(typeRef.getRawType()) && resolver.isCollectionDescriptor(d); - // invoke `copy` to avoid ObjectSerializer construct clear serializer by `clearSerializer`. + Serializer fieldSerializer; if (resolver.isMonomorphic(descriptor)) { typeInfo = resolver.getFieldTypeInfo(typeRef.getRawType()); + fieldSerializer = resolver.getConstructionSerializer(typeInfo.getType()); if (!resolver.isShareMeta() && !resolver.isCompatible() - && typeInfo.getSerializer() instanceof ReplaceResolveSerializer) { + && fieldSerializer instanceof ReplaceResolveSerializer) { // overwrite replace resolve serializer for final field - typeInfo.setSerializer( - new FinalFieldReplaceResolveSerializer(resolver, typeInfo.getType())); + fieldSerializer = new FinalFieldReplaceResolveSerializer(resolver, typeInfo.getType()); + resolver.setSerializer(typeInfo.getType(), fieldSerializer); } } else { typeInfo = null; + fieldSerializer = null; } useDeclaredTypeInfo = typeInfo != null && resolver.isMonomorphic(descriptor) && !primitiveListCollection; - if (typeInfo != null) { - serializer = typeInfo.getSerializer(); - } else { - serializer = null; - } + serializer = fieldSerializer; this.qualifiedFieldName = d.getDeclaringClass() + "." + d.getName(); if (d.getField() != null) { diff --git a/java/fory-core/src/main/java/org/apache/fory/serializer/ReplaceResolveSerializer.java b/java/fory-core/src/main/java/org/apache/fory/serializer/ReplaceResolveSerializer.java index 88dbc686ec..6ce2dd2eca 100644 --- a/java/fory-core/src/main/java/org/apache/fory/serializer/ReplaceResolveSerializer.java +++ b/java/fory-core/src/main/java/org/apache/fory/serializer/ReplaceResolveSerializer.java @@ -293,8 +293,7 @@ private static Class dataSerializerClass( private static Serializer createDataSerializer( TypeResolver typeResolver, Class cls, Class sc) { ClassResolver classResolver = (ClassResolver) typeResolver; - TypeInfo typeInfo = classResolver.getConstructionTypeInfo(cls); - Serializer prev = typeInfo == null ? null : typeInfo.getSerializer(); + Serializer prev = classResolver.getConstructionSerializer(cls); Serializer serializer = Serializers.newSerializer(typeResolver, cls, sc); classResolver.resetSerializer(cls, prev); return serializer; diff --git a/java/fory-core/src/main/java/org/apache/fory/serializer/Serializers.java b/java/fory-core/src/main/java/org/apache/fory/serializer/Serializers.java index 8efe8a40d3..9b57e51bf5 100644 --- a/java/fory-core/src/main/java/org/apache/fory/serializer/Serializers.java +++ b/java/fory-core/src/main/java/org/apache/fory/serializer/Serializers.java @@ -55,7 +55,6 @@ import org.apache.fory.platform.internal._JDKAccess; import org.apache.fory.reflect.ReflectionUtils; import org.apache.fory.resolver.ClassResolver; -import org.apache.fory.resolver.TypeInfo; import org.apache.fory.resolver.TypeResolver; import org.apache.fory.serializer.CodegenSerializer.LazyInitBeanSerializer; import org.apache.fory.serializer.collection.ChildContainerSerializers; @@ -108,8 +107,7 @@ public static Serializer newSerializer( */ public static Serializer newSerializer( TypeResolver typeResolver, Class type, Class serializerClass) { - TypeInfo typeInfo = typeResolver.getConstructionTypeInfo(type); - Serializer serializer = typeInfo == null ? null : typeInfo.getSerializer(); + Serializer serializer = typeResolver.getConstructionSerializer(type); try { return buildSerializer(typeResolver, type, serializerClass); } catch (Throwable t) { diff --git a/java/fory-core/src/main/java/org/apache/fory/serializer/StaticGeneratedStructSerializer.java b/java/fory-core/src/main/java/org/apache/fory/serializer/StaticGeneratedStructSerializer.java index 285b6be92e..e1ed8fc2f1 100644 --- a/java/fory-core/src/main/java/org/apache/fory/serializer/StaticGeneratedStructSerializer.java +++ b/java/fory-core/src/main/java/org/apache/fory/serializer/StaticGeneratedStructSerializer.java @@ -94,12 +94,13 @@ public StaticGeneratedStructSerializer( private void setSerializerIfAbsent(TypeResolver typeResolver, Class type) { TypeInfo typeInfo = typeResolver.getConstructionTypeInfo(type); + Serializer serializer = typeResolver.getConstructionSerializer(type); if (!typeResolver.isCrossLanguage() || typeInfo != null) { // Field-group construction resolves monomorphic field serializers. A generated serializer can // therefore encounter its own type before the subclass constructor has finished, just like // ObjectSerializer. The resolver routes combined registration to its construction owner so // recursive fields never observe an incomplete serializer in the runtime registry. - if (typeInfo != null && typeInfo.getSerializer() instanceof DeferedLazySerializer) { + if (typeInfo != null && serializer instanceof DeferedLazySerializer) { typeResolver.setSerializer(type, this); } else { typeResolver.setSerializerIfAbsent(type, this); diff --git a/java/fory-core/src/test/java/org/apache/fory/serializer/RegisterTest.java b/java/fory-core/src/test/java/org/apache/fory/serializer/RegisterTest.java index bdd8c9929d..62f0286e32 100644 --- a/java/fory-core/src/test/java/org/apache/fory/serializer/RegisterTest.java +++ b/java/fory-core/src/test/java/org/apache/fory/serializer/RegisterTest.java @@ -19,7 +19,11 @@ package org.apache.fory.serializer; +import java.io.Externalizable; +import java.io.ObjectInput; +import java.io.ObjectOutput; import java.util.ArrayList; +import java.util.Arrays; import java.util.Collections; import java.util.List; import java.util.Set; @@ -188,7 +192,7 @@ public static class ChildValue extends ParentValue { public int child; } - public static class RecursiveValue { + public static final class RecursiveValue { public int value; public RecursiveValue next; } @@ -284,14 +288,68 @@ public void testSerializerKeepsTypeOwner() { .requireClassRegistration(true) .withCompatible(false) .build(); - fory.register(MyExt.class); - TypeInfo typeInfo = fory.getTypeResolver().getTypeInfo(MyExt.class, false); + fory.register(RecursiveValue.class); + TypeInfo typeInfo = fory.getTypeResolver().getTypeInfo(RecursiveValue.class, false); - fory.registerSerializer(MyExt.class, ObjectSerializer.class); + fory.registerSerializer(RecursiveValue.class, ObjectSerializer.class); - Assert.assertSame(fory.getTypeResolver().getTypeInfo(MyExt.class, false), typeInfo); - Assert.assertTrue( - fory.getTypeResolver().getRawSerializer(MyExt.class) instanceof ObjectSerializer); + Assert.assertSame(fory.getTypeResolver().getTypeInfo(RecursiveValue.class, false), typeInfo); + ObjectSerializer serializer = + (ObjectSerializer) fory.getTypeResolver().getRawSerializer(RecursiveValue.class); + FieldGroups.SerializationFieldInfo[] fields = TestUtils.getFieldValue(serializer, "allFields"); + FieldGroups.SerializationFieldInfo nextField = + Arrays.stream(fields) + .filter(field -> field.descriptor.getName().equals("next")) + .findFirst() + .orElseThrow(AssertionError::new); + Assert.assertSame(nextField.typeInfo, typeInfo); + Assert.assertSame(nextField.serializer, serializer); + } + + @Test + public void testSharedSerializerKeepsLocalOwner() { + ForyBuilder builder = + Fory.builder() + .withSharedRegistry(new SharedRegistry()) + .withXlang(false) + .withCodegen(false) + .requireClassRegistration(true) + .withCompatible(false); + Fory first = builder.build(); + Fory second = builder.build(); + first.register(ExternalValue.class, 201); + second.register(ExternalValue.class, 201); + TypeInfo firstInfo = first.getTypeResolver().getTypeInfo(ExternalValue.class, false); + TypeInfo secondInfo = second.getTypeResolver().getTypeInfo(ExternalValue.class, false); + + first.registerSerializer(ExternalValue.class, ShareableExternalSerializer.class); + second.registerSerializer(ExternalValue.class, ShareableExternalSerializer.class); + + Assert.assertSame(first.getTypeResolver().getTypeInfo(ExternalValue.class, false), firstInfo); + Assert.assertSame(second.getTypeResolver().getTypeInfo(ExternalValue.class, false), secondInfo); + Assert.assertNotSame(firstInfo, secondInfo); + Assert.assertSame(firstInfo.getSerializer(), secondInfo.getSerializer()); + } + + @Test + public void testFailedConstructorKeepsOwner() { + Fory fory = + Fory.builder() + .withXlang(false) + .withCodegen(false) + .requireClassRegistration(true) + .withCompatible(false) + .build(); + fory.register(ExternalValue.class, 203); + TypeInfo typeInfo = fory.getTypeResolver().getTypeInfo(ExternalValue.class, false); + Assert.assertNull(typeInfo.getSerializer()); + + Assert.assertThrows( + IllegalStateException.class, + () -> fory.registerSerializer(ExternalValue.class, FailingExternalSerializer.class)); + + Assert.assertSame(fory.getTypeResolver().getTypeInfo(ExternalValue.class, false), typeInfo); + Assert.assertNull(typeInfo.getSerializer()); } @Test(dataProvider = "xlang") @@ -669,6 +727,45 @@ public SecondShareableSerializer(TypeResolver typeResolver) { } } + public static final class ExternalValue implements Externalizable { + @Override + public void writeExternal(ObjectOutput out) {} + + @Override + public void readExternal(ObjectInput in) {} + } + + public static final class ShareableExternalSerializer extends Serializer + implements Shareable { + public ShareableExternalSerializer(TypeResolver typeResolver) { + super(typeResolver.getConfig(), ExternalValue.class); + } + + @Override + public void write(WriteContext writeContext, ExternalValue value) {} + + @Override + public ExternalValue read(ReadContext readContext) { + return new ExternalValue(); + } + } + + public static final class FailingExternalSerializer extends Serializer { + public FailingExternalSerializer(TypeResolver typeResolver) { + super(typeResolver.getConfig(), ExternalValue.class); + typeResolver.setSerializer(ExternalValue.class, this); + throw new IllegalStateException("failed"); + } + + @Override + public void write(WriteContext writeContext, ExternalValue value) {} + + @Override + public ExternalValue read(ReadContext readContext) { + return new ExternalValue(); + } + } + public static class ObjectHolder { public ObjectField field; } From b8855d2fbbdaa6cd38716b08cedcdcf7eab7a649 Mon Sep 17 00:00:00 2001 From: chaokunyang Date: Fri, 28 Aug 2026 23:52:44 +0800 Subject: [PATCH 048/168] perf(javascript): reuse metadata owner counters --- javascript/packages/core/lib/context.ts | 15 ++++++--------- javascript/test/rootCleanup.test.ts | 18 ++++++++++-------- 2 files changed, 16 insertions(+), 17 deletions(-) diff --git a/javascript/packages/core/lib/context.ts b/javascript/packages/core/lib/context.ts index 023bb6beb3..a87156ea78 100644 --- a/javascript/packages/core/lib/context.ts +++ b/javascript/packages/core/lib/context.ts @@ -278,7 +278,6 @@ export class MetaStringWriter { private static readonly MAX_RETAINED_META_STRING_OWNERS = 8192; private disposeMetaStringBytes: MetaStringBytes[] = []; - private disposeMetaStringBytesSize = 0; private dynamicNameId = 0; private namespaceEncoder = new MetaStringEncoder(".", "_"); private typenameEncoder = new MetaStringEncoder("$", "_"); @@ -287,9 +286,10 @@ export class MetaStringWriter { if (bytes.dynamicWriteStringId !== -1) { writer.writeVarUInt32(((bytes.dynamicWriteStringId + 1) << 1) | 1); } else { - bytes.dynamicWriteStringId = this.dynamicNameId; + const index = this.dynamicNameId; + bytes.dynamicWriteStringId = index; this.dynamicNameId += 1; - this.disposeMetaStringBytes[this.disposeMetaStringBytesSize++] = bytes; + this.disposeMetaStringBytes[index] = bytes; const len = bytes.bytes.getBytes().byteLength; writer.writeVarUInt32(len << 1); if (len !== 0) { @@ -309,7 +309,7 @@ export class MetaStringWriter { reset() { const owners = this.disposeMetaStringBytes; - const size = this.disposeMetaStringBytesSize; + const size = this.dynamicNameId; for (let i = 0; i < size; i++) { owners[i].dynamicWriteStringId = -1; } @@ -318,7 +318,6 @@ export class MetaStringWriter { if (size > MetaStringWriter.MAX_RETAINED_META_STRING_OWNERS) { this.disposeMetaStringBytes = []; } - this.disposeMetaStringBytesSize = 0; this.dynamicNameId = 0; } } @@ -373,7 +372,6 @@ export class WriteContext { readonly metaStringWriter: MetaStringWriter; private disposeTypeMetaOwners: Array<{ dynamicTypeId: number }> = []; - private disposeTypeMetaOwnersSize = 0; private dynamicTypeId = 0; constructor( @@ -390,7 +388,7 @@ export class WriteContext { this.refWriter.reset(); this.metaStringWriter.reset(); const owners = this.disposeTypeMetaOwners; - const size = this.disposeTypeMetaOwnersSize; + const size = this.dynamicTypeId; for (let i = 0; i < size; i++) { owners[i].dynamicTypeId = -1; } @@ -399,7 +397,6 @@ export class WriteContext { if (size > WriteContext.MAX_RETAINED_TYPE_META_OWNERS) { this.disposeTypeMetaOwners = []; } - this.disposeTypeMetaOwnersSize = 0; this.dynamicTypeId = 0; } @@ -443,7 +440,7 @@ export class WriteContext { const index = this.dynamicTypeId; owner.dynamicTypeId = index; this.dynamicTypeId += 1; - this.disposeTypeMetaOwners[this.disposeTypeMetaOwnersSize++] = owner; + this.disposeTypeMetaOwners[index] = owner; this.writer.writeVarUInt32(index << 1); this.writer.buffer(bytes); } diff --git a/javascript/test/rootCleanup.test.ts b/javascript/test/rootCleanup.test.ts index 70fa6c02af..169b3ab6c1 100644 --- a/javascript/test/rootCleanup.test.ts +++ b/javascript/test/rootCleanup.test.ts @@ -133,8 +133,6 @@ test.each(["success", "failure"] as const)("restores root write state for %s", ( expect(fory.serialize(7)).toBeDefined(); } expect(writeContext.refWriter.writeObjects.size).toBe(0); - expect(writeContext.disposeTypeMetaOwnersSize).toBe(0); - expect(writeContext.metaStringWriter.disposeMetaStringBytesSize).toBe(0); expect(name.dynamicWriteStringId).toBe(-1); expect(typeMeta.dynamicTypeId).toBe(-1); }); @@ -152,12 +150,12 @@ test("reuses root write metastring owners", () => { expect(registered.serialize({})).toBeDefined(); const owners = writeContext.metaStringWriter.disposeMetaStringBytes; expect(owners).toHaveLength(1); - expect(writeContext.metaStringWriter.disposeMetaStringBytesSize).toBe(1); + expect(name.dynamicWriteStringId).toBe(0); expect(registered.serialize({})).toBeDefined(); expect(writeContext.metaStringWriter.disposeMetaStringBytes).toBe(owners); expect(owners).toHaveLength(1); - expect(writeContext.metaStringWriter.disposeMetaStringBytesSize).toBe(1); + expect(name.dynamicWriteStringId).toBe(0); }); test("reuses root write type metadata owners", () => { @@ -173,12 +171,12 @@ test("reuses root write type metadata owners", () => { expect(registered.serialize({})).toBeDefined(); const owners = writeContext.disposeTypeMetaOwners; expect(owners).toHaveLength(1); - expect(writeContext.disposeTypeMetaOwnersSize).toBe(1); + expect(typeMeta.dynamicTypeId).toBe(0); expect(registered.serialize({})).toBeDefined(); expect(writeContext.disposeTypeMetaOwners).toBe(owners); expect(owners).toHaveLength(1); - expect(writeContext.disposeTypeMetaOwnersSize).toBe(1); + expect(typeMeta.dynamicTypeId).toBe(0); }); test.each([8192, 8193])("bounds %s root write metastring owners", (ownerCount) => { @@ -193,13 +191,15 @@ test.each([8192, 8193])("bounds %s root write metastring owners", (ownerCount) = const owners = metaStringWriter.disposeMetaStringBytes; writeContext.reset(); - expect(metaStringWriter.disposeMetaStringBytesSize).toBe(0); if (ownerCount === 8192) { expect(metaStringWriter.disposeMetaStringBytes).toBe(owners); } else { expect(metaStringWriter.disposeMetaStringBytes).not.toBe(owners); expect(metaStringWriter.disposeMetaStringBytes).toHaveLength(0); } + const nextOwner = metaStringWriter.encodeTypeName("next-root"); + metaStringWriter.writeBytes(writeContext.writer, nextOwner); + expect(nextOwner.dynamicWriteStringId).toBe(0); }); test.each([8192, 8193])("bounds %s root write type metadata owners", (ownerCount) => { @@ -214,7 +214,6 @@ test.each([8192, 8193])("bounds %s root write type metadata owners", (ownerCount const owners = writeContext.disposeTypeMetaOwners; writeContext.reset(); - expect(writeContext.disposeTypeMetaOwnersSize).toBe(0); expect(typeMetaOwners.every((owner) => owner.dynamicTypeId === -1)).toBe(true); if (ownerCount === 8192) { expect(writeContext.disposeTypeMetaOwners).toBe(owners); @@ -222,6 +221,9 @@ test.each([8192, 8193])("bounds %s root write type metadata owners", (ownerCount expect(writeContext.disposeTypeMetaOwners).not.toBe(owners); expect(writeContext.disposeTypeMetaOwners).toHaveLength(0); } + const nextOwner = { dynamicTypeId: -1 }; + writeContext.writeTypeMeta(nextOwner, bytes); + expect(nextOwner.dynamicTypeId).toBe(0); }); test("releases a failed root write buffer before reuse", () => { From 565e0a85a3075a051bc7647eea8928a408311ba7 Mon Sep 17 00:00:00 2001 From: chaokunyang Date: Sat, 29 Aug 2026 01:47:52 +0800 Subject: [PATCH 049/168] perf(cpp): avoid repeated pool finalization --- cpp/fory/serialization/fory.h | 9 +++------ 1 file changed, 3 insertions(+), 6 deletions(-) diff --git a/cpp/fory/serialization/fory.h b/cpp/fory/serialization/fory.h index 71226c3d5f..c571a9cb5f 100644 --- a/cpp/fory/serialization/fory.h +++ b/cpp/fory/serialization/fory.h @@ -1061,8 +1061,10 @@ class ThreadSafeFory : public BaseFory { std::shared_ptr resolver) : BaseFory(config, std::move(resolver)), finalized_resolver_(), finalized_once_flag_(), fory_pool_([this]() { + // Every public root finalizes before pool acquisition, so a pool miss + // only clones the resolver already published by that root. return std::unique_ptr(new Fory( - config_, get_finalized_resolver(), Fory::PreFinalized{})); + config_, finalized_resolver_->clone(), Fory::PreFinalized{})); }) {} void ensure_finalized() const { @@ -1075,11 +1077,6 @@ class ThreadSafeFory : public BaseFory { }); } - std::shared_ptr get_finalized_resolver() const { - ensure_finalized(); - return finalized_resolver_->clone(); - } - mutable std::shared_ptr finalized_resolver_; mutable std::once_flag finalized_once_flag_; util::Pool fory_pool_; From 277cf6463cbab311ed6d6d8e75e4013ad6c9d79a Mon Sep 17 00:00:00 2001 From: chaokunyang Date: Sat, 29 Aug 2026 01:47:52 +0800 Subject: [PATCH 050/168] refactor(csharp): remove stale registry state --- csharp/src/Fory/ThreadSafeFory.cs | 2 +- csharp/src/Fory/TypeResolver.cs | 16 ++++------------ 2 files changed, 5 insertions(+), 13 deletions(-) diff --git a/csharp/src/Fory/ThreadSafeFory.cs b/csharp/src/Fory/ThreadSafeFory.cs index 6818695ad0..0674bb0c06 100644 --- a/csharp/src/Fory/ThreadSafeFory.cs +++ b/csharp/src/Fory/ThreadSafeFory.cs @@ -36,7 +36,7 @@ public sealed class ThreadSafeFory : IDisposable internal ThreadSafeFory(Config config) { _config = config; - _threadLocalFory = new ThreadLocal(CreatePerThreadFory, trackAllValues: true); + _threadLocalFory = new ThreadLocal(CreatePerThreadFory); } /// diff --git a/csharp/src/Fory/TypeResolver.cs b/csharp/src/Fory/TypeResolver.cs index 8aac5bae1b..385c561312 100644 --- a/csharp/src/Fory/TypeResolver.cs +++ b/csharp/src/Fory/TypeResolver.cs @@ -204,7 +204,7 @@ public Serializer GetSerializer() public TypeInfo GetTypeInfo(Type type) { - return GetOrCreateTypeInfo(type, null); + return GetOrCreateTypeInfo(type); } public TypeInfo GetTypeInfo() @@ -453,23 +453,15 @@ internal IReadOnlyList TypeMetaFields(TypeInfo typeInfo, bool return typeInfo.TypeMetaFields(trackRef); } - private TypeInfo GetOrCreateTypeInfo(Type type, TypeInfo? explicitTypeInfo) + private TypeInfo GetOrCreateTypeInfo(Type type) { ulong typeKey = TypeMapKey.Get(type); if (_typeInfos.TryGetValue(typeKey, out TypeInfo? existing)) { - if (explicitTypeInfo is null || ReferenceEquals(existing, explicitTypeInfo)) - { - return existing; - } - - if (existing.IsRegistered) - { - throw new InvalidDataException($"cannot override serializer for registered type {type}"); - } + return existing; } - TypeInfo typeInfo = explicitTypeInfo ?? CreateBindingCore(type); + TypeInfo typeInfo = CreateBindingCore(type); if (typeInfo.Type != type) { throw new InvalidDataException($"serializer type mismatch for {type}, got {typeInfo.Type}"); From 193773923ff94352b95cdde72dd8df949dcc6ac5 Mon Sep 17 00:00:00 2001 From: chaokunyang Date: Sat, 29 Aug 2026 01:47:52 +0800 Subject: [PATCH 051/168] refactor(python): restore read cleanup owner --- python/pyfory/serialization.pyx | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/python/pyfory/serialization.pyx b/python/pyfory/serialization.pyx index 477c65316c..112f5e9dcd 100644 --- a/python/pyfory/serialization.pyx +++ b/python/pyfory/serialization.pyx @@ -1315,7 +1315,7 @@ cdef class Fory: unsupported_objects=unsupported_objects, ) finally: - self.read_context.reset() + self.reset_read() cdef object _deserialize(self, buffer, buffers=None, unsupported_objects=None): cdef ReadContext read_context = self.read_context From a94052cc5cf49bf8acb5d70523dfaf861c625f69 Mon Sep 17 00:00:00 2001 From: chaokunyang Date: Sat, 29 Aug 2026 01:52:42 +0800 Subject: [PATCH 052/168] fix(java): publish late children atomically --- .../apache/fory/FacadeRegistrationGate.java | 25 ++-- .../java/org/apache/fory/ThreadLocalFory.java | 27 ++-- .../org/apache/fory/ThreadSafeForyTest.java | 117 +++++++++++++++++- 3 files changed, 139 insertions(+), 30 deletions(-) diff --git a/java/fory-core/src/main/java/org/apache/fory/FacadeRegistrationGate.java b/java/fory-core/src/main/java/org/apache/fory/FacadeRegistrationGate.java index bf410e2227..49f168b170 100644 --- a/java/fory-core/src/main/java/org/apache/fory/FacadeRegistrationGate.java +++ b/java/fory-core/src/main/java/org/apache/fory/FacadeRegistrationGate.java @@ -19,6 +19,7 @@ package org.apache.fory; +import java.util.function.Consumer; import java.util.function.Supplier; import org.apache.fory.annotation.Internal; import org.apache.fory.exception.ForyException; @@ -72,18 +73,28 @@ public void applyRegistration(Runnable prepare, Runnable publish) { } } - /** Initializes a child while registration callbacks cannot change. */ - public Fory initializeChild(Supplier initializer) { + /** Initializes and publishes a child while registration callbacks cannot change. */ + public Fory initializeChild(Supplier initializer, Consumer publisher) { synchronized (lock) { + if (state == RegistrationState.FAILED) { + throw registrationClosed(); + } // The monitor is reentrant, so an active initialization here is necessarily the same - // thread reentering the facade before its provisional child is ready. + // thread reentering the facade before the current child initialization completes. if (childInitializing) { throw new IllegalStateException( "ThreadSafeFory cannot start a root while a child is being initialized."); } childInitializing = true; try { - return initializer.get(); + Fory child = initializer.get(); + if (state == RegistrationState.FROZEN) { + child.getTypeResolver().finishRegistration(); + } else if (state != RegistrationState.OPEN) { + throw registrationClosed(); + } + publisher.accept(child); + return child; } catch (Throwable e) { state = RegistrationState.FAILED; throw ExceptionUtils.throwException(e); @@ -93,12 +104,6 @@ public Fory initializeChild(Supplier initializer) { } } - void finishChildIfFrozen(Fory child) { - if (state == RegistrationState.FROZEN) { - child.getTypeResolver().finishRegistration(); - } - } - public void freeze() { RegistrationState current = state; if (current == RegistrationState.FROZEN) { diff --git a/java/fory-core/src/main/java/org/apache/fory/ThreadLocalFory.java b/java/fory-core/src/main/java/org/apache/fory/ThreadLocalFory.java index 964aff00d9..51935ece79 100644 --- a/java/fory-core/src/main/java/org/apache/fory/ThreadLocalFory.java +++ b/java/fory-core/src/main/java/org/apache/fory/ThreadLocalFory.java @@ -35,7 +35,6 @@ import org.apache.fory.memory.MemoryBuffer; import org.apache.fory.resolver.SharedRegistry; import org.apache.fory.serializer.BufferCallback; -import org.apache.fory.util.ExceptionUtils; /** * A thread safe serialization entrance for {@link Fory} by binding a {@link Fory} for every thread. @@ -67,25 +66,15 @@ public ThreadLocalFory(Function factory) { private Fory newFory() { return registrationGate.initializeChild( () -> { - Fory fory = null; - try { - fory = foryFactory.get(); - allFory.put(fory, null); - factoryCallback.accept(fory); - if (fory.getTypeResolver().isRegistrationFrozen()) { - throw new IllegalStateException( - "A ThreadSafeFory child started a root operation during registration replay."); - } - // Keep finalization in this failure scope so an unusable late child is not retained. - registrationGate.finishChildIfFrozen(fory); - return fory; - } catch (Throwable e) { - if (fory != null) { - allFory.remove(fory); - } - throw ExceptionUtils.throwException(e); + Fory child = foryFactory.get(); + factoryCallback.accept(child); + if (child.getTypeResolver().isRegistrationFrozen()) { + throw new IllegalStateException( + "A ThreadSafeFory child started a root operation during registration replay."); } - }); + return child; + }, + child -> allFory.put(child, null)); } private void finishChildRegistration() { diff --git a/java/fory-core/src/test/java/org/apache/fory/ThreadSafeForyTest.java b/java/fory-core/src/test/java/org/apache/fory/ThreadSafeForyTest.java index 2234caeadf..7a2291fb64 100644 --- a/java/fory-core/src/test/java/org/apache/fory/ThreadSafeForyTest.java +++ b/java/fory-core/src/test/java/org/apache/fory/ThreadSafeForyTest.java @@ -690,6 +690,62 @@ public void testRegistrationGateLinearization() throws Exception { } } + @Test + public void testFreezeWaitsForChildPublish() throws Exception { + AtomicReference published = new AtomicReference<>(); + AtomicInteger finishSawChild = new AtomicInteger(); + FacadeRegistrationGate gate = + new FacadeRegistrationGate( + () -> { + Fory child = published.get(); + assertNotNull(child); + finishSawChild.incrementAndGet(); + child.getTypeResolver().finishRegistration(); + }); + Fory child = + Fory.builder() + .withXlang(false) + .requireClassRegistration(true) + .withCompatible(false) + .build(); + CountDownLatch publishEntered = new CountDownLatch(1); + CountDownLatch releasePublish = new CountDownLatch(1); + CountDownLatch freezeStarted = new CountDownLatch(1); + ExecutorService executor = Executors.newFixedThreadPool(2); + try { + Future initialization = + executor.submit( + () -> + gate.initializeChild( + () -> child, + value -> { + publishEntered.countDown(); + awaitUnchecked(releasePublish); + published.set(value); + })); + assertTrue(publishEntered.await(10, TimeUnit.SECONDS)); + Future freeze = + executor.submit( + () -> { + freezeStarted.countDown(); + gate.freeze(); + }); + assertTrue(freezeStarted.await(10, TimeUnit.SECONDS)); + Assert.assertThrows(TimeoutException.class, () -> freeze.get(100, TimeUnit.MILLISECONDS)); + + releasePublish.countDown(); + assertSame(initialization.get(10, TimeUnit.SECONDS), child); + freeze.get(10, TimeUnit.SECONDS); + assertSame(published.get(), child); + assertEquals(finishSawChild.get(), 1); + assertTrue(child.getTypeResolver().isRegistrationFinished()); + } finally { + releasePublish.countDown(); + executor.shutdownNow(); + executor.awaitTermination(10, TimeUnit.SECONDS); + } + } + @Test public void testFreezeWaitsForChildren() throws Exception { CountDownLatch finishEntered = new CountDownLatch(1); @@ -933,17 +989,19 @@ public void testReentrantReplayCleanup() throws Exception { .requireClassRegistration(true) .withCompatible(false) .buildThreadLocalFory(); + Map children = TestUtils.getFieldValue(facade, "allFory"); AtomicInteger callbackCalls = new AtomicInteger(); + AtomicInteger replayPublished = new AtomicInteger(-1); facade.registerCallback( child -> { if (callbackCalls.incrementAndGet() > 1) { + replayPublished.set(children.containsKey(child) ? 1 : 0); facade.serialize("nested"); } child.register(BeanA.class); }); facade.serialize("freeze"); - Map children = TestUtils.getFieldValue(facade, "allFory"); ExecutorService executor = Executors.newSingleThreadExecutor(); try { Assert.expectThrows( @@ -951,6 +1009,7 @@ public void testReentrantReplayCleanup() throws Exception { () -> executor.submit(() -> facade.execute(child -> child)).get(10, TimeUnit.SECONDS)); assertEquals(children.size(), 1); assertEquals(callbackCalls.get(), 2); + assertEquals(replayPublished.get(), 0); Assert.assertThrows(ForyException.class, () -> facade.serialize("closed")); assertEquals(callbackCalls.get(), 2); @@ -959,6 +1018,62 @@ public void testReentrantReplayCleanup() throws Exception { } } + @Test + public void testLateChildFailureRace() throws Exception { + ThreadLocalFory facade = + Fory.builder() + .withXlang(false) + .requireClassRegistration(true) + .withCompatible(false) + .buildThreadLocalFory(); + Map children = TestUtils.getFieldValue(facade, "allFory"); + AtomicInteger callbackCalls = new AtomicInteger(); + CountDownLatch replayEntered = new CountDownLatch(1); + CountDownLatch releaseReplay = new CountDownLatch(1); + facade.registerCallback( + child -> { + int call = callbackCalls.incrementAndGet(); + if (call == 2) { + replayEntered.countDown(); + awaitUnchecked(releaseReplay); + throw new IllegalStateException("failed replay"); + } + child.register(BeanA.class); + }); + facade.serialize("freeze"); + + ExecutorService executor = Executors.newFixedThreadPool(2); + AtomicReference waitingThread = new AtomicReference<>(); + CountDownLatch waitingStarted = new CountDownLatch(1); + try { + Future failing = executor.submit(() -> facade.execute(child -> child)); + assertTrue(replayEntered.await(10, TimeUnit.SECONDS)); + Future waiting = + executor.submit( + () -> { + waitingThread.set(Thread.currentThread()); + waitingStarted.countDown(); + return facade.execute(child -> child); + }); + assertTrue(waitingStarted.await(10, TimeUnit.SECONDS)); + awaitBlocked(waitingThread.get()); + + releaseReplay.countDown(); + ExecutionException replayFailure = + Assert.expectThrows(ExecutionException.class, () -> failing.get(10, TimeUnit.SECONDS)); + assertTrue(replayFailure.getCause() instanceof IllegalStateException); + ExecutionException waitingFailure = + Assert.expectThrows(ExecutionException.class, () -> waiting.get(10, TimeUnit.SECONDS)); + assertTrue(waitingFailure.getCause() instanceof ForyException); + assertEquals(callbackCalls.get(), 2); + assertEquals(children.size(), 1); + } finally { + releaseReplay.countDown(); + executor.shutdownNow(); + executor.awaitTermination(10, TimeUnit.SECONDS); + } + } + @Test public void testPoolGatePrecedesBorrow() throws Exception { ThreadPoolFory facade = From ef2db122225a810b5c871f61fefd326b1e536df2 Mon Sep 17 00:00:00 2001 From: chaokunyang Date: Sat, 29 Aug 2026 01:52:50 +0800 Subject: [PATCH 053/168] refactor(java): keep lifecycle logic in owners --- .../org/apache/fory/config/ForyBuilder.java | 17 +++-------------- .../org/apache/fory/pool/ThreadPoolFory.java | 7 +------ 2 files changed, 4 insertions(+), 20 deletions(-) diff --git a/java/fory-core/src/main/java/org/apache/fory/config/ForyBuilder.java b/java/fory-core/src/main/java/org/apache/fory/config/ForyBuilder.java index a4a2de1d22..c9c796b241 100644 --- a/java/fory-core/src/main/java/org/apache/fory/config/ForyBuilder.java +++ b/java/fory-core/src/main/java/org/apache/fory/config/ForyBuilder.java @@ -423,16 +423,14 @@ public ForyBuilder withSerializerFactory(SerializerFactory serializerFactory) { /** * Installs a runtime module into every Fory instance created by this builder. * - *

Repeated registration of the same module object is ignored. Dedupe uses identity, not {@link - * Object#equals(Object)}, so distinct module instances are installed independently. + *

Each created Fory instance ignores repeated registration of the same module object. Dedupe + * uses identity, not {@link Object#equals(Object)}, so distinct module instances are installed + * independently. * *

Thread-safe facades accept modules only through this builder configuration. */ public ForyBuilder withModule(ForyModule module) { ForyModule checkedModule = Objects.requireNonNull(module); - if (containsModule(checkedModule)) { - return this; - } modules.add(checkedModule); recordAction(b -> b.withModule(checkedModule)); return this; @@ -645,15 +643,6 @@ private void recordAction(Consumer action) { } } - private boolean containsModule(ForyModule module) { - for (int i = 0; i < modules.size(); i++) { - if (modules.get(i) == module) { - return true; - } - } - return false; - } - private void install(Fory fory) { for (int i = 0; i < serializerFactories.size(); i++) { fory.registerSerializerFactory(serializerFactories.get(i)); diff --git a/java/fory-core/src/main/java/org/apache/fory/pool/ThreadPoolFory.java b/java/fory-core/src/main/java/org/apache/fory/pool/ThreadPoolFory.java index 3b866ed10c..544d05d9d3 100644 --- a/java/fory-core/src/main/java/org/apache/fory/pool/ThreadPoolFory.java +++ b/java/fory-core/src/main/java/org/apache/fory/pool/ThreadPoolFory.java @@ -82,10 +82,6 @@ private void finishChildRegistration() { private PooledEntry acquire() { registrationGate.freeze(); - return acquireEntry(); - } - - private PooledEntry acquireEntry() { int slotIndex = slotIndexForCurrentThread(); PooledEntry entry = tryBorrowPreferredSlots(slotIndex); if (entry != null) { @@ -174,8 +170,7 @@ public void registerCallback(Consumer callback) { @Override public R execute(Function action) { - registrationGate.freeze(); - PooledEntry entry = acquireEntry(); + PooledEntry entry = acquire(); try { return action.apply(entry.fory); } finally { From a6f1802e967d52195a2a7bfbb7e3771ef8e58fff Mon Sep 17 00:00:00 2001 From: chaokunyang Date: Sat, 29 Aug 2026 01:58:46 +0800 Subject: [PATCH 054/168] fix(jvm): keep generated serializers unpublished --- .../apache/fory/resolver/TypeResolver.java | 27 ++++--- .../serializer/kotlin/KotlinSerializers.java | 25 ++++--- .../kotlin/KotlinDefaultValueSupport.kt | 2 +- .../kotlin/BuiltinClassSerializerTests.kt | 75 ++++++++++++++++++- .../serializer/kotlin/DefaultValueTest.kt | 2 +- .../apache/fory/scala/ForySerializer.scala | 20 ++--- .../scala/ForySerializerDerivationTest.scala | 35 +++++++-- 7 files changed, 141 insertions(+), 45 deletions(-) diff --git a/java/fory-core/src/main/java/org/apache/fory/resolver/TypeResolver.java b/java/fory-core/src/main/java/org/apache/fory/resolver/TypeResolver.java index 0bb4281305..780a367752 100644 --- a/java/fory-core/src/main/java/org/apache/fory/resolver/TypeResolver.java +++ b/java/fory-core/src/main/java/org/apache/fory/resolver/TypeResolver.java @@ -1803,8 +1803,12 @@ protected final TypeInfo bindConstructedSerializer(Class type, Serializer } TypeInfo currentInfo = classInfoMap.get(type); if (currentInfo != null - && currentInfo.typeId == preparedInfo.typeId - && currentInfo.userTypeId == preparedInfo.userTypeId) { + && (serializer instanceof StaticGeneratedStructSerializer + || (currentInfo.typeId == preparedInfo.typeId + && currentInfo.userTypeId == preparedInfo.userTypeId))) { + // Static-generated construction starts after canonical type registration. Its base + // constructor binds early for recursion, but that candidate must retain the registered + // Struct identity instead of being reclassified as an explicit EXT serializer. typeInfo = currentInfo; } else { typeInfo = preparedInfo; @@ -2127,10 +2131,9 @@ public final DescriptorGrouper groupDescriptors( } private List buildFieldDescriptors(Class clz, boolean searchParent) { - List registeredStaticDescriptors = - getRegisteredStaticGeneratedStructDescriptors(clz); - if (registeredStaticDescriptors != null) { - return normalizeFieldDescriptors(clz, searchParent, registeredStaticDescriptors); + List ownedStaticDescriptors = getOwnedStaticGeneratedDescriptors(clz); + if (ownedStaticDescriptors != null) { + return normalizeFieldDescriptors(clz, searchParent, ownedStaticDescriptors); } if (shouldPreferStaticGeneratedSerializer(clz)) { List staticDescriptors = getStaticGeneratedStructDescriptors(clz); @@ -2266,14 +2269,14 @@ private List getStaticGeneratedStructDescriptors(Class cls) { cls, isCrossLanguage()); } - private List getRegisteredStaticGeneratedStructDescriptors(Class cls) { - TypeInfo typeInfo = getTypeInfo(cls, false); - if (typeInfo == null - || !(typeInfo.getSerializer() instanceof StaticGeneratedStructSerializer)) { + private List getOwnedStaticGeneratedDescriptors(Class cls) { + Serializer serializer = getConstructionSerializer(cls); + if (!(serializer instanceof StaticGeneratedStructSerializer)) { return null; } - return ((StaticGeneratedStructSerializer) typeInfo.getSerializer()) - .getGeneratedDescriptors(); + // Generated descriptors are immutable constructor input. Let TypeDef construction see them + // without publishing the serializer candidate that owns the active construction. + return ((StaticGeneratedStructSerializer) serializer).getGeneratedDescriptors(); } private StaticGeneratedStructSerializer copyRegisteredStaticGeneratedStructSerializer( diff --git a/kotlin/fory-kotlin/src/main/java/org/apache/fory/serializer/kotlin/KotlinSerializers.java b/kotlin/fory-kotlin/src/main/java/org/apache/fory/serializer/kotlin/KotlinSerializers.java index c0e40e2ff7..35e69f0e52 100644 --- a/kotlin/fory-kotlin/src/main/java/org/apache/fory/serializer/kotlin/KotlinSerializers.java +++ b/kotlin/fory-kotlin/src/main/java/org/apache/fory/serializer/kotlin/KotlinSerializers.java @@ -20,12 +20,17 @@ package org.apache.fory.serializer.kotlin; import java.util.Objects; -import kotlin.*; +import kotlin.Result; import kotlin.UByteArray; import kotlin.UIntArray; import kotlin.ULongArray; import kotlin.UShortArray; -import kotlin.text.*; +import kotlin.text.CharCategory; +import kotlin.text.CharDirectionality; +import kotlin.text.HexFormat; +import kotlin.text.MatchGroup; +import kotlin.text.Regex; +import kotlin.text.RegexOption; import kotlin.time.Duration; import kotlin.time.DurationUnit; import kotlin.time.TimedValue; @@ -156,7 +161,7 @@ public static void installSerializers(Fory fory) { resolver.registerSerializer(Uuid.class, new UuidSerializer(config)); } checkRegistrationOpen(resolver); - DefaultValueUtils.setKotlinDefaultValueSupport(new KotlinDefaultValueSupport()); + DefaultValueUtils.setKotlinDefaultValueSupport(KotlinDefaultValueSupport.INSTANCE); } private static void registerIfAbsent(TypeResolver resolver, Class cls) { @@ -223,19 +228,15 @@ public static void register(Fory fory, Class cls, String namespace, String ty public static void registerSerializer(Fory fory, Class cls) { TypeResolver resolver = fory.getTypeResolver(); checkRegistrationOpen(resolver); - Serializer serializer = newGeneratedSerializer(resolver, cls); - checkRegistrationOpen(resolver); - if (resolver.isRegistered(cls)) { - resolver.setSerializer(cls, serializer); - } else { - resolver.registerSerializer(cls, serializer); + if (!resolver.isRegistered(cls) || resolver.getTypeInfo(cls, false) == null) { + throw new IllegalArgumentException( + "Generated Kotlin serializer requires registering the type first: " + cls.getName()); } + resolver.registerSerializer(cls, owner -> newGeneratedSerializer(owner, cls)); } private static void checkRegistrationOpen(TypeResolver resolver) { - // Resolver setSerializer remains available for lazy internal resolution, so this facade owns - // the public freeze checks around construction and before the final replacement. - if (resolver.isRegistrationFinished()) { + if (resolver.isRegistrationFrozen()) { throw new ForyException( "Cannot register class/serializer after registration has been frozen. Please register " + "all classes before invoking top-level `serialize/deserialize/copy` methods of " diff --git a/kotlin/fory-kotlin/src/main/kotlin/org/apache/fory/serializer/kotlin/KotlinDefaultValueSupport.kt b/kotlin/fory-kotlin/src/main/kotlin/org/apache/fory/serializer/kotlin/KotlinDefaultValueSupport.kt index dff2ebce21..6ac8f23960 100644 --- a/kotlin/fory-kotlin/src/main/kotlin/org/apache/fory/serializer/kotlin/KotlinDefaultValueSupport.kt +++ b/kotlin/fory-kotlin/src/main/kotlin/org/apache/fory/serializer/kotlin/KotlinDefaultValueSupport.kt @@ -41,7 +41,7 @@ import org.apache.fory.util.DefaultValueUtils * This class uses Kotlin native reflection to analyze data classes and extract default values from * their primary constructor parameters. */ -internal class KotlinDefaultValueSupport : DefaultValueUtils.DefaultValueSupport() { +internal object KotlinDefaultValueSupport : DefaultValueUtils.DefaultValueSupport() { private val LOG: Logger = LoggerFactory.getLogger(KotlinDefaultValueSupport::class.java) private val cachedKotlinDataClassDefaultValues = ClassValueCache.newClassKeyCache>(32) diff --git a/kotlin/fory-kotlin/src/test/kotlin/org/apache/fory/serializer/kotlin/BuiltinClassSerializerTests.kt b/kotlin/fory-kotlin/src/test/kotlin/org/apache/fory/serializer/kotlin/BuiltinClassSerializerTests.kt index 3ab0d14ec0..776ea32edc 100644 --- a/kotlin/fory-kotlin/src/test/kotlin/org/apache/fory/serializer/kotlin/BuiltinClassSerializerTests.kt +++ b/kotlin/fory-kotlin/src/test/kotlin/org/apache/fory/serializer/kotlin/BuiltinClassSerializerTests.kt @@ -34,32 +34,60 @@ import kotlin.time.Duration.Companion.seconds import kotlin.uuid.ExperimentalUuidApi import kotlin.uuid.Uuid import org.apache.fory.Fory +import org.apache.fory.context.CopyContext import org.apache.fory.context.ReadContext import org.apache.fory.context.WriteContext import org.apache.fory.exception.ForyException import org.apache.fory.kotlin.ForyKotlin +import org.apache.fory.resolver.TypeInfo import org.apache.fory.resolver.TypeResolver import org.apache.fory.serializer.Serializer +import org.apache.fory.serializer.StaticGeneratedStructSerializer +import org.apache.fory.type.Descriptor import org.apache.fory.util.DefaultValueUtils import org.testng.Assert import org.testng.Assert.assertThrows private object ReentrantRegistration { var fory: Fory? = null + var canonical: TypeInfo? = null + var canonicalSerializer: Serializer<*>? = null + var candidate: Serializer<*>? = null + var constructions: Int = 0 + + fun reset() { + fory = null + canonical = null + canonicalSerializer = null + candidate = null + constructions = 0 + } } class ReentrantStruct @Suppress("UNCHECKED_CAST") class ReentrantStruct_ForySerializer(resolver: TypeResolver, cls: Class<*>) : - Serializer(resolver.config, cls as Class) { + StaticGeneratedStructSerializer(resolver, cls as Class) { init { + ReentrantRegistration.constructions++ + ReentrantRegistration.canonical = resolver.getTypeInfo(cls, false) + ReentrantRegistration.canonicalSerializer = + ReentrantRegistration.canonical?.getSerializer() + ReentrantRegistration.candidate = this checkNotNull(ReentrantRegistration.fory).serialize("freeze") } override fun write(writeContext: WriteContext, value: ReentrantStruct) = Unit override fun read(readContext: ReadContext): ReentrantStruct = ReentrantStruct() + + override fun readCompatible(readContext: ReadContext): ReentrantStruct = ReentrantStruct() + + override fun copy(copyContext: CopyContext, value: ReentrantStruct): ReentrantStruct = + ReentrantStruct() + + override fun getGeneratedDescriptors(): List = emptyList() } class BuiltinClassSerializerTests { @@ -81,18 +109,59 @@ class BuiltinClassSerializerTests { .requireClassRegistration(true) .withRefTracking(false) .build() + ReentrantRegistration.reset() ReentrantRegistration.fory = fory try { assertThrows(ForyException::class.java) { KotlinSerializers.register(fory, ReentrantStruct::class.java, "kotlin.ReentrantStruct") } - Assert.assertTrue(fory.typeResolver.isRegistered(ReentrantStruct::class.java)) + val resolver = fory.typeResolver + val canonical = checkNotNull(ReentrantRegistration.canonical) + val candidate = checkNotNull(ReentrantRegistration.candidate) + Assert.assertEquals(ReentrantRegistration.constructions, 1) + Assert.assertTrue(candidate is StaticGeneratedStructSerializer<*>) + Assert.assertTrue(resolver.isRegistered(ReentrantStruct::class.java)) + Assert.assertSame(resolver.getTypeInfo(ReentrantStruct::class.java, false), canonical) + Assert.assertSame( + canonical.getSerializer(), + ReentrantRegistration.canonicalSerializer, + ) + Assert.assertNotSame(canonical.getSerializer(), candidate) } finally { - ReentrantRegistration.fory = null + ReentrantRegistration.reset() } } + @Test + fun testMissingGeneratedType() { + val fory = + ForyKotlin.builder() + .withXlang(true) + .requireClassRegistration(true) + .withRefTracking(false) + .build() + ReentrantRegistration.reset() + + assertThrows(IllegalArgumentException::class.java) { + KotlinSerializers.registerSerializer(fory, ReentrantStruct::class.java) + } + Assert.assertEquals(ReentrantRegistration.constructions, 0) + Assert.assertFalse(fory.typeResolver.isRegistered(ReentrantStruct::class.java)) + Assert.assertNull(fory.typeResolver.getTypeInfo(ReentrantStruct::class.java, false)) + } + + @Test + fun testSharedDefaultValueSupport() { + ForyKotlin.builder().withXlang(false).requireClassRegistration(true).build() + val support = checkNotNull(DefaultValueUtils.getKotlinDefaultValueSupport()) + Assert.assertEquals(support.getDefaultValue(ClassWithDefaults::class.java, "x"), 1) + + ForyKotlin.builder().withXlang(false).requireClassRegistration(true).build() + Assert.assertSame(DefaultValueUtils.getKotlinDefaultValueSupport(), support) + Assert.assertEquals(support.getDefaultValue(ClassWithDefaults::class.java, "x"), 1) + } + @Test fun testSerializePair() { val fory: Fory = ForyKotlin.builder().withXlang(false).requireClassRegistration(true).build() diff --git a/kotlin/fory-kotlin/src/test/kotlin/org/apache/fory/serializer/kotlin/DefaultValueTest.kt b/kotlin/fory-kotlin/src/test/kotlin/org/apache/fory/serializer/kotlin/DefaultValueTest.kt index 67c76a1ab2..771c6446f4 100644 --- a/kotlin/fory-kotlin/src/test/kotlin/org/apache/fory/serializer/kotlin/DefaultValueTest.kt +++ b/kotlin/fory-kotlin/src/test/kotlin/org/apache/fory/serializer/kotlin/DefaultValueTest.kt @@ -41,7 +41,7 @@ data class ClassObjectRequiredWithDefaults(val v: RegularClass, val x: Int = 3) class DefaultValueTest { - private val support = KotlinDefaultValueSupport() + private val support = KotlinDefaultValueSupport @Test fun testHasDefaultValues() { diff --git a/scala/fory-scala/src/main/scala-3/org/apache/fory/scala/ForySerializer.scala b/scala/fory-scala/src/main/scala-3/org/apache/fory/scala/ForySerializer.scala index 08e06703aa..88d271d76a 100644 --- a/scala/fory-scala/src/main/scala-3/org/apache/fory/scala/ForySerializer.scala +++ b/scala/fory-scala/src/main/scala-3/org/apache/fory/scala/ForySerializer.scala @@ -63,9 +63,7 @@ object ForySerializer { } private def checkRegistrationOpen(resolver: TypeResolver): Unit = { - // Public generated registration must freeze with the root facade. Resolver serializer - // mutation stays available for lazy internal resolution after registration has finished. - if resolver.isRegistrationFinished then { + if resolver.isRegistrationFrozen then { throw new ForyException( "Cannot register class/serializer after registration has been frozen. Please register " + "all classes before invoking top-level `serialize/deserialize/copy` methods of Fory.") @@ -124,9 +122,13 @@ object ForySerializer { if union then { throw new IllegalArgumentException("Use ForySerializer.register for Scala union serializers") } - val generatedSerializer = serializer.createSerializer(resolver) - checkRegistrationOpen(resolver) - resolver.setSerializer(cls, generatedSerializer) + if !resolver.isRegistered(cls) || resolver.getTypeInfo(cls, false) == null then { + throw new IllegalArgumentException( + "Generated Scala serializer requires registering the type first: " + cls.getName) + } + resolver.registerSerializer( + cls, + (owner: TypeResolver) => serializer.createSerializer(owner)) } private def register[T]( @@ -165,9 +167,9 @@ object ForySerializer { } } else { registerType(fory, cls, typeId, namespace, typeName) - val generatedSerializer = serializer.createSerializer(resolver) - checkRegistrationOpen(resolver) - resolver.setSerializer(cls, generatedSerializer) + resolver.registerSerializer( + cls, + (owner: TypeResolver) => serializer.createSerializer(owner)) } } diff --git a/scala/fory-scala/src/test/scala-3/org/apache/fory/serializer/scala/ForySerializerDerivationTest.scala b/scala/fory-scala/src/test/scala-3/org/apache/fory/serializer/scala/ForySerializerDerivationTest.scala index 0c643f8e03..1eb34f982d 100644 --- a/scala/fory-scala/src/test/scala-3/org/apache/fory/serializer/scala/ForySerializerDerivationTest.scala +++ b/scala/fory-scala/src/test/scala-3/org/apache/fory/serializer/scala/ForySerializerDerivationTest.scala @@ -35,10 +35,11 @@ import org.apache.fory.exception.{ForyException, InsecureException} import org.apache.fory.memory.MemoryBuffer import org.apache.fory.meta.TypeDef import org.apache.fory.reflect.{FieldAccessor, ObjectInstantiators} +import org.apache.fory.resolver.TypeInfo import org.apache.fory.scala.ForySerializer import org.apache.fory.scala.ForyScala import org.apache.fory.scala.register -import org.apache.fory.serializer.{GraphMemoryEstimates, StaticGeneratedStructSerializer} +import org.apache.fory.serializer.{GraphMemoryEstimates, Serializer, StaticGeneratedStructSerializer} import org.apache.fory.`type`.{Types, TypeUtils} import org.apache.fory.`type`.union.UnknownCase import org.scalatest.matchers.should.Matchers @@ -394,34 +395,50 @@ class ForySerializerDerivationTest extends AnyWordSpec with Matchers { "reject serializer replacement frozen during creation" in { val runtime = xlangFory() + val resolver = runtime.getTypeResolver val originalSerializer = runtime.getSerializer(classOf[Person]) - val replacementSerializer = - summon[ForySerializer[Person]].createSerializer(runtime.getTypeResolver) + val factory = summon[ForySerializer[Person]] + var canonical: TypeInfo = null + var canonicalSerializer: Serializer[Person] = null + var candidate: Serializer[Person] = null val reentrantSerializer = new ForySerializer[Person] { override def createSerializer( typeResolver: org.apache.fory.resolver.TypeResolver, typeDef: TypeDef): org.apache.fory.serializer.Serializer[Person] = { + canonical = typeResolver.getTypeInfo(classOf[Person], false) + canonicalSerializer = canonical.getSerializer + candidate = factory.createSerializer(typeResolver, typeDef) runtime.serialize(Person("Ada", 36, None)) - replacementSerializer + candidate } } intercept[ForyException] { ForySerializer.registerSerializer(runtime, classOf[Person])(using reentrantSerializer) } + candidate.isInstanceOf[StaticGeneratedStructSerializer[?]] shouldBe true + resolver.getTypeInfo(classOf[Person], false) shouldBe theSameInstanceAs(canonical) + canonical.getSerializer shouldBe theSameInstanceAs(canonicalSerializer) + canonical.getSerializer should not be theSameInstanceAs(candidate) runtime.getSerializer(classOf[Person]) shouldBe theSameInstanceAs(originalSerializer) } "reject combined registration frozen during creation" in { val runtime = xlangFory() + val resolver = runtime.getTypeResolver val factory = summon[ForySerializer[StoredState]] + var canonical: TypeInfo = null + var canonicalSerializer: Serializer[StoredState] = null + var candidate: Serializer[StoredState] = null val reentrantFactory = new ForySerializer[StoredState] { override def createSerializer( typeResolver: org.apache.fory.resolver.TypeResolver, typeDef: TypeDef): org.apache.fory.serializer.Serializer[StoredState] = { - val generatedSerializer = factory.createSerializer(typeResolver, typeDef) + canonical = typeResolver.getTypeInfo(classOf[StoredState], false) + canonicalSerializer = canonical.getSerializer + candidate = factory.createSerializer(typeResolver, typeDef) runtime.serialize("freeze") - generatedSerializer + candidate } } @@ -429,7 +446,11 @@ class ForySerializerDerivationTest extends AnyWordSpec with Matchers { ForySerializer.register(runtime, classOf[StoredState], "scala_test.ReentrantStoredState")( using reentrantFactory) } - runtime.getTypeResolver.isRegistered(classOf[StoredState]) shouldBe true + candidate.isInstanceOf[StaticGeneratedStructSerializer[?]] shouldBe true + resolver.isRegistered(classOf[StoredState]) shouldBe true + resolver.getTypeInfo(classOf[StoredState], false) shouldBe theSameInstanceAs(canonical) + canonical.getSerializer shouldBe theSameInstanceAs(canonicalSerializer) + canonical.getSerializer should not be theSameInstanceAs(candidate) } "serialize derived case classes with Scala collection fields" in { From 8661b79bcd32695a2b3a60d4560732a78faa5a5a Mon Sep 17 00:00:00 2001 From: chaokunyang Date: Sat, 29 Aug 2026 02:04:07 +0800 Subject: [PATCH 055/168] fix(python): replay semantic registrations --- .agents/languages/python.md | 22 +-- .../python/type-registration.md | 4 +- docs/security/deserialization.md | 12 +- python/pyfory/_fory.py | 81 +++++++++-- python/pyfory/registry.py | 9 +- python/pyfory/serializer.py | 2 +- python/pyfory/tests/test_thread_safe.py | 131 ++++++++++++++++-- 7 files changed, 221 insertions(+), 40 deletions(-) diff --git a/.agents/languages/python.md b/.agents/languages/python.md index 15e6fdfa20..4f3d73e172 100644 --- a/.agents/languages/python.md +++ b/.agents/languages/python.md @@ -18,17 +18,23 @@ Load this file when changing `python/`, Cython serialization, or Python xlang be registration conflicts and its frozen state after construction and before publishing type, serializer, name, or ID state. Allocate automatic type IDs only after those checks at the common publication point; do not reserve IDs before callbacks or maintain rollback state. - `ThreadSafeFory` validates registrations before retaining their replay callbacks, and it must not - execute application factories or callbacks while holding its pool lock. Its registration - linearization is reentrant so nested facade registrations share the same publication order. A - root started during registration must not reuse the staging instance, and root reentry from a - running user `fory_factory` or retained registration callback fails without recursively building + `ThreadSafeFory` validates registrations before retaining their semantic replay descriptors, and + it must not execute application factories or registrations while holding its pool lock. Its + registration linearization is reentrant so nested facade registrations share the same + publication order. A root started during registration must not reuse the staging instance, and + root reentry from a running user `fory_factory` or retained registration replay fails without + recursively building another instance. The build thread must be rejected before pool acquisition even when another instance becomes available during that build. The non-reentrant pool lock owns pool publication, root-started state, registration depth, and the staging instance; the separate instance-build - boundary covers the factory and complete callback replay. Retained callbacks may capture a - serializer class or factory, but never a resolver-bound serializer instance. Instance-specific - serializer configuration belongs in `fory_factory`, which creates and configures each child. + boundary covers the factory and complete registration replay. During child replay, a nested + facade registration is a no-op only when it exactly matches an accepted descriptor in the prefix + already applied to that child; reject every unknown or different request before child mutation. + Retained descriptors may contain a serializer class or factory, but never a resolver-bound + serializer instance. A serializer factory must return a supported serializer carrier bound to + the provided child resolver and normalized declared type; singleton serializers cannot be shared + across children. Instance-specific serializer configuration belongs in `fory_factory`, which + creates and configures each child. - Registry freeze prohibits type and serializer publication after the first root; it does not prohibit policy-authorized resolution of module-global classes or callables during a non-strict native read when that resolution does not mutate registry state. Do not describe these two diff --git a/docs/object-serialization/python/type-registration.md b/docs/object-serialization/python/type-registration.md index cb79c944fd..2c18806ad5 100644 --- a/docs/object-serialization/python/type-registration.md +++ b/docs/object-serialization/python/type-registration.md @@ -74,7 +74,9 @@ for model_class in [User, Order, Product, Invoice]: A direct `Fory` may receive a serializer instance. `ThreadSafeFory` accepts a serializer class or factory so every pooled child constructs a serializer against its own resolver. Use -`fory_factory` for serializer instances that need per-child configuration. +`fory_factory` for serializer instances that need per-child configuration. A serializer factory +must return a serializer for the resolver and declared type passed to that invocation; it must not +reuse one serializer instance across pooled children. ## Strict Mode Relationship diff --git a/docs/security/deserialization.md b/docs/security/deserialization.md index 7ffb4f9249..dfe4fd2540 100644 --- a/docs/security/deserialization.md +++ b/docs/security/deserialization.md @@ -619,11 +619,13 @@ that case, classify the behavior by concrete impact: The first root operation permanently closes type and serializer registration, including when that operation or registry finalization fails. Registration that invokes application code must recheck -the authoritative lifecycle before publishing callback-derived state. Thread-safe facades retain -only registrations that completed before the freeze. A thread-safe facade that cannot roll back an -opaque registration callback must become permanently unusable when that callback fails, rather -than expose children with partially applied registration. These rules prevent a failed or -reentrant registration from changing the accepted type surface after deserialization has begun. +the authoritative lifecycle before publishing application-derived state. Thread-safe facades +retain only registrations that completed before the freeze. During child construction, a semantic +replay log may reuse only an identical accepted registration that the child has already applied; +unknown or different requests fail before mutating the child. A facade that replays opaque +registration callbacks and cannot roll them back must become permanently unusable when a callback +fails rather than expose partially registered children. These rules prevent a failed or reentrant +registration from changing the accepted type surface after deserialization has begun. Runtime-specific publication ownership belongs in the implementation guide and language guidance. ## Metadata And Type Resolution diff --git a/python/pyfory/_fory.py b/python/pyfory/_fory.py index 5fe6b4068a..5534a49eea 100644 --- a/python/pyfory/_fory.py +++ b/python/pyfory/_fory.py @@ -25,9 +25,10 @@ "true", } +from pyfory.policy import DEFAULT_POLICY, DeserializationPolicy from pyfory.resolver import NOT_NULL_VALUE_FLAG +from pyfory.type_util import normalize_fory_type from pyfory.types import TypeId -from pyfory.policy import DeserializationPolicy, DEFAULT_POLICY DYNAMIC_TYPE_ID = -1 # preserve 0 as flag for type id not set in TypeInfo` @@ -632,6 +633,44 @@ def reset(self): self.reset_read() +class _Registration: + __slots__ = ( + "cls", + "declared_type", + "name", + "operation", + "serializer", + "type_id", + ) + + def __init__(self, operation, cls, type_id, name, serializer): + self.operation = operation + self.cls = cls + self.declared_type = normalize_fory_type(cls) + self.type_id = int(type_id) if isinstance(type_id, int) and not isinstance(type_id, bool) else type_id + self.name = str(name) if isinstance(name, str) else name + self.serializer = serializer + + def same_request(self, other): + return ( + self.operation == other.operation + and self.declared_type == other.declared_type + and type(self.type_id) is type(other.type_id) + and self.type_id == other.type_id + and type(self.name) is type(other.name) + and self.name == other.name + and self.serializer is other.serializer + ) + + def apply(self, fory): + getattr(fory, self.operation)( + self.cls, + type_id=self.type_id, + name=self.name, + serializer=self.serializer, + ) + + class ThreadSafeFory: """ Thread-safe wrapper for Fory using instance pooling. @@ -645,7 +684,8 @@ class ThreadSafeFory: deserialization attempt to ensure consistency across all pooled instances. Registration remains closed even when that first operation fails. Custom serializer registrations accept a serializer class or factory so every pooled instance owns a serializer bound to its own - resolver. Use ``fory_factory`` for configured serializer instances. + resolver and declared type. A serializer factory cannot reuse one serializer across children; + use ``fory_factory`` for configured serializer instances. Args: fory_factory (Callable): Optional factory that creates and configures each pooled Fory. @@ -692,12 +732,14 @@ class ThreadSafeFory: def __init__(self, fory_factory=None, **kwargs): self._config = kwargs self._fory_factory = fory_factory - self._callbacks = [] + self._registrations = [] self._lock = threading.Lock() self._registration_lock = threading.RLock() self._registration_depth = 0 self._registration_fory = None self._building_thread = None + # Number of accepted registrations already applied to the child being built. + self._replay_limit = 0 self._pool = [] if fory_factory is not None: self._fory_class = None @@ -715,15 +757,18 @@ def _build_fory(self): if self._building_thread == thread_id: raise RuntimeError("Cannot start a root serialization or deserialization operation while a Fory instance is being built.") self._building_thread = thread_id + self._replay_limit = 0 try: if self._fory_factory is not None: fory = self._fory_factory() else: fory = self._fory_class(**self._config) - for callback in self._callbacks: - callback(fory) + for registration in self._registrations: + registration.apply(fory) + self._replay_limit += 1 return fory finally: + self._replay_limit = 0 self._building_thread = None def _get_fory(self): @@ -739,7 +784,7 @@ def _get_fory(self): return self._pool.pop() self._root_started = True # Nested registrations share the staging instance, but a root may reuse it only - # after the outermost registration has published its callback. + # after the outermost registration has published its descriptor. if self._registration_depth == 0: fory = self._registration_fory else: @@ -748,7 +793,7 @@ def _get_fory(self): if fory is not None: # The validation instance already contains every published registration. return fory - # Factories and registration callbacks are application code. Keep them outside the + # Factories and registrations are application code. Keep them outside the # non-reentrant pool lock so a callback can enter the same facade root. return self._build_fory() @@ -756,10 +801,18 @@ def _return_fory(self, fory): with self._lock: self._pool.append(fory) - def _register_callback(self, callback): + def _register_registration(self, registration): # The reentrant lock gives nested facade registrations one publication order while the - # pool lock keeps a concurrently starting root atomic with callback publication. + # pool lock keeps a concurrently starting root atomic with registration publication. with self._registration_lock: + if self._building_thread == threading.get_ident(): + index = 0 + while index < self._replay_limit: + accepted = self._registrations[index] + if accepted.same_request(registration): + return + index += 1 + raise RuntimeError("A child may replay only a registration already accepted by this ThreadSafeFory.") with self._lock: self._check_registration_open() self._registration_depth += 1 @@ -770,10 +823,10 @@ def _register_callback(self, callback): with self._lock: self._check_registration_open() self._registration_fory = registration_fory - callback(registration_fory) + registration.apply(registration_fory) with self._lock: self._check_registration_open() - self._callbacks.append(callback) + self._registrations.append(registration) self._registration_fory = registration_fory except BaseException: with self._lock: @@ -806,7 +859,7 @@ def register( serializer=None, ): self._check_serializer_factory(serializer) - self._register_callback(lambda f: f.register(cls, type_id=type_id, name=name, serializer=serializer)) + self._register_registration(_Registration("register", cls, type_id, name, serializer)) def register_type( self, @@ -817,7 +870,7 @@ def register_type( serializer=None, ): self._check_serializer_factory(serializer) - self._register_callback(lambda f: f.register_type(cls, type_id=type_id, name=name, serializer=serializer)) + self._register_registration(_Registration("register_type", cls, type_id, name, serializer)) def register_union( self, @@ -828,7 +881,7 @@ def register_union( serializer=None, ): self._check_serializer_factory(serializer) - self._register_callback(lambda f: f.register_union(cls, type_id=type_id, name=name, serializer=serializer)) + self._register_registration(_Registration("register_union", cls, type_id, name, serializer)) def serialize( self, diff --git a/python/pyfory/registry.py b/python/pyfory/registry.py index 1fb5cfe7de..0ce7fa9ad4 100644 --- a/python/pyfory/registry.py +++ b/python/pyfory/registry.py @@ -223,7 +223,14 @@ def _construct_serializer(serializer_factory, type_resolver, cls): (0, ()), ): if _accepts_n_positional_args(serializer_factory, nargs): - return serializer_factory(*args) + serializer = serializer_factory(*args) + if not isinstance(serializer, (Serializer, CythonSerializer)): + raise TypeError("Serializer factory must return a supported Serializer carrier") + if serializer.type_resolver is not type_resolver: + raise TypeError("Serializer factory returned a serializer bound to a different resolver") + if normalize_fory_type(serializer.type_) != normalize_fory_type(cls): + raise TypeError("Serializer factory returned a serializer bound to a different type") + return serializer raise TypeError(f"Unsupported serializer constructor for {serializer_factory!r}; expected `(type_resolver, cls)`, `(type_resolver)`, or `()`.") diff --git a/python/pyfory/serializer.py b/python/pyfory/serializer.py index f06a3e124c..613df26743 100644 --- a/python/pyfory/serializer.py +++ b/python/pyfory/serializer.py @@ -382,7 +382,7 @@ def _resolve_validated_bound_method(policy, obj, method_name, is_local): class NoneSerializer(Serializer): def __init__(self, type_resolver): - super().__init__(type_resolver, None) + super().__init__(type_resolver, type(None)) self.need_to_write_ref = False def write(self, buffer, value): diff --git a/python/pyfory/tests/test_thread_safe.py b/python/pyfory/tests/test_thread_safe.py index 3f092c5796..917cf1de17 100644 --- a/python/pyfory/tests/test_thread_safe.py +++ b/python/pyfory/tests/test_thread_safe.py @@ -253,7 +253,7 @@ def fory_factory(): with pytest.raises(TypeError): getattr(fory, method)(Address, serializer=serializer) - assert fory._callbacks == [] + assert fory._registrations == [] assert fory._registration_fory is None assert not fory._root_started assert builds == 0 @@ -289,6 +289,67 @@ def serializer_factory(type_resolver, cls): assert fory.deserialize(fory.serialize(address)) == address +@pytest.mark.parametrize("result", ["value", "resolver", "type"]) +def test_serializer_factory_result(result): + class AddressSerializer(pyfory.Serializer): + def write(self, write_context, value): + write_context.write_string(value.city) + write_context.write_string(value.country) + + def read(self, read_context): + return Address(read_context.read_string(), read_context.read_string()) + + foreign = pyfory.Fory(xlang=False, compatible=False) + + def serializer_factory(type_resolver, cls): + if result == "value": + return object() + if result == "resolver": + return AddressSerializer(foreign.type_resolver, cls) + return AddressSerializer(type_resolver, Person) + + fory = ThreadSafeFory(xlang=False, compatible=False) + with pytest.raises(TypeError): + fory.register_type(Address, serializer=serializer_factory) + + assert not fory._registrations + assert fory._registration_fory is None + + +def test_singleton_serializer_factory(): + class AddressSerializer(pyfory.Serializer): + def write(self, write_context, value): + write_context.write_string(value.city) + write_context.write_string(value.country) + + def read(self, read_context): + return Address(read_context.read_string(), read_context.read_string()) + + children = [] + singleton = None + + def child_factory(): + child = pyfory.Fory(xlang=False, compatible=False) + children.append(child) + return child + + def serializer_factory(type_resolver, cls): + nonlocal singleton + if singleton is None: + singleton = AddressSerializer(type_resolver, cls) + return singleton + + fory = ThreadSafeFory(fory_factory=child_factory) + fory.register_type(Address, serializer=serializer_factory) + assert singleton.type_resolver is children[0].type_resolver + + with pytest.raises(TypeError): + fory._build_fory() + + assert len(children) == 2 + assert children[1].type_resolver.get_type_info(Address, create=False) is None + + def test_factory_serializer_owner(): class AddressSerializer(pyfory.Serializer): def write(self, write_context, value): @@ -353,7 +414,7 @@ def register(): assert constructions == 1 assert len(errors) == 1 assert isinstance(errors[0], RuntimeError) - assert not fory._callbacks + assert not fory._registrations assert fory._registration_fory is None assert fory.deserialize(fory.serialize(None)) is None with pytest.raises(RuntimeError): @@ -376,8 +437,7 @@ def read(self, read_context): def serializer_factory(type_resolver, cls): nonlocal constructions constructions += 1 - if constructions == 1: - fory.register_type(Person) + fory.register_type(Person) return AddressSerializer(type_resolver, cls) def register(): @@ -392,13 +452,64 @@ def register(): assert not thread.is_alive() assert not errors - resolver = fory._registration_fory.type_resolver - person_info = resolver.get_type_info(Person, create=False) - address_info = resolver.get_type_info(Address, create=False) - assert person_info.user_type_id + 1 == address_info.user_type_id + first = fory._registration_fory + second = fory._build_fory() + for child in (first, second): + resolver = child.type_resolver + person_info = resolver.get_type_info(Person, create=False) + address_info = resolver.get_type_info(Address, create=False) + assert person_info.user_type_id + 1 == address_info.user_type_id address = Address(city="Oslo", country="Norway") + assert second.deserialize(second.serialize(address)) == address assert fory.deserialize(fory.serialize(address)) == address - assert constructions == 1 + assert constructions == 2 + + +@pytest.mark.parametrize("scenario", ["unknown", "different"]) +def test_nested_replay_rejected(scenario): + class AddressSerializer(pyfory.Serializer): + def write(self, write_context, value): + write_context.write_string(value.city) + write_context.write_string(value.country) + + def read(self, read_context): + return Address(read_context.read_string(), read_context.read_string()) + + class Unknown: + pass + + children = [] + replay = False + + def child_factory(): + child = pyfory.Fory(xlang=True, compatible=False) + children.append(child) + return child + + def serializer_factory(type_resolver, cls): + if not replay: + fory.register_type(Person) + elif scenario == "unknown": + fory.register_type(Unknown) + else: + fory.register_type(Person, type_id=101) + return AddressSerializer(type_resolver, cls) + + fory = ThreadSafeFory(fory_factory=child_factory) + fory.register_type(Address, serializer=serializer_factory) + replay = True + + with pytest.raises(RuntimeError): + fory._build_fory() + + child = children[1] + person_info = child.type_resolver.get_type_info(Person, create=False) + assert person_info is not None + assert person_info.user_type_id != 101 + assert child.type_resolver.get_type_info(Address, create=False) is None + assert child.type_resolver.get_type_info(Unknown, create=False) is None + assert fory._replay_limit == 0 + assert fory._building_thread is None def test_factory_root_reentry(): @@ -417,7 +528,7 @@ def fory_factory(): assert constructions == 1 assert fory._root_started - assert not fory._callbacks + assert not fory._registrations assert fory._registration_fory is None with pytest.raises(RuntimeError): fory.register_type(Address) From 018f0343b8f5ff5f1144670c2afc51840e86bfb9 Mon Sep 17 00:00:00 2001 From: chaokunyang Date: Sat, 29 Aug 2026 02:06:57 +0800 Subject: [PATCH 056/168] docs(jvm): define generated serializer ownership --- .agents/languages/java.md | 15 +++++++++------ .agents/languages/kotlin.md | 17 ++++++++++------- .agents/languages/scala.md | 13 ++++++++----- docs/compiler/generated-code/kotlin.md | 3 +++ docs/compiler/generated-code/scala.md | 10 ++++++++-- 5 files changed, 38 insertions(+), 20 deletions(-) diff --git a/.agents/languages/java.md b/.agents/languages/java.md index bfab8d2d21..3c2481f61e 100644 --- a/.agents/languages/java.md +++ b/.agents/languages/java.md @@ -87,8 +87,8 @@ Load this file when changing anything under `java/` or when Java drives a cross- - `FacadeRegistrationGate` owns registration linearization for Java thread-local and pooled facades. Starting a root closes registration before child or pool access, then finishes every already-created child before exposing that root. A child created after closure must replay every - accepted registration, finish registration, and only then become visible; discard a provisional - child when replay or finalization fails. A callback registration is one facade transaction across + accepted registration, finish registration, and only then become visible; a child whose replay + or finalization fails must never be published. A callback registration is one facade transaction across all children: reject root or registration reentry while it is active and permanently fail the facade after any callback failure, rather than expose partial child mutation, divergent replay order, or rollback state. Keep the lock order gate before pool or child storage. @@ -99,10 +99,13 @@ Load this file when changing anything under `java/` or when Java drives a cross- owner immediately, while construction owners resolving recursive fields or candidate state use the construction-local serializer. Ordinary resolver lookups retain their runtime semantics. When wire and user IDs match, the final owner is the existing canonical `TypeInfo`. After - construction and the lifecycle recheck succeed, the Class/Xtype resolver's normal commit sink installs the candidate. - No constructor-specific publication path or nonpublishing serializer factory is allowed. Reject - static-generated serializer classes from the combined class overload because their construction - requires prior canonical type registration. `Fory.register(ForyModule)` owns module identity, + construction and the lifecycle recheck succeed, the Class/Xtype resolver's normal commit sink + installs the candidate. Static-generated construction starts from an already registered canonical + type, retains that type's identity, and exposes immutable generated descriptors only through the + same construction graph; it must not publish its early-bound serializer candidate. Do not add a + constructor-specific publication path. Reject static-generated serializer classes from the + combined class overload because their construction requires prior canonical type registration. + `Fory.register(ForyModule)` owns module identity, cycle breaking, and idempotence in one identity set: add the identity before the callback, remove it on failure, and retain it on success. Do not add separate installing/completed module states. Direct `Fory` accepts modules before its first root; thread-safe facades accept modules only diff --git a/.agents/languages/kotlin.md b/.agents/languages/kotlin.md index 66ec103a30..ebb299ba1a 100644 --- a/.agents/languages/kotlin.md +++ b/.agents/languages/kotlin.md @@ -15,16 +15,19 @@ Load this file when changing `kotlin/` or compiler code that generates Kotlin so wire format matches the previous serializer family and old-payload/new-runtime compatibility is tested. - Public registration helpers must check the registry freeze before constructing a serializer, - enum serializer, or union serializer. A registered-type serializer replacement must check again - after generated serializer construction and before the explicit replacement because - `TypeResolver.setSerializer` remains available for lazy internal resolution. + enum serializer, or union serializer. Generated serializer construction must enter the existing + `TypeResolver` construction graph so its candidate remains unpublished until the authoritative + lifecycle recheck and normal resolver commit. - Combined generated-struct registration must publish the canonical type before constructing its - serializer because generated construction resolves the canonical `TypeInfo`. Do not move that - construction before type registration or add rollback, staging, or a parallel registration path. + serializer because generated construction resolves the canonical `TypeInfo`. The serializer-only + helper must reject a missing canonical type rather than auto-register it. Do not move construction + before type registration or add direct replacement, rollback, staging, or a parallel registration + path. - `Fory.register(ForyModule)` is the only owner of bootstrap identity, cycle breaking, and idempotence. Kotlin bootstrap code must not add a marker, monitor, or separate reentry policy. - Keep the install body replay-safe until its final non-repeatable publication; publish global - Kotlin default-value support only after all per-runtime registrations succeed. + Keep the install body replay-safe until its final non-repeatable publication; publish the single + global Kotlin default-value support owner only after all per-runtime registrations succeed, and + never replace its class-value cache for each runtime. - Install modules for thread-safe facades through `ForyBuilder.withModule` before building them. Runtime registration extensions target concrete `Fory` instances and must not recreate a thread-safe module-registration wrapper. diff --git a/.agents/languages/scala.md b/.agents/languages/scala.md index 0e11d7ecfb..5a4fce815d 100644 --- a/.agents/languages/scala.md +++ b/.agents/languages/scala.md @@ -10,12 +10,15 @@ Load this file when changing `scala/`. sources, tests, resources, R8 metadata, compiler plugins, macros, dependencies, or compatibility design. - Public registration helpers must check the registry freeze before invoking generated serializer - construction or enum discovery. Registered-type replacement must check again after - `ForySerializer` callbacks and before mutation; Scala enum registration must likewise recheck - after companion-driven value discovery and reuse the values already owned by the serializer. + construction or enum discovery. Generated serializer construction must enter the existing + `TypeResolver` construction graph so its candidate remains unpublished until the authoritative + lifecycle recheck and normal resolver commit. Scala enum registration must recheck after + companion-driven value discovery and reuse the values already owned by the serializer. - Combined generated-struct registration must publish the canonical type before constructing its - serializer because generated construction resolves the canonical `TypeInfo`. Do not move that - construction before type registration or add rollback, staging, or a parallel registration path. + serializer because generated construction resolves the canonical `TypeInfo`. The serializer-only + helper must reject a missing canonical type rather than auto-register it. Do not move construction + before type registration or add direct replacement, rollback, staging, or a parallel registration + path. Union construction is the exception because it does not require canonical registration: finish its serializer-owned callbacks and recheck the freeze before publishing the union type. - `Fory.register(ForyModule)` is the only owner of bootstrap identity, cycle breaking, and diff --git a/docs/compiler/generated-code/kotlin.md b/docs/compiler/generated-code/kotlin.md index 910ddbf007..5ff697d8ff 100644 --- a/docs/compiler/generated-code/kotlin.md +++ b/docs/compiler/generated-code/kotlin.md @@ -153,6 +153,9 @@ public object AddressbookForyModule : ForyModule { } ``` +Generated modules register all message types before resolving their generated serializers. This +lets circular message schemas use the same module without application-managed registration order. + `registerUnion` discovers the generated `_ForySerializer`; callers do not pass a serializer instance. diff --git a/docs/compiler/generated-code/scala.md b/docs/compiler/generated-code/scala.md index aca455deaf..4c22e711f7 100644 --- a/docs/compiler/generated-code/scala.md +++ b/docs/compiler/generated-code/scala.md @@ -151,14 +151,20 @@ object AddressbookForyModule extends org.apache.fory.ForyModule { private[addressbook] def getFory: ThreadSafeFory = fory override def install(fory: Fory): Unit = { + ForySerializer.registerType(fory, classOf[Person.PhoneNumber], 102L) + ForySerializer.registerType(fory, classOf[Person], 100L) + ScalaSerializers.registerEnum(fory, classOf[Person.PhoneType], 101L) - ForySerializer.register(fory, classOf[Person.PhoneNumber], 102L) - ForySerializer.register(fory, classOf[Person], 100L) + ForySerializer.registerSerializer(fory, classOf[Person.PhoneNumber]) + ForySerializer.registerSerializer(fory, classOf[Person]) ForySerializer.register(fory, classOf[Animal], 106L) } } ``` +Generated modules register all message types before resolving their generated serializers. This +lets circular message schemas use the same module without application-managed registration order. + ## gRPC Service Companions With `--grpc`, Scala emits one `Grpc.scala` object per local service in the generated models' package. It exposes `SERVICE_NAME`, service and method descriptors, `ImplBase`, and `Client`. See [Scala gRPC](../../grpc/scala.md) for `RpcFuture`, `RpcIterator`, grpc-java variants, and lifecycle guidance. From de4fd8b8e3543c78253e252ef194a1799b4b52ae Mon Sep 17 00:00:00 2001 From: chaokunyang Date: Sat, 29 Aug 2026 02:13:39 +0800 Subject: [PATCH 057/168] fix(javascript): enforce one schema owner --- javascript/packages/core/lib/context.ts | 13 +- javascript/packages/core/lib/gen/index.ts | 234 ++++++++++++------ javascript/packages/core/lib/gen/router.ts | 20 +- javascript/packages/core/lib/gen/struct.ts | 29 ++- javascript/packages/core/lib/typeInfo.ts | 129 +++++----- javascript/packages/core/tsconfig.json | 2 +- javascript/test/fory.test.ts | 268 +++++++++++---------- javascript/test/map.test.ts | 48 +++- 8 files changed, 469 insertions(+), 274 deletions(-) diff --git a/javascript/packages/core/lib/context.ts b/javascript/packages/core/lib/context.ts index a87156ea78..6907c0f1cf 100644 --- a/javascript/packages/core/lib/context.ts +++ b/javascript/packages/core/lib/context.ts @@ -1623,11 +1623,14 @@ export class ReadContext { } if (originalSerializer === undefined) { originalTypeInfo ??= TypeId.isNamedType(typeId) - ? Type.struct({ - typeName: typeMeta.getTypeName(), - namespace: typeMeta.getNs(), - }) - : Type.struct(typeMeta.getUserTypeId()); + ? Type.struct( + { + typeName: typeMeta.getTypeName(), + namespace: typeMeta.getNs(), + }, + {}, + ) + : Type.struct(typeMeta.getUserTypeId(), {}); originalSerializer = this.typeResolver.generateReadSerializer(originalTypeInfo); } // This legacy direct-generation API accepts caller-owned metadata. It must diff --git a/javascript/packages/core/lib/gen/index.ts b/javascript/packages/core/lib/gen/index.ts index 10324ec186..e25d793d83 100644 --- a/javascript/packages/core/lib/gen/index.ts +++ b/javascript/packages/core/lib/gen/index.ts @@ -18,7 +18,7 @@ */ import { TypeId, Serializer } from "../type"; -import { TypeInfo } from "../typeInfo"; +import { sealTypeInfo, TypeInfo } from "../typeInfo"; import { CodegenRegistry } from "./router"; import { CodecBuilder, SerializerLookup } from "./builder"; import { Scope } from "./scope"; @@ -127,21 +127,10 @@ interface GeneratedRegistration { export class Gen { static external = CodegenRegistry.getExternal(); - private generatedRegistrations: GeneratedRegistration[] = []; - private readonly serializerLookup: SerializerLookup; - constructor( private typeResolver: TypeResolver, private regOptions: { [key: string]: any } = {}, - ) { - // Generator-time TypeInfo queries see initialized local serializers for codegen decisions. - // Factory-init ID/name queries instead return the stable owner captured by runtime closures. - this.serializerLookup = { - getSerializerByTypeInfo: (typeInfo) => this.getGeneratedSerializer(typeInfo), - getSerializerById: (id, userTypeId) => this.getCapturedSerializerById(id, userTypeId), - getSerializerByName: (name) => this.getCapturedSerializerByName(name), - }; - } + ) {} private prepare(typeInfo: TypeInfo, serializerLookup: SerializerLookup): Serializer { const InnerGeneratorClass = CodegenRegistry.get(typeInfo.typeId); @@ -186,77 +175,105 @@ export class Gen { return !!this.typeResolver.getSerializerByTypeInfo(typeInfo); } - private isFullyGenerated(typeInfo: TypeInfo) { - const ser = this.getGeneratedSerializer(typeInfo); + private isFullyGenerated(typeInfo: TypeInfo, registrations: GeneratedRegistration[]) { + const ser = this.getGeneratedSerializer(typeInfo, registrations); return ser && ser._initialized; } private sameRegistration(left: TypeInfo, right: TypeInfo) { const leftTypeId = this.typeResolver.computeTypeId(left); const rightTypeId = this.typeResolver.computeTypeId(right); - if (leftTypeId !== rightTypeId) { - return false; - } - if (TypeId.isNamedType(leftTypeId)) { + if (TypeId.isNamedType(leftTypeId) && TypeId.isNamedType(rightTypeId)) { return left.named === right.named; } - if (TypeId.needsUserTypeId(leftTypeId)) { + if ( + TypeId.needsUserTypeId(leftTypeId) && + TypeId.needsUserTypeId(rightTypeId) && + left.userTypeId !== -1 && + right.userTypeId !== -1 + ) { return left.userTypeId === right.userTypeId; } - return true; + return leftTypeId === rightTypeId; } - private findRegistration(typeInfo: TypeInfo) { - return this.generatedRegistrations.find((entry) => - this.sameRegistration(entry.typeInfo, typeInfo), + private sameStructDefinition(left: TypeInfo, right: TypeInfo) { + if (left === right) { + return true; + } + const leftOptions = left.options!; + const rightOptions = right.options!; + return ( + left.typeId === right.typeId && + left.named === right.named && + left.namespace === right.namespace && + left.typeName === right.typeName && + left.userTypeId === right.userTypeId && + left.evolving === right.evolving && + leftOptions.props === rightOptions.props && + leftOptions.fieldEntries === rightOptions.fieldEntries && + leftOptions.preserveFieldOrder === rightOptions.preserveFieldOrder && + leftOptions.withConstructor === rightOptions.withConstructor && + leftOptions.creator === rightOptions.creator ); } - private addRegistration(typeInfo: TypeInfo) { + private findRegistration(typeInfo: TypeInfo, registrations: GeneratedRegistration[]) { + return registrations.find((entry) => this.sameRegistration(entry.typeInfo, typeInfo)); + } + + private addRegistration(typeInfo: TypeInfo, registrations: GeneratedRegistration[]) { const owner = { ...uninitializedSerializer }; const entry: GeneratedRegistration = { typeInfo, serializer: owner, preparing: false, }; - this.generatedRegistrations.push(entry); + owner.getTypeInfo = () => entry.typeInfo; + registrations.push(entry); return entry; } - private getGeneratedSerializer(typeInfo: TypeInfo) { + private getGeneratedSerializer(typeInfo: TypeInfo, registrations: GeneratedRegistration[]) { return ( - this.findRegistration(typeInfo)?.serializer ?? - this.typeResolver.getSerializerByTypeInfo(typeInfo) + this.typeResolver.getSerializerByTypeInfo(typeInfo) ?? + this.findRegistration(typeInfo, registrations)?.serializer ); } - private getCapturedSerializerById(id: number, userTypeId?: number) { + private getCapturedSerializerById( + registrations: GeneratedRegistration[], + id: number, + userTypeId?: number, + ) { const published = this.typeResolver.getSerializerById(id, userTypeId); if (published !== undefined) { return published; } - const entry = this.generatedRegistrations.find((candidate) => { + const entry = registrations.find((candidate) => { const typeId = this.typeResolver.computeTypeId(candidate.typeInfo); - if (typeId !== id || TypeId.isNamedType(typeId)) { - return false; - } - if (TypeId.needsUserTypeId(typeId)) { - if (userTypeId !== undefined && userTypeId !== -1) { - return candidate.typeInfo.userTypeId === userTypeId; - } - return candidate.typeInfo.userTypeId === -1; + if ( + TypeId.needsUserTypeId(id) && + TypeId.needsUserTypeId(typeId) && + userTypeId !== undefined && + userTypeId !== -1 + ) { + return candidate.typeInfo.userTypeId === userTypeId; } - return true; + return typeId === id; }); return entry?.serializer as Serializer; } - private getCapturedSerializerByName(name: number | string) { + private getCapturedSerializerByName( + registrations: GeneratedRegistration[], + name: number | string, + ) { const published = this.typeResolver.getSerializerByName(name); if (published !== undefined) { return published; } - const entry = this.generatedRegistrations.find( + const entry = registrations.find( (candidate) => typeof name === "string" && TypeId.isNamedType(this.typeResolver.computeTypeId(candidate.typeInfo)) && @@ -265,31 +282,88 @@ export class Gen { return entry?.serializer; } - private prepareRegistration(typeInfo: TypeInfo, children: TypeInfo[]) { - let entry = this.findRegistration(typeInfo); + private prepareRegistration( + typeInfo: TypeInfo, + children: TypeInfo[], + registrations: GeneratedRegistration[], + serializerLookup: SerializerLookup, + ) { + let entry = this.findRegistration(typeInfo, registrations); if (entry?.serializer._initialized || entry?.preparing) { return; } if (entry === undefined) { - entry = this.addRegistration(typeInfo); + entry = this.addRegistration(typeInfo, registrations); } else { entry.typeInfo = typeInfo; } entry.preparing = true; try { for (const child of children) { - this.traversalContainer(child); + this.traversalContainer(child, registrations, serializerLookup); } - const serializer = this.prepare(typeInfo, this.serializerLookup); + const serializer = this.prepare(typeInfo, serializerLookup); Object.assign(entry.serializer, serializer); } finally { entry.preparing = false; } } - private traversalContainer(typeInfo: TypeInfo) { + private seedDefinitions(root: TypeInfo, registrations: GeneratedRegistration[]) { + const pending = [root]; + const seen = new Set(); + while (pending.length > 0) { + const typeInfo = pending.pop()!; + if (seen.has(typeInfo)) { + continue; + } + seen.add(typeInfo); + const options = typeInfo.options; + if ( + TypeId.structType(typeInfo.typeId) && + options?.props !== undefined && + !this.typeResolver.getSerializerByTypeInfo(typeInfo)?._initialized + ) { + const registration = this.findRegistration(typeInfo, registrations); + if (registration === undefined) { + this.addRegistration(typeInfo, registrations); + } else if (!this.sameStructDefinition(registration.typeInfo, typeInfo)) { + throw new Error("conflicting complete struct definitions for the same registry identity"); + } + } + if (options === undefined) { + continue; + } + if (options.props !== undefined) { + pending.push(...Object.values(options.props)); + } + if (options.cases !== undefined) { + pending.push(...Object.values(options.cases)); + } + if (options.fieldEntries !== undefined) { + for (const entry of options.fieldEntries) { + pending.push(entry.typeInfo); + } + } + if (options.inner !== undefined) { + pending.push(options.inner); + } + if (options.key !== undefined) { + pending.push(options.key); + } + if (options.value !== undefined) { + pending.push(options.value); + } + } + } + + private traversalContainer( + typeInfo: TypeInfo, + registrations: GeneratedRegistration[], + serializerLookup: SerializerLookup, + ) { if (TypeId.userDefinedType(typeInfo.typeId)) { - if (this.isFullyGenerated(typeInfo)) { + if (this.isFullyGenerated(typeInfo, registrations)) { return; } const options = typeInfo.options; @@ -298,34 +372,44 @@ export class Gen { typeInfo.typeId === TypeId.TYPED_UNION || typeInfo.typeId === TypeId.NAMED_UNION; if (unionType && options?.cases && Object.keys(options.cases).length > 0) { - this.prepareRegistration(typeInfo, Object.values(options.cases)); + this.prepareRegistration( + typeInfo, + Object.values(options.cases), + registrations, + serializerLookup, + ); return; - } else if (options?.props && Object.keys(options.props).length > 0) { - this.prepareRegistration(typeInfo, Object.values(options.props)); + } else if (options?.props !== undefined) { + this.prepareRegistration( + typeInfo, + Object.values(options.props), + registrations, + serializerLookup, + ); } else if (!this.isRegistered(typeInfo) && TypeId.structType(typeInfo.typeId)) { - if (this.findRegistration(typeInfo) === undefined) { + if (this.findRegistration(typeInfo, registrations) === undefined) { throw new Error("nested struct schema must be registered or defined before use"); } } else if (TypeId.enumType(typeInfo.typeId) && !this.isRegistered(typeInfo)) { - this.prepareRegistration(typeInfo, []); + this.prepareRegistration(typeInfo, [], registrations, serializerLookup); } } if (typeInfo.typeId === TypeId.LIST) { - this.traversalContainer(typeInfo.options!.inner!); + this.traversalContainer(typeInfo.options!.inner!, registrations, serializerLookup); } if (typeInfo.typeId === TypeId.SET) { - this.traversalContainer(typeInfo.options!.key!); + this.traversalContainer(typeInfo.options!.key!, registrations, serializerLookup); } if (typeInfo.typeId === TypeId.MAP) { if (!typeInfo.options?.key || !typeInfo.options?.value) { throw new Error("map type must have key and value"); } - this.traversalContainer(typeInfo.options!.key!); - this.traversalContainer(typeInfo.options!.value!); + this.traversalContainer(typeInfo.options!.key!, registrations, serializerLookup); + this.traversalContainer(typeInfo.options!.value!, registrations, serializerLookup); } if (typeInfo.options?.cases) { Object.values(typeInfo.options.cases).forEach((caseTypeInfo) => { - this.traversalContainer(caseTypeInfo); + this.traversalContainer(caseTypeInfo, registrations, serializerLookup); }); } } @@ -336,30 +420,42 @@ export class Gen { generateSerializer(typeInfo: TypeInfo) { this.typeResolver.ensureRegistrationOpen(); - typeInfo.freeze(); + sealTypeInfo(typeInfo); // TypeInfo freezing may invoke application-owned proxy traps. A root entered there closes the // resolver before code generation or publication can continue. this.typeResolver.ensureRegistrationOpen(); - if (!this.typeResolver.getSerializerByTypeInfo(typeInfo)?._initialized) { - // Seed the root owner before traversal so empty roots and self-recursive fields share the - // same transaction-local serializer without publishing an incomplete resolver entry. - this.addRegistration(typeInfo); + const registrations: GeneratedRegistration[] = []; + // Generator-time TypeInfo queries see initialized local serializers for codegen decisions. + // Factory-init ID/name queries instead return the stable owner captured by runtime closures. + const serializerLookup: SerializerLookup = { + getSerializerByTypeInfo: (fieldType) => this.getGeneratedSerializer(fieldType, registrations), + getSerializerById: (id, userTypeId) => + this.getCapturedSerializerById(registrations, id, userTypeId), + getSerializerByName: (name) => this.getCapturedSerializerByName(registrations, name), + }; + this.seedDefinitions(typeInfo, registrations); + if ( + !TypeId.structType(typeInfo.typeId) && + !this.typeResolver.getSerializerByTypeInfo(typeInfo)?._initialized && + this.findRegistration(typeInfo, registrations) === undefined + ) { + this.addRegistration(typeInfo, registrations); } - this.traversalContainer(typeInfo); + this.traversalContainer(typeInfo, registrations, serializerLookup); const serializer = this.typeResolver.getSerializerByTypeInfo(typeInfo); if (!serializer?._initialized) { - let registration = this.findRegistration(typeInfo); + let registration = this.findRegistration(typeInfo, registrations); if (registration === undefined) { - registration = this.addRegistration(typeInfo); + registration = this.addRegistration(typeInfo, registrations); } if (!registration.serializer._initialized) { - this.prepareRegistration(typeInfo, []); + this.prepareRegistration(typeInfo, [], registrations, serializerLookup); } } // Generated factories may execute application-transformed code, so every factory completes // against local owners before the resolver performs the only global publication step. - this.typeResolver.commitGeneratedSerializers(this.generatedRegistrations); + this.typeResolver.commitGeneratedSerializers(registrations); return this.typeResolver.getSerializerByTypeInfo(typeInfo)!; } } diff --git a/javascript/packages/core/lib/gen/router.ts b/javascript/packages/core/lib/gen/router.ts index 5492348aac..9eefe243e4 100644 --- a/javascript/packages/core/lib/gen/router.ts +++ b/javascript/packages/core/lib/gen/router.ts @@ -17,6 +17,7 @@ * under the License. */ +import { TypeId } from "../type"; import { TypeInfo } from "../typeInfo"; import { SerializerGenerator } from "./serializer"; import { CodecBuilder } from "./builder"; @@ -48,11 +49,26 @@ export class CodegenRegistry { } static newGeneratorByTypeInfo(typeInfo: TypeInfo, builder: CodecBuilder, scope: Scope) { - const constructor = CodegenRegistry.get(typeInfo.typeId); + let generatorTypeInfo = typeInfo; + if (TypeId.userDefinedType(typeInfo.typeId)) { + const ownerTypeInfo = builder.serializerLookup + .getSerializerByTypeInfo(typeInfo) + ?.getTypeInfo(); + if (ownerTypeInfo !== undefined && ownerTypeInfo !== typeInfo) { + // Schema comes from the authoritative serializer owner. Field occurrence modifiers remain + // local to the containing schema and must not be replaced with the owner's modifiers. + generatorTypeInfo = ownerTypeInfo.clone(); + generatorTypeInfo.nullable = typeInfo.nullable; + generatorTypeInfo.trackingRef = typeInfo.trackingRef; + generatorTypeInfo.id = typeInfo.id; + generatorTypeInfo.dynamic = typeInfo.dynamic; + } + } + const constructor = CodegenRegistry.get(generatorTypeInfo.typeId); if (!constructor) { throw new Error("type not registered"); } - return new constructor(typeInfo, builder, scope); + return new constructor(generatorTypeInfo, builder, scope); } static get(typeId: number) { diff --git a/javascript/packages/core/lib/gen/struct.ts b/javascript/packages/core/lib/gen/struct.ts index 4b730153ff..307a72189a 100644 --- a/javascript/packages/core/lib/gen/struct.ts +++ b/javascript/packages/core/lib/gen/struct.ts @@ -796,11 +796,11 @@ class StructSerializerGenerator extends BaseSerializerGenerator { continue; } } - const InnerGeneratorClass = CodegenRegistry.get(current.typeInfo.typeId); - if (!InnerGeneratorClass) { - throw new Error(`${current.typeInfo.typeId} generator not exists`); - } - const innerGenerator = new InnerGeneratorClass(current.typeInfo, this.builder, this.scope); + const innerGenerator = CodegenRegistry.newGeneratorByTypeInfo( + current.typeInfo, + this.builder, + this.scope, + ); const fieldAccessor = `${accessor}${CodecBuilder.safePropAccessor(current.key)}`; fieldWrites.push( this.writeField(current.key, current.typeInfo, fieldAccessor, innerGenerator.writeEmbed()), @@ -952,11 +952,11 @@ class StructSerializerGenerator extends BaseSerializerGenerator { ${this.maybeReference(result, refState)} ${this.sortedProps .map(({ key, typeInfo }) => { - const InnerGeneratorClass = CodegenRegistry.get(typeInfo.typeId); - if (!InnerGeneratorClass) { - throw new Error(`${typeInfo.typeId} generator not exists`); - } - const innerGenerator = new InnerGeneratorClass(typeInfo, this.builder, this.scope); + const innerGenerator = CodegenRegistry.newGeneratorByTypeInfo( + typeInfo, + this.builder, + this.scope, + ); return ` ${this.readField(key, typeInfo, (expr) => this.readFieldAssign(result, key, expr), innerGenerator.readEmbed())} `; @@ -1448,8 +1448,13 @@ class StructSerializerGenerator extends BaseSerializerGenerator { let fixedSize = 8; if (options!.props) { Object.values(options!.props).forEach((x) => { - const propGenerator = new (CodegenRegistry.get(x.typeId)!)(x, this.builder, this.scope); - fixedSize += propGenerator.getFixedSize(); + const serializer = this.builder.serializerLookup.getSerializerByTypeInfo(x); + if (TypeId.userDefinedType(x.typeId) && serializer !== undefined) { + fixedSize += serializer.fixedSize; + } else { + const propGenerator = CodegenRegistry.newGeneratorByTypeInfo(x, this.builder, this.scope); + fixedSize += propGenerator.getFixedSize(); + } }); } else { fixedSize += this.builder.serializerLookup.getSerializerByTypeInfo(typeInfo)!.fixedSize; diff --git a/javascript/packages/core/lib/typeInfo.ts b/javascript/packages/core/lib/typeInfo.ts index 77fb200d3a..8e8ad2d336 100644 --- a/javascript/packages/core/lib/typeInfo.ts +++ b/javascript/packages/core/lib/typeInfo.ts @@ -26,6 +26,20 @@ import { Decimal } from "./types/decimal"; const targetFields = new WeakMap any, { [key: string]: TypeInfo }>(); export const MAX_FIELD_ID = (1 << 29) - 1; +const sealedSchemaFields = { + options: { writable: false, configurable: false }, + named: { writable: false, configurable: false }, + namespace: { writable: false, configurable: false }, + typeName: { writable: false, configurable: false }, + userTypeId: { writable: false, configurable: false }, + evolving: { writable: false, configurable: false }, + _typeId: { writable: false, configurable: false }, + nullable: { writable: false, configurable: false }, + trackingRef: { writable: false, configurable: false }, + id: { writable: false, configurable: false }, + dynamic: { writable: false, configurable: false }, +}; + export function checkFieldId(fieldId: number) { if (Number.isFinite(fieldId) && fieldId < 0) { throw new Error("field id must be non-negative"); @@ -149,65 +163,6 @@ export class TypeInfo extends ExtensibleFunction { }); } - /** Freezes schema-owned state recursively while leaving root write IDs operation-local. */ - public freeze() { - const seen = new Set(); - const freezeTypeInfo = (typeInfo: TypeInfo) => { - if (seen.has(typeInfo)) { - return; - } - seen.add(typeInfo); - const options = typeInfo.options; - const children: TypeInfo[] = []; - if (options !== undefined) { - if (options.props !== undefined) { - children.push(...Object.values(options.props)); - Object.freeze(options.props); - } - if (options.cases !== undefined) { - children.push(...Object.values(options.cases)); - Object.freeze(options.cases); - } - if (options.fieldEntries !== undefined) { - for (const entry of options.fieldEntries) { - children.push(entry.typeInfo); - Object.freeze(entry); - } - Object.freeze(options.fieldEntries); - } - if (options.inner !== undefined) { - children.push(options.inner); - } - if (options.key !== undefined) { - children.push(options.key); - } - if (options.value !== undefined) { - children.push(options.value); - } - if (options.enumProps !== undefined) { - Object.freeze(options.enumProps); - } - Object.freeze(options); - } - Object.defineProperties(typeInfo, { - named: { writable: false, configurable: false }, - namespace: { writable: false, configurable: false }, - typeName: { writable: false, configurable: false }, - userTypeId: { writable: false, configurable: false }, - evolving: { writable: false, configurable: false }, - options: { writable: false, configurable: false }, - _typeId: { writable: false, configurable: false }, - nullable: { writable: false, configurable: false }, - trackingRef: { writable: false, configurable: false }, - id: { writable: false, configurable: false }, - dynamic: { writable: false, configurable: false }, - }); - // dynamicTypeId is operation-local writer state and remains mutable across roots. - children.forEach(freezeTypeInfo); - }; - freezeTypeInfo(this); - } - public constructor(typeId: number, userTypeId = -1) { super(function (target: any, key?: string | { name?: string }) { if (key === undefined) { @@ -372,7 +327,7 @@ export class TypeInfo extends ExtensibleFunction { } const typeInfo = new TypeInfo(finalTypeId, userTypeId); typeInfo.options = { - props: props || {}, + props, withConstructor, }; typeInfo.evolving = evolving; @@ -440,6 +395,60 @@ export class TypeInfo extends ExtensibleFunction { } } +/** @internal */ +export function sealTypeInfo(root: TypeInfo) { + const pending = [root]; + const seen = new Set(); + while (pending.length > 0) { + const typeInfo = pending.pop()!; + if (seen.has(typeInfo)) { + continue; + } + seen.add(typeInfo); + + // Lock the options pointer before any schema read. A proxy trap may replace the pointer while + // it is being locked, but later traps cannot replace the final value code generation observes. + Object.defineProperties(typeInfo, sealedSchemaFields); + const options = typeInfo.options; + if (options === undefined) { + continue; + } + Object.freeze(options); + + const props = options.props; + if (props !== undefined) { + Object.freeze(props); + pending.push(...Object.values(props)); + } + const cases = options.cases; + if (cases !== undefined) { + Object.freeze(cases); + pending.push(...Object.values(cases)); + } + const fieldEntries = options.fieldEntries; + if (fieldEntries !== undefined) { + Object.freeze(fieldEntries); + for (const entry of fieldEntries) { + Object.freeze(entry); + pending.push(entry.typeInfo); + } + } + if (options.inner !== undefined) { + pending.push(options.inner); + } + if (options.key !== undefined) { + pending.push(options.key); + } + if (options.value !== undefined) { + pending.push(options.value); + } + if (options.enumProps !== undefined) { + Object.freeze(options.enumProps); + } + } + // dynamicTypeId is operation-local writer state and remains mutable across roots. +} + export enum Dynamic { TRUE = "TRUE", FALSE = "FALSE", diff --git a/javascript/packages/core/tsconfig.json b/javascript/packages/core/tsconfig.json index 87cefde930..f0558acdfe 100644 --- a/javascript/packages/core/tsconfig.json +++ b/javascript/packages/core/tsconfig.json @@ -41,7 +41,7 @@ // "inlineSources": true, /* Include source code in the sourcemaps inside the emitted JavaScript. */ // "emitBOM": true, /* Emit a UTF-8 Byte Order Mark (BOM) in the beginning of output files. */ // "newLine": "crlf", /* Set the newline character for emitting files. */ - // "stripInternal": true, /* Disable emitting declarations that have '@internal' in their JSDoc comments. */ + "stripInternal": true, /* Disable emitting declarations that have '@internal' in their JSDoc comments. */ "noEmitHelpers": true, /* Disable generating custom helper functions like '__extends' in compiled output. */ "noEmitOnError": true, /* Disable emitting files if any type checking errors are reported. */ // "preserveConstEnums": true, /* Disable erasing 'const enum' declarations in generated code. */ diff --git a/javascript/test/fory.test.ts b/javascript/test/fory.test.ts index ee93e5f73f..0ba38905de 100644 --- a/javascript/test/fory.test.ts +++ b/javascript/test/fory.test.ts @@ -85,23 +85,20 @@ describe("fory", () => { testTypeInfo(typeinfo8, "123"); }); - test.each(["serialize", "deserialize"] as const)( - "freezes registration when %s starts and fails", - (operation) => { - const fory = new Fory({ compatible: false }); - fory.register(Type.struct(8101, {})); - - if (operation === "serialize") { - expect(() => fory.serialize(Symbol("unsupported"))).toThrow(); - } else { - expect(() => fory.deserialize(new Uint8Array([0]))).toThrow(); - } - - expect(() => fory.register(Type.struct(8102, {}))).toThrow(); - }, - ); - - test("keeps rejected descriptor mutable", () => { + test.each(["serialize", "deserialize"] as const)("freezes on failed %s", (operation) => { + const fory = new Fory({ compatible: false }); + fory.register(Type.struct(8101, {})); + + if (operation === "serialize") { + expect(() => fory.serialize(Symbol("unsupported"))).toThrow(); + } else { + expect(() => fory.deserialize(new Uint8Array([0]))).toThrow(); + } + + expect(() => fory.register(Type.struct(8102, {}))).toThrow(); + }); + + test("keeps rejected schema mutable", () => { const fory = new Fory({ compatible: false }); const typeInfo = Type.struct(8106, {}); fory.serialize(1); @@ -111,7 +108,7 @@ describe("fory", () => { expect(typeInfo.nullable).toBe(true); }); - test("keeps codegen callbacks from publishing registration", () => { + test("keeps callback failure local", () => { let reenterRoot = false; let fory: Fory; fory = new Fory({ @@ -146,7 +143,7 @@ describe("fory", () => { expect(() => rootType.setNullable(true)).toThrow(); }); - test("keeps failed generated factories local", () => { + test("keeps factory failure local", () => { let failFactory = false; const fory = new Fory({ compatible: false, @@ -182,7 +179,7 @@ describe("fory", () => { expect(() => rootType.setNullable(true)).toThrow(); }); - test("rejects an unresolved nested schema", () => { + test("rejects unresolved schema", () => { const fory = new Fory({ compatible: false }); const typeResolver = fory.typeResolver as any; const internalBefore = Array.from(typeResolver.internalSerializer); @@ -211,13 +208,13 @@ describe("fory", () => { expect(parent.deserialize(parent.serialize(value))).toEqual(value); }); - test("registers an empty root schema", () => { + test("registers empty roots", () => { const registered = new Fory({ compatible: false }).register(Type.struct(8122, {})); expect(registered.deserialize(registered.serialize({}))).toEqual({}); }); - test("registers a self-recursive schema", () => { + test("registers self recursion", () => { const nodeType = Type.struct(8123, { value: Type.int32(), next: Type.struct(8123).setNullable(true).setTrackingRef(true), @@ -231,7 +228,7 @@ describe("fory", () => { expect(result.next).toBe(result); }); - test("registers a mutually recursive schema", () => { + test("registers mutual recursion", () => { const rightType = Type.struct(8125, { value: Type.string(), left: Type.struct(8124).setNullable(true), @@ -246,56 +243,53 @@ describe("fory", () => { expect(registered.deserialize(registered.serialize(value))).toEqual(value); }); - test.each(["userTypeId", "name", "options"] as const)( - "rejects root %s changes during codegen", - (change) => { - let mutateDescriptor: (() => void) | undefined; - const fory = new Fory({ - compatible: false, - hooks: { - afterCodeGenerated(code) { - const mutate = mutateDescriptor; - mutateDescriptor = undefined; - mutate?.(); - return code; - }, + test.each(["userTypeId", "name", "options"] as const)("rejects root %s mutation", (change) => { + let mutateDescriptor: (() => void) | undefined; + const fory = new Fory({ + compatible: false, + hooks: { + afterCodeGenerated(code) { + const mutate = mutateDescriptor; + mutateDescriptor = undefined; + mutate?.(); + return code; }, - }); - const typeInfo = - change === "name" - ? Type.struct("stable.Root", { value: Type.int32() }) - : Type.struct(8115, { value: Type.int32() }); - const typeResolver = fory.typeResolver as any; - const internalBefore = Array.from(typeResolver.internalSerializer); - const customBefore = Array.from(typeResolver.customSerializer.entries()); - if (change === "userTypeId") { - mutateDescriptor = () => { - typeInfo.userTypeId = 9115; - }; - } else if (change === "name") { - mutateDescriptor = () => { - typeInfo.named = "changed$Root"; - }; - } else { - mutateDescriptor = () => { - typeInfo.options!.props!.extra = Type.string(); - }; - } - - expect(() => fory.register(typeInfo)).toThrow(); - - expect(Array.from(typeResolver.internalSerializer)).toEqual(internalBefore); - expect(Array.from(typeResolver.customSerializer.entries())).toEqual(customBefore); - expect(() => typeInfo.setNullable(true)).toThrow(); - if (change === "options") { - expect(() => { - typeInfo.options!.props!.afterFailure = Type.bool(); - }).toThrow(); - } - }, - ); - - test("rejects nested schema changes during codegen", () => { + }, + }); + const typeInfo = + change === "name" + ? Type.struct("stable.Root", { value: Type.int32() }) + : Type.struct(8115, { value: Type.int32() }); + const typeResolver = fory.typeResolver as any; + const internalBefore = Array.from(typeResolver.internalSerializer); + const customBefore = Array.from(typeResolver.customSerializer.entries()); + if (change === "userTypeId") { + mutateDescriptor = () => { + typeInfo.userTypeId = 9115; + }; + } else if (change === "name") { + mutateDescriptor = () => { + typeInfo.named = "changed$Root"; + }; + } else { + mutateDescriptor = () => { + typeInfo.options!.props!.extra = Type.string(); + }; + } + + expect(() => fory.register(typeInfo)).toThrow(); + + expect(Array.from(typeResolver.internalSerializer)).toEqual(internalBefore); + expect(Array.from(typeResolver.customSerializer.entries())).toEqual(customBefore); + expect(() => typeInfo.setNullable(true)).toThrow(); + if (change === "options") { + expect(() => { + typeInfo.options!.props!.afterFailure = Type.bool(); + }).toThrow(); + } + }); + + test("rejects nested mutation", () => { let mutateDescriptor: (() => void) | undefined; const fory = new Fory({ compatible: false, @@ -327,70 +321,102 @@ describe("fory", () => { expect(() => childType.setNullable(true)).toThrow(); }); - test("captures a reentrant same-key owner", () => { - let registerSameKey = false; - let reentrant: ReturnType; - let fory: Fory; - fory = new Fory({ - compatible: false, - hooks: { - afterCodeGenerated(code) { - if (registerSameKey) { - registerSameKey = false; - reentrant = fory.register(Type.struct(8118, { innerValue: Type.string() })); - } - return code; + test("resolves later definitions", () => { + const itemType = Type.struct(8118, { value: Type.int32() }); + const registered = new Fory({ compatible: false }).register( + Type.struct(8119, { + first: Type.struct(8118), + definition: itemType, + }), + ); + const value = { + first: { value: 1 }, + definition: { value: 2 }, + }; + + expect(registered.deserialize(registered.serialize(value as any))).toEqual(value); + }); + + test("rejects conflicting definitions", () => { + for (const reversed of [false, true]) { + let generated = 0; + const fory = new Fory({ + compatible: false, + hooks: { + afterCodeGenerated(code) { + generated++; + return code; + }, }, + }); + generated = 0; + const first = Type.struct(8127, { firstValue: Type.int32() }); + const second = Type.struct(8127, { secondValue: Type.string() }); + const props = reversed ? { second, first } : { first, second }; + const root = Type.struct(8128, props); + + expect(() => fory.register(root)).toThrow(); + expect(generated).toBe(0); + expect(fory.typeResolver.getSerializerById(TypeId.STRUCT, 8127)).toBeUndefined(); + expect(fory.typeResolver.getSerializerById(TypeId.STRUCT, 8128)).toBeUndefined(); + } + }); + + test("seals replaced schema options", () => { + const original = Type.struct(8126, { oldValue: Type.int32() }); + const replacement: { + props: Record; + withConstructor: boolean; + } = { + props: { value: Type.string() }, + withConstructor: false, + }; + let replaceOptions = true; + const typeInfo = new Proxy(original, { + defineProperty(target, property, descriptor) { + if (property === "options" && replaceOptions) { + replaceOptions = false; + target.options = replacement; + } + return Reflect.defineProperty(target, property, descriptor); }, }); - const childType = Type.struct(8118, { outerValue: Type.int32() }); - const parentType = Type.struct(8119, { child: childType }); - - registerSameKey = true; - const parent = fory.register(parentType); - const owner = fory.typeResolver.getSerializerById(TypeId.STRUCT, childType.userTypeId); - expect(reentrant!.serializer).toBe(owner); - expect(owner.getTypeInfo()).toBe(reentrant!.serializer.getTypeInfo()); - const write = owner.write; - let childWrites = 0; - owner.write = (value) => { - childWrites++; - write(value); - }; + const registered = new Fory({ compatible: false }).register(typeInfo); + const value = { value: "sealed" }; - const value = { child: { innerValue: "kept" } }; - expect(parent.deserialize(parent.serialize(value as any))).toEqual(value); - expect(childWrites).toBeGreaterThan(0); + expect(registered.deserialize(registered.serialize(value as any))).toEqual(value); + expect(() => { + replacement.props.extra = Type.bool(); + }).toThrow(); }); - test("freezes a recursive schema graph", () => { + test("seals recursive schemas", () => { const left = Type.struct(8120, {}); const right = Type.struct(8121, { left }); left.options!.props!.right = right; - left.freeze(); + new Fory({ compatible: false }).register(left); expect(() => left.setNullable(true)).toThrow(); expect(() => right.setTrackingRef(true)).toThrow(); + left.dynamicTypeId = 7; + expect(left.dynamicTypeId).toBe(7); }); - test.each(["serialize", "deserialize"] as const)( - "freezes registration after registered %s succeeds", - (operation) => { - const typeInfo = Type.struct(8103, {}); - const source = new Fory({ compatible: false }).register(typeInfo.clone()); - const fory = new Fory({ compatible: false }); - const registered = fory.register(typeInfo); - - if (operation === "serialize") { - registered.serialize({}); - } else { - registered.deserialize(source.serialize({})); - } - - expect(() => fory.register(Type.struct(8104, {}))).toThrow(); - }, - ); + test.each(["serialize", "deserialize"] as const)("freezes after successful %s", (operation) => { + const typeInfo = Type.struct(8103, {}); + const source = new Fory({ compatible: false }).register(typeInfo.clone()); + const fory = new Fory({ compatible: false }); + const registered = fory.register(typeInfo); + + if (operation === "serialize") { + registered.serialize({}); + } else { + registered.deserialize(source.serialize({})); + } + + expect(() => fory.register(Type.struct(8104, {}))).toThrow(); + }); function testTypeInfo(typeinfo: TypeInfo, input: any, expected?: any) { const fory = new Fory({ compatible: false }); diff --git a/javascript/test/map.test.ts b/javascript/test/map.test.ts index c65cdc5e47..dac77e69a7 100644 --- a/javascript/test/map.test.ts +++ b/javascript/test/map.test.ts @@ -118,7 +118,7 @@ describe("map", () => { test.each([ ["fixed", false, 320], ["evolving", true, 321], - ])("round-trips %s map sides beside null", (_, evolving, itemId) => { + ])("round-trips nullable %s maps", (_, evolving, itemId) => { const fory = new Fory({ compatible: true, ref: true }); const itemType = Type.struct( { typeId: itemId, evolving }, @@ -149,7 +149,7 @@ describe("map", () => { ]); }); - test("preserves compatible struct map framing", () => { + test("keeps compatible map framing", () => { const serializeMap = ( compatible: boolean, evolving: boolean, @@ -186,7 +186,47 @@ describe("map", () => { expect((native.header >> 3) & 0b100).toBe(0b100); }); - test("rejects invalid runtime chunks before type detection", () => { + test("uses reentrant map owner", () => { + let registerSameKey = false; + let reentrant: ReturnType; + let fory: Fory; + fory = new Fory({ + compatible: true, + ref: true, + hooks: { + afterCodeGenerated(code) { + if (registerSameKey) { + registerSameKey = false; + reentrant = fory.register( + Type.struct({ typeId: 350, evolving: true }, { innerValue: Type.string() }), + ); + } + return code; + }, + }, + }); + const outerType = Type.struct({ typeId: 350, evolving: false }, { outerValue: Type.int32() }); + registerSameKey = true; + const registered = fory.register( + Type.struct(351, { + outer: outerType.setId(2), + values: Type.map(Type.struct(350), Type.struct(350)).setId(1), + }), + ); + const value = { + outer: { innerValue: "outer" }, + values: new Map([[{ innerValue: "key" }, { innerValue: "value" }]]), + }; + const bytes = registered.serialize(value as any); + const { header } = structMapHeader(fory, bytes, true, 351); + + expect(fory.typeResolver.getSerializerById(TypeId.STRUCT, 350)).toBe(reentrant!.serializer); + expect(header & 0b100).toBe(0); + expect((header >> 3) & 0b100).toBe(0); + expect(registered.deserialize(bytes)).toEqual(value); + }); + + test("rejects invalid runtime chunks", () => { const fory = new Fory({ compatible: false, ref: true }); const MapAnySerializer = CodegenRegistry.getExternal().MapAnySerializer; const serializer = new MapAnySerializer(fory.writeContext, fory.readContext, null, null); @@ -197,7 +237,7 @@ describe("map", () => { } }); - test("rejects invalid generated chunks and reuses the root", () => { + test("rejects invalid generated chunks", () => { const fory = new Fory({ compatible: false, ref: true }); const serializer = fory.register(Type.map(Type.string(), Type.int32())); const value = new Map([["key", 1]]); From 3d9ac1f0da4ec4e080e8a27aff026193c563f182 Mon Sep 17 00:00:00 2001 From: chaokunyang Date: Sat, 29 Aug 2026 02:13:46 +0800 Subject: [PATCH 058/168] docs: define generated schema ownership --- .agents/languages/javascript.md | 21 ++++++---- .../javascript/type-registration.md | 8 ++-- .../xlang_implementation_guide.md | 40 ++++++++++++------- 3 files changed, 43 insertions(+), 26 deletions(-) diff --git a/.agents/languages/javascript.md b/.agents/languages/javascript.md index 789f3a3955..2f1dca891e 100644 --- a/.agents/languages/javascript.md +++ b/.agents/languages/javascript.md @@ -18,14 +18,19 @@ Load this file when changing `javascript/`. their own logical size: reset active owner IDs and that table's logical size without clearing bounded backing, and replace either backing only after its root has more than 8192 owners. - Generated registration must initialize the complete recursive serializer graph against - generation-local owners before one `TypeResolver` batch publication. Freeze the complete - `TypeInfo` schema graph before code generation; schema fields and occurrence modifiers are - immutable afterward, while `dynamicTypeId` remains operation-local writer state. Factory-init - serializer lookup may resolve local owners for fixed captures; runtime and dynamic lookup must - keep the real resolver. A nested identity-only Struct must already be registered or resolve to an - owner in the current complete recursive schema graph. Reject any unresolved nested identity - before resolver publication. Never publish placeholders, nested serializers, descriptors, or - cache state before every generated factory and application code hook succeeds. + generation-local owners before one `TypeResolver` batch publication. The package-internal schema + seal must lock each `TypeInfo` schema pointer before reading or traversing it; schema fields and + occurrence modifiers are immutable afterward, while `dynamicTypeId` remains operation-local + writer state. Seed every complete Struct definition in the sealed graph before resolving + identity-only occurrences, so definition order cannot affect recursive resolution. Each resolver + identity has one complete schema owner in that graph. Repeated references and clones may share + the same immutable definition containers and settings; reject a second conflicting definition + before code generation without deep schema comparison. One authoritative serializer owner + supplies both generator-time schema/progress facts and fixed factory captures; an initialized + owner published by a nested registration wins over an outer generation-local owner, while field + occurrence modifiers remain field-owned. Reject any unresolved nested identity before resolver + publication. Never publish generated serializers, descriptors, or cache state before every + generated factory and application code hook succeeds. - Runtime value carriers such as decimal or reduced-precision numeric types belong under the core `types/` ownership boundary, with imports, exports, and codegen externals updated together. - Keep `TypeInfo` as schema metadata. Compatibility-sensitive decisions belong on `TypeResolver` or explicit operations, not as retained resolver state on metadata objects. - Normalize optional boolean config values at config construction; do not carry `null` through runtime paths when it means `false`. diff --git a/docs/object-serialization/javascript/type-registration.md b/docs/object-serialization/javascript/type-registration.md index a18e3d6bc2..690fd1417a 100644 --- a/docs/object-serialization/javascript/type-registration.md +++ b/docs/object-serialization/javascript/type-registration.md @@ -152,9 +152,11 @@ const wrapperType = Type.struct("example.wrapper", { }); ``` -A nested Struct that supplies only an ID or name must already be registered. Alternatively, define -the complete nested schema in the same recursive `TypeInfo` graph. Registration rejects an -unresolved nested identity without publishing either the parent or a placeholder. +A nested Struct that supplies only an ID or name must already be registered. Alternatively, include +a complete definition with the same identity anywhere in the recursive `TypeInfo` graph; the +definition may appear before or after the identity-only use. Registration rejects an unresolved +nested identity. Each identity can have only one complete definition in that graph. Repeated uses +may share that definition, but separate conflicting definitions are rejected. ## Field Metadata diff --git a/docs/specification/xlang_implementation_guide.md b/docs/specification/xlang_implementation_guide.md index d111cec231..8009d133dc 100644 --- a/docs/specification/xlang_implementation_guide.md +++ b/docs/specification/xlang_implementation_guide.md @@ -87,8 +87,11 @@ callbacks. Complete the callback before publishing the registry entry it prepare starts a root operation, registration must recheck the authoritative per-instance freeze owner when the callback returns and reject that publication. Kotlin and Scala combined generated-struct registration are the type-first exception: publish the canonical type required by generated -serializer construction, then recheck after construction and before replacing its serializer. -Do not add rollback, staging, or a parallel registration path for that exception. Module +serializer construction, then construct the serializer in the resolver's existing construction +graph. The candidate is visible only to that construction and reaches the normal resolver commit +only after the authoritative freeze recheck succeeds. A serializer-only helper rejects a missing +canonical type instead of auto-registering it. Do not use direct serializer replacement or add +rollback, staging, or a parallel registration path for that exception. Module installation may consist of complete nested registrations. `Fory.register(ForyModule)` alone owns module identity, cycle breaking, and idempotence; language bootstrap helpers must not add markers, monitors, or separate reentry policies. Keep a retryable install body replay-safe until its final @@ -118,23 +121,30 @@ resolve recursive fields or candidate state use the construction-local serialize resolver lookups retain their runtime semantics. When wire and user IDs match, the final owner is the existing canonical `TypeInfo`, so generated serializers and field metadata never retain temporary metadata. After construction and the registry lifecycle recheck succeed, the normal -Class/Xtype resolver commit path installs the candidate. There is no constructor-specific publication path or -nonpublishing serializer factory. Static-generated serializer classes require an already +Class/Xtype resolver commit path installs the candidate. Static-generated construction retains the +already registered canonical type identity and exposes its immutable generated descriptors only to +the same construction graph; its early-bound serializer candidate is never published. There is no +constructor-specific publication path. Static-generated serializer classes require an already registered canonical type and are therefore rejected by the combined class overload. Direct Java `Fory` instances may install a module before their first root operation; thread-safe facades install modules only through `ForyBuilder.withModule` during construction. -JavaScript generated registration freezes the complete `TypeInfo` schema graph before code -generation, including nested schemas and field occurrence modifiers. The writer-owned -`dynamicTypeId` remains mutable because it is reset per root. Code generation then constructs and -initializes the complete recursive serializer graph against generation-local owners. Generated -factories may use a construction-only lookup for fixed serializer captures, while runtime and -dynamic dispatch retain the real `TypeResolver`. After every factory and application code hook -succeeds, the resolver performs one guarded batch publication. A nested identity-only Struct must -already have a fully initialized registered owner or belong to the current complete recursive -schema graph. An unresolved nested identity fails registration before resolver publication. An -initialized owner published by a nested registration is authoritative and must not be overwritten -by the outer registration. +JavaScript generated registration seals the complete `TypeInfo` schema graph before code +generation, including nested schemas and field occurrence modifiers. The package-internal seal +locks each schema-owned pointer before reading or traversing it. The writer-owned `dynamicTypeId` +remains mutable because it is reset per root. Code generation seeds every complete Struct +definition by registry identity before resolving identity-only occurrences, so recursive schema +resolution does not depend on field order. Each resolver identity has one complete schema owner in +the graph. Repeated references and clones may share that owner's immutable definition containers +and settings, while a second conflicting complete definition fails before code generation without +a deep structural comparison. Code generation then constructs and initializes the complete +serializer graph against generation-local owners. The same authoritative owner supplies +generator-time schema and progress facts and fixed factory captures; field occurrence modifiers +remain owned by the containing schema. Runtime and dynamic dispatch retain the real +`TypeResolver`. After every factory and application code hook succeeds, the resolver performs one +guarded batch publication. An unresolved nested identity fails registration before resolver +publication. An initialized owner published by a nested registration is authoritative and must not +be overwritten or contradicted by the outer registration's generated decisions. Nested serializers must not call back into root `serialize(...)` or `deserialize(...)` entry points. From ac1826bdd7b19d1b3ed0e9415b5a9189c0a42f3f Mon Sep 17 00:00:00 2001 From: chaokunyang Date: Sat, 29 Aug 2026 02:19:45 +0800 Subject: [PATCH 059/168] test(javascript): shorten lifecycle names --- javascript/test/depthLimit.test.ts | 2 +- javascript/test/rootCleanup.test.ts | 10 +++++----- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/javascript/test/depthLimit.test.ts b/javascript/test/depthLimit.test.ts index 8e6689d379..d75ad025f3 100644 --- a/javascript/test/depthLimit.test.ts +++ b/javascript/test/depthLimit.test.ts @@ -298,7 +298,7 @@ describe("depth-limit", () => { expect(readerFory.readContext.depth).toBe(0); }); - test("should reset depth before each deserialization", () => { + test("resets depth between roots", () => { const fory = new Fory({ compatible: false, maxDepth: 50 }); const typeInfo = Type.struct( { diff --git a/javascript/test/rootCleanup.test.ts b/javascript/test/rootCleanup.test.ts index 169b3ab6c1..9c2fe95ed4 100644 --- a/javascript/test/rootCleanup.test.ts +++ b/javascript/test/rootCleanup.test.ts @@ -158,7 +158,7 @@ test("reuses root write metastring owners", () => { expect(name.dynamicWriteStringId).toBe(0); }); -test("reuses root write type metadata owners", () => { +test("reuses write metadata owners", () => { const fory = new Fory({ compatible: true }); const registered = fory.register(Type.struct(7610, {})); const writeContext = (fory as any).writeContext; @@ -179,7 +179,7 @@ test("reuses root write type metadata owners", () => { expect(typeMeta.dynamicTypeId).toBe(0); }); -test.each([8192, 8193])("bounds %s root write metastring owners", (ownerCount) => { +test.each([8192, 8193])("bounds %s metastring owners", (ownerCount) => { const fory = new Fory({ compatible: true }); const writeContext = (fory as any).writeContext; const metaStringWriter = writeContext.metaStringWriter; @@ -202,7 +202,7 @@ test.each([8192, 8193])("bounds %s root write metastring owners", (ownerCount) = expect(nextOwner.dynamicWriteStringId).toBe(0); }); -test.each([8192, 8193])("bounds %s root write type metadata owners", (ownerCount) => { +test.each([8192, 8193])("bounds %s type metadata owners", (ownerCount) => { const fory = new Fory({ compatible: true }); const writeContext = (fory as any).writeContext; const typeMetaOwners = Array.from({ length: ownerCount }, () => ({ dynamicTypeId: -1 })); @@ -226,7 +226,7 @@ test.each([8192, 8193])("bounds %s root write type metadata owners", (ownerCount expect(nextOwner.dynamicTypeId).toBe(0); }); -test("releases a failed root write buffer before reuse", () => { +test("releases failed write buffer", () => { const fory = new Fory({ compatible: true }); const registered = fory.register(Type.struct(7608, {})); const writer = (fory as any).writeContext.writer; @@ -241,7 +241,7 @@ test("releases a failed root write buffer before reuse", () => { expect(writer.getPlatformBuffer().byteLength).toBeLessThan(4 * 1024 * 1024); }); -test("clears failed root references before reuse", () => { +test("clears failed write refs", () => { const fory = new Fory({ compatible: false, ref: true }); const registered = fory.register(Type.struct(7612, {})); const refReader = (fory as any).readContext.refReader; From 3ad15744ffdcf0c33feef54a9e6ebfa01b54e83a Mon Sep 17 00:00:00 2001 From: chaokunyang Date: Sat, 29 Aug 2026 02:37:56 +0800 Subject: [PATCH 060/168] fix(python): preserve reduce iterator carriers --- .agents/languages/python.md | 3 +++ docs/object-serialization/python/native.md | 4 ++++ python/pyfory/serializer.py | 7 ------- python/pyfory/tests/test_reduce_serializer.py | 8 +++++++- 4 files changed, 14 insertions(+), 8 deletions(-) diff --git a/.agents/languages/python.md b/.agents/languages/python.md index 4f3d73e172..26d9d5f4b6 100644 --- a/.agents/languages/python.md +++ b/.agents/languages/python.md @@ -46,6 +46,9 @@ Load this file when changing `python/`, Cython serialization, or Python xlang be - Function serialization writes captured globals as a data-only exact `dict`. Keep the reader's exact-type check before sizing or merging the namespace; a dict subclass or other mapping must not introduce runtime behavior into function reconstruction. +- Python reduction list-item and dict-item iterators remain native carrier values. Register their + concrete iterator types before the first root; do not materialize them into lists in the serializer, + which changes the established carrier path and allocates storage proportional to their contents. - Pandas `RangeIndex` owns its dtype wire slot. Encode `dtype.str` and reconstruct it with `numpy.dtype`; do not serialize the dtype object as a reference because concrete NumPy dtype classes vary across versions and would make the wire depend on version-specific registration. diff --git a/docs/object-serialization/python/native.md b/docs/object-serialization/python/native.md index a6062a9e5e..8ba371bfe9 100644 --- a/docs/object-serialization/python/native.md +++ b/docs/object-serialization/python/native.md @@ -58,6 +58,10 @@ permanently freezes the instance's registry even when it fails. With `strict=Fal policy may still authorize module-global classes and callables resolved while reading a native payload; that lookup does not add a type or serializer to the frozen registry. +For an application type whose `__reduce__` or `__reduce_ex__` result contains list-item or +dict-item iterators, register the concrete iterator carrier types before the first root operation. +Fory preserves those carriers without copying their contents into temporary lists. + ## Common Usage ```python diff --git a/python/pyfory/serializer.py b/python/pyfory/serializer.py index 613df26743..a2aeadc18d 100644 --- a/python/pyfory/serializer.py +++ b/python/pyfory/serializer.py @@ -1349,17 +1349,10 @@ def write(self, write_context, value): elif len(reduce_result) == 4: # Case 4: (callable, args, state, listitems) callable_obj, args, state, listitems = reduce_result - # Reduce item iterators carry contents, not runtime iterator identity. - if listitems is not None: - listitems = list(listitems) reduce_data = (1, callable_obj, args, state, listitems) elif len(reduce_result) == 5: # Case 5: (callable, args, state, listitems, dictitems) callable_obj, args, state, listitems, dictitems = reduce_result - if listitems is not None: - listitems = list(listitems) - if dictitems is not None: - dictitems = list(dictitems) reduce_data = ( 1, callable_obj, diff --git a/python/pyfory/tests/test_reduce_serializer.py b/python/pyfory/tests/test_reduce_serializer.py index 49e170f8d0..8b5fea5e7b 100644 --- a/python/pyfory/tests/test_reduce_serializer.py +++ b/python/pyfory/tests/test_reduce_serializer.py @@ -27,7 +27,13 @@ def register_reduce_types(fory, *classes): - for cls in (type, types.BuiltinFunctionType, *classes): + for cls in ( + type, + types.BuiltinFunctionType, + type(iter([])), + type(iter({}.items())), + *classes, + ): fory.register_type(cls) From 7802abb9336a2c329113048f3950362291d9cbb1 Mon Sep 17 00:00:00 2001 From: chaokunyang Date: Sat, 29 Aug 2026 02:38:03 +0800 Subject: [PATCH 061/168] refactor(javascript): remove legacy metadata adapter --- javascript/packages/core/lib/context.ts | 34 ------------------------- javascript/test/typemeta.test.ts | 23 ----------------- 2 files changed, 57 deletions(-) diff --git a/javascript/packages/core/lib/context.ts b/javascript/packages/core/lib/context.ts index 6907c0f1cf..7b71c08e6d 100644 --- a/javascript/packages/core/lib/context.ts +++ b/javascript/packages/core/lib/context.ts @@ -1605,40 +1605,6 @@ export class ReadContext { ); } - genSerializerByTypeMetaRuntime( - typeMeta: TypeMeta, - original?: Serializer | TypeInfo, - expectedLocalHash?: number, - ) { - void expectedLocalHash; - const typeId = typeMeta.getTypeId(); - if (!TypeId.structType(typeId)) { - throw new Error("only support reconstructor struct type"); - } - let originalSerializer = original instanceof TypeInfo ? undefined : original; - let originalTypeInfo = original instanceof TypeInfo ? original : original?.getTypeInfo(); - if (originalSerializer === undefined && originalTypeInfo === undefined) { - originalSerializer = this.serializerByTypeMeta(typeMeta); - originalTypeInfo = originalSerializer?.getTypeInfo(); - } - if (originalSerializer === undefined) { - originalTypeInfo ??= TypeId.isNamedType(typeId) - ? Type.struct( - { - typeName: typeMeta.getTypeName(), - namespace: typeMeta.getNs(), - }, - {}, - ) - : Type.struct(typeMeta.getUserTypeId(), {}); - originalSerializer = this.typeResolver.generateReadSerializer(originalTypeInfo); - } - // This legacy direct-generation API accepts caller-owned metadata. It must - // not publish into the checked wire cache; only the validated read path can - // do that after binding a concrete local TypeMeta owner. - return this.generateTypeMetaSerializer(typeMeta, originalSerializer); - } - private generateTypeMetaSerializer(typeMeta: TypeMeta, original: Serializer) { const typeId = typeMeta.getTypeId(); if (!TypeId.structType(typeId)) { diff --git a/javascript/test/typemeta.test.ts b/javascript/test/typemeta.test.ts index 2038f28094..5f4117f82f 100644 --- a/javascript/test/typemeta.test.ts +++ b/javascript/test/typemeta.test.ts @@ -620,29 +620,6 @@ describe("typemeta", () => { ); }); - test("direct TypeMeta generation keeps its public adapters", () => { - const writerFory = new Fory({ compatible: true }); - const readerFory = new Fory({ compatible: true }); - const remoteTypeMeta = TypeMeta.fromTypeInfo( - Type.struct(7020, { value: Type.string().setId(1) }), - (writerFory as any).typeResolver, - ); - const localTypeInfo = Type.struct(7020, { value: Type.int32().setId(1) }); - const context = (readerFory as any).readContext as ReadContext; - - expect( - context.genSerializerByTypeMetaRuntime(remoteTypeMeta, localTypeInfo, 123), - ).toBeDefined(); - expect(context.genSerializerByTypeMetaRuntime(remoteTypeMeta)).toBeDefined(); - expect((context as any).compatibleReadSerializers.size).toBe(0); - - const localTypeMeta = TypeMeta.fromTypeInfo(localTypeInfo, (readerFory as any).typeResolver); - context.reset(typeMetaRecord(remoteTypeMeta)); - expect( - context.readCompatibleStructSerializer(localTypeMeta.getHash(), localTypeInfo), - ).toBeDefined(); - }); - test("generated named enum validates TypeMeta owner", () => { const colorInfo = Type.enum({ namespace: "example", typeName: "Color" }, { Red: 0 }); const otherInfo = Type.enum({ namespace: "example", typeName: "Other" }, { Blue: 0 }); From eb8f1b5f49aabe53ed494f42e2cfe065a9937934 Mon Sep 17 00:00:00 2001 From: chaokunyang Date: Sat, 29 Aug 2026 02:38:11 +0800 Subject: [PATCH 062/168] fix(javascript): reject schema identity conflicts --- .agents/languages/javascript.md | 9 +- .../javascript/type-registration.md | 3 +- javascript/packages/core/lib/gen/index.ts | 111 +++++++++++-- javascript/test/fory.test.ts | 146 +++++++++++++++--- 4 files changed, 232 insertions(+), 37 deletions(-) diff --git a/.agents/languages/javascript.md b/.agents/languages/javascript.md index 2f1dca891e..32e933e61b 100644 --- a/.agents/languages/javascript.md +++ b/.agents/languages/javascript.md @@ -21,11 +21,14 @@ Load this file when changing `javascript/`. generation-local owners before one `TypeResolver` batch publication. The package-internal schema seal must lock each `TypeInfo` schema pointer before reading or traversing it; schema fields and occurrence modifiers are immutable afterward, while `dynamicTypeId` remains operation-local - writer state. Seed every complete Struct definition in the sealed graph before resolving - identity-only occurrences, so definition order cannot affect recursive resolution. Each resolver + writer state. Seed every complete Struct, enum, and union definition in the sealed graph before + resolving identity-only occurrences, so definition order cannot affect recursive resolution. + One numeric ID or name cannot identify different user-defined type families. Each resolver identity has one complete schema owner in that graph. Repeated references and clones may share the same immutable definition containers and settings; reject a second conflicting definition - before code generation without deep schema comparison. One authoritative serializer owner + before code generation without deep schema comparison. Anonymous user-defined values without a + name or user ID do not share registry identity merely because their raw type IDs match. One + authoritative serializer owner supplies both generator-time schema/progress facts and fixed factory captures; an initialized owner published by a nested registration wins over an outer generation-local owner, while field occurrence modifiers remain field-owned. Reject any unresolved nested identity before resolver diff --git a/docs/object-serialization/javascript/type-registration.md b/docs/object-serialization/javascript/type-registration.md index 690fd1417a..f2a267635c 100644 --- a/docs/object-serialization/javascript/type-registration.md +++ b/docs/object-serialization/javascript/type-registration.md @@ -156,7 +156,8 @@ A nested Struct that supplies only an ID or name must already be registered. Alt a complete definition with the same identity anywhere in the recursive `TypeInfo` graph; the definition may appear before or after the identity-only use. Registration rejects an unresolved nested identity. Each identity can have only one complete definition in that graph. Repeated uses -may share that definition, but separate conflicting definitions are rejected. +may share that definition, but separate conflicting definitions are rejected. One numeric ID or +name also cannot identify different Struct, enum, extension, or union families. ## Field Metadata diff --git a/javascript/packages/core/lib/gen/index.ts b/javascript/packages/core/lib/gen/index.ts index e25d793d83..b1e9e1a41a 100644 --- a/javascript/packages/core/lib/gen/index.ts +++ b/javascript/packages/core/lib/gen/index.ts @@ -194,23 +194,70 @@ export class Gen { ) { return left.userTypeId === right.userTypeId; } + if (TypeId.userDefinedType(leftTypeId) || TypeId.userDefinedType(rightTypeId)) { + return left === right; + } return leftTypeId === rightTypeId; } - private sameStructDefinition(left: TypeInfo, right: TypeInfo) { + private sameTypeFamily(left: TypeInfo, right: TypeInfo) { + const leftTypeId = left.typeId; + const rightTypeId = right.typeId; + if (TypeId.structType(leftTypeId) && TypeId.structType(rightTypeId)) { + return true; + } + if (TypeId.enumType(leftTypeId) && TypeId.enumType(rightTypeId)) { + return true; + } + if (TypeId.extType(leftTypeId) && TypeId.extType(rightTypeId)) { + return true; + } + const leftUnion = + leftTypeId === TypeId.UNION || + leftTypeId === TypeId.TYPED_UNION || + leftTypeId === TypeId.NAMED_UNION; + const rightUnion = + rightTypeId === TypeId.UNION || + rightTypeId === TypeId.TYPED_UNION || + rightTypeId === TypeId.NAMED_UNION; + return leftUnion && rightUnion; + } + + private hasCompleteDefinition(typeInfo: TypeInfo) { + const options = typeInfo.options; + if (TypeId.structType(typeInfo.typeId)) { + return options?.props !== undefined; + } + if (TypeId.enumType(typeInfo.typeId)) { + return options?.enumProps !== undefined; + } + if (TypeId.extType(typeInfo.typeId)) { + return options !== undefined; + } + return ( + (typeInfo.typeId === TypeId.UNION || + typeInfo.typeId === TypeId.TYPED_UNION || + typeInfo.typeId === TypeId.NAMED_UNION) && + options?.cases !== undefined + ); + } + + private sameDefinition(left: TypeInfo, right: TypeInfo) { if (left === right) { return true; } const leftOptions = left.options!; const rightOptions = right.options!; return ( - left.typeId === right.typeId && + this.sameTypeFamily(left, right) && left.named === right.named && left.namespace === right.namespace && left.typeName === right.typeName && left.userTypeId === right.userTypeId && left.evolving === right.evolving && leftOptions.props === rightOptions.props && + leftOptions.enumProps === rightOptions.enumProps && + leftOptions.cases === rightOptions.cases && leftOptions.fieldEntries === rightOptions.fieldEntries && leftOptions.preserveFieldOrder === rightOptions.preserveFieldOrder && leftOptions.withConstructor === rightOptions.withConstructor && @@ -218,8 +265,37 @@ export class Gen { ); } + private checkTypeFamily(owner: TypeInfo, typeInfo: TypeInfo) { + if ( + TypeId.userDefinedType(typeInfo.typeId) && + (!TypeId.userDefinedType(owner.typeId) || !this.sameTypeFamily(owner, typeInfo)) + ) { + throw new Error("conflicting type families for the same registry identity"); + } + } + + private checkDefinitionOwner(owner: TypeInfo, typeInfo: TypeInfo) { + if (!TypeId.userDefinedType(typeInfo.typeId)) { + return; + } + this.checkTypeFamily(owner, typeInfo); + if ( + this.hasCompleteDefinition(owner) && + this.hasCompleteDefinition(typeInfo) && + !this.sameDefinition(owner, typeInfo) + ) { + throw new Error("conflicting complete definitions for the same registry identity"); + } + } + private findRegistration(typeInfo: TypeInfo, registrations: GeneratedRegistration[]) { - return registrations.find((entry) => this.sameRegistration(entry.typeInfo, typeInfo)); + const registration = registrations.find((entry) => + this.sameRegistration(entry.typeInfo, typeInfo), + ); + if (registration !== undefined) { + this.checkDefinitionOwner(registration.typeInfo, typeInfo); + } + return registration; } private addRegistration(typeInfo: TypeInfo, registrations: GeneratedRegistration[]) { @@ -235,10 +311,12 @@ export class Gen { } private getGeneratedSerializer(typeInfo: TypeInfo, registrations: GeneratedRegistration[]) { - return ( - this.typeResolver.getSerializerByTypeInfo(typeInfo) ?? - this.findRegistration(typeInfo, registrations)?.serializer - ); + const published = this.typeResolver.getSerializerByTypeInfo(typeInfo); + if (published !== undefined) { + this.checkTypeFamily(published.getTypeInfo(), typeInfo); + return published; + } + return this.findRegistration(typeInfo, registrations)?.serializer; } private getCapturedSerializerById( @@ -320,15 +398,13 @@ export class Gen { seen.add(typeInfo); const options = typeInfo.options; if ( - TypeId.structType(typeInfo.typeId) && - options?.props !== undefined && - !this.typeResolver.getSerializerByTypeInfo(typeInfo)?._initialized + !TypeId.extType(typeInfo.typeId) && + this.hasCompleteDefinition(typeInfo) && + !this.getGeneratedSerializer(typeInfo, registrations)?._initialized ) { const registration = this.findRegistration(typeInfo, registrations); if (registration === undefined) { this.addRegistration(typeInfo, registrations); - } else if (!this.sameStructDefinition(registration.typeInfo, typeInfo)) { - throw new Error("conflicting complete struct definitions for the same registry identity"); } } if (options === undefined) { @@ -355,6 +431,11 @@ export class Gen { pending.push(options.value); } } + for (const typeInfo of seen) { + if (TypeId.userDefinedType(typeInfo.typeId)) { + this.findRegistration(typeInfo, registrations); + } + } } private traversalContainer( @@ -455,6 +536,12 @@ export class Gen { // Generated factories may execute application-transformed code, so every factory completes // against local owners before the resolver performs the only global publication step. + for (const registration of registrations) { + const published = this.typeResolver.getSerializerByTypeInfo(registration.typeInfo); + if (published !== undefined) { + this.checkTypeFamily(published.getTypeInfo(), registration.typeInfo); + } + } this.typeResolver.commitGeneratedSerializers(registrations); return this.typeResolver.getSerializerByTypeInfo(typeInfo)!; } diff --git a/javascript/test/fory.test.ts b/javascript/test/fory.test.ts index 0ba38905de..fe08c52606 100644 --- a/javascript/test/fory.test.ts +++ b/javascript/test/fory.test.ts @@ -338,30 +338,134 @@ describe("fory", () => { }); test("rejects conflicting definitions", () => { - for (const reversed of [false, true]) { - let generated = 0; - const fory = new Fory({ - compatible: false, - hooks: { - afterCodeGenerated(code) { - generated++; - return code; - }, - }, - }); - generated = 0; - const first = Type.struct(8127, { firstValue: Type.int32() }); - const second = Type.struct(8127, { secondValue: Type.string() }); - const props = reversed ? { second, first } : { first, second }; - const root = Type.struct(8128, props); - - expect(() => fory.register(root)).toThrow(); - expect(generated).toBe(0); - expect(fory.typeResolver.getSerializerById(TypeId.STRUCT, 8127)).toBeUndefined(); - expect(fory.typeResolver.getSerializerById(TypeId.STRUCT, 8128)).toBeUndefined(); + const identities: (number | { namespace: string; typeName: string })[] = [ + 8127, + { namespace: "test", typeName: "Conflict" }, + ]; + for (const identity of identities) { + const definitionPairs = [ + [ + Type.struct(identity, { firstValue: Type.int32() }), + Type.struct(identity, { secondValue: Type.string() }), + ], + [Type.enum(identity, { FIRST: 1 }), Type.enum(identity, { FIRST: 1 })], + [Type.union(identity, { 1: Type.int32() }), Type.union(identity, { 1: Type.int32() })], + ]; + for (const [first, second] of definitionPairs) { + for (const reversed of [false, true]) { + let generated = 0; + const fory = new Fory({ + compatible: false, + hooks: { + afterCodeGenerated(code) { + generated++; + return code; + }, + }, + }); + generated = 0; + const props = reversed ? { second, first } : { first, second }; + + expect(() => fory.register(Type.struct(8128, props))).toThrow(); + expect(generated).toBe(0); + expect(fory.typeResolver.getSerializerById(TypeId.STRUCT, 8128)).toBeUndefined(); + } + } + } + }); + + test("rejects mixed type families", () => { + const identities: (number | { namespace: string; typeName: string })[] = [ + 8129, + { namespace: "test", typeName: "Mixed" }, + ]; + for (const identity of identities) { + const types = [ + Type.struct(identity, { value: Type.int32() }), + Type.enum(identity, { VALUE: 1 }), + Type.ext(identity), + Type.union(identity, { 1: Type.string() }), + ]; + for (let left = 0; left < types.length; left++) { + for (let right = left + 1; right < types.length; right++) { + for (const reversed of [false, true]) { + let generated = 0; + const fory = new Fory({ + compatible: false, + hooks: { + afterCodeGenerated(code) { + generated++; + return code; + }, + }, + }); + generated = 0; + const first = types[reversed ? right : left]; + const second = types[reversed ? left : right]; + + expect(() => fory.register(Type.struct(8130, { first, second }))).toThrow(); + expect(generated).toBe(0); + expect(fory.typeResolver.getSerializerById(TypeId.STRUCT, 8130)).toBeUndefined(); + } + } + } } }); + test("reuses shared definitions", () => { + const struct = Type.struct(8131, { value: Type.int32() }); + const enumType = Type.enum(8132, { VALUE: 1 }); + const union = Type.union(8133, { 1: Type.string() }); + const registered = new Fory({ compatible: false }).register( + Type.struct(8134, { + firstStruct: struct, + secondStruct: struct.clone(), + firstEnum: enumType, + secondEnum: enumType.clone(), + firstUnion: union, + secondUnion: union.clone(), + }), + ); + const value = { + firstStruct: { value: 1 }, + secondStruct: { value: 2 }, + firstEnum: 1, + secondEnum: 1, + firstUnion: { case: 1, value: "first" }, + secondUnion: { case: 1, value: "second" }, + }; + + expect(registered.deserialize(registered.serialize(value as any))).toEqual(value); + }); + + test("rejects reentrant family conflict", () => { + let publishConflict = false; + let fory: Fory; + fory = new Fory({ + compatible: false, + hooks: { + afterCodeGenerated(code) { + if (publishConflict) { + publishConflict = false; + fory.register(Type.enum(8135, { VALUE: 1 })); + } + return code; + }, + }, + }); + publishConflict = true; + + expect(() => + fory.register( + Type.struct(8136, { + value: Type.struct(8135, { value: Type.int32() }), + }), + ), + ).toThrow("conflicting type families"); + expect(fory.typeResolver.getSerializerById(TypeId.ENUM, 8135)).toBeDefined(); + expect(fory.typeResolver.getSerializerById(TypeId.STRUCT, 8136)).toBeUndefined(); + }); + test("seals replaced schema options", () => { const original = Type.struct(8126, { oldValue: Type.int32() }); const replacement: { From f98e52239733813204dd47d18f82e9e5a8c24f52 Mon Sep 17 00:00:00 2001 From: chaokunyang Date: Sat, 29 Aug 2026 02:38:47 +0800 Subject: [PATCH 063/168] docs: align lifecycle ownership rules --- AGENTS.md | 33 ++++++++++++------- .../python/type-registration.md | 5 +-- docs/security/deserialization.md | 2 +- .../xlang_implementation_guide.md | 25 ++++++++------ 4 files changed, 41 insertions(+), 24 deletions(-) diff --git a/AGENTS.md b/AGENTS.md index 987887c55f..aec2f072fb 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -166,10 +166,16 @@ This is the entry point for AI guidance in Apache Fory. Read this file first, th - JavaScript generated registration must build and initialize the complete recursive serializer graph against generation-local owners before one `TypeResolver` batch publication. Factory-init serializer lookup may see those local owners, but runtime and dynamic lookup must retain the real - resolver. A nested identity-only Struct must already have a fully initialized registered owner or - resolve to an owner in the current complete recursive schema graph; otherwise registration fails - before resolver publication. Do not publish placeholders, nested serializers, descriptors, or - cache state before every generated factory and application code hook succeeds. + resolver. Seal schema definitions before traversal and preseed complete definitions by identity, + so field order cannot affect recursive resolution. One numeric ID or name cannot identify + different user-defined type families. Each identity has one complete schema owner; repeated clones + are valid only when they share that owner's definition containers and settings. A nested + identity-only user-defined type must already have a fully initialized registered owner or resolve + to an owner in the current complete recursive schema graph; otherwise registration fails before + resolver publication. Anonymous user-defined values without a registry key remain distinct. Once + initialized, the published nested owner is authoritative. Do not publish placeholders, nested + serializers, descriptors, or cache state before every generated factory and application code hook + succeeds. - Root failure exceptions must not copy or retain the operation reference table or materialized object graph for diagnostics. Root cleanup owns releasing that graph, and failure reporting must remain bounded independently of graph size. @@ -198,9 +204,11 @@ This is the entry point for AI guidance in Apache Fory. Read this file first, th prepares, then recheck the authoritative per-instance freeze owner immediately before publication. Kotlin and Scala combined generated-struct registration are the sole type-first exception: publish the canonical type needed by generated - serializer construction, then recheck after construction and before replacing - its serializer. Do not add rollback, staging, or a parallel registration path - for this exception. A module installation may perform complete nested + serializer construction, then enter the existing resolver construction graph. + Keep the candidate unpublished until the authoritative lifecycle recheck and + install it through the normal Class/Xtype commit sink. Do not add direct + replacement, rollback, staging, or a parallel registration path for this exception. + A module installation may perform complete nested registrations. Java `Fory.register(ForyModule)` owns cycle breaking and idempotence in one identity set: add the identity before installation, remove it on failure, and retain it on success; do not add parallel installing and @@ -223,10 +231,13 @@ This is the entry point for AI guidance in Apache Fory. Read this file first, th but the `Fory` facade must not mirror that state; Cython roots call the resolver owner directly. Allocate automatic type IDs only after callback preparation and the final freeze recheck, at the common registry publication point; do not reserve IDs early or maintain counter rollback state. - `ThreadSafeFory` validates registrations before publishing replay callbacks and never invokes an - application factory or callback under its non-reentrant pool lock. Reject root reentry from the - active instance-build thread before looking in the pool, including when another thread returned - an instance during the build. + `ThreadSafeFory` validates registrations before retaining semantic replay descriptors and never + invokes an application factory or callback under its non-reentrant pool lock. During child replay, + a nested request is a no-op only when it exactly matches the accepted prefix already applied to + that child; reject unknown or different requests before that request mutates the child. Serializer + factories must return carriers bound to the current child resolver and normalized declared type. + Reject root reentry from the active instance-build thread before looking in the pool, including + when another thread returned an instance during the build. - Use semantic naming only. Name things after protocol or domain concepts, not history, runtime origin, or workaround style; avoid vague names such as `Internal`, `java_style_*`, `Runtime`, `Session`, `Plan`, `Payload`, or `Binding` when they do not name the real concept. Keep class, method, function, and variable names concise; do not encode the whole scenario or implementation history into one identifier. Never name a class or method with a `Plan` suffix; use the real domain concept instead. For Fory codec/read APIs, do not use generic `payload` naming; name the exact owner and data shape, such as bytes, body, frame, field, string, list, map, compressed bytes, or primitive-array encoding. - Keep one implementation path. Do not keep parallel helpers, serializers, harnesses, wrappers, or registration flows for the same concept; extend the existing owner path instead of inventing another one. - Follow current scope exactly. The latest explicit user instruction overrides earlier plans, and when scope narrows, remove leaked out-of-scope edits immediately. diff --git a/docs/object-serialization/python/type-registration.md b/docs/object-serialization/python/type-registration.md index 2c18806ad5..3a4ca96959 100644 --- a/docs/object-serialization/python/type-registration.md +++ b/docs/object-serialization/python/type-registration.md @@ -75,8 +75,9 @@ for model_class in [User, Order, Product, Invoice]: A direct `Fory` may receive a serializer instance. `ThreadSafeFory` accepts a serializer class or factory so every pooled child constructs a serializer against its own resolver. Use `fory_factory` for serializer instances that need per-child configuration. A serializer factory -must return a serializer for the resolver and declared type passed to that invocation; it must not -reuse one serializer instance across pooled children. +may accept `(resolver, type)`, `(resolver)`, or no arguments, but it must return a serializer bound +to that invocation's child resolver and normalized registered type. It must not reuse one serializer +instance across pooled children. ## Strict Mode Relationship diff --git a/docs/security/deserialization.md b/docs/security/deserialization.md index dfe4fd2540..5e0cdbfbc2 100644 --- a/docs/security/deserialization.md +++ b/docs/security/deserialization.md @@ -622,7 +622,7 @@ operation or registry finalization fails. Registration that invokes application the authoritative lifecycle before publishing application-derived state. Thread-safe facades retain only registrations that completed before the freeze. During child construction, a semantic replay log may reuse only an identical accepted registration that the child has already applied; -unknown or different requests fail before mutating the child. A facade that replays opaque +unknown or different requests fail before that request mutates the child. A facade that replays opaque registration callbacks and cannot roll them back must become permanently unusable when a callback fails rather than expose partially registered children. These rules prevent a failed or reentrant registration from changing the accepted type surface after deserialization has begun. diff --git a/docs/specification/xlang_implementation_guide.md b/docs/specification/xlang_implementation_guide.md index 8009d133dc..0abe101700 100644 --- a/docs/specification/xlang_implementation_guide.md +++ b/docs/specification/xlang_implementation_guide.md @@ -109,10 +109,13 @@ C# `ThreadSafeFory` validates registration on its staging runtime and publishes replay action. The resolver prepares serializer bindings and encoded names before one map commit; a failed callback does not require rebuilding or replacing the staging runtime. -Python `ThreadSafeFory` likewise validates callbacks before retaining them. Retained callbacks may -capture a serializer class or factory so every child constructs a serializer against its own -resolver; they must not replay one resolver-bound serializer instance. Use the existing -`fory_factory` when each child needs a separately configured serializer instance. +Python `ThreadSafeFory` validates registrations before retaining semantic replay descriptors. A +later child applies the accepted descriptor prefix in order. A nested replay request is a no-op +only when it exactly matches an accepted descriptor already applied to that child; an unknown or +different request fails before that request mutates the child. Retained descriptors may contain a +serializer class or factory, but every result must belong to the current child resolver and +normalized declared type; they must not reuse one resolver-bound serializer instance. Use the +existing `fory_factory` when each child needs a separately configured serializer instance. Java `TypeResolver` owns one construction-local graph for serializer constructors, including self and mutual recursion. The graph separates final `TypeInfo` owners from unpublished serializer @@ -132,12 +135,14 @@ install modules only through `ForyBuilder.withModule` during construction. JavaScript generated registration seals the complete `TypeInfo` schema graph before code generation, including nested schemas and field occurrence modifiers. The package-internal seal locks each schema-owned pointer before reading or traversing it. The writer-owned `dynamicTypeId` -remains mutable because it is reset per root. Code generation seeds every complete Struct -definition by registry identity before resolving identity-only occurrences, so recursive schema -resolution does not depend on field order. Each resolver identity has one complete schema owner in -the graph. Repeated references and clones may share that owner's immutable definition containers -and settings, while a second conflicting complete definition fails before code generation without -a deep structural comparison. Code generation then constructs and initializes the complete +remains mutable because it is reset per root. Code generation seeds every complete Struct, enum, +and union definition by registry identity before resolving identity-only occurrences, so recursive +schema resolution does not depend on field order. One numeric ID or name cannot identify different +user-defined type families. Each resolver identity has one complete schema owner in the graph. +Repeated references and clones may share that owner's immutable definition containers and settings, +while a second conflicting complete definition fails before code generation without a deep +structural comparison. Anonymous user-defined values without a name or user ID remain distinct. +Code generation then constructs and initializes the complete serializer graph against generation-local owners. The same authoritative owner supplies generator-time schema and progress facts and fixed factory captures; field occurrence modifiers remain owned by the containing schema. Runtime and dynamic dispatch retain the real From ef7176303ff4b1aca181be9ae1178e2217d07a4d Mon Sep 17 00:00:00 2001 From: chaokunyang Date: Sat, 29 Aug 2026 02:38:59 +0800 Subject: [PATCH 064/168] chore(scala): restore unchanged test spacing --- .../test/scala/org/apache/fory/serializer/scala/ScalaTest.scala | 1 + 1 file changed, 1 insertion(+) diff --git a/scala/fory-scala/src/test/scala/org/apache/fory/serializer/scala/ScalaTest.scala b/scala/fory-scala/src/test/scala/org/apache/fory/serializer/scala/ScalaTest.scala index ab03794c02..fccc21e8e1 100644 --- a/scala/fory-scala/src/test/scala/org/apache/fory/serializer/scala/ScalaTest.scala +++ b/scala/fory-scala/src/test/scala/org/apache/fory/serializer/scala/ScalaTest.scala @@ -74,6 +74,7 @@ class ScalaTest extends AnyWordSpec with Matchers { } } + package object PkgObject { case class Id(value: Int) case class IdAnyVal(value: Int) extends AnyVal From d24d85f951a390a981ef99ba33e73c34c95faf90 Mon Sep 17 00:00:00 2001 From: chaokunyang Date: Sat, 29 Aug 2026 02:58:23 +0800 Subject: [PATCH 065/168] fix(javascript): keep schema owners authoritative --- .agents/languages/javascript.md | 13 +- AGENTS.md | 10 +- .../javascript/type-registration.md | 5 + .../xlang_implementation_guide.md | 11 +- javascript/packages/core/lib/gen/index.ts | 41 ++++-- javascript/packages/core/lib/type.ts | 2 + javascript/test/fory.test.ts | 127 ++++++++++++++++++ javascript/test/map.test.ts | 5 +- javascript/test/union.test.ts | 26 ++++ 9 files changed, 218 insertions(+), 22 deletions(-) diff --git a/.agents/languages/javascript.md b/.agents/languages/javascript.md index 32e933e61b..a1ba6f035a 100644 --- a/.agents/languages/javascript.md +++ b/.agents/languages/javascript.md @@ -26,14 +26,17 @@ Load this file when changing `javascript/`. One numeric ID or name cannot identify different user-defined type families. Each resolver identity has one complete schema owner in that graph. Repeated references and clones may share the same immutable definition containers and settings; reject a second conflicting definition - before code generation without deep schema comparison. Anonymous user-defined values without a - name or user ID do not share registry identity merely because their raw type IDs match. One + before code generation without deep schema comparison. Complete anonymous definitions without a + name or user ID do not share registry identity merely because their raw type IDs match. The + definition-free generic enum or union serializer remains the canonical raw-type owner. One authoritative serializer owner supplies both generator-time schema/progress facts and fixed factory captures; an initialized owner published by a nested registration wins over an outer generation-local owner, while field - occurrence modifiers remain field-owned. Reject any unresolved nested identity before resolver - publication. Never publish generated serializers, descriptors, or cache state before every - generated factory and application code hook succeeds. + occurrence modifiers remain field-owned. Reject an unresolved nested Struct identity before + resolver publication. An enum without a mapping and a union without cases keep their existing + generic definitions; an extension reference resolves through its registered custom serializer + owner. Never publish generated serializers, descriptors, or cache state before every generated + factory and application code hook succeeds. - Runtime value carriers such as decimal or reduced-precision numeric types belong under the core `types/` ownership boundary, with imports, exports, and codegen externals updated together. - Keep `TypeInfo` as schema metadata. Compatibility-sensitive decisions belong on `TypeResolver` or explicit operations, not as retained resolver state on metadata objects. - Normalize optional boolean config values at config construction; do not carry `null` through runtime paths when it means `false`. diff --git a/AGENTS.md b/AGENTS.md index aec2f072fb..58b933b5c5 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -170,10 +170,12 @@ This is the entry point for AI guidance in Apache Fory. Read this file first, th so field order cannot affect recursive resolution. One numeric ID or name cannot identify different user-defined type families. Each identity has one complete schema owner; repeated clones are valid only when they share that owner's definition containers and settings. A nested - identity-only user-defined type must already have a fully initialized registered owner or resolve - to an owner in the current complete recursive schema graph; otherwise registration fails before - resolver publication. Anonymous user-defined values without a registry key remain distinct. Once - initialized, the published nested owner is authoritative. Do not publish placeholders, nested + identity-only Struct must already have a fully initialized registered owner or resolve to an owner + in the current complete recursive schema graph; otherwise registration fails before resolver + publication. Enum without a mapping and union without cases retain the canonical generic owner + for their raw wire type. Complete anonymous definitions without a registry key remain + generation-local and distinct. Once initialized, the published nested owner is authoritative. Do + not publish placeholders, nested serializers, descriptors, or cache state before every generated factory and application code hook succeeds. - Root failure exceptions must not copy or retain the operation reference table or materialized diff --git a/docs/object-serialization/javascript/type-registration.md b/docs/object-serialization/javascript/type-registration.md index f2a267635c..6105e78457 100644 --- a/docs/object-serialization/javascript/type-registration.md +++ b/docs/object-serialization/javascript/type-registration.md @@ -159,6 +159,11 @@ nested identity. Each identity can have only one complete definition in that gra may share that definition, but separate conflicting definitions are rejected. One numeric ID or name also cannot identify different Struct, enum, extension, or union families. +An anonymous union with declared cases has no registry identity. Keep and use the serializer pair +returned by `fory.register(...)`; registering another anonymous union does not replace or reuse the +earlier union's serializer. A union declared without cases remains an open union with the generic +union owner and reads each value from the type information carried by that union case. + ## Field Metadata Field nullability, reference tracking, dynamic field behavior, numeric widths, and per-struct diff --git a/docs/specification/xlang_implementation_guide.md b/docs/specification/xlang_implementation_guide.md index 0abe101700..c693b540aa 100644 --- a/docs/specification/xlang_implementation_guide.md +++ b/docs/specification/xlang_implementation_guide.md @@ -141,15 +141,20 @@ schema resolution does not depend on field order. One numeric ID or name cannot user-defined type families. Each resolver identity has one complete schema owner in the graph. Repeated references and clones may share that owner's immutable definition containers and settings, while a second conflicting complete definition fails before code generation without a deep -structural comparison. Anonymous user-defined values without a name or user ID remain distinct. +structural comparison. Complete anonymous definitions without a name or user ID remain distinct. +They stay in the current generation graph rather than publishing under their raw wire type ID. An +enum without a mapping and a union without cases use the canonical generic serializer for their raw +wire type; they are definitions, not unresolved schema references. An extension occurrence without +class metadata resolves through its registered custom serializer owner. Code generation then constructs and initializes the complete serializer graph against generation-local owners. The same authoritative owner supplies generator-time schema and progress facts and fixed factory captures; field occurrence modifiers remain owned by the containing schema. Runtime and dynamic dispatch retain the real `TypeResolver`. After every factory and application code hook succeeds, the resolver performs one guarded batch publication. An unresolved nested identity fails registration before resolver -publication. An initialized owner published by a nested registration is authoritative and must not -be overwritten or contradicted by the outer registration's generated decisions. +publication when its type family requires a separate definition, such as Struct. An initialized +owner published by a nested registration is authoritative and must not be overwritten or +contradicted by the outer registration's generated decisions. Nested serializers must not call back into root `serialize(...)` or `deserialize(...)` entry points. diff --git a/javascript/packages/core/lib/gen/index.ts b/javascript/packages/core/lib/gen/index.ts index b1e9e1a41a..94a06218a0 100644 --- a/javascript/packages/core/lib/gen/index.ts +++ b/javascript/packages/core/lib/gen/index.ts @@ -232,7 +232,7 @@ export class Gen { return options?.enumProps !== undefined; } if (TypeId.extType(typeInfo.typeId)) { - return options !== undefined; + return options?.props !== undefined || options?.creator !== undefined; } return ( (typeInfo.typeId === TypeId.UNION || @@ -242,6 +242,19 @@ export class Gen { ); } + private hasRegistryIdentity(typeInfo: TypeInfo) { + const typeId = this.typeResolver.computeTypeId(typeInfo); + if (!TypeId.userDefinedType(typeId) || TypeId.isNamedType(typeId)) { + return true; + } + if (TypeId.needsUserTypeId(typeId) && typeInfo.userTypeId !== -1) { + return true; + } + // A complete anonymous schema belongs to this generation graph. Only the definition-free + // generic serializer can be the canonical owner of a raw user-defined wire type ID. + return !this.hasCompleteDefinition(typeInfo); + } + private sameDefinition(left: TypeInfo, right: TypeInfo) { if (left === right) { return true; @@ -311,9 +324,11 @@ export class Gen { } private getGeneratedSerializer(typeInfo: TypeInfo, registrations: GeneratedRegistration[]) { - const published = this.typeResolver.getSerializerByTypeInfo(typeInfo); + const published = this.hasRegistryIdentity(typeInfo) + ? this.typeResolver.getSerializerByTypeInfo(typeInfo) + : undefined; if (published !== undefined) { - this.checkTypeFamily(published.getTypeInfo(), typeInfo); + this.checkDefinitionOwner(published.getTypeInfo(), typeInfo); return published; } return this.findRegistration(typeInfo, registrations)?.serializer; @@ -328,6 +343,9 @@ export class Gen { if (published !== undefined) { return published; } + if (id === TypeId.TYPED_UNION && (userTypeId === undefined || userTypeId === -1)) { + throw new Error("anonymous union serializer requires its TypeInfo owner"); + } const entry = registrations.find((candidate) => { const typeId = this.typeResolver.computeTypeId(candidate.typeInfo); if ( @@ -471,6 +489,10 @@ export class Gen { if (this.findRegistration(typeInfo, registrations) === undefined) { throw new Error("nested struct schema must be registered or defined before use"); } + } else if (TypeId.extType(typeInfo.typeId)) { + if (this.findRegistration(typeInfo, registrations) === undefined) { + throw new Error("nested extension serializer must be registered before use"); + } } else if (TypeId.enumType(typeInfo.typeId) && !this.isRegistered(typeInfo)) { this.prepareRegistration(typeInfo, [], registrations, serializerLookup); } @@ -523,8 +545,8 @@ export class Gen { this.addRegistration(typeInfo, registrations); } this.traversalContainer(typeInfo, registrations, serializerLookup); - const serializer = this.typeResolver.getSerializerByTypeInfo(typeInfo); - if (!serializer?._initialized) { + const publishedRoot = this.typeResolver.getSerializerByTypeInfo(typeInfo); + if (!publishedRoot?._initialized) { let registration = this.findRegistration(typeInfo, registrations); if (registration === undefined) { registration = this.addRegistration(typeInfo, registrations); @@ -539,10 +561,13 @@ export class Gen { for (const registration of registrations) { const published = this.typeResolver.getSerializerByTypeInfo(registration.typeInfo); if (published !== undefined) { - this.checkTypeFamily(published.getTypeInfo(), registration.typeInfo); + this.checkDefinitionOwner(published.getTypeInfo(), registration.typeInfo); } } - this.typeResolver.commitGeneratedSerializers(registrations); - return this.typeResolver.getSerializerByTypeInfo(typeInfo)!; + const serializer = this.getGeneratedSerializer(typeInfo, registrations)!; + this.typeResolver.commitGeneratedSerializers( + registrations.filter((registration) => this.hasRegistryIdentity(registration.typeInfo)), + ); + return serializer; } } diff --git a/javascript/packages/core/lib/type.ts b/javascript/packages/core/lib/type.ts index 3f2ecc998a..73f0ab34c6 100644 --- a/javascript/packages/core/lib/type.ts +++ b/javascript/packages/core/lib/type.ts @@ -156,6 +156,8 @@ export const TypeId = { TypeId.NAMED_COMPATIBLE_STRUCT, TypeId.EXT, TypeId.NAMED_EXT, + TypeId.TYPED_UNION, + TypeId.NAMED_UNION, ].includes(id as any); }, structType(id: number) { diff --git a/javascript/test/fory.test.ts b/javascript/test/fory.test.ts index fe08c52606..0de7e7e8f6 100644 --- a/javascript/test/fory.test.ts +++ b/javascript/test/fory.test.ts @@ -208,6 +208,15 @@ describe("fory", () => { expect(parent.deserialize(parent.serialize(value))).toEqual(value); }); + test("rejects unresolved extension", () => { + const fory = new Fory({ compatible: false }); + const root = Type.struct(8151, { value: Type.ext(8152) }); + + expect(() => fory.register(root)).toThrow(); + expect(fory.typeResolver.getSerializerById(TypeId.STRUCT, 8151)).toBeUndefined(); + expect(fory.typeResolver.getSerializerById(TypeId.EXT, 8152)).toBeUndefined(); + }); + test("registers empty roots", () => { const registered = new Fory({ compatible: false }).register(Type.struct(8122, {})); @@ -466,6 +475,124 @@ describe("fory", () => { expect(fory.typeResolver.getSerializerById(TypeId.STRUCT, 8136)).toBeUndefined(); }); + test("rejects published schema conflicts", () => { + const sharedProps = { value: Type.int32() }; + const definitionPairs: [TypeInfo, TypeInfo][] = [ + [Type.struct(8137, { first: Type.int32() }), Type.struct(8137, { second: Type.string() })], + [ + Type.struct({ typeId: 8138, evolving: false }, sharedProps), + Type.struct({ typeId: 8138, evolving: true }, sharedProps), + ], + [Type.enum(8139, { FIRST: 1 }), Type.enum(8139, { SECOND: 2 })], + [Type.union(8140, { 1: Type.int32() }), Type.union(8140, { 2: Type.string() })], + ]; + + for (const [first, second] of definitionPairs) { + let generated = 0; + const fory = new Fory({ + compatible: false, + hooks: { + afterCodeGenerated(code) { + generated++; + return code; + }, + }, + }); + const registered = fory.register(first); + generated = 0; + + expect(() => fory.register(second)).toThrow("conflicting complete definitions"); + expect(generated).toBe(0); + expect(fory.typeResolver.getSerializerByTypeInfo(first)).toBe(registered.serializer); + } + }); + + test("keeps extension owner", () => { + class FirstExtension { + value = 0; + } + class SecondExtension { + value = 0; + } + Type.ext(8144)(FirstExtension); + Type.ext(8144)(SecondExtension); + const customSerializer = { + write(writeContext: any, value: FirstExtension | SecondExtension) { + writeContext.writeVarInt32(value.value); + }, + read(readContext: any, value: FirstExtension | SecondExtension) { + value.value = readContext.readVarInt32(); + }, + }; + const fory = new Fory({ compatible: false }); + const extension = fory.register(FirstExtension, customSerializer); + const wrapper = fory.register(Type.struct(8145, { value: Type.ext(8144) })); + + expect(() => fory.register(SecondExtension, customSerializer)).toThrow( + "conflicting complete definitions", + ); + expect(fory.typeResolver.getSerializerById(TypeId.EXT, 8144)).toBe(extension.serializer); + const value = new FirstExtension(); + value.value = 7; + expect(wrapper.serializer).toBeDefined(); + expect(extension.deserialize(extension.serialize(value))).toEqual(value); + }); + + test("uses published schema owners", () => { + const fory = new Fory({ compatible: false }); + fory.register(Type.enum(8146, { VALUE: 7 })); + fory.register(Type.union(8147, { 1: Type.string() })); + const registered = fory.register( + Type.struct(8148, { + enumValue: Type.enum(8146), + unionValue: Type.union(8147), + }), + ); + const value = { enumValue: 7, unionValue: { case: 1, value: "value" } }; + + expect(registered.deserialize(registered.serialize(value))).toEqual(value); + }); + + test("keeps open enum and union", () => { + const fory = new Fory({ compatible: false }); + const enumType = fory.register(Type.enum(8149)); + const unionType = fory.register(Type.union(8150)); + const unionValue = { case: 1, value: "value" }; + + expect(enumType.deserialize(enumType.serialize(7))).toBe(7); + expect(unionType.deserialize(unionType.serialize(unionValue))).toEqual(unionValue); + }); + + test("rejects reentrant schema conflict", () => { + let publishConflict = false; + let reentrant: ReturnType; + let fory: Fory; + fory = new Fory({ + compatible: false, + hooks: { + afterCodeGenerated(code) { + if (publishConflict) { + publishConflict = false; + reentrant = fory.register(Type.struct(8141, { inner: Type.string() })); + } + return code; + }, + }, + }); + publishConflict = true; + + expect(() => + fory.register( + Type.struct(8142, { + trigger: Type.struct(8143, { value: Type.int32() }), + value: Type.struct(8141, { outer: Type.int32() }), + }), + ), + ).toThrow("conflicting complete definitions"); + expect(fory.typeResolver.getSerializerById(TypeId.STRUCT, 8141)).toBe(reentrant!.serializer); + expect(fory.typeResolver.getSerializerById(TypeId.STRUCT, 8142)).toBeUndefined(); + }); + test("seals replaced schema options", () => { const original = Type.struct(8126, { oldValue: Type.int32() }); const replacement: { diff --git a/javascript/test/map.test.ts b/javascript/test/map.test.ts index dac77e69a7..59e4c0ddd8 100644 --- a/javascript/test/map.test.ts +++ b/javascript/test/map.test.ts @@ -205,15 +205,16 @@ describe("map", () => { }, }, }); - const outerType = Type.struct({ typeId: 350, evolving: false }, { outerValue: Type.int32() }); registerSameKey = true; const registered = fory.register( Type.struct(351, { - outer: outerType.setId(2), + trigger: Type.struct(352, { value: Type.int32() }).setId(3), + outer: Type.struct(350).setId(2), values: Type.map(Type.struct(350), Type.struct(350)).setId(1), }), ); const value = { + trigger: { value: 1 }, outer: { innerValue: "outer" }, values: new Map([[{ innerValue: "key" }, { innerValue: "value" }]]), }; diff --git a/javascript/test/union.test.ts b/javascript/test/union.test.ts index a8b4c11df4..cf46630041 100644 --- a/javascript/test/union.test.ts +++ b/javascript/test/union.test.ts @@ -18,6 +18,7 @@ */ import Fory, { Type } from "../packages/core/index"; +import { TypeId } from "../packages/core/lib/type"; import { describe, expect, test } from "@jest/globals"; describe("union", () => { @@ -219,4 +220,29 @@ describe("union", () => { expect(result.value).toBe(result); expect(readContext.getReadRef(0)).toBe(result); }); + + test("keeps anonymous union owners", () => { + const fory = new Fory({ compatible: false, ref: true }); + const first = fory.register(Type.union({ 1: Type.string() })); + const second = fory.register(Type.union({ 2: Type.int32() })); + expect((fory as any).typeResolver.getSerializerById(TypeId.TYPED_UNION)).toBeUndefined(); + const open = fory.register(Type.union()); + const combined = fory.register( + Type.struct(702, { + first: Type.union({ 1: Type.string() }), + second: Type.union({ 2: Type.int32() }), + }), + ); + const firstValue = { case: 1, value: "first" }; + const secondValue = { case: 2, value: 42 }; + const openValue = { case: 3, value: "open" }; + const combinedValue = { first: firstValue, second: secondValue }; + + expect(first.serializer).not.toBe(second.serializer); + expect((fory as any).typeResolver.getSerializerById(TypeId.TYPED_UNION)).toBe(open.serializer); + expect(first.deserialize(first.serialize(firstValue))).toEqual(firstValue); + expect(second.deserialize(second.serialize(secondValue))).toEqual(secondValue); + expect(open.deserialize(open.serialize(openValue))).toEqual(openValue); + expect(combined.deserialize(combined.serialize(combinedValue))).toEqual(combinedValue); + }); }); From 7d9eba68f4fe09a9a9cff37cd559d610bfe5fa7e Mon Sep 17 00:00:00 2001 From: chaokunyang Date: Sat, 29 Aug 2026 03:00:54 +0800 Subject: [PATCH 066/168] docs: align root lifecycle ownership --- .agents/languages/python.md | 3 ++- docs/security/deserialization.md | 5 +++-- 2 files changed, 5 insertions(+), 3 deletions(-) diff --git a/.agents/languages/python.md b/.agents/languages/python.md index 26d9d5f4b6..322b08d80e 100644 --- a/.agents/languages/python.md +++ b/.agents/languages/python.md @@ -29,7 +29,8 @@ Load this file when changing `python/`, Cython serialization, or Python xlang be root-started state, registration depth, and the staging instance; the separate instance-build boundary covers the factory and complete registration replay. During child replay, a nested facade registration is a no-op only when it exactly matches an accepted descriptor in the prefix - already applied to that child; reject every unknown or different request before child mutation. + already applied to that child; reject every unknown or different request before that request + mutates the child. Retained descriptors may contain a serializer class or factory, but never a resolver-bound serializer instance. A serializer factory must return a supported serializer carrier bound to the provided child resolver and normalized declared type; singleton serializers cannot be shared diff --git a/docs/security/deserialization.md b/docs/security/deserialization.md index 5e0cdbfbc2..5904b45ec6 100644 --- a/docs/security/deserialization.md +++ b/docs/security/deserialization.md @@ -383,8 +383,9 @@ derived from input size, and stream budgeting should not depend on dynamic bytes Graph budget accounting should: -- be initialized in top-level read state, with cleanup owned by the top-level deserialization - `finally`; +- be initialized in top-level read state, with restoration owned by the runtime's root lifecycle + boundary before that read state is reused; runtimes may restore it in the root `finally` or in + the next root-entry reset according to their established context lifecycle; - account only for Fory-created objects or storage that are retained by the returned value graph; temporary helper objects used only during construction are outside the graph budget; From 24f02c9745f94c360388c7c679b9edd6f97fa1a1 Mon Sep 17 00:00:00 2001 From: chaokunyang Date: Sat, 29 Aug 2026 03:02:56 +0800 Subject: [PATCH 067/168] test(javascript): cover named schema conflicts --- javascript/test/fory.test.ts | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/javascript/test/fory.test.ts b/javascript/test/fory.test.ts index 0de7e7e8f6..9d6acf5327 100644 --- a/javascript/test/fory.test.ts +++ b/javascript/test/fory.test.ts @@ -485,6 +485,10 @@ describe("fory", () => { ], [Type.enum(8139, { FIRST: 1 }), Type.enum(8139, { SECOND: 2 })], [Type.union(8140, { 1: Type.int32() }), Type.union(8140, { 2: Type.string() })], + [ + Type.struct("test.PublishedOwner", { first: Type.int32() }), + Type.struct("test.PublishedOwner", { second: Type.string() }), + ], ]; for (const [first, second] of definitionPairs) { From efaf9315ec480886c78a0e46b1e7f34c9974a924 Mon Sep 17 00:00:00 2001 From: chaokunyang Date: Sat, 29 Aug 2026 03:11:52 +0800 Subject: [PATCH 068/168] fix(csharp): preserve registry identity owners --- .agents/languages/csharp.md | 4 + csharp/src/Fory/Fory.cs | 38 +++ csharp/src/Fory/TypeResolver.cs | 79 ++++++ .../tests/Fory.Tests/RuntimeEdgeCaseTests.cs | 241 ++++++++++++++++++ .../csharp/type-registration.md | 4 + 5 files changed, 366 insertions(+) diff --git a/.agents/languages/csharp.md b/.agents/languages/csharp.md index 6c68f1fc32..449a70451f 100644 --- a/.agents/languages/csharp.md +++ b/.agents/languages/csharp.md @@ -17,6 +17,10 @@ Load this file when changing `csharp/` or C# xlang behavior. `ThreadSafeFory` must recheck disposal and freeze after staging registration and before replay-log publication. Resolver registration must prepare all serializer and MetaString state before its single map commit, so failed validation needs no staging rebuild or identity workaround. + Before serializer resolution and again before commit, the resolver must enforce a one-to-one + mapping between CLR types and wire IDs or names. The same mapping is idempotent, including the + same concrete custom serializer when one is explicit; neither identity side nor an explicit + serializer may be rebound to a different owner. New per-thread runtimes replay that log; do not mutate existing runtimes or introduce another freeze owner. - Generated C# gRPC service companions are compiler-owned files that depend on application-provided gRPC packages, not `csharp/src/Fory`. Keep gRPC package references out of the Fory runtime package. diff --git a/csharp/src/Fory/Fory.cs b/csharp/src/Fory/Fory.cs index 2e86dae908..e5d8222cba 100644 --- a/csharp/src/Fory/Fory.cs +++ b/csharp/src/Fory/Fory.cs @@ -73,6 +73,11 @@ public static ForyBuilder Builder() public Fory Register(uint typeId) { EnsureRegistrationOpen(); + if (_typeResolver.CheckRegistration(typeof(T), typeId)) + { + return this; + } + TypeInfo typeInfo = PrepareRegistration(typeof(T)); _typeResolver.Register(typeof(T), typeId, typeInfo); return this; @@ -90,6 +95,11 @@ public Fory Register(string name) { EnsureRegistrationOpen(); (string namespaceName, string typeName) = TypeResolver.SplitTypeName(name); + if (_typeResolver.CheckRegistration(typeof(T), namespaceName, typeName)) + { + return this; + } + TypeInfo typeInfo = PrepareRegistration(typeof(T)); _typeResolver.Register(typeof(T), namespaceName, typeName, typeInfo); return this; @@ -108,6 +118,11 @@ public Fory Register(string typeNamespace, string typeName) { EnsureRegistrationOpen(); TypeResolver.ValidateSplitTypeName(typeNamespace, typeName); + if (_typeResolver.CheckRegistration(typeof(T), typeNamespace, typeName)) + { + return this; + } + TypeInfo typeInfo = PrepareRegistration(typeof(T)); _typeResolver.Register(typeof(T), typeNamespace, typeName, typeInfo); return this; @@ -126,6 +141,11 @@ public Fory Register(uint typeId) where TSerializer : Serializer, new() { EnsureRegistrationOpen(); + if (_typeResolver.CheckRegistration(typeof(T), typeId, typeof(TSerializer))) + { + return this; + } + TypeInfo typeInfo = PrepareRegistration(); _typeResolver.Register(typeof(T), typeId, typeInfo); return this; @@ -145,6 +165,15 @@ public Fory Register(string name) { EnsureRegistrationOpen(); (string namespaceName, string typeName) = TypeResolver.SplitTypeName(name); + if (_typeResolver.CheckRegistration( + typeof(T), + namespaceName, + typeName, + typeof(TSerializer))) + { + return this; + } + TypeInfo typeInfo = PrepareRegistration(); _typeResolver.Register(typeof(T), namespaceName, typeName, typeInfo); return this; @@ -165,6 +194,15 @@ public Fory Register(string typeNamespace, string typeName) { EnsureRegistrationOpen(); TypeResolver.ValidateSplitTypeName(typeNamespace, typeName); + if (_typeResolver.CheckRegistration( + typeof(T), + typeNamespace, + typeName, + typeof(TSerializer))) + { + return this; + } + TypeInfo typeInfo = PrepareRegistration(); _typeResolver.Register(typeof(T), typeNamespace, typeName, typeInfo); return this; diff --git a/csharp/src/Fory/TypeResolver.cs b/csharp/src/Fory/TypeResolver.cs index 385c561312..6e036dbe5a 100644 --- a/csharp/src/Fory/TypeResolver.cs +++ b/csharp/src/Fory/TypeResolver.cs @@ -501,13 +501,49 @@ internal TypeInfo PrepareRegistration() return TypeInfo.Create(typeof(T), new TSerializer()); } + internal bool CheckRegistration(Type type, uint id, Type? serializerType = null) + { + if (_byUserTypeId.TryGetValue(id, out TypeInfo? wireOwner) && wireOwner.Type != type) + { + throw new InvalidDataException($"type ID {id} is already registered for {wireOwner.Type}"); + } + + if (!_typeInfos.TryGetValue(TypeMapKey.Get(type), out TypeInfo? typeInfo) || !typeInfo.IsRegistered) + { + return false; + } + + if (!typeInfo.RegisterByName && typeInfo.UserTypeId == id) + { + if (serializerType is not null && typeInfo.SerializerType != serializerType) + { + throw new InvalidDataException( + $"type {type} is already registered with serializer {typeInfo.SerializerType}"); + } + + return true; + } + + throw new InvalidDataException($"type {type} is already registered with a different wire identity"); + } + internal void Register(Type type, uint id) { + if (CheckRegistration(type, id)) + { + return; + } + Register(type, id, PrepareRegistration(type)); } internal void Register(Type type, uint id, TypeInfo typeInfo) { + if (CheckRegistration(type, id, typeInfo.SerializerType)) + { + return; + } + typeInfo = PrepareRegistration(type, typeInfo).WithTypeIdRegistration(id); _typeInfos.Set(TypeMapKey.Get(type), typeInfo); _byUserTypeId[id] = typeInfo; @@ -548,15 +584,58 @@ internal static void ValidateSplitTypeName(string namespaceName, string typeName } } + internal bool CheckRegistration( + Type type, + string namespaceName, + string typeName, + Type? serializerType = null) + { + if (_byTypeName.TryGetValue((namespaceName, typeName), out TypeInfo? wireOwner) && wireOwner.Type != type) + { + throw new InvalidDataException( + $"type name {namespaceName}.{typeName} is already registered for {wireOwner.Type}"); + } + + if (!_typeInfos.TryGetValue(TypeMapKey.Get(type), out TypeInfo? typeInfo) || !typeInfo.IsRegistered) + { + return false; + } + + if (typeInfo.RegisterByName && + typeInfo.NamespaceName?.Value == namespaceName && + typeInfo.TypeName?.Value == typeName) + { + if (serializerType is not null && typeInfo.SerializerType != serializerType) + { + throw new InvalidDataException( + $"type {type} is already registered with serializer {typeInfo.SerializerType}"); + } + + return true; + } + + throw new InvalidDataException($"type {type} is already registered with a different wire identity"); + } + internal void Register(Type type, string namespaceName, string typeName) { ValidateSplitTypeName(namespaceName, typeName); + if (CheckRegistration(type, namespaceName, typeName)) + { + return; + } + Register(type, namespaceName, typeName, PrepareRegistration(type)); } internal void Register(Type type, string namespaceName, string typeName, TypeInfo typeInfo) { ValidateSplitTypeName(namespaceName, typeName); + if (CheckRegistration(type, namespaceName, typeName, typeInfo.SerializerType)) + { + return; + } + MetaString namespaceMeta = MetaStringEncoder.Namespace.Encode(namespaceName, TypeMetaEncodings.NamespaceMetaStringEncodings); MetaString typeNameMeta = MetaStringEncoder.TypeName.Encode(typeName, TypeMetaEncodings.TypeNameMetaStringEncodings); typeInfo = PrepareRegistration(type, typeInfo).WithTypeNameRegistration(namespaceMeta, typeNameMeta); diff --git a/csharp/tests/Fory.Tests/RuntimeEdgeCaseTests.cs b/csharp/tests/Fory.Tests/RuntimeEdgeCaseTests.cs index 1b5e6d69c4..e64d082525 100644 --- a/csharp/tests/Fory.Tests/RuntimeEdgeCaseTests.cs +++ b/csharp/tests/Fory.Tests/RuntimeEdgeCaseTests.cs @@ -109,6 +109,29 @@ public override FrozenPayload ReadData(ReadContext context) } } +public sealed class AlternateFrozenSerializer : Serializer +{ + public static int Constructions; + + public AlternateFrozenSerializer() + { + Interlocked.Increment(ref Constructions); + } + + public override FrozenPayload DefaultValue => null!; + + public override void WriteData(WriteContext context, in FrozenPayload value, bool hasGenerics) + { + _ = hasGenerics; + context.Writer.WriteVarInt32(value.Value); + } + + public override FrozenPayload ReadData(ReadContext context) + { + return new FrozenPayload { Value = context.Reader.ReadVarInt32() }; + } +} + public enum GeneratedFrozenValue { Zero, @@ -928,6 +951,224 @@ public void FrozenRegistryRejectsBeforeMutation() Assert.Equal(0, FrozenPayloadSerializer.Constructions); } + [Fact] + public void DirectIdIdentityOwners() + { + ForyRuntime fory = ForyRuntime.Builder().Build(); + FrozenPayloadSerializer.Constructions = 0; + AlternateFrozenSerializer.Constructions = 0; + int atomicConstructions = 0; + AtomicRegistrationSerializer.ConstructionAction = () => atomicConstructions++; + + try + { + fory.Register(740); + fory.Register(740); + Assert.Equal(1, FrozenPayloadSerializer.Constructions); + Assert.ThrowsAny( + () => fory.Register(740)); + Assert.Equal(0, AlternateFrozenSerializer.Constructions); + + Assert.ThrowsAny( + () => fory.Register(740)); + Assert.Equal(0, atomicConstructions); + Assert.ThrowsAny( + () => fory.Register(741)); + Assert.Equal(1, FrozenPayloadSerializer.Constructions); + + fory.Register(742); + Assert.Equal(1, atomicConstructions); + FrozenPayload value = new() { Value = 11 }; + Assert.Equal(value.Value, fory.Deserialize(fory.Serialize(value)).Value); + Assert.Equal( + AtomicRegistrationValue.One, + fory.Deserialize(fory.Serialize(AtomicRegistrationValue.One))); + } + finally + { + AtomicRegistrationSerializer.ConstructionAction = null; + } + } + + [Fact] + public void DirectNameIdentityOwners() + { + ForyRuntime fory = ForyRuntime.Builder().Build(); + FrozenPayloadSerializer.Constructions = 0; + AlternateFrozenSerializer.Constructions = 0; + int atomicConstructions = 0; + AtomicRegistrationSerializer.ConstructionAction = () => atomicConstructions++; + + try + { + fory.Register("identity.direct"); + fory.Register("identity", "direct"); + Assert.Equal(1, FrozenPayloadSerializer.Constructions); + Assert.ThrowsAny( + () => fory.Register("identity.direct")); + Assert.Equal(0, AlternateFrozenSerializer.Constructions); + + Assert.ThrowsAny( + () => fory.Register("identity.direct")); + Assert.Equal(0, atomicConstructions); + Assert.ThrowsAny( + () => fory.Register("identity.changed")); + Assert.Equal(1, FrozenPayloadSerializer.Constructions); + + fory.Register("identity.atomic"); + Assert.Equal(1, atomicConstructions); + FrozenPayload value = new() { Value = 12 }; + Assert.Equal(value.Value, fory.Deserialize(fory.Serialize(value)).Value); + Assert.Equal( + AtomicRegistrationValue.One, + fory.Deserialize(fory.Serialize(AtomicRegistrationValue.One))); + } + finally + { + AtomicRegistrationSerializer.ConstructionAction = null; + } + } + + [Fact] + public void ThreadSafeIdIdentityOwners() + { + using ThreadSafeFory fory = ForyRuntime.Builder().BuildThreadSafe(); + FrozenPayloadSerializer.Constructions = 0; + AlternateFrozenSerializer.Constructions = 0; + int atomicConstructions = 0; + AtomicRegistrationSerializer.ConstructionAction = () => atomicConstructions++; + + try + { + fory.Register(743); + fory.Register(743); + Assert.Equal(1, FrozenPayloadSerializer.Constructions); + Assert.ThrowsAny( + () => fory.Register(743)); + Assert.Equal(0, AlternateFrozenSerializer.Constructions); + + Assert.ThrowsAny( + () => fory.Register(743)); + Assert.Equal(0, atomicConstructions); + Assert.ThrowsAny( + () => fory.Register(744)); + Assert.Equal(1, FrozenPayloadSerializer.Constructions); + + fory.Register(745); + Assert.Equal(1, atomicConstructions); + FrozenPayload value = new() { Value = 13 }; + Assert.Equal(value.Value, fory.Deserialize(fory.Serialize(value)).Value); + Assert.Equal( + AtomicRegistrationValue.One, + fory.Deserialize(fory.Serialize(AtomicRegistrationValue.One))); + Assert.Equal(2, FrozenPayloadSerializer.Constructions); + Assert.Equal(2, atomicConstructions); + } + finally + { + AtomicRegistrationSerializer.ConstructionAction = null; + } + } + + [Fact] + public void ThreadSafeNameIdentityOwners() + { + using ThreadSafeFory fory = ForyRuntime.Builder().BuildThreadSafe(); + FrozenPayloadSerializer.Constructions = 0; + AlternateFrozenSerializer.Constructions = 0; + int atomicConstructions = 0; + AtomicRegistrationSerializer.ConstructionAction = () => atomicConstructions++; + + try + { + fory.Register("identity.thread_safe"); + fory.Register("identity", "thread_safe"); + Assert.Equal(1, FrozenPayloadSerializer.Constructions); + Assert.ThrowsAny( + () => fory.Register("identity.thread_safe")); + Assert.Equal(0, AlternateFrozenSerializer.Constructions); + + Assert.ThrowsAny( + () => fory.Register("identity.thread_safe")); + Assert.Equal(0, atomicConstructions); + Assert.ThrowsAny( + () => fory.Register("identity.changed")); + Assert.Equal(1, FrozenPayloadSerializer.Constructions); + + fory.Register("identity.atomic"); + Assert.Equal(1, atomicConstructions); + FrozenPayload value = new() { Value = 14 }; + Assert.Equal(value.Value, fory.Deserialize(fory.Serialize(value)).Value); + Assert.Equal( + AtomicRegistrationValue.One, + fory.Deserialize(fory.Serialize(AtomicRegistrationValue.One))); + Assert.Equal(2, FrozenPayloadSerializer.Constructions); + Assert.Equal(2, atomicConstructions); + } + finally + { + AtomicRegistrationSerializer.ConstructionAction = null; + } + } + + [Fact] + public void DirectReentryKeepsOwner() + { + ForyRuntime fory = ForyRuntime.Builder().Build(); + bool callbackStarted = false; + FrozenPayloadSerializer.ConstructionAction = () => + { + callbackStarted = true; + fory.Register(746); + }; + + try + { + Assert.ThrowsAny( + () => fory.Register(746)); + } + finally + { + FrozenPayloadSerializer.ConstructionAction = null; + } + + Assert.True(callbackStarted); + fory.Register(747); + TimeEnvelope first = new() { Date = new DateOnly(2026, 8, 29) }; + Assert.Equal(first.Date, fory.Deserialize(fory.Serialize(first)).Date); + FrozenPayload second = new() { Value = 15 }; + Assert.Equal(second.Value, fory.Deserialize(fory.Serialize(second)).Value); + } + + [Fact] + public void ThreadSafeReentryKeepsOwner() + { + using ThreadSafeFory fory = ForyRuntime.Builder().BuildThreadSafe(); + bool callbackStarted = false; + FrozenPayloadSerializer.ConstructionAction = () => + { + callbackStarted = true; + fory.Register(748); + }; + + try + { + Assert.ThrowsAny( + () => fory.Register(748)); + } + finally + { + FrozenPayloadSerializer.ConstructionAction = null; + } + + Assert.True(callbackStarted); + fory.Register(749); + TimeEnvelope first = new() { Date = new DateOnly(2026, 8, 29) }; + Assert.Equal(first.Date, fory.Deserialize(fory.Serialize(first)).Date); + FrozenPayload second = new() { Value = 16 }; + Assert.Equal(second.Value, fory.Deserialize(fory.Serialize(second)).Value); + } + [Fact] public void ReentrantRegistrationFreezes() { diff --git a/docs/object-serialization/csharp/type-registration.md b/docs/object-serialization/csharp/type-registration.md index 7f275c6e47..7a42403bb3 100644 --- a/docs/object-serialization/csharp/type-registration.md +++ b/docs/object-serialization/csharp/type-registration.md @@ -92,6 +92,10 @@ fory.Register(101); - Register user-defined types on both writer and reader sides. - Keep ID/name mappings consistent across services and languages. +- Within one `Fory` instance, each numeric ID or full type name belongs to one CLR type, and each + CLR type has one wire identity. Repeating the same mapping is idempotent. Reusing either side for + a different mapping fails and leaves the existing registration unchanged. Repeating an explicit + custom serializer registration must use the same concrete serializer type. - For external-type serialization, register the third-party target, such as `fory.Register(100)`, not the local serializer declaration. - Register a concrete derived class by its concrete type. Annotated abstract From c46bb057abb0405b1e5fb53e098a37e58ff9f330 Mon Sep 17 00:00:00 2001 From: chaokunyang Date: Sat, 29 Aug 2026 03:17:21 +0800 Subject: [PATCH 069/168] fix(go): preserve registry identity owners --- .agents/languages/go.md | 15 +- .../go/type-registration.md | 4 + go/fory/registry_freeze_lifecycle_test.go | 108 +++++++++ go/fory/threadsafe/fory.go | 4 +- .../registry_freeze_lifecycle_test.go | 45 +++- go/fory/type_resolver.go | 209 +++++++++++------- 6 files changed, 294 insertions(+), 91 deletions(-) diff --git a/.agents/languages/go.md b/.agents/languages/go.md index ce0f3071f9..3222546237 100644 --- a/.agents/languages/go.md +++ b/.agents/languages/go.md @@ -11,12 +11,15 @@ Load this file when changing `go/fory/` or Go xlang behavior. failed root. Exported resolver registration entries recheck that facade-owned state before mutation. `threadsafe.Fory` owns the cross-pool boundary with one frozen state, one prepared validation instance, and one log of successful named-struct registrations. Failed registrations - are not logged; pool misses after freeze replay the immutable successful log. Its custom factory - runs without the registration mutex because application code may reenter a root; after the - factory returns, registration rechecks the frozen state before publishing prepared or replay - state. Resolver duplicate diagnostics identify application serializers by concrete type only; - they must not invoke application string or format methods while registration holds the lifecycle - mutex. + are failure-atomic and are not logged, so they leave the prepared instance intact; pool misses + after freeze replay the immutable successful log. Its custom factory runs without the + registration mutex because application code may reenter a root; after the factory returns, + registration rechecks the frozen state before publishing prepared or replay state. Numeric IDs + and registered names are bidirectional identities: each identity owns one registered Go value + type, and passing that type's pointer form refers to the same registration. Resolver registration + must validate both directions before changing serializers or identity indexes. Resolver duplicate + diagnostics identify application serializers by concrete type only; they must not invoke + application string or format methods while registration holds the lifecycle mutex. - Go `ReadContext` intentionally defers codec errors to existing `HasError` or `CheckError` boundaries. After an error, work may continue only while it remains panic- and bounds-safe and cannot cause disproportionate work or allocation, publish state that survives root cleanup, or diff --git a/docs/object-serialization/go/type-registration.md b/docs/object-serialization/go/type-registration.md index f794f694e9..70188fe466 100644 --- a/docs/object-serialization/go/type-registration.md +++ b/docs/object-serialization/go/type-registration.md @@ -126,6 +126,10 @@ f1.RegisterStruct(User{}, 1) f2.RegisterStruct(User{}, 1) ``` +Within one Fory instance, each numeric ID or registered name identifies one registered Go value +type. Passing a pointer value for that type uses the same registration. A conflicting ID, name, or +type registration returns an error without replacing the first registration. + ## Registration Timing Register types after creating a Fory instance and before the first serialization or deserialization diff --git a/go/fory/registry_freeze_lifecycle_test.go b/go/fory/registry_freeze_lifecycle_test.go index 980706e5f4..5baed3aaea 100644 --- a/go/fory/registry_freeze_lifecycle_test.go +++ b/go/fory/registry_freeze_lifecycle_test.go @@ -34,6 +34,8 @@ type registryFreezeUnion struct{} type registryFreezeEnum int32 +type registryIdentityEnum int32 + type registryFreezeExtension struct { Value int32 } @@ -138,6 +140,112 @@ func TestNamedEncoderPreflight(t *testing.T) { f.RegisterStructByName(registryFreezeStruct{}, "test.RegistryFreezeStruct")) } +func TestNamedRegistryIdentity(t *testing.T) { + f := New(WithXlang(false), WithCompatible(false)) + const name = "test.RegistryIdentity" + require.NoError(t, f.RegisterStructByName(registryFreezeStruct{}, name)) + + nameKey := namedTypeKey{"test", "RegistryIdentity"} + owner := f.typeResolver.namedTypeToTypeInfo[nameKey] + require.NotNil(t, owner) + hashKey := nsTypeKey{owner.PkgPathBytes.Hashcode, owner.NameBytes.Hashcode} + hashOwner := f.typeResolver.nsTypeToTypeInfo[hashKey] + require.NotNil(t, hashOwner) + before := takeRegistryFreezeSnapshot(f.typeResolver) + attempts := []struct { + name string + call func() error + }{ + {"same name struct", func() error { + return f.RegisterStructByName(registryFreezeUnion{}, name) + }}, + {"same name enum", func() error { + return f.RegisterEnumByName(registryFreezeEnum(0), name) + }}, + {"same name union", func() error { + return f.RegisterUnionByName(registryFreezeUnion{}, name, NewUnionSerializer( + UnionCase{ID: 0, Type: reflect.TypeOf(int32(0)), TypeID: INT32})) + }}, + {"same name extension", func() error { + return f.RegisterExtensionByName( + registryFreezeExtension{}, name, registryPanicSerializer{}) + }}, + {"same type name", func() error { + return f.RegisterStructByName(®istryFreezeStruct{}, "test.OtherIdentity") + }}, + {"same type ID", func() error { + return f.RegisterStruct(®istryFreezeStruct{}, 7110) + }}, + } + for _, attempt := range attempts { + t.Run(attempt.name, func(t *testing.T) { + require.Error(t, attempt.call()) + require.Equal(t, before, takeRegistryFreezeSnapshot(f.typeResolver)) + require.Same(t, owner, f.typeResolver.namedTypeToTypeInfo[nameKey]) + require.Same(t, hashOwner, f.typeResolver.nsTypeToTypeInfo[hashKey]) + }) + } + + want := registryFreezeStruct{Value: 7} + data, err := f.Serialize(&want) + require.NoError(t, err) + var got registryFreezeStruct + require.NoError(t, f.Deserialize(data, &got)) + require.Equal(t, want, got) +} + +func TestNumericRegistryIdentity(t *testing.T) { + f := New(WithXlang(false), WithCompatible(false)) + const typeID = 7111 + require.NoError(t, f.RegisterEnum(registryFreezeEnum(0), typeID)) + + owner := f.typeResolver.userTypeIdToTypeInfo[typeID] + require.NotNil(t, owner) + before := takeRegistryFreezeSnapshot(f.typeResolver) + attempts := []struct { + name string + call func() error + }{ + {"same type new ID", func() error { + value := registryFreezeEnum(0) + return f.RegisterEnum(&value, typeID+1) + }}, + {"same ID new type", func() error { + return f.RegisterEnum(registryIdentityEnum(0), typeID) + }}, + {"same ID struct", func() error { + return f.RegisterStruct(registryFreezeStruct{}, typeID) + }}, + {"same ID union", func() error { + return f.RegisterUnion(registryFreezeUnion{}, typeID, NewUnionSerializer( + UnionCase{ID: 0, Type: reflect.TypeOf(int32(0)), TypeID: INT32})) + }}, + {"same ID extension", func() error { + return f.RegisterExtension( + registryFreezeExtension{}, typeID, registryPanicSerializer{}) + }}, + {"same type name", func() error { + value := registryFreezeEnum(0) + return f.RegisterEnumByName(&value, "test.RegistryIdentityEnum") + }}, + } + for _, attempt := range attempts { + t.Run(attempt.name, func(t *testing.T) { + require.Error(t, attempt.call()) + require.Equal(t, before, takeRegistryFreezeSnapshot(f.typeResolver)) + require.Same(t, owner, f.typeResolver.userTypeIdToTypeInfo[typeID]) + require.Same(t, owner, f.typeResolver.typesInfo[reflect.TypeOf(registryFreezeEnum(0))]) + }) + } + + want := registryFreezeEnum(7) + data, err := f.Serialize(want) + require.NoError(t, err) + var got registryFreezeEnum + require.NoError(t, f.Deserialize(data, &got)) + require.Equal(t, want, got) +} + func TestRegistryFreezeRoots(t *testing.T) { badDecimal := Decimal{Scale: maxDecimalScale + 1} tests := []struct { diff --git a/go/fory/threadsafe/fory.go b/go/fory/threadsafe/fory.go index b7f291f3d8..f6dc0d741b 100644 --- a/go/fory/threadsafe/fory.go +++ b/go/fory/threadsafe/fory.go @@ -178,9 +178,7 @@ func (f *Fory) RegisterStructByName(type_ any, name string) error { func (f *Fory) registerPrepared(type_ any, name string) error { registration := structRegistration{name: name} if err := f.prepared.RegisterStructByName(type_, name); err != nil { - // A failed registration is not part of the facade registry. Rebuild from - // the successful log before the next registration or first root. - f.prepared = nil + // Direct registration errors leave the prepared resolver unchanged. return err } if registeredType, ok := type_.(reflect.Type); ok { diff --git a/go/fory/threadsafe/registry_freeze_lifecycle_test.go b/go/fory/threadsafe/registry_freeze_lifecycle_test.go index 530d2ab7a3..7d58d67d33 100644 --- a/go/fory/threadsafe/registry_freeze_lifecycle_test.go +++ b/go/fory/threadsafe/registry_freeze_lifecycle_test.go @@ -36,6 +36,15 @@ type registryFreezeRace struct { Value int32 } +type registryIdentityPooled struct { + Value int32 +} + +type registryInvalidPooled struct { + First int32 `fory:"id=0"` + Second int32 `fory:"id=0"` +} + type reentrantStringSerializer struct { f *Fory called chan struct{} @@ -73,14 +82,15 @@ func TestDuplicateSerializerFormatting(t *testing.T) { called := make(chan struct{}, 1) serializer := &reentrantStringSerializer{called: called} var f *Fory + var prepared *fory.Fory f = NewWithFactory(func() *fory.Fory { - inner := fory.New(fory.WithXlang(false), fory.WithCompatible(false)) - if err := inner.RegisterUnionByName( + prepared = fory.New(fory.WithXlang(false), fory.WithCompatible(false)) + if err := prepared.RegisterUnionByName( registryFreezePooled{}, "test.DuplicateSerializer", serializer, ); err != nil { panic(err) } - return inner + return prepared }) serializer.f = f @@ -104,7 +114,7 @@ func TestDuplicateSerializerFormatting(t *testing.T) { } require.False(t, f.registryFrozen.Load()) require.Empty(t, f.registrations) - require.Nil(t, f.prepared) + require.Same(t, prepared, f.prepared) } func TestFactoryRootReentry(t *testing.T) { @@ -162,6 +172,33 @@ func TestRegistryFreezePropagation(t *testing.T) { } } +func TestPreparedSurvivesConflict(t *testing.T) { + f := New(fory.WithXlang(false), fory.WithCompatible(false)) + const name = "test.PreparedIdentity" + require.NoError(t, f.RegisterStructByName(registryFreezePooled{}, name)) + prepared := f.prepared + require.NotNil(t, prepared) + + require.Error(t, f.RegisterStructByName(registryIdentityPooled{}, name)) + require.Same(t, prepared, f.prepared) + require.Len(t, f.registrations, 1) + require.Error(t, + f.RegisterStructByName(registryFreezePooled{}, "test.OtherPreparedIdentity")) + require.Same(t, prepared, f.prepared) + require.Len(t, f.registrations, 1) + require.Error(t, + f.RegisterStructByName(registryInvalidPooled{}, "test.InvalidPrepared")) + require.Same(t, prepared, f.prepared) + require.Len(t, f.registrations, 1) + + want := registryFreezePooled{Value: 7} + data, err := f.Serialize(&want) + require.NoError(t, err) + var got registryFreezePooled + require.NoError(t, f.Deserialize(data, &got)) + require.Equal(t, want, got) +} + func TestRegistryFreezeOnFailure(t *testing.T) { f := New(fory.WithXlang(false), fory.WithCompatible(false)) _, err := f.Serialize(fory.Decimal{Scale: 10_001}) diff --git a/go/fory/type_resolver.go b/go/fory/type_resolver.go index 56cb370cfc..ddfb5a51ef 100644 --- a/go/fory/type_resolver.go +++ b/go/fory/type_resolver.go @@ -131,6 +131,50 @@ func (r *TypeResolver) validateNamedRegistration(namespace, typeName string) err return nil } +func hasWireIdentity(info *TypeInfo) bool { + return info != nil && + (info.UserTypeID != invalidUserTypeID || info.NameBytes != nil) +} + +// A registered Go value type and its pointer form share one wire identity. +// Check both indexes before serializer creation so failure cannot alter registration. +func (r *TypeResolver) checkUserTypeIDOwner( + type_ reflect.Type, + typeID TypeId, + userTypeID uint32, +) (bool, error) { + typeInfo := r.typesInfo[type_] + if idInfo, ok := r.userTypeIdToTypeInfo[userTypeID]; ok { + if idInfo.Type == type_ && TypeId(idInfo.TypeID) == typeID && typeInfo == idInfo { + return true, nil + } + return false, fmt.Errorf( + "wire type ID %d already identifies %s", userTypeID, idInfo.Type) + } + if hasWireIdentity(typeInfo) { + return false, fmt.Errorf("type %s already has a different wire identity", type_) + } + return false, nil +} + +func (r *TypeResolver) checkNamedTypeOwner( + type_ reflect.Type, + namespace string, + typeName string, +) error { + typeInfo := r.typesInfo[type_] + nameKey := namedTypeKey{namespace, typeName} + if nameInfo, ok := r.namedTypeToTypeInfo[nameKey]; ok { + return fmt.Errorf( + "wire name %q already identifies %s", + joinRegisteredName(namespace, typeName), nameInfo.Type) + } + if hasWireIdentity(typeInfo) { + return fmt.Errorf("type %s already has a different wire identity", type_) + } + return nil +} + type TypeInfo struct { Type reflect.Type FullNameBytes []byte @@ -512,56 +556,47 @@ func (r *TypeResolver) RegisterStruct(type_ reflect.Type, typeID TypeId, userTyp if err := r.fory.checkRegistrationOpen(); err != nil { return err } - // Check if already registered - if info, ok := r.userTypeIdToTypeInfo[userTypeID]; ok { - if info.Type == type_ { - return nil - } - return fmt.Errorf("type %s with id %d has been registered", info.Type, userTypeID) + if type_.Kind() != reflect.Struct { + return fmt.Errorf("unsupported type for ID registration: %v (use RegisterEnum for enum types)", type_.Kind()) + } + if err := validateOptionalFields(type_); err != nil { + return err + } + alreadyRegistered, err := r.checkUserTypeIDOwner(type_, typeID, userTypeID) + if err != nil { + return err + } + if alreadyRegistered { + return nil } - switch type_.Kind() { - case reflect.Struct: - if err := validateOptionalFields(type_); err != nil { - return err - } - // For struct types, check if serializer already registered - if prev, ok := r.typeToSerializers[type_]; ok { - return fmt.Errorf("type %s already has a serializer of type %T registered", type_, prev) - } - - // Create struct serializer - tag := type_.Name() - serializer := newStructSerializer(type_, tag) - r.typeToSerializers[type_] = serializer - r.typeToTypeInfo[type_] = "@" + tag - - // Create pointer serializer - ptrType := reflect.PtrTo(type_) - ptrSerializer, ok := r.typeToSerializers[ptrType] - if !ok { - ptrSerializer = &ptrToValueSerializer{ - valueSerializer: serializer, - valueBytes: serializer.valueBytes, - } - r.typeToSerializers[ptrType] = ptrSerializer - } - r.typeToTypeInfo[ptrType] = "*@" + tag + if prev, ok := r.typeToSerializers[type_]; ok { + return fmt.Errorf("type %s already has a serializer of type %T registered", type_, prev) + } - // Register value type with fullTypeID - _, err := r.registerType(type_, uint32(typeID), userTypeID, "", "", serializer, false) - if err != nil { - return fmt.Errorf("failed to register type by ID: %w", err) + tag := type_.Name() + serializer := newStructSerializer(type_, tag) + ptrType := reflect.PtrTo(type_) + ptrSerializer, ok := r.typeToSerializers[ptrType] + if !ok { + ptrSerializer = &ptrToValueSerializer{ + valueSerializer: serializer, + valueBytes: serializer.valueBytes, } + } - // Register pointer type with same fullTypeID (Java treats value and pointer types the same) - _, err = r.registerType(ptrType, uint32(typeID), userTypeID, "", "", ptrSerializer, false) - if err != nil { - return fmt.Errorf("failed to register pointer type by ID: %w", err) - } + r.typeToSerializers[type_] = serializer + r.typeToTypeInfo[type_] = "@" + tag + r.typeToSerializers[ptrType] = ptrSerializer + r.typeToTypeInfo[ptrType] = "*@" + tag - default: - return fmt.Errorf("unsupported type for ID registration: %v (use RegisterEnum for enum types)", type_.Kind()) + if _, err = r.registerType( + type_, uint32(typeID), userTypeID, "", "", serializer, false); err != nil { + return fmt.Errorf("failed to register type by ID: %w", err) + } + if _, err = r.registerType( + ptrType, uint32(typeID), userTypeID, "", "", ptrSerializer, false); err != nil { + return fmt.Errorf("failed to register pointer type by ID: %w", err) } return nil @@ -575,12 +610,15 @@ func (r *TypeResolver) RegisterUnion(type_ reflect.Type, userTypeID uint32, seri if serializer == nil { return fmt.Errorf("RegisterUnion requires a non-nil serializer") } - if info, ok := r.userTypeIdToTypeInfo[userTypeID]; ok { - return fmt.Errorf("type %s with id %d has been registered", info.Type, userTypeID) - } if type_.Kind() != reflect.Struct { return fmt.Errorf("RegisterUnion only supports struct types; got: %v", type_.Kind()) } + if alreadyRegistered, err := r.checkUserTypeIDOwner( + type_, TYPED_UNION, userTypeID); err != nil { + return err + } else if alreadyRegistered { + return fmt.Errorf("type %s with id %d has been registered", type_, userTypeID) + } if prev, ok := r.typeToSerializers[type_]; ok { return fmt.Errorf("type %s already has a serializer of type %T registered", type_, prev) } @@ -610,12 +648,6 @@ func (r *TypeResolver) RegisterEnum(type_ reflect.Type, userTypeID uint32) error if err := r.fory.checkRegistrationOpen(); err != nil { return err } - // Check if already registered - if info, ok := r.userTypeIdToTypeInfo[userTypeID]; ok { - return fmt.Errorf("type %s with id %d has been registered", info.Type, userTypeID) - } - - // Verify it's a numeric type switch type_.Kind() { case reflect.Int, reflect.Int8, reflect.Int16, reflect.Int32, reflect.Int64, reflect.Uint, reflect.Uint8, reflect.Uint16, reflect.Uint32, reflect.Uint64: @@ -623,6 +655,14 @@ func (r *TypeResolver) RegisterEnum(type_ reflect.Type, userTypeID uint32) error default: return fmt.Errorf("RegisterEnum only supports numeric types; got: %v", type_.Kind()) } + if alreadyRegistered, err := r.checkUserTypeIDOwner(type_, ENUM, userTypeID); err != nil { + return err + } else if alreadyRegistered { + return fmt.Errorf("type %s with id %d has been registered", type_, userTypeID) + } + if prev, ok := r.typeToSerializers[type_]; ok { + return fmt.Errorf("type %s already has a serializer of type %T registered", type_, prev) + } // Create enum serializer serializer := &enumSerializer{type_: type_, typeID: uint32(ENUM)} @@ -649,10 +689,6 @@ func (r *TypeResolver) RegisterEnum(type_ reflect.Type, userTypeID uint32) error } func (r *TypeResolver) registerEnumByName(type_ reflect.Type, namespace, typeName string) error { - // Check if already registered - if prev, ok := r.typeToSerializers[type_]; ok { - return fmt.Errorf("type %s already has a serializer of type %T registered", type_, prev) - } if typeName == "" { return fmt.Errorf("typeName must be non-empty") } @@ -668,12 +704,16 @@ func (r *TypeResolver) registerEnumByName(type_ reflect.Type, namespace, typeNam default: return fmt.Errorf("RegisterEnumByName only supports numeric types; got: %v", type_.Kind()) } - - // Compute type ID for NAMED_ENUM - typeId := uint32(NAMED_ENUM) + if err := r.checkNamedTypeOwner(type_, namespace, typeName); err != nil { + return err + } + if prev, ok := r.typeToSerializers[type_]; ok { + return fmt.Errorf("type %s already has a serializer of type %T registered", type_, prev) + } // Create enum serializer - serializer := &enumSerializer{type_: type_, typeID: typeId} + typeID := uint32(NAMED_ENUM) + serializer := &enumSerializer{type_: type_, typeID: typeID} tag := joinRegisteredName(namespace, typeName) @@ -681,7 +721,7 @@ func (r *TypeResolver) registerEnumByName(type_ reflect.Type, namespace, typeNam r.typeToTypeInfo[type_] = "@" + tag // Register the type - _, err := r.registerType(type_, typeId, invalidUserTypeID, namespace, typeName, serializer, false) + _, err := r.registerType(type_, typeID, invalidUserTypeID, namespace, typeName, serializer, false) if err != nil { return fmt.Errorf("failed to register enum by name: %w", err) } @@ -690,9 +730,6 @@ func (r *TypeResolver) registerEnumByName(type_ reflect.Type, namespace, typeNam } func (r *TypeResolver) registerStructByName(type_ reflect.Type, namespace, typeName string) error { - if prev, ok := r.typeToSerializers[type_]; ok { - return fmt.Errorf("type %s already has a serializer of type %T registered", type_, prev) - } if typeName == "" { return fmt.Errorf("typeName must be non-empty") } @@ -702,20 +739,25 @@ func (r *TypeResolver) registerStructByName(type_ reflect.Type, namespace, typeN if err := validateOptionalFields(type_); err != nil { return err } + internalTypeID := r.structTypeID(type_, true) + if err := r.checkNamedTypeOwner(type_, namespace, typeName); err != nil { + return err + } + if prev, ok := r.typeToSerializers[type_]; ok { + return fmt.Errorf("type %s already has a serializer of type %T registered", type_, prev) + } tag := joinRegisteredName(namespace, typeName) serializer := newStructSerializer(type_, tag) - r.typeToSerializers[type_] = serializer - // multiple struct with same name defined inside function will have same `type_.String()`, but they are - // different types. so we use tag to encode type info. - // tagged type encode as `@$tag`/`*@$tag`. - r.typeToTypeInfo[type_] = "@" + tag - ptrType := reflect.PtrTo(type_) ptrSerializer := &ptrToValueSerializer{valueSerializer: serializer, valueBytes: int(type_.Size())} + + r.typeToSerializers[type_] = serializer + // Distinct Go types can have the same display name, so registered wire names + // own the encoded type information. + r.typeToTypeInfo[type_] = "@" + tag r.typeToSerializers[ptrType] = ptrSerializer // use `ptrToValueSerializer` as default deserializer when deserializing data from other languages. r.typeToTypeInfo[ptrType] = "*@" + tag - internalTypeID := r.structTypeID(type_, true) userTypeID := invalidUserTypeID // For structs registered by name, directly register both their value and pointer types. _, err := r.registerType(type_, uint32(internalTypeID), userTypeID, namespace, typeName, nil, false) @@ -738,9 +780,6 @@ func (r *TypeResolver) registerUnionByName( if serializer == nil { return fmt.Errorf("RegisterUnionByName requires a non-nil serializer") } - if prev, ok := r.typeToSerializers[type_]; ok { - return fmt.Errorf("type %s already has a serializer of type %T registered", type_, prev) - } if type_.Kind() != reflect.Struct { return fmt.Errorf("RegisterUnionByName only supports struct types; got: %v", type_.Kind()) } @@ -750,6 +789,12 @@ func (r *TypeResolver) registerUnionByName( if err := r.validateNamedRegistration(namespace, typeName); err != nil { return err } + if err := r.checkNamedTypeOwner(type_, namespace, typeName); err != nil { + return err + } + if prev, ok := r.typeToSerializers[type_]; ok { + return fmt.Errorf("type %s already has a serializer of type %T registered", type_, prev) + } tag := joinRegisteredName(namespace, typeName) r.typeToSerializers[type_] = serializer r.typeToTypeInfo[type_] = "@" + tag @@ -780,15 +825,18 @@ func (r *TypeResolver) registerExtensionByName( if userSerializer == nil { return fmt.Errorf("serializer cannot be nil for extension type %s", type_) } - if prev, ok := r.typeToSerializers[type_]; ok { - return fmt.Errorf("type %s already has a serializer of type %T registered", type_, prev) - } if typeName == "" { return fmt.Errorf("typeName must be non-empty") } if err := r.validateNamedRegistration(namespace, typeName); err != nil { return err } + if err := r.checkNamedTypeOwner(type_, namespace, typeName); err != nil { + return err + } + if prev, ok := r.typeToSerializers[type_]; ok { + return fmt.Errorf("type %s already has a serializer of type %T registered", type_, prev) + } tag := joinRegisteredName(namespace, typeName) // Create adapter wrapping the user's ExtensionSerializer @@ -831,6 +879,11 @@ func (r *TypeResolver) RegisterExtension( if userSerializer == nil { return fmt.Errorf("serializer cannot be nil for extension type %s", type_) } + if alreadyRegistered, err := r.checkUserTypeIDOwner(type_, EXT, userTypeID); err != nil { + return err + } else if alreadyRegistered { + return fmt.Errorf("type %s with id %d has been registered", type_, userTypeID) + } if prev, ok := r.typeToSerializers[type_]; ok { return fmt.Errorf("type %s already has a serializer of type %T registered", type_, prev) } From 96e25dd5d232c31dc78dc4e07d7773f4c868f508 Mon Sep 17 00:00:00 2001 From: chaokunyang Date: Sat, 29 Aug 2026 03:26:53 +0800 Subject: [PATCH 070/168] docs(javascript): describe open union behavior --- docs/object-serialization/javascript/type-registration.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/object-serialization/javascript/type-registration.md b/docs/object-serialization/javascript/type-registration.md index 6105e78457..d8ae4fbe9b 100644 --- a/docs/object-serialization/javascript/type-registration.md +++ b/docs/object-serialization/javascript/type-registration.md @@ -162,7 +162,7 @@ name also cannot identify different Struct, enum, extension, or union families. An anonymous union with declared cases has no registry identity. Keep and use the serializer pair returned by `fory.register(...)`; registering another anonymous union does not replace or reuse the earlier union's serializer. A union declared without cases remains an open union with the generic -union owner and reads each value from the type information carried by that union case. +union encoding and reads each value from the type information carried by that union case. ## Field Metadata From 16572ab992bc42f65855b989ba74d0b1c0799375 Mon Sep 17 00:00:00 2001 From: chaokunyang Date: Sat, 29 Aug 2026 03:38:33 +0800 Subject: [PATCH 071/168] fix(cpp): preserve registry identity owners --- .agents/languages/cpp.md | 5 + cpp/fory/serialization/serialization_test.cc | 118 ++++++++++++++++++ cpp/fory/serialization/type_resolver.h | 39 +++--- .../cpp/type-registration.md | 5 +- 4 files changed, 143 insertions(+), 24 deletions(-) diff --git a/.agents/languages/cpp.md b/.agents/languages/cpp.md index f2d658e94f..aaeeaf1870 100644 --- a/.agents/languages/cpp.md +++ b/.agents/languages/cpp.md @@ -27,6 +27,11 @@ Load this file when changing `cpp/`, Cython build plumbing, or C++ xlang behavio registration-thread checks. Do not add a layout-preserving helper, padding, or call-shape workaround. Resolver finalization prepares metadata completely and publishes it only after the completed resolver clone succeeds; failed finalization must not expose partial metadata. +- `TypeResolver::register_type_internal` owns the bidirectional C++ type-to-wire identity preflight + for struct, enum, extension, and union registration. Reject an existing compile-time type owner + or wire ID/name before publication; numeric user type IDs are unique across all four families, + and exact repeated registration remains rejected. Do not add rollback, rebuild, or parallel + identity state. - Put private methods last in class definitions, immediately before private fields. - Do not redesign alias-based or low-level public type shapes to add convenience methods unless the user explicitly asks for that API change. - For cross-language feature ports, match protocol behavior but use idiomatic C++ ownership and layering instead of mirroring Java structure literally. diff --git a/cpp/fory/serialization/serialization_test.cc b/cpp/fory/serialization/serialization_test.cc index 17a705d99e..e2e4605be2 100644 --- a/cpp/fory/serialization/serialization_test.cc +++ b/cpp/fory/serialization/serialization_test.cc @@ -35,6 +35,7 @@ #include #include #include +#include #include #include #include @@ -192,6 +193,22 @@ inline std::vector buffer_bytes(Buffer &buffer) { buffer.data() + buffer.writer_index()); } +template +void expect_numeric_owner(TypeResolver &resolver, uint32_t user_type_id, + const TypeInfo *owner) { + auto by_type = resolver.get_type_info(); + ASSERT_TRUE(by_type.ok()); + EXPECT_EQ(by_type.value(), owner); + + auto by_id = resolver.get_user_type_info_by_id(owner->type_id, user_type_id); + ASSERT_TRUE(by_id.ok()); + EXPECT_EQ(by_id.value(), owner); + + auto by_runtime = resolver.get_type_info(std::type_index(typeid(T))); + ASSERT_TRUE(by_runtime.ok()); + EXPECT_EQ(by_runtime.value(), owner); +} + class RegistryProbeInputStream final : public InputStream { public: explicit RegistryProbeInputStream(Fory &fory) : fory_(fory) {} @@ -1427,6 +1444,107 @@ TEST(SerializationTest, RegistrationByNameFailureDoesNotLeakTypeInfo) { EXPECT_EQ(dotted_type_name.error().code(), ErrorCode::Invalid); } +TEST(SerializationTest, TypeIdentityConflictsAreAtomic) { + using IdentityUnion = std::variant; + using IdentityRoot = std::tuple<::SimpleStruct, ::SignedScopedStatus, + ::IdLimitExt, IdentityUnion>; + + auto fory = + Fory::builder().xlang(true).compatible(false).track_ref(false).build(); + TypeResolver &resolver = fory.type_resolver(); + + ASSERT_TRUE(fory.register_struct<::SimpleStruct>(1).ok()); + ASSERT_TRUE(fory.register_enum<::SignedScopedStatus>(2).ok()); + ASSERT_TRUE(fory.register_extension_type<::IdLimitExt>(3).ok()); + ASSERT_TRUE(fory.register_union(4).ok()); + + auto struct_info = resolver.get_type_info<::SimpleStruct>(); + auto enum_info = resolver.get_type_info<::SignedScopedStatus>(); + auto ext_info = resolver.get_type_info<::IdLimitExt>(); + auto union_info = resolver.get_type_info(); + ASSERT_TRUE(struct_info.ok()); + ASSERT_TRUE(enum_info.ok()); + ASSERT_TRUE(ext_info.ok()); + ASSERT_TRUE(union_info.ok()); + + const TypeInfo *struct_owner = struct_info.value(); + const TypeInfo *enum_owner = enum_info.value(); + const TypeInfo *ext_owner = ext_info.value(); + const TypeInfo *union_owner = union_info.value(); + + EXPECT_FALSE(fory.register_struct<::SimpleStruct>(1).ok()); + EXPECT_FALSE(fory.register_struct<::SimpleStruct>("conflict", "Struct").ok()); + EXPECT_FALSE( + fory.register_enum<::SignedScopedStatus>("conflict", "Enum").ok()); + EXPECT_FALSE( + fory.register_extension_type<::IdLimitExt>("conflict", "Ext").ok()); + EXPECT_FALSE(fory.register_union("conflict", "Union").ok()); + + expect_numeric_owner<::SimpleStruct>(resolver, 1, struct_owner); + expect_numeric_owner<::SignedScopedStatus>(resolver, 2, enum_owner); + expect_numeric_owner<::IdLimitExt>(resolver, 3, ext_owner); + expect_numeric_owner(resolver, 4, union_owner); + + EXPECT_FALSE(resolver.get_type_info_by_name("conflict", "Struct").ok()); + EXPECT_FALSE(resolver.get_type_info_by_name("conflict", "Enum").ok()); + EXPECT_FALSE(resolver.get_type_info_by_name("conflict", "Ext").ok()); + EXPECT_FALSE(resolver.get_type_info_by_name("conflict", "Union").ok()); + + IdentityRoot original{::SimpleStruct{7, 9}, ::SignedScopedStatus::LARGE, + ::IdLimitExt{42}, IdentityUnion{std::string("value")}}; + auto bytes = fory.serialize(original); + ASSERT_TRUE(bytes.ok()) << bytes.error().to_string(); + auto decoded = fory.deserialize(bytes.value()); + ASSERT_TRUE(decoded.ok()) << decoded.error().to_string(); + EXPECT_EQ(decoded.value(), original); +} + +TEST(SerializationTest, NumericIdentityConflictsAreAtomic) { + using IdentityUnion = std::variant; + + auto fory = + Fory::builder().xlang(true).compatible(false).track_ref(false).build(); + TypeResolver &resolver = fory.type_resolver(); + + ASSERT_TRUE(fory.register_struct<::SimpleStruct>(1).ok()); + auto struct_info = resolver.get_type_info<::SimpleStruct>(); + ASSERT_TRUE(struct_info.ok()); + const TypeInfo *struct_owner = struct_info.value(); + + EXPECT_FALSE(fory.register_enum<::SignedScopedStatus>(1).ok()); + EXPECT_FALSE(fory.register_extension_type<::IdLimitExt>(1).ok()); + EXPECT_FALSE(fory.register_union(1).ok()); + + expect_numeric_owner<::SimpleStruct>(resolver, 1, struct_owner); + EXPECT_FALSE(resolver.get_type_info<::SignedScopedStatus>().ok()); + EXPECT_FALSE(resolver.get_type_info<::IdLimitExt>().ok()); + EXPECT_FALSE(resolver.get_type_info().ok()); + EXPECT_FALSE( + resolver.get_user_type_info_by_id(static_cast(TypeId::ENUM), 1) + .ok()); + EXPECT_FALSE( + resolver.get_user_type_info_by_id(static_cast(TypeId::EXT), 1) + .ok()); + EXPECT_FALSE(resolver + .get_user_type_info_by_id( + static_cast(TypeId::TYPED_UNION), 1) + .ok()); + EXPECT_FALSE( + resolver.get_type_info(std::type_index(typeid(::SignedScopedStatus))) + .ok()); + EXPECT_FALSE( + resolver.get_type_info(std::type_index(typeid(::IdLimitExt))).ok()); + EXPECT_FALSE( + resolver.get_type_info(std::type_index(typeid(IdentityUnion))).ok()); + + ::SimpleStruct original{7, 9}; + auto bytes = fory.serialize(original); + ASSERT_TRUE(bytes.ok()) << bytes.error().to_string(); + auto decoded = fory.deserialize<::SimpleStruct>(bytes.value()); + ASSERT_TRUE(decoded.ok()) << decoded.error().to_string(); + EXPECT_EQ(decoded.value(), original); +} + static std::vector make_remote_type_meta(const std::string &type_name, const std::string &field) { std::vector fields; diff --git a/cpp/fory/serialization/type_resolver.h b/cpp/fory/serialization/type_resolver.h index 6d90c7165e..67e1dff0f2 100644 --- a/cpp/fory/serialization/type_resolver.h +++ b/cpp/fory/serialization/type_resolver.h @@ -1509,7 +1509,6 @@ class TypeResolver { static std::string make_name_key(const std::string &ns, const std::string &name); - static uint64_t make_user_type_key(uint32_t type_id, uint32_t user_type_id); /// Register a TypeInfo, taking ownership and storing in primary storage. /// Returns pointer to the stored TypeInfo (owned by TypeResolver). @@ -2351,12 +2350,6 @@ inline std::string TypeResolver::make_name_key(const std::string &ns, return key; } -inline uint64_t TypeResolver::make_user_type_key(uint32_t type_id, - uint32_t user_type_id) { - return (static_cast(type_id) << 32) | - static_cast(user_type_id); -} - inline Result TypeResolver::register_type_internal(uint64_t ctid, std::unique_ptr info) { @@ -2365,13 +2358,16 @@ TypeResolver::register_type_internal(uint64_t ctid, Error::invalid("TypeInfo or harness is invalid during registration")); } - // Validate all uniqueness constraints before mutating resolver state so - // failed registration leaves no partial entries behind. + // Validate both directions of the C++ type-to-wire identity before mutating + // resolver state so failed registration leaves every owner index unchanged. TypeInfo *raw_ptr = info.get(); + TypeInfo *existing_type = type_info_by_ctid_.get_or_default(ctid, nullptr); + if (existing_type != nullptr) { + return Unexpected(Error::invalid("C++ type already registered")); + } const bool is_internal = ::fory::is_internal_type(raw_ptr->type_id); - const bool has_user_type_key = + const bool has_user_type_id = !raw_ptr->register_by_name && raw_ptr->user_type_id != kInvalidUserTypeId; - uint64_t user_type_key = 0; std::string name_key; if (is_internal) { @@ -2381,14 +2377,14 @@ TypeResolver::register_type_internal(uint64_t ctid, return Unexpected(Error::invalid("Type id already registered: " + std::to_string(raw_ptr->type_id))); } - } else if (has_user_type_key) { - user_type_key = make_user_type_key(raw_ptr->type_id, raw_ptr->user_type_id); + } else if (has_user_type_id) { + // Numeric user IDs share one registry namespace across all user type + // families. TypeInfo retains the family for validation during lookup. TypeInfo *existing = - user_type_info_by_id_.get_or_default(user_type_key, nullptr); + user_type_info_by_id_.get_or_default(raw_ptr->user_type_id, nullptr); if (existing != nullptr) { - return Unexpected(Error::invalid( - "Type id already registered: " + std::to_string(raw_ptr->type_id) + - "/" + std::to_string(raw_ptr->user_type_id))); + return Unexpected(Error::invalid("User type id already registered: " + + std::to_string(raw_ptr->user_type_id))); } } @@ -2409,8 +2405,8 @@ TypeResolver::register_type_internal(uint64_t ctid, if (is_internal) { type_info_by_id_.put(stored_ptr->type_id, stored_ptr); - } else if (has_user_type_key) { - user_type_info_by_id_.put(user_type_key, stored_ptr); + } else if (has_user_type_id) { + user_type_info_by_id_.put(stored_ptr->user_type_id, stored_ptr); } if (stored_ptr->register_by_name) { @@ -2440,9 +2436,8 @@ TypeResolver::get_type_info_by_id(uint32_t type_id) const { inline Result TypeResolver::get_user_type_info_by_id(uint32_t type_id, uint32_t user_type_id) const { - uint64_t key = make_user_type_key(type_id, user_type_id); - TypeInfo *info = user_type_info_by_id_.get_or_default(key, nullptr); - if (info != nullptr) { + TypeInfo *info = user_type_info_by_id_.get_or_default(user_type_id, nullptr); + if (info != nullptr && info->type_id == type_id) { return info; } return Unexpected(Error::type_error( diff --git a/docs/object-serialization/cpp/type-registration.md b/docs/object-serialization/cpp/type-registration.md index 13169fc4bf..1d6f2b34ef 100644 --- a/docs/object-serialization/cpp/type-registration.md +++ b/docs/object-serialization/cpp/type-registration.md @@ -66,8 +66,9 @@ int main() { Type IDs must be: -1. **Unique**: Each type must have a unique ID within a Fory instance -2. **Consistent**: Same ID must be used across all languages and versions +1. **Unique**: Each numeric ID identifies one C++ type across structs, enums, extension types, and unions within a Fory instance. Each registered name also identifies one C++ type +2. **Single binding**: Register each C++ type once. It cannot be rebound to another ID or name, and repeating the same registration also returns an error +3. **Consistent**: Use the same ID across all languages and versions User-registered type IDs are in a separate namespace from built-in type IDs, so you can start from 0: From 41690d44c89bd76dc8bd057aacb4ec0e7a4a1054 Mon Sep 17 00:00:00 2001 From: chaokunyang Date: Sat, 29 Aug 2026 03:42:23 +0800 Subject: [PATCH 072/168] refactor(cpp): match registry key storage --- cpp/fory/serialization/type_resolver.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cpp/fory/serialization/type_resolver.h b/cpp/fory/serialization/type_resolver.h index 67e1dff0f2..4844170dd2 100644 --- a/cpp/fory/serialization/type_resolver.h +++ b/cpp/fory/serialization/type_resolver.h @@ -1541,7 +1541,7 @@ class TypeResolver { // FlatIntMap is optimized for integer keys with minimal overhead util::U64PtrMap type_info_by_ctid_{256}; util::U32PtrMap type_info_by_id_{256}; - util::U64PtrMap user_type_info_by_id_{256}; + util::U32PtrMap user_type_info_by_id_{256}; fory::flat_hash_map type_info_by_name_; util::U64PtrMap partial_type_infos_{256}; From 7a41bd390a042f4110e36614ce6c4c0e743a4c98 Mon Sep 17 00:00:00 2001 From: chaokunyang Date: Sat, 29 Aug 2026 06:25:59 +0800 Subject: [PATCH 073/168] perf(cpp): avoid repeated pool finalization --- cpp/fory/serialization/fory.h | 18 ++++++++++++------ 1 file changed, 12 insertions(+), 6 deletions(-) diff --git a/cpp/fory/serialization/fory.h b/cpp/fory/serialization/fory.h index c571a9cb5f..520d465f6b 100644 --- a/cpp/fory/serialization/fory.h +++ b/cpp/fory/serialization/fory.h @@ -753,10 +753,15 @@ class Fory : public BaseFory { struct PreFinalized {}; explicit Fory(const Config &config, std::shared_ptr resolver, PreFinalized) - : BaseFory(config, std::move(resolver)), finalized_(false), + : BaseFory(config, std::move(resolver)), finalized_(true), precomputed_header_(compute_header(config.xlang)) { - // Pre-finalized, immediately create contexts - ensure_finalized(); + // The facade only retains finalized registration metadata. Runtime caches + // belong to the context clones, which stay distinct per pooled Fory. + // Sharing the published facade resolver avoids a third deep clone on every + // pool miss and must not be replaced with another finalization pass. + registration_frozen_ = true; + write_ctx_.emplace(config_, type_resolver_->clone()); + read_ctx_.emplace(config_, type_resolver_->clone()); } /// Finalize the type resolver on first use. @@ -1062,9 +1067,10 @@ class ThreadSafeFory : public BaseFory { : BaseFory(config, std::move(resolver)), finalized_resolver_(), finalized_once_flag_(), fory_pool_([this]() { // Every public root finalizes before pool acquisition, so a pool miss - // only clones the resolver already published by that root. - return std::unique_ptr(new Fory( - config_, finalized_resolver_->clone(), Fory::PreFinalized{})); + // can share the facade resolver published by that root. The pooled + // Fory constructor owns its context clones. + return std::unique_ptr( + new Fory(config_, finalized_resolver_, Fory::PreFinalized{})); }) {} void ensure_finalized() const { From 23c0cfcb76d523a425e26882e750111ca5f736e6 Mon Sep 17 00:00:00 2001 From: chaokunyang Date: Sat, 29 Aug 2026 06:26:07 +0800 Subject: [PATCH 074/168] perf(cpp): isolate registration-only synchronization --- .agents/languages/cpp.md | 4 ++++ cpp/fory/serialization/type_resolver.cc | 2 +- cpp/fory/serialization/type_resolver.h | 18 ++++++++++-------- 3 files changed, 15 insertions(+), 9 deletions(-) diff --git a/.agents/languages/cpp.md b/.agents/languages/cpp.md index aaeeaf1870..6c516e3c69 100644 --- a/.agents/languages/cpp.md +++ b/.agents/languages/cpp.md @@ -27,6 +27,10 @@ Load this file when changing `cpp/`, Cython build plumbing, or C++ xlang behavio registration-thread checks. Do not add a layout-preserving helper, padding, or call-shape workaround. Resolver finalization prepares metadata completely and publishes it only after the completed resolver clone succeeds; failed finalization must not expose partial metadata. +- `TypeResolver` owns its registration mutex indirectly because runtime serializers retain and + query the same resolver object after registration closes. Keep this cold synchronization object + off the resolver's hot lookup footprint; do not replace it with padding, alignment fields, + platform locks, or benchmark-shape logic. - `TypeResolver::register_type_internal` owns the bidirectional C++ type-to-wire identity preflight for struct, enum, extension, and union registration. Reject an existing compile-time type owner or wire ID/name before publication; numeric user type IDs are unique across all four families, diff --git a/cpp/fory/serialization/type_resolver.cc b/cpp/fory/serialization/type_resolver.cc index f97578489c..b98acbc611 100644 --- a/cpp/fory/serialization/type_resolver.cc +++ b/cpp/fory/serialization/type_resolver.cc @@ -1775,7 +1775,7 @@ Result TypeResolver::check_registration() { Result, Error> TypeResolver::build_final_type_resolver() { - std::lock_guard lock(registration_mutex_); + std::lock_guard lock(*registration_mutex_); // Freeze the source before building so even failed finalization permanently // rejects later registration. Holding the registration mutex makes first use // linearizable with direct registration helpers. ThreadSafeFory retains this diff --git a/cpp/fory/serialization/type_resolver.h b/cpp/fory/serialization/type_resolver.h index 4844170dd2..f3a93a9341 100644 --- a/cpp/fory/serialization/type_resolver.h +++ b/cpp/fory/serialization/type_resolver.h @@ -1531,7 +1531,9 @@ class TypeResolver { std::thread::id registration_thread_id_; bool registry_frozen_; - std::mutex registration_mutex_; + // Keep registration-only synchronization out of the hot resolver footprint. + std::unique_ptr registration_mutex_{ + std::make_unique()}; // Primary storage - owns all TypeInfo objects std::vector> type_infos_; @@ -1749,7 +1751,7 @@ get_type_info_with_resolver(TypeResolver &resolver) { } template Result TypeResolver::register_any_type() { - std::lock_guard lock(registration_mutex_); + std::lock_guard lock(*registration_mutex_); FORY_RETURN_IF_ERROR(check_registration()); using ChronoTimestamp = std::chrono::time_point; @@ -1788,7 +1790,7 @@ template Result TypeResolver::register_any_type() { template Result TypeResolver::register_by_id(uint32_t type_id) { - std::lock_guard lock(registration_mutex_); + std::lock_guard lock(*registration_mutex_); FORY_RETURN_IF_ERROR(check_registration()); if (type_id == kInvalidUserTypeId) { return Unexpected(Error::invalid( @@ -1844,7 +1846,7 @@ template Result TypeResolver::register_by_name(const std::string &ns, const std::string &type_name) { - std::lock_guard lock(registration_mutex_); + std::lock_guard lock(*registration_mutex_); FORY_RETURN_IF_ERROR(check_registration()); if (type_name.empty()) { return Unexpected( @@ -1897,7 +1899,7 @@ TypeResolver::register_by_name(const std::string &ns, template Result TypeResolver::register_ext_type_by_id(uint32_t type_id) { - std::lock_guard lock(registration_mutex_); + std::lock_guard lock(*registration_mutex_); FORY_RETURN_IF_ERROR(check_registration()); if (type_id == kInvalidUserTypeId) { return Unexpected(Error::invalid("type_id must be in range [0, 0xfffffffe] " @@ -1922,7 +1924,7 @@ template Result TypeResolver::register_ext_type_by_name(const std::string &ns, const std::string &type_name) { - std::lock_guard lock(registration_mutex_); + std::lock_guard lock(*registration_mutex_); FORY_RETURN_IF_ERROR(check_registration()); if (type_name.empty()) { return Unexpected(Error::invalid( @@ -1947,7 +1949,7 @@ TypeResolver::register_ext_type_by_name(const std::string &ns, template Result TypeResolver::register_union_by_id(uint32_t type_id) { - std::lock_guard lock(registration_mutex_); + std::lock_guard lock(*registration_mutex_); FORY_RETURN_IF_ERROR(check_registration()); if (type_id == kInvalidUserTypeId) { return Unexpected(Error::invalid( @@ -1971,7 +1973,7 @@ template Result TypeResolver::register_union_by_name(const std::string &ns, const std::string &type_name) { - std::lock_guard lock(registration_mutex_); + std::lock_guard lock(*registration_mutex_); FORY_RETURN_IF_ERROR(check_registration()); if (type_name.empty()) { return Unexpected(Error::invalid( From 8e7daf6e80405461b2596333d4b172546e21c3fe Mon Sep 17 00:00:00 2001 From: chaokunyang Date: Sat, 29 Aug 2026 06:26:17 +0800 Subject: [PATCH 075/168] test(kotlin): keep freeze probes owner-local --- .../fory/kotlin/xlang/KotlinXlangPeer.kt | 5 ----- .../kotlin/BuiltinClassSerializerTests.kt | 22 +++++++++++++++++++ 2 files changed, 22 insertions(+), 5 deletions(-) diff --git a/kotlin/fory-kotlin-tests/src/main/kotlin/org/apache/fory/kotlin/xlang/KotlinXlangPeer.kt b/kotlin/fory-kotlin-tests/src/main/kotlin/org/apache/fory/kotlin/xlang/KotlinXlangPeer.kt index 6a6d8f17c3..a46ef3f764 100644 --- a/kotlin/fory-kotlin-tests/src/main/kotlin/org/apache/fory/kotlin/xlang/KotlinXlangPeer.kt +++ b/kotlin/fory-kotlin-tests/src/main/kotlin/org/apache/fory/kotlin/xlang/KotlinXlangPeer.kt @@ -781,10 +781,6 @@ private fun serializerRegistrationFreezes() { check(runCatching { failedRootFory.deserialize(byteArrayOf()) }.isFailure) for (frozenFory in listOf(unregisteredFory, failedRootFory)) { val resolver = frozenFory.typeResolver - val cacheField = resolver.sharedRegistry.javaClass.getDeclaredField("objectInstantiatorCache") - cacheField.isAccessible = true - val cache = cacheField.get(resolver.sharedRegistry) as Map<*, *> - check(KotlinPet::class.java !in cache) check( runCatching { KotlinSerializers.registerUnion( @@ -795,7 +791,6 @@ private fun serializerRegistrationFreezes() { } .isFailure ) - check(KotlinPet::class.java !in cache) check(!resolver.isRegistered(KotlinPet::class.java)) } diff --git a/kotlin/fory-kotlin/src/test/kotlin/org/apache/fory/serializer/kotlin/BuiltinClassSerializerTests.kt b/kotlin/fory-kotlin/src/test/kotlin/org/apache/fory/serializer/kotlin/BuiltinClassSerializerTests.kt index 776ea32edc..684a4b285c 100644 --- a/kotlin/fory-kotlin/src/test/kotlin/org/apache/fory/serializer/kotlin/BuiltinClassSerializerTests.kt +++ b/kotlin/fory-kotlin/src/test/kotlin/org/apache/fory/serializer/kotlin/BuiltinClassSerializerTests.kt @@ -151,6 +151,28 @@ class BuiltinClassSerializerTests { Assert.assertNull(fory.typeResolver.getTypeInfo(ReentrantStruct::class.java, false)) } + @Test + fun testFrozenUnionSkipsConstruction() { + val fory = + ForyKotlin.builder() + .withXlang(true) + .requireClassRegistration(true) + .withRefTracking(false) + .build() + fory.serialize("freeze") + ReentrantRegistration.reset() + + assertThrows(ForyException::class.java) { + KotlinSerializers.registerUnion( + fory, + ReentrantStruct::class.java, + "kotlin.ReentrantStruct", + ) + } + Assert.assertEquals(ReentrantRegistration.constructions, 0) + Assert.assertFalse(fory.typeResolver.isRegistered(ReentrantStruct::class.java)) + } + @Test fun testSharedDefaultValueSupport() { ForyKotlin.builder().withXlang(false).requireClassRegistration(true).build() From 595531f6c8e18e085cd59ab7519096baeaca23ab Mon Sep 17 00:00:00 2001 From: chaokunyang Date: Sat, 29 Aug 2026 06:26:36 +0800 Subject: [PATCH 076/168] docs: clarify metadata logical reset --- docs/security/deserialization.md | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/docs/security/deserialization.md b/docs/security/deserialization.md index 5904b45ec6..cf90e1b384 100644 --- a/docs/security/deserialization.md +++ b/docs/security/deserialization.md @@ -655,10 +655,11 @@ Metadata readers should: - Reset or release metadata state at the correct root-operation boundary. Operation-local metadata occurrences and writer IDs must be reset before the context is reused, -including after a failed root. The reset must make prior-root entries unreachable and release -unusual high-water backing without adding allocation or slot-clearing work to normal roots. -Runtime-specific retention thresholds and reset ownership belong in the implementation guide and -language guidance. +including after a failed root. The reset must make prior-root entries invisible through the current +logical size and release unusual high-water backing without adding allocation or slot-clearing work +to normal roots. Bounded backing may retain inactive slot references when the runtime-specific +retention rule permits it. Runtime-specific thresholds and reset ownership belong in the +implementation guide and language guidance. A class-resolution cache reachable from untrusted deserialization may publish an entry only from explicit trusted configuration or after the active class From 3f62b42e70dc5f761ef4ff009621a8f1cee888bb Mon Sep 17 00:00:00 2001 From: chaokunyang Date: Sat, 29 Aug 2026 06:27:18 +0800 Subject: [PATCH 077/168] fix(scala): reject callbacks after failed freeze --- .../serializer/scala/ScalaSerializers.java | 2 +- .../fory/serializer/scala/ScalaEnumTest.scala | 23 +++++++++++++++++++ 2 files changed, 24 insertions(+), 1 deletion(-) diff --git a/scala/fory-scala/src/main/java/org/apache/fory/serializer/scala/ScalaSerializers.java b/scala/fory-scala/src/main/java/org/apache/fory/serializer/scala/ScalaSerializers.java index 77cb7ca3ad..e9a9ff3459 100644 --- a/scala/fory-scala/src/main/java/org/apache/fory/serializer/scala/ScalaSerializers.java +++ b/scala/fory-scala/src/main/java/org/apache/fory/serializer/scala/ScalaSerializers.java @@ -244,7 +244,7 @@ private static void registerEnumRuntimeAliases(Fory fory, Class cls, Object[] } private static void checkRegistrationOpen(TypeResolver resolver) { - if (resolver.isRegistrationFinished()) { + if (resolver.isRegistrationFrozen()) { throw new ForyException( "Cannot register class/serializer after registration has been frozen. Please register " + "all classes before invoking top-level `serialize/deserialize/copy` methods of " diff --git a/scala/fory-scala/src/test/scala-3/org/apache/fory/serializer/scala/ScalaEnumTest.scala b/scala/fory-scala/src/test/scala-3/org/apache/fory/serializer/scala/ScalaEnumTest.scala index b77139476c..35190ef863 100644 --- a/scala/fory-scala/src/test/scala-3/org/apache/fory/serializer/scala/ScalaEnumTest.scala +++ b/scala/fory-scala/src/test/scala-3/org/apache/fory/serializer/scala/ScalaEnumTest.scala @@ -23,6 +23,7 @@ import org.apache.fory.Fory import org.apache.fory.scala.ForyScala import org.apache.fory.annotation.ForyEnumId import org.apache.fory.exception.ForyException +import org.apache.fory.resolver.TypeResolver import org.scalatest.matchers.should.Matchers import org.scalatest.wordspec.AnyWordSpec @@ -109,5 +110,27 @@ class ScalaEnumTest extends AnyWordSpec with Matchers { } EnumDiscoveryProbe.initialized shouldBe 0 } + "reject discovery after failed finalization" in { + val failed = ForyScala.builder() + .withXlang(false) + .requireClassRegistration(false) + .build() + intercept[ForyException] { + failed.registerSerializerAndType( + classOf[Colors], + (_: TypeResolver) => { + failed.serialize("freeze") + null + }) + } + failed.getTypeResolver.isRegistrationFrozen shouldBe true + failed.getTypeResolver.isRegistrationFinished shouldBe false + + EnumDiscoveryProbe.initialized = 0 + intercept[ForyException] { + ScalaSerializers.registerEnum(failed, classOf[CountingEnum], 713L) + } + EnumDiscoveryProbe.initialized shouldBe 0 + } } } From be97f4a7ab9f1399f819d7d931aff17abf4101f8 Mon Sep 17 00:00:00 2001 From: chaokunyang Date: Sat, 29 Aug 2026 06:30:31 +0800 Subject: [PATCH 078/168] fix(go): normalize registration type owners --- .agents/languages/go.md | 11 +- go/fory/fory.go | 80 +-------- go/fory/registry_freeze_lifecycle_test.go | 207 ++++++++++++++++++++++ go/fory/type_resolver.go | 25 ++- 4 files changed, 244 insertions(+), 79 deletions(-) diff --git a/.agents/languages/go.md b/.agents/languages/go.md index 3222546237..98d1efce8b 100644 --- a/.agents/languages/go.md +++ b/.agents/languages/go.md @@ -16,10 +16,13 @@ Load this file when changing `go/fory/` or Go xlang behavior. registration mutex because application code may reenter a root; after the factory returns, registration rechecks the frozen state before publishing prepared or replay state. Numeric IDs and registered names are bidirectional identities: each identity owns one registered Go value - type, and passing that type's pointer form refers to the same registration. Resolver registration - must validate both directions before changing serializers or identity indexes. Resolver duplicate - diagnostics identify application serializers by concrete type only; they must not invoke - application string or format methods while registration holds the lifecycle mutex. + type, and passing that type's pointer form refers to the same registration. Facade and exported + resolver registration entries normalize values and `reflect.Type` inputs to that non-pointer + value owner before type validation or resolver mutation; only resolver publication creates its + single pointer companion. Resolver registration must validate both directions before changing + serializers or identity indexes. Resolver duplicate diagnostics identify application serializers + by concrete type only; they must not invoke application string or format methods while + registration holds the lifecycle mutex. - Go `ReadContext` intentionally defers codec errors to existing `HasError` or `CheckError` boundaries. After an error, work may continue only while it remains panic- and bounds-safe and cannot cause disproportionate work or allocation, publish state that survives root cleanup, or diff --git a/go/fory/fory.go b/go/fory/fory.go index 5797d9712d..eb22be7ac4 100644 --- a/go/fory/fory.go +++ b/go/fory/fory.go @@ -327,15 +327,7 @@ func (f *Fory) RegisterStruct(type_ any, typeID uint32) error { if err := validateUserTypeID(typeID); err != nil { return err } - var t reflect.Type - if rt, ok := type_.(reflect.Type); ok { - t = rt - } else { - t = reflect.TypeOf(type_) - if t.Kind() == reflect.Ptr { - t = t.Elem() - } - } + t := valueRegistrationType(type_) // Only struct types are supported via RegisterStruct // For enums, use RegisterEnum @@ -366,15 +358,7 @@ func (f *Fory) RegisterUnion(type_ any, typeID uint32, serializer Serializer) er if err := validateUserTypeID(typeID); err != nil { return err } - var t reflect.Type - if rt, ok := type_.(reflect.Type); ok { - t = rt - } else { - t = reflect.TypeOf(type_) - if t.Kind() == reflect.Ptr { - t = t.Elem() - } - } + t := valueRegistrationType(type_) if t.Kind() != reflect.Struct { return fmt.Errorf("RegisterUnion only supports struct types; got: %v", t.Kind()) } @@ -394,15 +378,7 @@ func (f *Fory) RegisterUnionByName(type_ any, name string, serializer Serializer if serializer == nil { return fmt.Errorf("RegisterUnionByName requires a non-nil serializer") } - var t reflect.Type - if rt, ok := type_.(reflect.Type); ok { - t = rt - } else { - t = reflect.TypeOf(type_) - if t.Kind() == reflect.Ptr { - t = t.Elem() - } - } + t := valueRegistrationType(type_) if t.Kind() != reflect.Struct { return fmt.Errorf("RegisterUnionByName only supports struct types; got: %v", t.Kind()) } @@ -423,15 +399,7 @@ func (f *Fory) RegisterStructByName(type_ any, name string) error { if err := f.checkRegistrationOpen(); err != nil { return err } - var t reflect.Type - if rt, ok := type_.(reflect.Type); ok { - t = rt - } else { - t = reflect.TypeOf(type_) - if t.Kind() == reflect.Ptr { - t = t.Elem() - } - } + t := valueRegistrationType(type_) if t.Kind() != reflect.Struct { return fmt.Errorf("RegisterStructByName only supports struct types; for enum types use RegisterEnumByName. Got: %v", t.Kind()) } @@ -456,15 +424,7 @@ func (f *Fory) RegisterEnum(type_ any, typeID uint32) error { if err := validateUserTypeID(typeID); err != nil { return err } - var t reflect.Type - if rt, ok := type_.(reflect.Type); ok { - t = rt - } else { - t = reflect.TypeOf(type_) - if t.Kind() == reflect.Ptr { - t = t.Elem() - } - } + t := valueRegistrationType(type_) // Verify it's a numeric type (Go enums are int-based) switch t.Kind() { @@ -488,15 +448,7 @@ func (f *Fory) RegisterEnumByName(type_ any, name string) error { if err := f.checkRegistrationOpen(); err != nil { return err } - var t reflect.Type - if rt, ok := type_.(reflect.Type); ok { - t = rt - } else { - t = reflect.TypeOf(type_) - if t.Kind() == reflect.Ptr { - t = t.Elem() - } - } + t := valueRegistrationType(type_) // Verify it's a numeric type (Go enums are int-based) switch t.Kind() { @@ -526,15 +478,7 @@ func (f *Fory) RegisterExtension(type_ any, typeID uint32, serializer ExtensionS if err := validateUserTypeID(typeID); err != nil { return err } - var t reflect.Type - if rt, ok := type_.(reflect.Type); ok { - t = rt - } else { - t = reflect.TypeOf(type_) - if t.Kind() == reflect.Ptr { - t = t.Elem() - } - } + t := valueRegistrationType(type_) return f.typeResolver.RegisterExtension(t, typeID, serializer) } @@ -565,15 +509,7 @@ func (f *Fory) RegisterExtensionByName(type_ any, name string, serializer Extens if err := f.checkRegistrationOpen(); err != nil { return err } - var t reflect.Type - if rt, ok := type_.(reflect.Type); ok { - t = rt - } else { - t = reflect.TypeOf(type_) - if t.Kind() == reflect.Ptr { - t = t.Elem() - } - } + t := valueRegistrationType(type_) namespace, typeName, err := splitRegisteredName(name) if err != nil { return err diff --git a/go/fory/registry_freeze_lifecycle_test.go b/go/fory/registry_freeze_lifecycle_test.go index 5baed3aaea..68c1e851c5 100644 --- a/go/fory/registry_freeze_lifecycle_test.go +++ b/go/fory/registry_freeze_lifecycle_test.go @@ -36,6 +36,48 @@ type registryFreezeEnum int32 type registryIdentityEnum int32 +type pointerRegistrationStruct struct { + Value int32 +} + +type pointerRegistrationEnum int32 + +type pointerRegistrationUnion struct { + caseID uint32 + value any +} + +func (pointerRegistrationUnion) ForyUnionMarker() {} + +func (u pointerRegistrationUnion) ForyUnionGet() (uint32, any) { + return u.caseID, u.value +} + +func (u *pointerRegistrationUnion) ForyUnionSet(caseID uint32, value any) { + u.caseID = caseID + u.value = value +} + +type pointerRegistrationExtension struct { + Value int32 +} + +type pointerExtensionSerializer struct{} + +func (pointerExtensionSerializer) WriteData(ctx *WriteContext, value reflect.Value) { + if value.Kind() == reflect.Ptr { + value = value.Elem() + } + ctx.Buffer().WriteInt32(int32(value.FieldByName("Value").Int())) +} + +func (pointerExtensionSerializer) ReadData(ctx *ReadContext, value reflect.Value) { + if value.Kind() == reflect.Ptr { + value = value.Elem() + } + value.FieldByName("Value").SetInt(int64(ctx.Buffer().ReadInt32(ctx.Err()))) +} + type registryFreezeExtension struct { Value int32 } @@ -246,6 +288,171 @@ func TestNumericRegistryIdentity(t *testing.T) { require.Equal(t, want, got) } +func requireRegisteredRoundTrip[T any](t *testing.T, f *Fory, want T) { + t.Helper() + data, err := f.Serialize(&want) + require.NoError(t, err) + var got T + require.NoError(t, f.Deserialize(data, &got)) + require.Equal(t, want, got) +} + +func requireValueRegistrationOwner( + t *testing.T, + f *Fory, + valueType reflect.Type, + owner *TypeInfo, +) { + t.Helper() + pointerType := reflect.PointerTo(valueType) + doublePointerType := reflect.PointerTo(pointerType) + require.NotNil(t, owner) + require.Equal(t, valueType, owner.Type) + valueInfo := f.typeResolver.typesInfo[valueType] + pointerInfo := f.typeResolver.typesInfo[pointerType] + require.NotNil(t, valueInfo) + require.NotNil(t, pointerInfo) + require.Equal(t, valueInfo.TypeID, pointerInfo.TypeID) + require.Equal(t, valueInfo.UserTypeID, pointerInfo.UserTypeID) + require.Contains(t, f.typeResolver.typeToSerializers, valueType) + require.NotContains(t, f.typeResolver.typeToSerializers, doublePointerType) + require.NotContains(t, f.typeResolver.typeToTypeInfo, doublePointerType) + require.NotContains(t, f.typeResolver.typesInfo, doublePointerType) + require.NotContains(t, f.typeResolver.typeToTypeDef, doublePointerType) + require.NotContains(t, f.typeResolver.unionTypeCache, doublePointerType) + require.NotContains(t, f.typeResolver.typePointerCache, typePointer(doublePointerType)) +} + +func TestPointerReflectTypeRegistration(t *testing.T) { + type registrationFamily struct { + name string + valueType reflect.Type + pointerType reflect.Type + userTypeID uint32 + wireName string + registerID func(*Fory, reflect.Type) error + registerName func(*Fory, reflect.Type) error + registerResolver func(*Fory, reflect.Type) error + roundTrip func(*testing.T, *Fory) + } + unionSerializer := func() *UnionSerializer { + return NewUnionSerializer( + UnionCase{ID: 0, Type: reflect.TypeOf(int32(0)), TypeID: INT32}) + } + families := []registrationFamily{ + { + name: "struct", + valueType: reflect.TypeOf(pointerRegistrationStruct{}), + pointerType: reflect.TypeOf((*pointerRegistrationStruct)(nil)), + userTypeID: 7120, + wireName: "test.PointerRegistrationStruct", + registerID: func(f *Fory, type_ reflect.Type) error { + return f.RegisterStruct(type_, 7120) + }, + registerName: func(f *Fory, type_ reflect.Type) error { + return f.RegisterStructByName(type_, "test.PointerRegistrationStruct") + }, + registerResolver: func(f *Fory, type_ reflect.Type) error { + return f.GetTypeResolver().RegisterStruct(type_, STRUCT, 7120) + }, + roundTrip: func(t *testing.T, f *Fory) { + requireRegisteredRoundTrip(t, f, pointerRegistrationStruct{Value: 7}) + }, + }, + { + name: "enum", + valueType: reflect.TypeOf(pointerRegistrationEnum(0)), + pointerType: reflect.TypeOf((*pointerRegistrationEnum)(nil)), + userTypeID: 7121, + wireName: "test.PointerRegistrationEnum", + registerID: func(f *Fory, type_ reflect.Type) error { + return f.RegisterEnum(type_, 7121) + }, + registerName: func(f *Fory, type_ reflect.Type) error { + return f.RegisterEnumByName(type_, "test.PointerRegistrationEnum") + }, + registerResolver: func(f *Fory, type_ reflect.Type) error { + return f.GetTypeResolver().RegisterEnum(type_, 7121) + }, + roundTrip: func(t *testing.T, f *Fory) { + requireRegisteredRoundTrip(t, f, pointerRegistrationEnum(7)) + }, + }, + { + name: "union", + valueType: reflect.TypeOf(pointerRegistrationUnion{}), + pointerType: reflect.TypeOf((*pointerRegistrationUnion)(nil)), + userTypeID: 7122, + wireName: "test.PointerRegistrationUnion", + registerID: func(f *Fory, type_ reflect.Type) error { + return f.RegisterUnion(type_, 7122, unionSerializer()) + }, + registerName: func(f *Fory, type_ reflect.Type) error { + return f.RegisterUnionByName( + type_, "test.PointerRegistrationUnion", unionSerializer()) + }, + registerResolver: func(f *Fory, type_ reflect.Type) error { + return f.GetTypeResolver().RegisterUnion(type_, 7122, unionSerializer()) + }, + roundTrip: func(t *testing.T, f *Fory) { + requireRegisteredRoundTrip(t, f, pointerRegistrationUnion{ + caseID: 0, + value: int32(7), + }) + }, + }, + { + name: "extension", + valueType: reflect.TypeOf(pointerRegistrationExtension{}), + pointerType: reflect.TypeOf((*pointerRegistrationExtension)(nil)), + userTypeID: 7123, + wireName: "test.PointerRegistrationExtension", + registerID: func(f *Fory, type_ reflect.Type) error { + return f.RegisterExtension(type_, 7123, pointerExtensionSerializer{}) + }, + registerName: func(f *Fory, type_ reflect.Type) error { + return f.RegisterExtensionByName( + type_, "test.PointerRegistrationExtension", pointerExtensionSerializer{}) + }, + registerResolver: func(f *Fory, type_ reflect.Type) error { + return f.GetTypeResolver().RegisterExtension( + type_, 7123, pointerExtensionSerializer{}) + }, + roundTrip: func(t *testing.T, f *Fory) { + requireRegisteredRoundTrip(t, f, pointerRegistrationExtension{Value: 7}) + }, + }, + } + + for _, family := range families { + pointerType := family.pointerType + require.Equal(t, family.valueType, pointerType.Elem()) + t.Run(family.name+" facade ID", func(t *testing.T) { + f := New(WithXlang(true), WithCompatible(false)) + require.NoError(t, family.registerID(f, pointerType)) + family.roundTrip(t, f) + requireValueRegistrationOwner( + t, f, family.valueType, f.typeResolver.userTypeIdToTypeInfo[family.userTypeID]) + }) + t.Run(family.name+" facade name", func(t *testing.T) { + f := New(WithXlang(true), WithCompatible(false)) + require.NoError(t, family.registerName(f, pointerType)) + family.roundTrip(t, f) + namespace, typeName, err := splitRegisteredName(family.wireName) + require.NoError(t, err) + requireValueRegistrationOwner(t, f, family.valueType, + f.typeResolver.namedTypeToTypeInfo[namedTypeKey{namespace, typeName}]) + }) + t.Run(family.name+" resolver", func(t *testing.T) { + f := New(WithXlang(true), WithCompatible(false)) + require.NoError(t, family.registerResolver(f, pointerType)) + family.roundTrip(t, f) + requireValueRegistrationOwner( + t, f, family.valueType, f.typeResolver.userTypeIdToTypeInfo[family.userTypeID]) + }) + } +} + func TestRegistryFreezeRoots(t *testing.T) { badDecimal := Decimal{Scale: maxDecimalScale + 1} tests := []struct { diff --git a/go/fory/type_resolver.go b/go/fory/type_resolver.go index ddfb5a51ef..ac30dec83b 100644 --- a/go/fory/type_resolver.go +++ b/go/fory/type_resolver.go @@ -516,13 +516,24 @@ func (r *TypeResolver) registerSerializer(type_ reflect.Type, typeId TypeId, s S return nil } +// valueRegistrationType returns the canonical value owner represented by an +// instance or reflect.Type. Registration publishes that value type and creates +// at most one pointer companion from it. +func valueRegistrationType(type_ any) reflect.Type { + registeredType, ok := type_.(reflect.Type) + if !ok { + registeredType = reflect.TypeOf(type_) + } + for registeredType != nil && registeredType.Kind() == reflect.Ptr { + registeredType = registeredType.Elem() + } + return registeredType +} + func validateOptionalFields(type_ reflect.Type) error { if type_ == nil { return nil } - if type_.Kind() == reflect.Ptr { - type_ = type_.Elem() - } if type_.Kind() != reflect.Struct { return nil } @@ -556,6 +567,7 @@ func (r *TypeResolver) RegisterStruct(type_ reflect.Type, typeID TypeId, userTyp if err := r.fory.checkRegistrationOpen(); err != nil { return err } + type_ = valueRegistrationType(type_) if type_.Kind() != reflect.Struct { return fmt.Errorf("unsupported type for ID registration: %v (use RegisterEnum for enum types)", type_.Kind()) } @@ -607,6 +619,7 @@ func (r *TypeResolver) RegisterUnion(type_ reflect.Type, userTypeID uint32, seri if err := r.fory.checkRegistrationOpen(); err != nil { return err } + type_ = valueRegistrationType(type_) if serializer == nil { return fmt.Errorf("RegisterUnion requires a non-nil serializer") } @@ -648,6 +661,7 @@ func (r *TypeResolver) RegisterEnum(type_ reflect.Type, userTypeID uint32) error if err := r.fory.checkRegistrationOpen(); err != nil { return err } + type_ = valueRegistrationType(type_) switch type_.Kind() { case reflect.Int, reflect.Int8, reflect.Int16, reflect.Int32, reflect.Int64, reflect.Uint, reflect.Uint8, reflect.Uint16, reflect.Uint32, reflect.Uint64: @@ -689,6 +703,7 @@ func (r *TypeResolver) RegisterEnum(type_ reflect.Type, userTypeID uint32) error } func (r *TypeResolver) registerEnumByName(type_ reflect.Type, namespace, typeName string) error { + type_ = valueRegistrationType(type_) if typeName == "" { return fmt.Errorf("typeName must be non-empty") } @@ -730,6 +745,7 @@ func (r *TypeResolver) registerEnumByName(type_ reflect.Type, namespace, typeNam } func (r *TypeResolver) registerStructByName(type_ reflect.Type, namespace, typeName string) error { + type_ = valueRegistrationType(type_) if typeName == "" { return fmt.Errorf("typeName must be non-empty") } @@ -777,6 +793,7 @@ func (r *TypeResolver) registerUnionByName( typeName string, serializer Serializer, ) error { + type_ = valueRegistrationType(type_) if serializer == nil { return fmt.Errorf("RegisterUnionByName requires a non-nil serializer") } @@ -822,6 +839,7 @@ func (r *TypeResolver) registerExtensionByName( typeName string, userSerializer ExtensionSerializer, ) error { + type_ = valueRegistrationType(type_) if userSerializer == nil { return fmt.Errorf("serializer cannot be nil for extension type %s", type_) } @@ -873,6 +891,7 @@ func (r *TypeResolver) RegisterExtension( if err := r.fory.checkRegistrationOpen(); err != nil { return err } + type_ = valueRegistrationType(type_) if userTypeID > maxUserTypeID { return fmt.Errorf("typeID must be in range [0, 0xfffffffe], got %d", userTypeID) } From 60398e13d38a869978488c06b1dcefa808418353 Mon Sep 17 00:00:00 2001 From: chaokunyang Date: Sat, 29 Aug 2026 08:23:14 +0800 Subject: [PATCH 079/168] fix(java): gate resolver registration entries --- .agents/languages/java.md | 5 + .../apache/fory/resolver/ClassResolver.java | 94 +++++++-------- .../org/apache/fory/resolver/TypeInfo.java | 2 + .../apache/fory/resolver/TypeResolver.java | 12 +- .../apache/fory/resolver/XtypeResolver.java | 7 ++ .../apache/fory/serializer/RegisterTest.java | 110 ++++++++++++++++++ 6 files changed, 175 insertions(+), 55 deletions(-) diff --git a/.agents/languages/java.md b/.agents/languages/java.md index 3c2481f61e..22607ff6dc 100644 --- a/.agents/languages/java.md +++ b/.agents/languages/java.md @@ -110,6 +110,11 @@ Load this file when changing anything under `java/` or when Java drives a cross- it on failure, and retain it on success. Do not add separate installing/completed module states. Direct `Fory` accepts modules before its first root; thread-safe facades accept modules only through `ForyBuilder.withModule` before construction. +- Every explicit resolver registration or initialization entry must call the authoritative + registration gate as its first executable statement, before argument validation, class loading, + no-op return, callback invocation, or publication. Serializer completion methods used by lazy, + JIT, and generated serializers are internal resolver-owned operations, remain valid after + registration freezes, and must not be treated or repurposed as registration APIs. - For GraalVM, use `fory codegen` to generate serializers when building native images. Do not add reflection configuration except for JDK `proxy`. - In Java native mode (`xlang=false`), only `Types.BOOL` through `Types.STRING` share type IDs with xlang mode. Other native-mode type IDs differ. - Choose one serializer ownership location per logical Java type family. Add native/xlang serializer variants only when the wire format or constructor contract truly differs. diff --git a/java/fory-core/src/main/java/org/apache/fory/resolver/ClassResolver.java b/java/fory-core/src/main/java/org/apache/fory/resolver/ClassResolver.java index f39bc47c3c..644d1c5bbe 100644 --- a/java/fory-core/src/main/java/org/apache/fory/resolver/ClassResolver.java +++ b/java/fory-core/src/main/java/org/apache/fory/resolver/ClassResolver.java @@ -266,7 +266,9 @@ private void clearTypeInfoCache() { } @Override + @Internal public void initialize() { + checkRegisterAllowed(); extRegistry.objectGenericType = buildGenericType(OBJECT_TYPE); registerInternal(LambdaSerializer.ReplaceStub.class, LAMBDA_STUB_ID); registerInternal(JdkProxySerializer.ReplaceStub.class, JDK_PROXY_STUB_ID); @@ -484,33 +486,10 @@ public void register(Class cls) { while (containsUserTypeId(extRegistry.userIdGenerator)) { extRegistry.userIdGenerator++; } - register(cls, extRegistry.userIdGenerator); + registerUserImpl(cls, extRegistry.userIdGenerator); } } - /** - * Registers a class by its fully qualified name with an auto-assigned user ID. - * - * @param className the fully qualified class name - * @see #register(Class) - */ - @Override - public void register(String className) { - register(loadClassFromLoader(className)); - } - - /** - * Registers a class by its fully qualified name with a specified user ID. - * - * @param className the fully qualified class name - * @param classId the user ID to assign (0-based, in user ID space) - * @see #register(Class, long) - */ - @Override - public void register(String className, long classId) { - register(loadClassFromLoader(className), classId); - } - /** * Registers a class with a user-specified ID. * @@ -523,6 +502,7 @@ public void register(String className, long classId) { */ @Override public void register(Class cls, long id) { + checkRegisterAllowed(); registerUserImpl(cls, toUserTypeId(id)); } @@ -668,9 +648,11 @@ public void registerEnum(Class cls, String namespace, String name, Serializer * * @param classes the classes to register */ + @Internal public void registerInternal(Class... classes) { + checkRegisterAllowed(); for (Class cls : classes) { - registerInternal(cls); + registerInternalType(cls); } } @@ -682,22 +664,10 @@ public void registerInternal(Class... classes) { * * @param cls the class to register */ + @Internal public void registerInternal(Class cls) { - if (!extRegistry.registeredClassIdMap.containsKey(cls)) { - Preconditions.checkArgument( - extRegistry.classIdGenerator < INTERNAL_NATIVE_ID_LIMIT, - "Internal type id overflow: %s", - extRegistry.classIdGenerator); - while (extRegistry.classIdGenerator < typeIdToTypeInfo.length - && typeIdToTypeInfo[extRegistry.classIdGenerator] != null) { - extRegistry.classIdGenerator++; - } - Preconditions.checkArgument( - extRegistry.classIdGenerator < INTERNAL_NATIVE_ID_LIMIT, - "Internal type id overflow: %s", - extRegistry.classIdGenerator); - registerInternal(cls, extRegistry.classIdGenerator); - } + checkRegisterAllowed(); + registerInternalType(cls); } /** @@ -712,13 +682,32 @@ public void registerInternal(Class cls) { * @param classId the internal ID, must be in range [0, 255] * @throws IllegalArgumentException if the ID is out of range or already in use */ + @Internal public void registerInternal(Class cls, int classId) { + checkRegisterAllowed(); Preconditions.checkArgument(classId >= 0 && classId < INTERNAL_NATIVE_ID_LIMIT); registerInternalImpl(cls, classId); } + private void registerInternalType(Class cls) { + if (!extRegistry.registeredClassIdMap.containsKey(cls)) { + Preconditions.checkArgument( + extRegistry.classIdGenerator < INTERNAL_NATIVE_ID_LIMIT, + "Internal type id overflow: %s", + extRegistry.classIdGenerator); + while (extRegistry.classIdGenerator < typeIdToTypeInfo.length + && typeIdToTypeInfo[extRegistry.classIdGenerator] != null) { + extRegistry.classIdGenerator++; + } + Preconditions.checkArgument( + extRegistry.classIdGenerator < INTERNAL_NATIVE_ID_LIMIT, + "Internal type id overflow: %s", + extRegistry.classIdGenerator); + registerInternalImpl(cls, extRegistry.classIdGenerator); + } + } + private void registerInternalImpl(Class cls, int typeId) { - checkRegisterAllowed(); Preconditions.checkArgument(typeId >= 0 && typeId < INTERNAL_NATIVE_ID_LIMIT); checkRegistration(cls, typeId, cls.getName(), true); extRegistry.registeredClassIdMap.put(cls, typeId); @@ -734,7 +723,6 @@ private void registerInternalImpl(Class cls, int typeId) { } private void registerUserImpl(Class cls, int userId) { - checkRegisterAllowed(); Preconditions.checkArgument(userId != -1, "User type id 0xffffffff is reserved"); checkRegistration(cls, userId, cls.getName(), false); extRegistry.registeredClassIdMap.put(cls, userId); @@ -944,6 +932,7 @@ public String getTypeAlias(Class cls) { * Compute the typeId used in TypeDef without forcing serializer creation. This avoids recursive * serializer construction while building class metadata. */ + @Internal public int getTypeIdForTypeDef(Class cls) { TypeInfo typeInfo = classInfoMap.get(cls); if (typeInfo != null) { @@ -970,6 +959,7 @@ && checkType(cls.getName()) return typeId; } + @Internal public int getTypeDefRootTypeId(Class cls, boolean hasFieldMetadata) { if (hasFieldMetadata) { // Preserve the normal TypeInfo/name cache so locally generated or dynamically registered @@ -1236,7 +1226,9 @@ public void registerSerializer(Class type, Serializer serializer) { * @param serializer serializer for object of {@code type} */ @Override + @Internal public void registerInternalSerializer(Class type, Serializer serializer) { + checkRegisterAllowed(); Integer classId = extRegistry.registeredClassIdMap.get(type); if (classId != null && !isInternalRegisteredClassId(type, classId)) { throw new IllegalArgumentException( @@ -1251,7 +1243,7 @@ public void registerInternalSerializer(Class type, Serializer serializer) classId); } if (classId == null) { - registerInternal(type); + registerInternalType(type); } // Internal serializers are owned by the resolver path, not by their runtime package name. // Android/R8 may obfuscate Fory packages, so package text is not a stable internal marker. @@ -1259,7 +1251,6 @@ public void registerInternalSerializer(Class type, Serializer serializer) } private void registerSerializerImpl(Class type, Serializer serializer) { - checkRegisterAllowed(); TypeInfo typeInfo = newSerializerTypeInfo(type, serializer, false); publishSerializerTypeInfo(typeInfo, false, true); } @@ -1328,6 +1319,7 @@ protected TypeInfo newSerializerTypeInfo( } @Override + @Internal protected TypeInfo publishSerializerTypeInfo( TypeInfo typeInfo, boolean registerType, boolean explicitRegistration) { Class type = typeInfo.type; @@ -1397,6 +1389,7 @@ protected TypeInfo publishSerializerTypeInfo( * classinfo. */ @Override + @Internal public void setSerializer(Class cls, Serializer serializer) { if (isConstructingSerializer()) { bindConstructedSerializer(cls, serializer); @@ -1412,6 +1405,7 @@ public void setSerializer(Class cls, Serializer serializer) { * creating a data serializer for serialization of parts fields of a class. */ @Override + @Internal public void setSerializerIfAbsent(Class cls, Serializer serializer) { if (isConstructingSerializer()) { if (!hasConstructedSerializer(cls)) { @@ -1425,16 +1419,7 @@ public void setSerializerIfAbsent(Class cls, Serializer serializer) { } } - /** Clear serializer associated with cls if not null. */ - public void clearSerializer(Class cls) { - TypeInfo typeInfo = classInfoMap.get(cls); - if (typeInfo != null) { - typeInfo.setSerializer(this, null); - } - } - - /** Add serializer for specified class. */ - public void addSerializer(Class type, Serializer serializer) { + private void addSerializer(Class type, Serializer serializer) { Preconditions.checkNotNull(serializer); TypeInfo typeInfo = newAutomaticTypeInfo(type, serializer); publishSerializerTypeInfo(typeInfo, false, false); @@ -1708,6 +1693,7 @@ public TypeInfo getTypeInfo(Class cls) { return typeInfo; } + @Internal public TypeInfo getTypeInfo(short classId) { TypeInfo typeInfo = typeIdToTypeInfo[classId]; assert typeInfo != null : classId; diff --git a/java/fory-core/src/main/java/org/apache/fory/resolver/TypeInfo.java b/java/fory-core/src/main/java/org/apache/fory/resolver/TypeInfo.java index 862c24c3a8..874ff091b2 100644 --- a/java/fory-core/src/main/java/org/apache/fory/resolver/TypeInfo.java +++ b/java/fory-core/src/main/java/org/apache/fory/resolver/TypeInfo.java @@ -22,6 +22,7 @@ import static org.apache.fory.meta.Encoders.PACKAGE_DECODER; import static org.apache.fory.meta.Encoders.TYPE_NAME_DECODER; +import org.apache.fory.annotation.Internal; import org.apache.fory.collection.Tuple2; import org.apache.fory.meta.EncodedMetaString; import org.apache.fory.meta.Encoders; @@ -163,6 +164,7 @@ public Serializer getSerializer() { return (Serializer) serializer; } + @Internal public void setSerializer(Serializer serializer) { this.serializer = serializer; } diff --git a/java/fory-core/src/main/java/org/apache/fory/resolver/TypeResolver.java b/java/fory-core/src/main/java/org/apache/fory/resolver/TypeResolver.java index 780a367752..bfd022c21b 100644 --- a/java/fory-core/src/main/java/org/apache/fory/resolver/TypeResolver.java +++ b/java/fory-core/src/main/java/org/apache/fory/resolver/TypeResolver.java @@ -290,11 +290,13 @@ protected final void checkRegisterAllowed() { /** Registers a class by name with an auto-assigned user ID. */ public void register(String className) { + checkRegisterAllowed(); register(loadClassFromLoader(className)); } /** Registers a class by name with a user-specified ID. */ public void register(String className, long classId) { + checkRegisterAllowed(); register(loadClassFromLoader(className), classId); } @@ -302,6 +304,7 @@ public void register(String className, long classId) { * Registers a class by name with a namespace and type name. The type name must not contain `.`. */ public void register(String className, String namespace, String typeName) { + checkRegisterAllowed(); register(loadClassFromLoader(className), namespace, typeName); } @@ -315,12 +318,12 @@ public void register(String className, String namespace, String typeName) { */ @Internal public final void registerRuntimeTypeAlias(Class runtimeType, Class canonicalType) { + checkRegisterAllowed(); Preconditions.checkNotNull(runtimeType, "runtimeType"); Preconditions.checkNotNull(canonicalType, "canonicalType"); if (runtimeType == canonicalType) { return; } - checkRegisterAllowed(); TypeInfo canonicalInfo = classInfoMap.get(canonicalType); Preconditions.checkArgument( canonicalInfo != null, @@ -418,6 +421,7 @@ public final void registerSerializer( * @param type the class to register * @param serializer the serializer to use */ + @Internal public abstract void registerInternalSerializer(Class type, Serializer serializer); /** @@ -582,6 +586,7 @@ protected TypeInfo newAutomaticTypeInfo(Class type, Serializer serializer) *

When the wire and user IDs are unchanged, the resolver must update the existing {@link * TypeInfo} owner because generated serializers and field metadata may already retain it. */ + @Internal protected abstract TypeInfo publishSerializerTypeInfo( TypeInfo typeInfo, boolean registerType, boolean explicitRegistration); @@ -1746,8 +1751,10 @@ private Serializer getNativeTypedValueSerializer(int typeId, Class rawType public abstract Serializer getRawSerializer(Class cls); + @Internal public abstract void setSerializer(Class cls, Serializer serializer); + @Internal public abstract void setSerializerIfAbsent(Class cls, Serializer serializer); /** Returns the final metadata owner for a declared field during serializer construction. */ @@ -1833,6 +1840,7 @@ protected final TypeInfo stageConstructedTypeInfo(Class type, TypeInfo typeIn /** * Reset serializer if {@code serializer} is not null, otherwise clear serializer for {@code cls}. */ + @Internal public void resetSerializer(Class cls, Serializer serializer) { TypeInfo constructedTypeInfo = getConstructedTypeInfo(cls); if (constructedTypeInfo != null) { @@ -1913,6 +1921,7 @@ public GenericType getGenericTypeInStruct(Class cls, String genericTypeStr) { return map.getOrDefault(genericTypeStr, OBJECT_GENERIC_TYPE); } + @Internal public abstract void initialize(); public abstract void ensureSerializersCompiled(); @@ -2580,6 +2589,7 @@ final void clearCheckerCache() { } public void registerSerializerFactory(SerializerFactory serializerFactory) { + checkRegisterAllowed(); extRegistry.serializerFactories.add(Preconditions.checkNotNull(serializerFactory)); } diff --git a/java/fory-core/src/main/java/org/apache/fory/resolver/XtypeResolver.java b/java/fory-core/src/main/java/org/apache/fory/resolver/XtypeResolver.java index be16895e4d..5c2a8a51fd 100644 --- a/java/fory-core/src/main/java/org/apache/fory/resolver/XtypeResolver.java +++ b/java/fory-core/src/main/java/org/apache/fory/resolver/XtypeResolver.java @@ -168,7 +168,9 @@ public XtypeResolver( } @Override + @Internal public void initialize() { + checkRegisterAllowed(); registerDefaultTypes(); Serializers.registerDefaultSerializers(this); if (shareMeta) { @@ -465,6 +467,7 @@ public void registerEnum( */ @Internal public void registerForyType(Class type, Serializer serializer, int typeId) { + checkRegisterAllowed(); Preconditions.checkArgument(typeId < MAX_TYPE_ID, "Too big type id %s", typeId); register( type, @@ -610,6 +613,7 @@ protected TypeInfo newAutomaticTypeInfo(Class type, Serializer serializer) } @Override + @Internal protected TypeInfo publishSerializerTypeInfo( TypeInfo typeInfo, boolean registerType, boolean explicitRegistration) { Class type = typeInfo.type; @@ -657,6 +661,7 @@ protected TypeInfo publishSerializerTypeInfo( } @Override + @Internal public void registerInternalSerializer(Class type, Serializer serializer) { checkRegisterAllowed(); Class unwrapped = TypeUtils.unwrap(type); @@ -1347,6 +1352,7 @@ public Serializer getRawSerializer(Class cls) { } @Override + @Internal public void setSerializer(Class cls, Serializer serializer) { if (isConstructingSerializer()) { bindConstructedSerializer(cls, serializer); @@ -1356,6 +1362,7 @@ public void setSerializer(Class cls, Serializer serializer) { } @Override + @Internal public void setSerializerIfAbsent(Class cls, Serializer serializer) { if (isConstructingSerializer()) { if (!hasConstructedSerializer(cls)) { diff --git a/java/fory-core/src/test/java/org/apache/fory/serializer/RegisterTest.java b/java/fory-core/src/test/java/org/apache/fory/serializer/RegisterTest.java index 62f0286e32..4e7594e01a 100644 --- a/java/fory-core/src/test/java/org/apache/fory/serializer/RegisterTest.java +++ b/java/fory-core/src/test/java/org/apache/fory/serializer/RegisterTest.java @@ -40,10 +40,13 @@ import org.apache.fory.context.WriteContext; import org.apache.fory.exception.ForyException; import org.apache.fory.meta.TypeDef; +import org.apache.fory.resolver.ClassResolver; import org.apache.fory.resolver.SharedRegistry; import org.apache.fory.resolver.TypeInfo; import org.apache.fory.resolver.TypeResolver; +import org.apache.fory.resolver.XtypeResolver; import org.apache.fory.type.Descriptor; +import org.apache.fory.type.Types; import org.apache.fory.util.ExceptionUtils; import org.testng.Assert; import org.testng.annotations.Test; @@ -539,6 +542,98 @@ public void testFrozenFacadeRegistration() { Assert.assertFalse(creatorCalled.get()); } + @Test(dataProvider = "xlang") + public void testFrozenResolverRegistration(boolean xlang) { + Fory fory = + Fory.builder() + .withXlang(xlang) + .withCodegen(false) + .requireClassRegistration(true) + .withCompatible(false) + .build(); + TypeResolver resolver = fory.getTypeResolver(); + TypeInfo stringTypeInfo = resolver.getTypeInfo(String.class, false); + Serializer stringSerializer = stringTypeInfo.getSerializer(); + fory.serialize("freeze"); + int factoryCount = serializerFactoryCount(resolver); + + Assert.assertThrows( + ForyException.class, () -> resolver.registerSerializerFactory((r, type) -> null)); + Assert.assertEquals(serializerFactoryCount(resolver), factoryCount); + Assert.assertThrows( + ForyException.class, () -> resolver.registerRuntimeTypeAlias(String.class, String.class)); + Assert.assertThrows(ForyException.class, () -> resolver.register("missing.FrozenType")); + Assert.assertThrows(ForyException.class, () -> resolver.register(ObjectField.class, -1L)); + Assert.assertThrows(ForyException.class, resolver::initialize); + Assert.assertSame(resolver.getTypeInfo(String.class, false), stringTypeInfo); + Assert.assertSame(stringTypeInfo.getSerializer(), stringSerializer); + + if (xlang) { + Assert.assertThrows( + ForyException.class, + () -> + resolver.registerInternalSerializer(char.class, new ObjectFieldSerializer(resolver))); + Assert.assertNull(resolver.getTypeInfo(ObjectField.class, false)); + Assert.assertThrows( + ForyException.class, + () -> + ((XtypeResolver) resolver) + .registerForyType( + ObjectField.class, new ObjectFieldSerializer(resolver), Types.EXT)); + Assert.assertNull(resolver.getTypeInfo(ObjectField.class, false)); + } else { + ClassResolver classResolver = (ClassResolver) resolver; + Object extRegistry = TestUtils.getFieldValue(resolver, "extRegistry"); + int classIdGenerator = TestUtils.getFieldValue(extRegistry, "classIdGenerator"); + Assert.assertThrows( + ForyException.class, () -> classResolver.registerInternal(ObjectField.class)); + Assert.assertThrows(ForyException.class, () -> classResolver.registerInternal(String.class)); + Assert.assertThrows( + ForyException.class, () -> classResolver.registerInternal(new Class[0])); + Assert.assertThrows( + ForyException.class, + () -> + classResolver.registerInternalSerializer( + String.class, new ObjectFieldSerializer(resolver))); + int currentClassIdGenerator = TestUtils.getFieldValue(extRegistry, "classIdGenerator"); + Assert.assertEquals(currentClassIdGenerator, classIdGenerator); + Assert.assertNull(resolver.getTypeInfo(ObjectField.class, false)); + } + } + + @Test(dataProvider = "xlang") + public void testFactoryRegistrationReentry(boolean xlang) { + Fory fory = + Fory.builder() + .withXlang(xlang) + .withCodegen(false) + .requireClassRegistration(true) + .withCompatible(false) + .build(); + TypeResolver resolver = fory.getTypeResolver(); + int factoryCount = serializerFactoryCount(resolver); + FactoryRegisteringSerializer.ATTEMPTED.set(false); + FactoryRegisteringSerializer.FACTORY.set((r, type) -> null); + try { + Assert.assertThrows( + ForyException.class, + () -> fory.registerSerializerAndType(MyExt.class, FactoryRegisteringSerializer.class)); + } finally { + FactoryRegisteringSerializer.FACTORY.set(null); + } + + Assert.assertTrue(FactoryRegisteringSerializer.ATTEMPTED.get()); + Assert.assertEquals(serializerFactoryCount(resolver), factoryCount); + Assert.assertFalse(resolver.isRegistered(MyExt.class)); + Assert.assertNull(resolver.getTypeInfo(MyExt.class, false)); + } + + private static int serializerFactoryCount(TypeResolver resolver) { + Object extRegistry = TestUtils.getFieldValue(resolver, "extRegistry"); + List factories = TestUtils.getFieldValue(extRegistry, "serializerFactories"); + return factories.size(); + } + @Test public void testReentrantModuleFreeze() { Fory fory = @@ -705,6 +800,21 @@ public ReentrantSerializer(TypeResolver typeResolver) { } } + public static class FactoryRegisteringSerializer extends MyExtSerializer { + private static final AtomicBoolean ATTEMPTED = new AtomicBoolean(); + private static final AtomicReference FACTORY = new AtomicReference<>(); + + public FactoryRegisteringSerializer(TypeResolver typeResolver) { + super(typeResolver); + ATTEMPTED.set(true); + try { + typeResolver.registerSerializerFactory(FACTORY.get()); + } catch (ForyException ignored) { + // The enclosing registration must remain rejected after this callback returns. + } + } + } + public static class FailingSerializer extends MyExtSerializer { public FailingSerializer(TypeResolver typeResolver) { super(typeResolver); From 328d55adad5f62e04c6c978feca566db9b619c83 Mon Sep 17 00:00:00 2001 From: chaokunyang Date: Sat, 29 Aug 2026 08:23:20 +0800 Subject: [PATCH 080/168] fix(javascript): reconcile generated serializer owners --- .agents/languages/javascript.md | 17 ++- AGENTS.md | 23 +-- .../xlang_implementation_guide.md | 23 +-- javascript/packages/core/lib/fory.ts | 9 +- javascript/packages/core/lib/gen/index.ts | 135 +++++++++++++----- .../packages/core/lib/gen/serializer.ts | 40 ++++-- javascript/packages/core/lib/typeResolver.ts | 5 +- javascript/test/fory.test.ts | 77 +++++++++- 8 files changed, 252 insertions(+), 77 deletions(-) diff --git a/.agents/languages/javascript.md b/.agents/languages/javascript.md index a1ba6f035a..9882615ad9 100644 --- a/.agents/languages/javascript.md +++ b/.agents/languages/javascript.md @@ -17,7 +17,7 @@ Load this file when changing `javascript/`. occurrence arrays use native replacement reset. The MetaString and TypeMeta writer owner tables each have their own logical size: reset active owner IDs and that table's logical size without clearing bounded backing, and replace either backing only after its root has more than 8192 owners. -- Generated registration must initialize the complete recursive serializer graph against +- Generated registration must build the complete recursive serializer source graph against generation-local owners before one `TypeResolver` batch publication. The package-internal schema seal must lock each `TypeInfo` schema pointer before reading or traversing it; schema fields and occurrence modifiers are immutable afterward, while `dynamicTypeId` remains operation-local @@ -28,11 +28,16 @@ Load this file when changing `javascript/`. the same immutable definition containers and settings; reject a second conflicting definition before code generation without deep schema comparison. Complete anonymous definitions without a name or user ID do not share registry identity merely because their raw type IDs match. The - definition-free generic enum or union serializer remains the canonical raw-type owner. One - authoritative serializer owner - supplies both generator-time schema/progress facts and fixed factory captures; an initialized - owner published by a nested registration wins over an outer generation-local owner, while field - occurrence modifiers remain field-owned. Reject an unresolved nested Struct identity before + definition-free generic enum or union serializer remains the canonical raw-type owner. Each + transaction entry stores the schema/progress facts needed by later code generation. Within one + registration transaction, run all of its application code hooks before instantiating any of its + runtime serializer factories. Then reconcile each same-definition identity with any nested + published winner and instantiate each remaining factory once in dependency completion order, with + fixed direct captures of the final published-or-local owners. Batch-publish only the remaining + local owners. Do not reject a valid late winner, rerun code generation or a hook, rebuild a factory + after publication, or retain a transaction lookup, cell, callback, or wrapper in runtime + serializers. Field occurrence modifiers remain field-owned, and conflicting families or complete + definitions still fail before outer publication. Reject an unresolved nested Struct identity before resolver publication. An enum without a mapping and a union without cases keep their existing generic definitions; an extension reference resolves through its registered custom serializer owner. Never publish generated serializers, descriptors, or cache state before every generated diff --git a/AGENTS.md b/AGENTS.md index 58b933b5c5..94ef9c3b14 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -163,10 +163,18 @@ This is the entry point for AI guidance in Apache Fory. Read this file first, th occurrence arrays use native replacement reset. The MetaString and TypeMeta writer owner tables each have their own logical size: reset active owner IDs and that table's logical size without clearing bounded backing, and replace either backing only after its root has more than 8192 owners. -- JavaScript generated registration must build and initialize the complete recursive serializer - graph against generation-local owners before one `TypeResolver` batch publication. Factory-init - serializer lookup may see those local owners, but runtime and dynamic lookup must retain the real - resolver. Seal schema definitions before traversal and preseed complete definitions by identity, +- JavaScript generated registration must build the complete recursive serializer source graph + against generation-local owners before one `TypeResolver` batch publication. Each transaction + entry owns the schema and progress facts used by later code generation. Within one registration + transaction, run all of its application code hooks before instantiating any of its runtime + serializer factories. After that transaction's hooks complete, reconcile same-definition + identities with any nested published winner, then instantiate every remaining factory once in + dependency completion order so its fixed captures point directly to the final published-or-local + owners. Batch-publish only the remaining local owners. Do not reject a valid late same-definition + winner, rerun code generation or a hook, rebuild a factory after publication, or retain a + transaction lookup, cell, callback, or wrapper in runtime serializers. Runtime and dynamic lookup + must retain the real resolver. Seal schema definitions before traversal and preseed complete + definitions by identity, so field order cannot affect recursive resolution. One numeric ID or name cannot identify different user-defined type families. Each identity has one complete schema owner; repeated clones are valid only when they share that owner's definition containers and settings. A nested @@ -174,10 +182,9 @@ This is the entry point for AI guidance in Apache Fory. Read this file first, th in the current complete recursive schema graph; otherwise registration fails before resolver publication. Enum without a mapping and union without cases retain the canonical generic owner for their raw wire type. Complete anonymous definitions without a registry key remain - generation-local and distinct. Once initialized, the published nested owner is authoritative. Do - not publish placeholders, nested - serializers, descriptors, or cache state before every generated factory and application code hook - succeeds. + generation-local and distinct. A conflicting family or complete definition still fails before + outer publication. Do not publish placeholders, nested serializers, descriptors, or cache state + before every generated factory and application code hook succeeds. - Root failure exceptions must not copy or retain the operation reference table or materialized object graph for diagnostics. Root cleanup owns releasing that graph, and failure reporting must remain bounded independently of graph size. diff --git a/docs/specification/xlang_implementation_guide.md b/docs/specification/xlang_implementation_guide.md index c693b540aa..a22f23ff63 100644 --- a/docs/specification/xlang_implementation_guide.md +++ b/docs/specification/xlang_implementation_guide.md @@ -146,15 +146,20 @@ They stay in the current generation graph rather than publishing under their raw enum without a mapping and a union without cases use the canonical generic serializer for their raw wire type; they are definitions, not unresolved schema references. An extension occurrence without class metadata resolves through its registered custom serializer owner. -Code generation then constructs and initializes the complete -serializer graph against generation-local owners. The same authoritative owner supplies -generator-time schema and progress facts and fixed factory captures; field occurrence modifiers -remain owned by the containing schema. Runtime and dynamic dispatch retain the real -`TypeResolver`. After every factory and application code hook succeeds, the resolver performs one -guarded batch publication. An unresolved nested identity fails registration before resolver -publication when its type family requires a separate definition, such as Struct. An initialized -owner published by a nested registration is authoritative and must not be overwritten or -contradicted by the outer registration's generated decisions. +Code generation first builds the complete serializer source graph against generation-local owners. +Each transaction entry stores the schema and progress facts used by later code generation, while +field occurrence modifiers remain owned by the containing schema. Within one registration +transaction, all of its application code hooks run before any of its runtime serializer factories +are instantiated. After those hooks complete, a same-definition owner published by nested +registration becomes the final owner for that identity. Each remaining factory is instantiated once +in dependency completion order so its fixed captures point directly to the final published-or-local +owners, and the resolver then batch-publishes only the remaining local owners. This does not rerun +code generation or hooks, rebuild a factory after publication, or leave a transaction lookup, cell, +callback, or wrapper in a runtime serializer. Runtime and dynamic dispatch retain the real +`TypeResolver`. An unresolved nested identity fails registration before resolver publication when +its type family requires a separate definition, such as Struct. An initialized owner published by a +nested registration is authoritative and must not be overwritten. A conflicting family or complete +definition still fails before outer publication. Nested serializers must not call back into root `serialize(...)` or `deserialize(...)` entry points. diff --git a/javascript/packages/core/lib/fory.ts b/javascript/packages/core/lib/fory.ts index 9e2a4a2a78..d62293d8e2 100644 --- a/javascript/packages/core/lib/fory.ts +++ b/javascript/packages/core/lib/fory.ts @@ -154,15 +154,10 @@ export default class Fory { if (constructor.prototype?.[ForyTypeInfoSymbol]) { const typeInfo: TypeInfo = (constructor.prototype[ForyTypeInfoSymbol] as WithForyClsInfo) .structTypeInfo; - serializer = new Gen(this.typeResolver, { - creator: constructor, - customSerializer, - }).generateSerializer(typeInfo); + serializer = new Gen(this.typeResolver, customSerializer).generateSerializer(typeInfo); } else { const typeInfo = constructor; - serializer = new Gen(this.typeResolver, { - customSerializer, - }).generateSerializer(typeInfo); + serializer = new Gen(this.typeResolver, customSerializer).generateSerializer(typeInfo); } return { serializer, diff --git a/javascript/packages/core/lib/gen/index.ts b/javascript/packages/core/lib/gen/index.ts index 94a06218a0..93724ee62e 100644 --- a/javascript/packages/core/lib/gen/index.ts +++ b/javascript/packages/core/lib/gen/index.ts @@ -17,7 +17,7 @@ * under the License. */ -import { TypeId, Serializer } from "../type"; +import { CustomSerializer, TypeId, Serializer } from "../type"; import { sealTypeInfo, TypeInfo } from "../typeInfo"; import { CodegenRegistry } from "./router"; import { CodecBuilder, SerializerLookup } from "./builder"; @@ -53,13 +53,22 @@ type SerializerFactoryBuilder = () => ( serializerLookup: SerializerLookup, external: unknown, typeInfo: TypeInfo, - options: { [key: string]: unknown }, + options: object | undefined, localTypeMeta: TypeMeta | undefined, localTypeMetaSymbol: symbol, checkedTypeMetaSerializerSymbol: symbol, checkedTypeMetaWireTypeIdSymbol: symbol, ) => Serializer; +type SerializerFactory = ReturnType; + +interface GeneratedFactory { + create: SerializerFactory; + localTypeMeta: TypeMeta | undefined; + fixedSize: number; + readDataAlwaysAdvances: boolean; +} + const uninitializedSerializer: Serializer = { _initialized: false, fixedSize: 0, @@ -122,6 +131,7 @@ interface GeneratedRegistration { typeInfo: TypeInfo; serializer: Serializer; preparing: boolean; + factory?: GeneratedFactory; } export class Gen { @@ -129,10 +139,13 @@ export class Gen { constructor( private typeResolver: TypeResolver, - private regOptions: { [key: string]: any } = {}, + private rootCustomSerializer?: CustomSerializer, ) {} - private prepare(typeInfo: TypeInfo, serializerLookup: SerializerLookup): Serializer { + private generateFactory( + typeInfo: TypeInfo, + serializerLookup: SerializerLookup, + ): GeneratedFactory { const InnerGeneratorClass = CodegenRegistry.get(typeInfo.typeId); if (!InnerGeneratorClass) { throw new Error(`${typeInfo.typeId} generator not exists`); @@ -144,27 +157,43 @@ export class Gen { scope, ); - const funcString = generator.toSerializer(); + const generated = generator.toSerializer(); let factoryBuilder: SerializerFactoryBuilder; if (this.typeResolver.config && this.typeResolver.config.hooks) { const afterCodeGenerated = this.typeResolver.config.hooks.afterCodeGenerated; if (typeof afterCodeGenerated === "function") { - factoryBuilder = new Function(afterCodeGenerated(funcString)) as SerializerFactoryBuilder; + factoryBuilder = new Function( + afterCodeGenerated(generated.source), + ) as SerializerFactoryBuilder; } else { - factoryBuilder = new Function(funcString) as SerializerFactoryBuilder; + factoryBuilder = new Function(generated.source) as SerializerFactoryBuilder; } } else { - factoryBuilder = new Function(funcString) as SerializerFactoryBuilder; + factoryBuilder = new Function(generated.source) as SerializerFactoryBuilder; } - const factory = factoryBuilder(); - const localTypeMeta = generator.getLocalTypeMeta(); - return factory( + return { + create: factoryBuilder(), + localTypeMeta: generated.localTypeMeta, + fixedSize: generated.fixedSize, + readDataAlwaysAdvances: generated.readDataAlwaysAdvances, + }; + } + + private createSerializer( + typeInfo: TypeInfo, + serializerLookup: SerializerLookup, + factory: GeneratedFactory, + ): Serializer { + const options = TypeId.extType(typeInfo.typeId) + ? { ...typeInfo.options, customSerializer: this.rootCustomSerializer } + : typeInfo.options; + return factory.create( this.typeResolver, serializerLookup, Gen.external, typeInfo, - this.regOptions, - localTypeMeta, + options, + factory.localTypeMeta, localTypeMetaSymbol, checkedTypeMetaSerializerSymbol, checkedTypeMetaWireTypeIdSymbol, @@ -382,6 +411,7 @@ export class Gen { typeInfo: TypeInfo, children: TypeInfo[], registrations: GeneratedRegistration[], + factories: GeneratedRegistration[], serializerLookup: SerializerLookup, ) { let entry = this.findRegistration(typeInfo, registrations); @@ -396,10 +426,15 @@ export class Gen { entry.preparing = true; try { for (const child of children) { - this.traversalContainer(child, registrations, serializerLookup); + this.traversalContainer(child, registrations, factories, serializerLookup); } - const serializer = this.prepare(typeInfo, serializerLookup); - Object.assign(entry.serializer, serializer); + entry.factory = this.generateFactory(typeInfo, serializerLookup); + // This local owner is still unreachable by the resolver. Expose only the completed static + // facts to later code generation; the final pass installs its runtime methods after hooks. + entry.serializer.fixedSize = entry.factory.fixedSize; + entry.serializer.readDataAlwaysAdvances = entry.factory.readDataAlwaysAdvances; + entry.serializer._initialized = true; + factories.push(entry); } finally { entry.preparing = false; } @@ -459,6 +494,7 @@ export class Gen { private traversalContainer( typeInfo: TypeInfo, registrations: GeneratedRegistration[], + factories: GeneratedRegistration[], serializerLookup: SerializerLookup, ) { if (TypeId.userDefinedType(typeInfo.typeId)) { @@ -470,11 +506,26 @@ export class Gen { typeInfo.typeId === TypeId.UNION || typeInfo.typeId === TypeId.TYPED_UNION || typeInfo.typeId === TypeId.NAMED_UNION; - if (unionType && options?.cases && Object.keys(options.cases).length > 0) { + // Extension generation belongs only to an explicit root registration. Check it before the + // generic props path so a decorated nested extension cannot create a second local owner. + if (TypeId.extType(typeInfo.typeId)) { + if (this.findRegistration(typeInfo, registrations) === undefined) { + throw new Error("nested extension serializer must be registered before use"); + } + this.prepareRegistration( + typeInfo, + Object.values(options?.props ?? {}), + registrations, + factories, + serializerLookup, + ); + return; + } else if (unionType && options?.cases && Object.keys(options.cases).length > 0) { this.prepareRegistration( typeInfo, Object.values(options.cases), registrations, + factories, serializerLookup, ); return; @@ -483,42 +534,40 @@ export class Gen { typeInfo, Object.values(options.props), registrations, + factories, serializerLookup, ); } else if (!this.isRegistered(typeInfo) && TypeId.structType(typeInfo.typeId)) { if (this.findRegistration(typeInfo, registrations) === undefined) { throw new Error("nested struct schema must be registered or defined before use"); } - } else if (TypeId.extType(typeInfo.typeId)) { - if (this.findRegistration(typeInfo, registrations) === undefined) { - throw new Error("nested extension serializer must be registered before use"); - } } else if (TypeId.enumType(typeInfo.typeId) && !this.isRegistered(typeInfo)) { - this.prepareRegistration(typeInfo, [], registrations, serializerLookup); + this.prepareRegistration(typeInfo, [], registrations, factories, serializerLookup); } } if (typeInfo.typeId === TypeId.LIST) { - this.traversalContainer(typeInfo.options!.inner!, registrations, serializerLookup); + this.traversalContainer(typeInfo.options!.inner!, registrations, factories, serializerLookup); } if (typeInfo.typeId === TypeId.SET) { - this.traversalContainer(typeInfo.options!.key!, registrations, serializerLookup); + this.traversalContainer(typeInfo.options!.key!, registrations, factories, serializerLookup); } if (typeInfo.typeId === TypeId.MAP) { if (!typeInfo.options?.key || !typeInfo.options?.value) { throw new Error("map type must have key and value"); } - this.traversalContainer(typeInfo.options!.key!, registrations, serializerLookup); - this.traversalContainer(typeInfo.options!.value!, registrations, serializerLookup); + this.traversalContainer(typeInfo.options!.key!, registrations, factories, serializerLookup); + this.traversalContainer(typeInfo.options!.value!, registrations, factories, serializerLookup); } if (typeInfo.options?.cases) { Object.values(typeInfo.options.cases).forEach((caseTypeInfo) => { - this.traversalContainer(caseTypeInfo, registrations, serializerLookup); + this.traversalContainer(caseTypeInfo, registrations, factories, serializerLookup); }); } } reGenerateSerializer(typeInfo: TypeInfo) { - return this.prepare(typeInfo, this.typeResolver); + const factory = this.generateFactory(typeInfo, this.typeResolver); + return this.createSerializer(typeInfo, this.typeResolver, factory); } generateSerializer(typeInfo: TypeInfo) { @@ -528,6 +577,7 @@ export class Gen { // resolver before code generation or publication can continue. this.typeResolver.ensureRegistrationOpen(); const registrations: GeneratedRegistration[] = []; + const factories: GeneratedRegistration[] = []; // Generator-time TypeInfo queries see initialized local serializers for codegen decisions. // Factory-init ID/name queries instead return the stable owner captured by runtime closures. const serializerLookup: SerializerLookup = { @@ -544,7 +594,7 @@ export class Gen { ) { this.addRegistration(typeInfo, registrations); } - this.traversalContainer(typeInfo, registrations, serializerLookup); + this.traversalContainer(typeInfo, registrations, factories, serializerLookup); const publishedRoot = this.typeResolver.getSerializerByTypeInfo(typeInfo); if (!publishedRoot?._initialized) { let registration = this.findRegistration(typeInfo, registrations); @@ -552,21 +602,40 @@ export class Gen { registration = this.addRegistration(typeInfo, registrations); } if (!registration.serializer._initialized) { - this.prepareRegistration(typeInfo, [], registrations, serializerLookup); + this.prepareRegistration(typeInfo, [], registrations, factories, serializerLookup); } } - // Generated factories may execute application-transformed code, so every factory completes - // against local owners before the resolver performs the only global publication step. + // Hooks may publish an owner after earlier code generation used an equivalent local schema. + // Reconcile every identity before invoking any factory so fixed captures use the final owner. for (const registration of registrations) { + if (!this.hasRegistryIdentity(registration.typeInfo)) { + continue; + } const published = this.typeResolver.getSerializerByTypeInfo(registration.typeInfo); if (published !== undefined) { this.checkDefinitionOwner(published.getTypeInfo(), registration.typeInfo); } } + for (const registration of factories) { + const published = this.hasRegistryIdentity(registration.typeInfo) + ? this.typeResolver.getSerializerByTypeInfo(registration.typeInfo) + : undefined; + if (published !== undefined) { + continue; + } + Object.assign( + registration.serializer, + this.createSerializer(registration.typeInfo, serializerLookup, registration.factory!), + ); + } const serializer = this.getGeneratedSerializer(typeInfo, registrations)!; this.typeResolver.commitGeneratedSerializers( - registrations.filter((registration) => this.hasRegistryIdentity(registration.typeInfo)), + registrations.filter( + (registration) => + this.hasRegistryIdentity(registration.typeInfo) && + this.typeResolver.getSerializerByTypeInfo(registration.typeInfo) === undefined, + ), ); return serializer; } diff --git a/javascript/packages/core/lib/gen/serializer.ts b/javascript/packages/core/lib/gen/serializer.ts index 7da7c3490c..aa621bf38b 100644 --- a/javascript/packages/core/lib/gen/serializer.ts +++ b/javascript/packages/core/lib/gen/serializer.ts @@ -40,7 +40,12 @@ export interface SerializerGenerator { write(accessor: string): string; writeEmbed(): any; - toSerializer(): string; + toSerializer(): { + source: string; + localTypeMeta: TypeMeta | undefined; + fixedSize: number; + readDataAlwaysAdvances: boolean; + }; getFixedSize(): number; needToWriteRef(): boolean; @@ -332,12 +337,14 @@ export abstract class BaseSerializerGenerator implements SerializerGenerator { ? "" : `[localTypeMetaSymbol]: localTypeMeta, [checkedTypeMetaWireTypeIdSymbol]: localTypeMeta.getTypeId(),`; + const hash = this.getHash(); + const typeMetaBytes = this.getTypeMetaBytes(); const declare = ` const getHash = () => { - return ${this.getHash()}; + return ${hash}; } const getTypeMetaBytes = () => { - return ${this.getTypeMetaBytes()}; + return ${typeMetaBytes}; } const write = (v) => { ${this.write("v")} @@ -370,19 +377,26 @@ export abstract class BaseSerializerGenerator implements SerializerGenerator { ${this.readTypeInfo()} }; `; + const scope = this.scope.generate(); + const fixedSize = this.getFixedSize(); + const needToWriteRef = this.needToWriteRef(); + const typeId = this.getTypeId(); + const userTypeId = this.getUserTypeId(); + const readDataAlwaysAdvances = this.readDataAlwaysAdvances(); // Append read-only capability metadata so existing writer properties keep // their object-layout order on serialization hot paths. - return ` + return { + source: ` return function (typeResolver, serializerLookup, external, typeInfo, options${localTypeMetaParams}) { - ${this.scope.generate()} + ${scope} ${serializerDeclaration} ${declare} ${serializerAssignment} { _initialized: true, - fixedSize: ${this.getFixedSize()}, - needToWriteRef: () => ${this.needToWriteRef()}, - getTypeId: () => ${this.getTypeId()}, - getUserTypeId: () => ${this.getUserTypeId()}, + fixedSize: ${fixedSize}, + needToWriteRef: () => ${needToWriteRef}, + getTypeId: () => ${typeId}, + getUserTypeId: () => ${userTypeId}, getTypeInfo: () => typeInfo, getHash, getTypeMetaBytes, @@ -398,11 +412,15 @@ export abstract class BaseSerializerGenerator implements SerializerGenerator { readRefWithoutTypeInfo, readNoRef, readTypeInfo, - readDataAlwaysAdvances: ${this.readDataAlwaysAdvances()}, + readDataAlwaysAdvances: ${readDataAlwaysAdvances}, ${localTypeMetaProperty} }; ${serializerReturn} } - `; + `, + localTypeMeta, + fixedSize, + readDataAlwaysAdvances, + }; } } diff --git a/javascript/packages/core/lib/typeResolver.ts b/javascript/packages/core/lib/typeResolver.ts index 33d819f075..7db8d75293 100644 --- a/javascript/packages/core/lib/typeResolver.ts +++ b/javascript/packages/core/lib/typeResolver.ts @@ -213,18 +213,21 @@ export default class TypeResolver { this.initInternalSerializer(); } + /** @internal */ freezeRegistration() { if (!this.registrationFrozen) { this.registrationFrozen = true; } } + /** @internal */ ensureRegistrationOpen() { if (this.registrationFrozen) { throw new Error("types and serializers must be registered before the first root operation"); } } + /** @internal */ commitGeneratedSerializers(entries: readonly { typeInfo: TypeInfo; serializer: Serializer }[]) { this.ensureRegistrationOpen(); const publications = entries.map((entry) => { @@ -267,7 +270,7 @@ export default class TypeResolver { } generateReadSerializer(typeInfo: TypeInfo) { - return new Gen(this, { creator: typeInfo.options?.creator }).reGenerateSerializer(typeInfo); + return new Gen(this).reGenerateSerializer(typeInfo); } getSerializerByTypeInfo(typeInfo: TypeInfo) { diff --git a/javascript/test/fory.test.ts b/javascript/test/fory.test.ts index 9d6acf5327..8f3a0d4796 100644 --- a/javascript/test/fory.test.ts +++ b/javascript/test/fory.test.ts @@ -208,15 +208,43 @@ describe("fory", () => { expect(parent.deserialize(parent.serialize(value))).toEqual(value); }); - test("rejects unresolved extension", () => { + test("rejects nested extension", () => { + const extensionType = Type.ext(8152); + @extensionType + class NestedExtension { + @Type.int32() + value = 0; + } const fory = new Fory({ compatible: false }); - const root = Type.struct(8151, { value: Type.ext(8152) }); + const root = Type.struct(8151, { value: extensionType }); expect(() => fory.register(root)).toThrow(); expect(fory.typeResolver.getSerializerById(TypeId.STRUCT, 8151)).toBeUndefined(); expect(fory.typeResolver.getSerializerById(TypeId.EXT, 8152)).toBeUndefined(); }); + test("uses decorated constructors", () => { + const childType = Type.struct(8153); + @childType + class Child { + @Type.int32() + value = 0; + } + const parentType = Type.struct(8154, { child: childType }); + @parentType + class Parent { + child = new Child(); + } + const registered = new Fory({ compatible: false }).register(Parent); + const value = new Parent(); + value.child.value = 7; + + const result = registered.deserialize(registered.serialize(value)); + expect(result).toBeInstanceOf(Parent); + expect(result!.child).toBeInstanceOf(Child); + expect(result!.child.value).toBe(7); + }); + test("registers empty roots", () => { const registered = new Fory({ compatible: false }).register(Type.struct(8122, {})); @@ -597,6 +625,51 @@ describe("fory", () => { expect(fory.typeResolver.getSerializerById(TypeId.STRUCT, 8142)).toBeUndefined(); }); + test("uses a late reentrant owner", () => { + const child = Type.struct(8155, { value: Type.int32() }); + const early = Type.struct(8156, { child }); + const trigger = Type.struct(8157, { value: Type.string() }); + const root = Type.struct(8158, { early, trigger }); + let hookCount = 0; + let generateReentrant = false; + let reentrant: ReturnType; + let fory: Fory; + fory = new Fory({ + compatible: false, + hooks: { + afterCodeGenerated(code) { + hookCount++; + if (generateReentrant && hookCount === 3) { + generateReentrant = false; + reentrant = fory.register(child); + } + return code; + }, + }, + }); + hookCount = 0; + generateReentrant = true; + + const registered = fory.register(root); + expect(hookCount).toBe(5); + expect(fory.typeResolver.getSerializerById(TypeId.STRUCT, 8155)).toBe(reentrant!.serializer); + expect(fory.typeResolver.getSerializerById(TypeId.STRUCT, 8158)).toBe(registered.serializer); + + let winnerReads = 0; + const winnerRead = reentrant!.serializer.read; + reentrant!.serializer.read = (fromRef) => { + winnerReads++; + return winnerRead(fromRef); + }; + const value = { + early: { child: { value: 7 } }, + trigger: { value: "trigger" }, + }; + + expect(registered.deserialize(registered.serialize(value))).toEqual(value); + expect(winnerReads).toBe(1); + }); + test("seals replaced schema options", () => { const original = Type.struct(8126, { oldValue: Type.int32() }); const replacement: { From cd156f0763baab48d489ea9b7e6abc34937470c4 Mon Sep 17 00:00:00 2001 From: chaokunyang Date: Sat, 29 Aug 2026 08:23:30 +0800 Subject: [PATCH 081/168] fix(python): enforce registry finalization ownership --- .agents/languages/python.md | 20 ++-- docs/security/deserialization.md | 20 ++-- python/pyfory/_fory.py | 6 +- python/pyfory/registry.py | 14 ++- python/pyfory/serialization.pyx | 23 ++-- python/pyfory/tests/test_serializer.py | 140 +++++++++++++++++++++++- python/pyfory/tests/test_thread_safe.py | 36 ++++++ 7 files changed, 223 insertions(+), 36 deletions(-) diff --git a/.agents/languages/python.md b/.agents/languages/python.md index 322b08d80e..5efcb8ce04 100644 --- a/.agents/languages/python.md +++ b/.agents/languages/python.md @@ -11,9 +11,14 @@ Load this file when changing `python/`, Cython serialization, or Python xlang be - Python mode is the pure-Python xlang implementation and is mainly for debugging and testing. - Cython mode is the default high-performance implementation. - Cython mode owns the hot runtime path. Do not duplicate core runtime types between Python and Cython, tunnel Python facade methods into hidden Cython internals, or keep dead shims unless the user explicitly needs a compatibility module path. -- Python `TypeResolver` owns registry freeze and finalization state. Its Cython companion may cache - completion of the one Python-owner dispatch needed to populate native resolver tables, but the - `Fory` facade must not mirror that state. Cython roots call the resolver owner directly. +- Python `TypeResolver` separately owns permanent registry freeze, active finalization, and + successful finalization. A root may bypass that owner only after successful completion; roots + entered during finalization or after failed finalization must fail before codec work. Its Cython + companion may cache completion only after the Python owner succeeds and every native resolver + table is synchronized; the `Fory` facade must not mirror that state. Cython roots call the + resolver owner directly until then. The Python owner permanently rejects its own incomplete + finalization; if native synchronization fails, the companion records only permanent failure, + never the exception, and rejects later roots without retrying partial synchronization. Serializer construction may reenter registration or a root, so the resolver rechecks both registration conflicts and its frozen state after construction and before publishing type, serializer, name, or ID state. Allocate automatic type IDs only after those checks at the common @@ -23,11 +28,10 @@ Load this file when changing `python/`, Cython serialization, or Python xlang be registration linearization is reentrant so nested facade registrations share the same publication order. A root started during registration must not reuse the staging instance, and root reentry from a running user `fory_factory` or retained registration replay fails without - recursively building - another instance. The build thread must be rejected before pool acquisition even when another - instance becomes available during that build. The non-reentrant pool lock owns pool publication, - root-started state, registration depth, and the staging instance; the separate instance-build - boundary covers the factory and complete registration replay. During child replay, a nested + recursively building another instance. The build thread must be rejected before pool acquisition + even when another instance becomes available during that build. The non-reentrant pool lock owns + pool publication, root-started state, registration depth, and the staging instance; the separate + instance-build boundary covers the factory and complete registration replay. During child replay, a nested facade registration is a no-op only when it exactly matches an accepted descriptor in the prefix already applied to that child; reject every unknown or different request before that request mutates the child. diff --git a/docs/security/deserialization.md b/docs/security/deserialization.md index cf90e1b384..1e6afdf3d1 100644 --- a/docs/security/deserialization.md +++ b/docs/security/deserialization.md @@ -619,14 +619,18 @@ that case, classify the behavior by concrete impact: ## Registry Lifecycle The first root operation permanently closes type and serializer registration, including when that -operation or registry finalization fails. Registration that invokes application code must recheck -the authoritative lifecycle before publishing application-derived state. Thread-safe facades -retain only registrations that completed before the freeze. During child construction, a semantic -replay log may reuse only an identical accepted registration that the child has already applied; -unknown or different requests fail before that request mutates the child. A facade that replays opaque -registration callbacks and cannot roll them back must become permanently unusable when a callback -fails rather than expose partially registered children. These rules prevent a failed or reentrant -registration from changing the accepted type surface after deserialization has begun. +operation or registry finalization fails. Permanent freeze and successful finalization are distinct: +a root entered during finalization or after failed finalization must fail before serializer or read +work, and an accelerator must neither cache nor retry incomplete finalization. Failure state must +not retain the exception or traceback graph. Registration that invokes application code must +recheck the authoritative lifecycle before publishing application-derived state. Thread-safe +facades retain only registrations that completed before the freeze. During child +construction, a semantic replay log may reuse only an identical accepted registration that the +child has already applied; unknown or different requests fail before that request mutates the child. +A facade that replays opaque registration callbacks and cannot roll them back must become +permanently unusable when a callback fails rather than expose partially registered children. These +rules prevent a failed or reentrant registration from changing the accepted type surface after +deserialization has begun. Runtime-specific publication ownership belongs in the implementation guide and language guidance. ## Metadata And Type Resolution diff --git a/python/pyfory/_fory.py b/python/pyfory/_fory.py index 5534a49eea..a504fed858 100644 --- a/python/pyfory/_fory.py +++ b/python/pyfory/_fory.py @@ -431,7 +431,7 @@ def dump(self, obj, stream): the passed object (or a view of it) is unsupported. If your sink needs retention, copy bytes inside ``write``. """ - if not self.type_resolver._registry_frozen: + if not self.type_resolver._registry_finalization_complete: self.type_resolver._freeze_registry() try: self.buffer.set_writer_index(0) @@ -488,7 +488,7 @@ def serialize( >>> print(type(data)) """ - if not self.type_resolver._registry_frozen: + if not self.type_resolver._registry_finalization_complete: self.type_resolver._freeze_registry() try: write_buffer = self._serialize( @@ -567,7 +567,7 @@ def deserialize( >>> print(obj) {'key': 'value'} """ - if not self.type_resolver._registry_frozen: + if not self.type_resolver._registry_finalization_complete: self.type_resolver._freeze_registry() try: return self._deserialize(buffer, buffers, unsupported_objects) diff --git a/python/pyfory/registry.py b/python/pyfory/registry.py index 0ce7fa9ad4..60672c196b 100644 --- a/python/pyfory/registry.py +++ b/python/pyfory/registry.py @@ -220,7 +220,6 @@ def _construct_serializer(serializer_factory, type_resolver, cls): for nargs, args in ( (2, (type_resolver, cls)), (1, (type_resolver,)), - (0, ()), ): if _accepts_n_positional_args(serializer_factory, nargs): serializer = serializer_factory(*args) @@ -231,7 +230,7 @@ def _construct_serializer(serializer_factory, type_resolver, cls): if normalize_fory_type(serializer.type_) != normalize_fory_type(cls): raise TypeError("Serializer factory returned a serializer bound to a different type") return serializer - raise TypeError(f"Unsupported serializer constructor for {serializer_factory!r}; expected `(type_resolver, cls)`, `(type_resolver)`, or `()`.") + raise TypeError(f"Unsupported serializer constructor for {serializer_factory!r}; expected `(type_resolver, cls)` or `(type_resolver)`.") def _split_registration_name(name: str): @@ -379,6 +378,7 @@ class TypeResolver: "meta_share", "_internal_py_serializer_map", "_actual_type_resolver", + "_registry_finalization_complete", "_registry_frozen", "_registry_finalizing", ) @@ -422,7 +422,9 @@ def __init__(self, config, *, shared_registry): self._internal_py_serializer_map = {} self._actual_type_resolver = self # Fory exposes this resolver, so the resolver must own the root-use gate; - # facade-only state would leave direct registration methods mutable. + # facade-only state would leave direct registration methods mutable. Freeze + # is permanent, while completion records only a successful finalization. + self._registry_finalization_complete = False self._registry_frozen = False self._registry_finalizing = False @@ -441,17 +443,19 @@ def _needs_registration_finalization(self, type_info): return self.meta_share and type_info.type_def is None and TypeId.is_type_share_meta(type_info.type_id) def _freeze_registry(self): + if self._registry_finalization_complete: + return if self._registry_frozen: - return False + raise RuntimeError("Registry finalization did not complete") self._registry_frozen = True self._registry_finalizing = True try: for type_info in self._types_info.values(): if self._needs_registration_finalization(type_info): self._set_type_info(type_info) + self._registry_finalization_complete = True finally: self._registry_finalizing = False - return True def _set_actual_resolver(self, type_resolver): # Cython mode injects the compiled companion before initialize() so all diff --git a/python/pyfory/serialization.pyx b/python/pyfory/serialization.pyx index 112f5e9dcd..4a909a1d75 100644 --- a/python/pyfory/serialization.pyx +++ b/python/pyfory/serialization.pyx @@ -269,9 +269,11 @@ cdef class TypeResolver: cdef flat_hash_map[uint32_t, PyObject *] _c_user_type_id_to_type_info cdef flat_hash_map[uint64_t, PyObject *] _c_types_info cdef flat_hash_map[pair[int64_t, int64_t], PyObject *] _c_meta_hash_to_type_info - # The Python resolver owns registry mutability. This monotonic native cache only - # avoids re-entering Python after that owner has completed its first freeze attempt. - cdef bint registry_freeze_complete + # The Python resolver owns registry mutability. Native completion is published only + # after that owner succeeds and every native table is synchronized. Failure is + # retained without its exception so a partial native cache is never retried or used. + cdef bint registry_finalization_complete + cdef bint registry_finalization_failed def __init__(self, Config config, *, shared_registry): """ @@ -303,7 +305,8 @@ cdef class TypeResolver: self._ns_type_to_type_info = resolver._ns_type_to_type_info self._local_type_info_by_hash = resolver._local_type_info_by_hash self._meta_shared_type_info = resolver._meta_shared_type_info - self.registry_freeze_complete = False + self.registry_finalization_complete = False + self.registry_finalization_failed = False for typeinfo in resolver._types_info.values(): self._populate_type_info(typeinfo) @@ -316,11 +319,17 @@ cdef class TypeResolver: cdef inline void _freeze_registry(self): cdef object typeinfo - if not self.registry_freeze_complete: - if self.resolver._freeze_registry(): + if not self.registry_finalization_complete: + if self.registry_finalization_failed: + raise RuntimeError("Registry finalization did not complete") + self.resolver._freeze_registry() + try: for typeinfo in self.resolver._types_info.values(): self._populate_type_info(typeinfo) - self.registry_freeze_complete = True + except BaseException: + self.registry_finalization_failed = True + raise + self.registry_finalization_complete = True def register_type( self, diff --git a/python/pyfory/tests/test_serializer.py b/python/pyfory/tests/test_serializer.py index bd102ff8d5..8d54b35ed2 100644 --- a/python/pyfory/tests/test_serializer.py +++ b/python/pyfory/tests/test_serializer.py @@ -1286,10 +1286,27 @@ def test_named_serializer_id_map(): def test_failed_finalization_freezes(monkeypatch): writer = Fory(xlang=True, compatible=True) - writer.register_type(FrozenChild, name="test.PendingFinalization") - pending_data = writer.serialize(FrozenChild(7)) + writer.register_type( + FrozenExt, + name="test.FinalizedRoot", + serializer=FrozenExtSerializer(writer.type_resolver, FrozenExt), + ) + data = writer.serialize(FrozenExt(7)) fory = Fory(xlang=True, compatible=True) + read_calls = 0 + + class CountingSerializer(FrozenExtSerializer): + def read(self, read_context): + nonlocal read_calls + read_calls += 1 + return super().read(read_context) + + fory.register_type( + FrozenExt, + name="test.FinalizedRoot", + serializer=CountingSerializer(fory.type_resolver, FrozenExt), + ) type_info = fory.register_type( BrokenFinalization, name="test.BrokenFinalization", @@ -1299,12 +1316,15 @@ def test_failed_finalization_freezes(monkeypatch): name="test.PendingFinalization", ) encode_type_def = registry_module.encode_typedef + finalization_calls = 0 class FinalizationAbort(BaseException): pass def fail_finalization(resolver, cls): + nonlocal finalization_calls if cls is type_info.cls: + finalization_calls += 1 assert type_info.serializer is not None raise FinalizationAbort return encode_type_def(resolver, cls) @@ -1316,14 +1336,18 @@ def fail_finalization(resolver, cls): ) with pytest.raises(FinalizationAbort): - fory.serialize(None) + fory.deserialize(data) + assert read_calls == 0 + assert finalization_calls == 1 assert type_info.serializer is None assert type_info.type_def is None assert pending_info.serializer is None assert pending_info.type_def is None - with pytest.raises(Exception): - fory.deserialize(pending_data) + with pytest.raises(RuntimeError): + fory.deserialize(data) + assert read_calls == 0 + assert finalization_calls == 1 assert pending_info.serializer is None assert pending_info.type_def is None @@ -1336,6 +1360,112 @@ def fail_finalization(resolver, cls): assert fory.type_resolver.get_type_info(RejectedRegistration, create=False) is None +def test_reentrant_finalization_freezes(monkeypatch): + writer = Fory(xlang=True, compatible=True) + writer.register_type( + FrozenExt, + name="test.ReentrantRoot", + serializer=FrozenExtSerializer(writer.type_resolver, FrozenExt), + ) + data = writer.serialize(FrozenExt(7)) + + fory = Fory(xlang=True, compatible=True) + read_calls = 0 + + class CountingSerializer(FrozenExtSerializer): + def read(self, read_context): + nonlocal read_calls + read_calls += 1 + return super().read(read_context) + + fory.register_type( + FrozenExt, + name="test.ReentrantRoot", + serializer=CountingSerializer(fory.type_resolver, FrozenExt), + ) + type_info = fory.register_type( + BrokenFinalization, + name="test.ReentrantFinalization", + ) + encode_type_def = registry_module.encode_typedef + finalization_calls = 0 + + def reenter_root(resolver, cls): + nonlocal finalization_calls + if cls is type_info.cls: + finalization_calls += 1 + fory.deserialize(data) + return encode_type_def(resolver, cls) + + monkeypatch.setattr( + registry_module, + "encode_typedef", + reenter_root, + ) + + with pytest.raises(RuntimeError): + fory.deserialize(data) + assert read_calls == 0 + assert finalization_calls == 1 + assert type_info.serializer is None + assert type_info.type_def is None + + with pytest.raises(RuntimeError): + fory.deserialize(data) + assert read_calls == 0 + assert finalization_calls == 1 + with pytest.raises(Exception): + fory.register_type(RejectedRegistration, name="test.Rejected") + assert fory.type_resolver.get_type_info(RejectedRegistration, create=False) is None + + +@pytest.mark.skipif( + not pyfory.ENABLE_FORY_CYTHON_SERIALIZATION, + reason="Requires the Cython resolver cache", +) +def test_native_finalization_failure(): + write_calls = 0 + + class CountingSerializer(FrozenExtSerializer): + def write(self, write_context, value): + nonlocal write_calls + write_calls += 1 + super().write(write_context, value) + + fory = Fory(xlang=True, compatible=False) + type_info = fory.register_type( + FrozenExt, + name="test.NativeFinalization", + serializer=CountingSerializer(fory.type_resolver, FrozenExt), + ) + hash_reads = 0 + + class NativeSyncAbort(BaseException): + pass + + class BrokenMetaString: + @property + def hashcode(self): + nonlocal hash_reads + hash_reads += 1 + raise NativeSyncAbort + + type_info.namespace_bytes = BrokenMetaString() + + with pytest.raises(NativeSyncAbort): + fory.serialize(FrozenExt(7)) + assert hash_reads == 1 + assert write_calls == 0 + + with pytest.raises(RuntimeError): + fory.serialize(FrozenExt(7)) + assert hash_reads == 1 + assert write_calls == 0 + with pytest.raises(Exception): + fory.register_type(RejectedRegistration, name="test.Rejected") + assert fory.type_resolver.get_type_info(RejectedRegistration, create=False) is None + + def test_native_carrier_registration(): writer = Fory(xlang=False, strict=False, compatible=False) reader = Fory(xlang=False, strict=False, compatible=False) diff --git a/python/pyfory/tests/test_thread_safe.py b/python/pyfory/tests/test_thread_safe.py index 917cf1de17..29c1bbce0f 100644 --- a/python/pyfory/tests/test_thread_safe.py +++ b/python/pyfory/tests/test_thread_safe.py @@ -229,6 +229,42 @@ def serializer_factory(type_resolver, cls): assert valid_constructions == 1 +def test_zero_arg_serializer_rejected(): + class AddressSerializer(pyfory.Serializer): + def write(self, write_context, value): + write_context.write_string(value.city) + write_context.write_string(value.country) + + def read(self, read_context): + return Address(read_context.read_string(), read_context.read_string()) + + children = [] + constructions = 0 + + def fory_factory(): + child = pyfory.Fory(xlang=False, compatible=False) + children.append(child) + return child + + def serializer_factory(): + nonlocal constructions + constructions += 1 + child = children[-1] + return AddressSerializer(child.type_resolver, Address) + + fory = ThreadSafeFory(fory_factory=fory_factory) + with pytest.raises(TypeError): + fory.register_type(Address, serializer=serializer_factory) + + assert constructions == 0 + assert len(children) == 1 + assert children[0].type_resolver.get_type_info(Address, create=False) is None + + fory.register_type(Address, serializer=AddressSerializer) + value = Address(city="Oslo", country="Norway") + assert fory.deserialize(fory.serialize(value)) == value + + @pytest.mark.parametrize("method", ["register", "register_type", "register_union"]) def test_serializer_instance_rejected(method): class AddressSerializer(pyfory.Serializer): From 6316f6c1718e6fe1c28b94543274d64381c6ae14 Mon Sep 17 00:00:00 2001 From: chaokunyang Date: Sat, 29 Aug 2026 08:23:36 +0800 Subject: [PATCH 082/168] refactor(python): remove unused range index state --- python/pyfory/serializer.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/python/pyfory/serializer.py b/python/pyfory/serializer.py index a2aeadc18d..a2146b91c2 100644 --- a/python/pyfory/serializer.py +++ b/python/pyfory/serializer.py @@ -525,8 +525,6 @@ def read(self, read_context): class PandasRangeIndexSerializer(Serializer): - __slots__ = "_cached" - def __init__(self, type_resolver): import pandas as pd From 83e4893bf2f3b42793f11b02cf5f6c9872fe2b02 Mon Sep 17 00:00:00 2001 From: chaokunyang Date: Sat, 29 Aug 2026 08:23:46 +0800 Subject: [PATCH 083/168] docs(python): align callable and serializer setup --- .../python/configuration.md | 94 ++++---- .../python/custom-serializers.md | 17 +- .../python/functions-classes-methods.md | 209 +++++++++++++--- docs/object-serialization/python/index.md | 8 +- .../python/type-registration.md | 7 +- python/README.md | 228 +----------------- 6 files changed, 251 insertions(+), 312 deletions(-) diff --git a/docs/object-serialization/python/configuration.md b/docs/object-serialization/python/configuration.md index dc8673acaa..a099a2f6fe 100644 --- a/docs/object-serialization/python/configuration.md +++ b/docs/object-serialization/python/configuration.md @@ -59,23 +59,23 @@ class ThreadSafeFory: ## Parameters -| Parameter | Type | Default | Description | -| -------------------------------------- | ------------------------------- | ----------- | --------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | -| `xlang` | `bool` | `True` | Use xlang mode. Set `False` for Python native mode. | -| `ref` | `bool` | `False` | Enable reference tracking for shared/circular references. Disable for better performance if your data has no shared references. | -| `strict` | `bool` | `True` | Require registration before loading application classes. Compatible unknown Structs use `UnknownStruct`. `False` permits policy-authorized native module-global resolution. | -| `compatible` | `bool \| None` | `None` | Schema evolution mode. `None` enables compatible mode in both xlang and native mode. Set `False` only when every reader and writer uses the same schema. | -| `max_depth` | `int` | `50` | Maximum deserialization depth for security, preventing stack overflow attacks. | -| `max_type_fields` | `int` | `512` | Maximum fields accepted in one received remote struct metadata body. | -| `max_type_meta_bytes` | `int` | `4096` | Maximum encoded body bytes accepted for one received TypeDef body, excluding the 8-byte header and any extended-size varint. | -| `max_schema_versions_per_type` | `int` | `10` | Maximum accepted remote metadata versions for one logical type. | -| `max_average_schema_versions_per_type` | `int` | `3` | Average accepted remote metadata versions across accepted remote types. The effective global floor is `8192` schemas. | -| `max_graph_memory_bytes` | `int` | `134217728` | Approximate graph-memory gate for one root deserialization. Explicit non-positive values are rejected. | -| `max_unbacked_container_items` | `int` | `8192` | Maximum collection elements and map entries whose repeated reads are not backed by input progress. Zero is strict. | -| `policy` | `DeserializationPolicy \| None` | `None` | Deserialization policy used for security checks. Strongly recommended when `strict=False`. | -| `field_nullable` | `bool` | `False` | Treat dataclass fields as nullable by default. | -| `meta_compressor` | `Any` | `None` | Optional metadata compressor used for compatible-mode metadata encoding. | -| `fory_factory` | `Callable \| None` | `None` | `ThreadSafeFory` factory hook. When set, `ThreadSafeFory` creates instances via this callback; otherwise it forwards `**kwargs` to `Fory` construction. | +| Parameter | Type | Default | Description | +| -------------------------------------- | ------------------------------- | ----------- | --------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| `xlang` | `bool` | `True` | Use xlang mode. Set `False` for Python native mode. | +| `ref` | `bool` | `False` | Enable reference tracking for shared/circular references. Disable for better performance if your data has no shared references. | +| `strict` | `bool` | `True` | Require registration before loading application classes. Compatible unknown Structs use `UnknownStruct`. `False` permits policy-authorized native module-global resolution but does not replace carrier registration. | +| `compatible` | `bool \| None` | `None` | Schema evolution mode. `None` enables compatible mode in both xlang and native mode. Set `False` only when every reader and writer uses the same schema. | +| `max_depth` | `int` | `50` | Maximum deserialization depth for security, preventing stack overflow attacks. | +| `max_type_fields` | `int` | `512` | Maximum fields accepted in one received remote struct metadata body. | +| `max_type_meta_bytes` | `int` | `4096` | Maximum encoded body bytes accepted for one received TypeDef body, excluding the 8-byte header and any extended-size varint. | +| `max_schema_versions_per_type` | `int` | `10` | Maximum accepted remote metadata versions for one logical type. | +| `max_average_schema_versions_per_type` | `int` | `3` | Average accepted remote metadata versions across accepted remote types. The effective global floor is `8192` schemas. | +| `max_graph_memory_bytes` | `int` | `134217728` | Approximate graph-memory gate for one root deserialization. Explicit non-positive values are rejected. | +| `max_unbacked_container_items` | `int` | `8192` | Maximum collection elements and map entries whose repeated reads are not backed by input progress. Zero is strict. | +| `policy` | `DeserializationPolicy \| None` | `None` | Deserialization policy used for security checks. Strongly recommended when `strict=False`. | +| `field_nullable` | `bool` | `False` | Treat dataclass fields as nullable by default. | +| `meta_compressor` | `Any` | `None` | Optional metadata compressor used for compatible-mode metadata encoding. | +| `fory_factory` | `Callable \| None` | `None` | `ThreadSafeFory` factory hook. When set, `ThreadSafeFory` creates instances via this callback; otherwise it forwards `**kwargs` to `Fory` construction. | ## Key Methods @@ -89,7 +89,7 @@ fory.register(MyClass) # fory.register(MyClass, name="my.package.MyClass") # Direct Fory accepts an instance; ThreadSafeFory requires a serializer class or factory. -# fory.register(MyClass, serializer=MySerializer) +# fory.register(MyClass, serializer=MySerializer(fory.type_resolver, MyClass)) # Serialization (serialize/deserialize are identical to dumps/loads) data: bytes = fory.serialize(obj) @@ -109,15 +109,15 @@ a value. See ## Xlang And Native Mode Comparison -| Feature | Native mode (`xlang=False`) | Xlang mode (default) | -| ------------------- | -------------------------------------------- | -------------------------------------------------------------------------------- | -| Use case | Python-only applications | Multi-language systems | -| Compatibility | Python only | Java, C++, Go, Rust, JavaScript/TypeScript, C#, Swift, Dart, Scala, Kotlin, etc. | -| Supported types | Configured Python object surface | Cross-language compatible types | -| Functions/lambdas | Supported when their carriers are registered | Not allowed | -| Local classes | Supported when their carriers are registered | Not allowed | -| Class objects | Supported when their carriers are registered | Not allowed | -| Schema mode default | Compatible | Compatible | +| Feature | Native mode (`xlang=False`) | Xlang mode (default) | +| ------------------- | ----------------------------------------------------- | -------------------------------------------------------------------------------- | +| Use case | Python-only applications | Multi-language systems | +| Compatibility | Python only | Java, C++, Go, Rust, JavaScript/TypeScript, C#, Swift, Dart, Scala, Kotlin, etc. | +| Supported types | Configured Python object surface | Cross-language compatible types | +| Functions/lambdas | Require carrier registration and policy authorization | Not allowed | +| Local classes | Require carrier registration and policy authorization | Not allowed | +| Class objects | Require type registration and policy authorization | Not allowed | +| Schema mode default | Compatible | Compatible | ## Xlang Mode @@ -136,18 +136,25 @@ Use `compatible=False` for xlang payloads only when every reader and writer alwa ## Native Mode ```python -import types +from dataclasses import dataclass import pyfory -fory = pyfory.Fory(xlang=False, ref=True, strict=False) -fory.register_type(types.FunctionType) + +@dataclass +class Event: + name: str + + +fory = pyfory.Fory(xlang=False, ref=True, strict=True) +fory.register_type(Event) ``` Native mode supports Python-specific object features such as functions, local classes, methods, `__reduce__`, and `__getstate__` when their application and carrier types are registered before the -first root attempt. Compatible mode is still enabled by default. Set `compatible=False` only when -every reader and writer always uses the same Python class schema. +first root attempt and the configured policy authorizes their deserialization. Compatible mode is +still enabled by default. Set `compatible=False` only when every reader and writer always uses the +same Python class schema. ## Compatible Mode @@ -177,24 +184,29 @@ fory.register(UserModel, name="example.User") ### Native Mode With Configured Python Types ```python -import types +from dataclasses import dataclass import pyfory + +@dataclass +class Event: + name: str + + fory = pyfory.Fory( xlang=False, ref=True, - strict=False, - max_depth=1000, + strict=True, ) - -fory.register_type(types.FunctionType) +fory.register_type(Event) ``` -Use `strict=False` only for trusted data, preferably with a `policy=` deserialization policy. -Register every application and Python-native carrier type whose serializer must be installed before -the first root attempt. Policy-authorized module-global classes and callables may still be resolved -while reading a native payload; that lookup does not reopen or mutate the frozen registry. +Use `strict=False` only for trusted data and configure a `policy=` deserialization policy for +functions, local classes, class objects, or other dynamic native values. Register every application +and Python-native carrier type whose serializer must be installed before the first root attempt. +Policy-authorized module-global classes and callables may still be resolved while reading a native +payload; that lookup does not reopen or mutate the frozen registry. ## Security diff --git a/docs/object-serialization/python/custom-serializers.md b/docs/object-serialization/python/custom-serializers.md index 615a6b129f..a641bd02f8 100644 --- a/docs/object-serialization/python/custom-serializers.md +++ b/docs/object-serialization/python/custom-serializers.md @@ -134,26 +134,15 @@ fory.register(MyClass, name="com.example.MyClass", serializer=MySerializer(fory. ### Thread-safe registration `ThreadSafeFory` accepts a serializer class or factory rather than an already constructed -serializer. Each pooled `Fory` then owns a serializer bound to its own resolver: +serializer: ```python thread_safe = pyfory.ThreadSafeFory(xlang=False) thread_safe.register(Foo, type_id=100, serializer=FooSerializer) ``` -When construction needs instance-specific settings, configure each child through the existing -`fory_factory`: - -```python -def create_fory(): - child = pyfory.Fory(xlang=False) - serializer = FooSerializer(child.type_resolver, Foo) - serializer.application_option = "configured" - child.register(Foo, type_id=100, serializer=serializer) - return child - -thread_safe = pyfory.ThreadSafeFory(fory_factory=create_fory) -``` +A serializer factory passed to `ThreadSafeFory.register` must accept `(resolver, type)` or +`(resolver)` and return a serializer for that resolver and registered type. ## Related Topics diff --git a/docs/object-serialization/python/functions-classes-methods.md b/docs/object-serialization/python/functions-classes-methods.md index 108aae1d9d..69907c6a55 100644 --- a/docs/object-serialization/python/functions-classes-methods.md +++ b/docs/object-serialization/python/functions-classes-methods.md @@ -20,8 +20,8 @@ license: | --- Python native mode serializes Python-specific callable and type values that are outside the xlang -type system. Use `strict=False` only for trusted payloads and apply a deserialization policy when -the accepted dynamic surface must be restricted. +type system. Use `strict=False` only for trusted payloads. Configure a deserialization policy that +authorizes the callable and class references accepted by the application. Register every callable carrier and application type whose serializer must be installed before the first root operation. The first serialization or deserialization attempt permanently freezes the @@ -31,21 +31,37 @@ install a type or serializer. ## Serialize Global Functions -Capture and serialize functions defined at module level. Fory deserializes and returns the same -function object: +Functions imported from a module deserialize to the same function object. Register the function +carrier and authorize the expected module-level function before the first root operation: ```python -import pyfory +import statistics import types -fory = pyfory.Fory(xlang=False, ref=True, strict=False) +import pyfory +from pyfory import DeserializationPolicy + + +class TrustedFunctionPolicy(DeserializationPolicy): + def validate_module(self, module_name, is_local, **kwargs): + if module_name != "statistics": + raise ValueError(f"Blocked module: {module_name}") -def my_global_function(x): - return 10 * x + def validate_function(self, func, is_local, **kwargs): + if func is not statistics.mean or is_local: + raise ValueError(f"Blocked function: {func!r}") + +fory = pyfory.Fory( + xlang=False, + ref=True, + strict=False, + policy=TrustedFunctionPolicy(), +) fory.register_type(types.FunctionType) -data = fory.dumps(my_global_function) -print(fory.loads(data)(10)) # 100 +restored = fory.loads(fory.dumps(statistics.mean)) +assert restored is statistics.mean +assert restored([10, 20, 30]) == 20 ``` ## Serialize Local Functions/Lambdas @@ -54,25 +70,136 @@ Serialize functions with closures and lambda expressions. Fory captures the clos automatically: ```python -import pyfory import types -fory = pyfory.Fory(xlang=False, ref=True, strict=False) +import pyfory +from pyfory import DeserializationPolicy + -# Local functions with closures -def my_function(): - local_var = 10 - def local_func(x): - return x * local_var - return local_func +class TrustedLocalFunctionPolicy(DeserializationPolicy): + def authorize_instantiation(self, cls, **kwargs): + if cls is not types.FunctionType: + raise ValueError(f"Blocked materialization: {cls!r}") + def validate_function(self, func, is_local, **kwargs): + if not is_local or func.__name__ not in {"multiply", ""}: + raise ValueError(f"Blocked function: {func!r}") + + +fory = pyfory.Fory( + xlang=False, + ref=True, + strict=False, + policy=TrustedLocalFunctionPolicy(), +) fory.register_type(types.FunctionType) -data = fory.dumps(my_function()) -print(fory.loads(data)(10)) # 100 -# Lambdas -data = fory.dumps(lambda x: 10 * x) -print(fory.loads(data)(10)) # 100 +def make_multiplier(factor): + def multiply(value): + return factor * value + + return multiply + + +restored = fory.loads(fory.dumps(make_multiplier(10))) +assert restored(10) == 100 + +restored_lambda = fory.loads(fory.dumps(lambda x: 10 * x)) +assert restored_lambda(10) == 100 +``` + +## Serialize Class Objects + +Register the `type` carrier before serializing a class object, and authorize class resolution in the +deserialization policy. Register the concrete application class separately when its instances also +appear in the payload: + +```python +from collections import Counter + +import pyfory +from pyfory import DeserializationPolicy + + +class TrustedClassPolicy(DeserializationPolicy): + def validate_module(self, module_name, is_local, **kwargs): + if module_name != "collections": + raise ValueError(f"Blocked module: {module_name}") + + def validate_class(self, cls, is_local, **kwargs): + if cls is not Counter or is_local: + raise ValueError(f"Blocked class: {cls!r}") + + +fory = pyfory.Fory( + xlang=False, + ref=True, + strict=False, + policy=TrustedClassPolicy(), +) +fory.register_type(type) + +restored = fory.loads(fory.dumps(Counter)) +assert restored is Counter +``` + +## Serialize Local Classes And Class Methods + +Local classes are reconstructed from their definition, so use `ref=True` and a policy that +authorizes construction of the class, its functions, and its bound class methods. Register all +carriers before the first root operation: + +```python +import types + +import pyfory +from pyfory import DeserializationPolicy + + +class TrustedLocalTypePolicy(DeserializationPolicy): + allowed_materialization = {type, types.FunctionType, types.MethodType} + + def authorize_instantiation(self, cls, **kwargs): + if cls not in self.allowed_materialization: + raise ValueError(f"Blocked materialization: {cls!r}") + + def validate_class(self, cls, is_local, **kwargs): + if cls is object and not is_local: + return + if not is_local or cls.__name__ != "LocalMessage": + raise ValueError(f"Blocked class: {cls!r}") + + def validate_function(self, func, is_local, **kwargs): + if not is_local or func.__name__ != "label": + raise ValueError(f"Blocked function: {func!r}") + + def validate_method(self, method, is_local, **kwargs): + if not is_local or method.__name__ != "label": + raise ValueError(f"Blocked method: {method!r}") + + +def make_local_class(): + class LocalMessage: + kind = "local" + + @classmethod + def label(cls, value): + return f"{cls.kind}: {value}" + + return LocalMessage + + +fory = pyfory.Fory( + xlang=False, + ref=True, + strict=False, + policy=TrustedLocalTypePolicy(), +) +for carrier in (type, types.FunctionType, types.MethodType, staticmethod, classmethod): + fory.register_type(carrier) + +restored = fory.loads(fory.dumps(make_local_class())) +assert restored.label("hello") == "local: hello" ``` ## Serialize Methods @@ -81,10 +208,11 @@ Register the method carriers and receiver class before serializing bound instanc static method is serialized as its underlying function: ```python -import pyfory import types -fory = pyfory.Fory(xlang=False, ref=True, strict=False) +import pyfory +from pyfory import DeserializationPolicy + class Calculator: def scale(self, x): @@ -94,12 +222,33 @@ class Calculator: def double(x): return 2 * x + +class TrustedMethodPolicy(DeserializationPolicy): + def authorize_instantiation(self, cls, **kwargs): + if cls not in (Calculator, types.FunctionType, types.MethodType): + raise ValueError(f"Blocked materialization: {cls!r}") + + def validate_function(self, func, is_local, **kwargs): + if func is Calculator.double and not is_local: + return + if is_local and func.__qualname__ == "Calculator.double": + return + raise ValueError(f"Blocked function: {func!r}") + + def validate_method(self, method, is_local, **kwargs): + if type(method.__self__) is not Calculator or method.__name__ != "scale": + raise ValueError(f"Blocked method: {method!r}") + + +fory = pyfory.Fory( + xlang=False, + ref=True, + strict=False, + policy=TrustedMethodPolicy(), +) for carrier in (types.FunctionType, types.MethodType, Calculator): fory.register_type(carrier) -# Serialize instance method -print(fory.loads(fory.dumps(Calculator().scale))(10)) # 30 - -# Serialize static method -print(fory.loads(fory.dumps(Calculator.double))(10)) # 20 +assert fory.loads(fory.dumps(Calculator().scale))(10) == 30 +assert fory.loads(fory.dumps(Calculator.double))(10) == 20 ``` diff --git a/docs/object-serialization/python/index.md b/docs/object-serialization/python/index.md index 1f8eb91f62..4e51a77b05 100644 --- a/docs/object-serialization/python/index.md +++ b/docs/object-serialization/python/index.md @@ -80,7 +80,7 @@ pip install -e ".[dev]" ## Thread Safety -`pyfory` provides `ThreadSafeFory` for thread-safe serialization using a pooled wrapper: +`pyfory` provides `ThreadSafeFory` for sharing one configured serialization facade across threads: ```python import threading @@ -109,10 +109,10 @@ for t in threads: t.start() for t in threads: t.join() ``` -**Key Features:** +**Key Behavior:** -- **Instance Pool**: Maintains a pool of `Fory` instances protected by a lock for thread safety -- **Shared Configuration**: All registrations must be done upfront and are applied to all instances +- **Thread-safe use**: Root serialization and deserialization may be called from multiple threads +- **Shared Configuration**: Complete every registration before the first root attempt - **Matching Operations**: Exposes the corresponding root and registration methods - **Registration Safety**: The first root attempt permanently freezes registration, even if the operation fails diff --git a/docs/object-serialization/python/type-registration.md b/docs/object-serialization/python/type-registration.md index 3a4ca96959..768ec5b3c1 100644 --- a/docs/object-serialization/python/type-registration.md +++ b/docs/object-serialization/python/type-registration.md @@ -73,11 +73,8 @@ for model_class in [User, Order, Product, Invoice]: ``` A direct `Fory` may receive a serializer instance. `ThreadSafeFory` accepts a serializer class or -factory so every pooled child constructs a serializer against its own resolver. Use -`fory_factory` for serializer instances that need per-child configuration. A serializer factory -may accept `(resolver, type)`, `(resolver)`, or no arguments, but it must return a serializer bound -to that invocation's child resolver and normalized registered type. It must not reuse one serializer -instance across pooled children. +factory. The factory must accept `(resolver, type)` or `(resolver)` and return a serializer for that +resolver and registered type. ## Strict Mode Relationship diff --git a/python/README.md b/python/README.md index 50c0457edc..6460a5b11a 100644 --- a/python/README.md +++ b/python/README.md @@ -177,78 +177,13 @@ data = fory.dumps(person) print(fory.loads(data)) # Person(name='Bob', age=25) ``` -### Serialize Global Functions +### Functions, Classes, And Methods -Capture and get functions defined at module level. Fory deserialize and return same function object: - -```python -import pyfory -import types - -# Create Fory instance -fory = pyfory.Fory(xlang=False, ref=True, strict=False) - -# serialize global functions -def my_global_function(x): - return 10 * x - -fory.register_type(types.FunctionType) -data = fory.dumps(my_global_function) -print(fory.loads(data)(10)) # 100 -``` - -#### Serialize Local Functions/Lambdas - -Serialize functions with closures and lambda expressions. Fory captures the closure variables automatically: - -```python -import pyfory -import types - -# Create Fory instance -fory = pyfory.Fory(xlang=False, ref=True, strict=False) - -# serialize local functions with closures -def my_function(): - local_var = 10 - def local_func(x): - return x * local_var - return local_func - -fory.register_type(types.FunctionType) -data = fory.dumps(my_function()) -print(fory.loads(data)(10)) # 100 - -# serialize lambdas -data = fory.dumps(lambda x: 10 * x) -print(fory.loads(data)(10)) # 100 -``` - -#### Serialize Methods - -Register method carriers and receiver classes before serializing bound instance methods. A static -method is serialized as its underlying function: - -```python -import pyfory -import types - -fory = pyfory.Fory(xlang=False, ref=True, strict=False) - -class Calculator: - def scale(self, x): - return 3 * x - - @staticmethod - def double(x): - return 2 * x - -for carrier in (types.FunctionType, types.MethodType, Calculator): - fory.register_type(carrier) - -print(fory.loads(fory.dumps(Calculator().scale))(10)) # 30 -print(fory.loads(fory.dumps(Calculator.double))(10)) # 20 -``` +Python native mode supports configured global and local functions, lambdas, class objects, and +methods. Register their application and carrier types and authorize them with a deserialization +policy before the first root operation. See +[Functions, Classes, and Methods](https://fory.apache.org/docs/object-serialization/python/functions-classes-methods) +for the supported shapes and complete examples. ### Out-of-Band Buffer Serialization @@ -486,154 +421,11 @@ std::string str = bar10->get_string(0); // Access bar.f1 - **Cross-language layout**: Share Standard Row Format data between supported runtimes - **Partial deserialization**: Deserialize only the elements the application needs -## Core API Reference - -### Fory Class - -The main serialization interface: - -```python -class Fory: - def __init__( - self, - xlang: bool = True, - ref: bool = False, - strict: bool = True, - compatible: bool | None = None, - max_depth: int = 50 - ) -``` - -### ThreadSafeFory Class - -Thread-safe serialization interface for sharing one configured facade across threads: - -```python -class ThreadSafeFory: - def __init__(self, fory_factory=None, **kwargs) -``` - -Without `fory_factory`, keyword arguments are forwarded to each pooled `Fory`. Use -`fory_factory` when each pooled instance needs custom instance-level configuration. +## API Reference -Register all types before the first serialization or deserialization attempt. That first attempt -permanently freezes registration, even when it fails. Every later registration attempt raises an -error. - -**Thread Safety Example:** - -```python -import pyfory -import threading -from dataclasses import dataclass - -@dataclass -class Person: - name: str - age: int - -# Create thread-safe Fory instance -fory = pyfory.ThreadSafeFory(xlang=False, ref=True) -fory.register(Person) - -# Use in multiple threads safely -def serialize_in_thread(thread_id): - person = Person(name=f"User{thread_id}", age=25 + thread_id) - data = fory.serialize(person) - result = fory.deserialize(data) - print(f"Thread {thread_id}: {result}") - -threads = [threading.Thread(target=serialize_in_thread, args=(i,)) for i in range(10)] -for t in threads: t.start() -for t in threads: t.join() -``` - -**Key Features:** - -- **Thread-safe use**: One configured facade can be shared across threads -- **Shared Configuration**: Complete all registrations before the first root attempt -- **Same root API**: Provides the same serialization and deserialization methods as `Fory` -- **Registration Safety**: The first root attempt permanently freezes registration, even if it fails - -**When to Use:** - -- **Multi-threaded Applications**: Web servers, concurrent workers, parallel processing -- **Shared Fory Instances**: When multiple threads need to serialize/deserialize data -- **Thread Pools**: Applications using thread pools or concurrent.futures - -**Parameters:** - -- **`xlang`** (`bool`, default=`True`): Use xlang mode. Set `False` for Python native mode supporting Python-specific objects. -- **`ref`** (`bool`, default=`False`): Enable reference tracking for shared/circular references. Disable for better performance if your data has no shared references. -- **`strict`** (`bool`, default=`True`): Require type registration for security. **Highly recommended** for production. Only disable in trusted environments. -- **`compatible`** (`bool | None`, default `None`): Enable schema evolution. `None` enables compatible mode in both xlang and native mode. Set `False` only when every reader and writer always uses the same Python class schema and you want faster serialization and smaller size. -- **`max_depth`** (`int`, default=`50`): Maximum deserialization depth for security, preventing stack overflow attacks. - -**Key Methods:** - -```python -# Complete registration before the first root API call. -fory.register(MyClass, type_id=123) -# Alternatively, register by name or provide a custom serializer. -# fory.register(MyClass, name="my.package.MyClass") -# fory.register(MyClass, type_id=123, serializer=MySerializer) - -# serialize/deserialize are identical to dumps/loads. -data: bytes = fory.serialize(obj) -obj = fory.deserialize(data) -data = fory.dumps(obj) -obj = fory.loads(data) -``` - -### Xlang And Native Mode Comparison - -| Feature | Native mode (`xlang=False`) | Xlang mode (default) | -| ------------------- | ----------------------------------------- | ------------------------------------- | -| Use case | Pure Python applications | Multi-language systems | -| Compatibility | Python only | Java, Go, Rust, C++, JavaScript, etc. | -| Supported types | Configured Python type surface | Cross-language compatible types | -| Functions/lambdas | Registered and policy-authorized carriers | Not allowed | -| Instance methods | Registered and policy-authorized carriers | Not allowed | -| Stateful/reduce | Registered and policy-authorized types | Not allowed | -| Schema mode default | Compatible | Compatible | - -#### Native Mode (`xlang=False`) - -Python native mode supports Python-specific objects such as functions and closures. Configure the -complete type surface before the first root operation: - -```python -import pyfory -import types - -# Python native mode -fory = pyfory.Fory(xlang=False, ref=True, strict=False) -fory.register_type(types.FunctionType) - -# Every carrier is registered before this first root operation. -data = fory.dumps({ - 'function': lambda x: x * 2, - 'values': [1, 2, 3], -}) -result = fory.loads(data) -assert result['function'](4) == 8 -``` - -#### Xlang Mode - -Xlang mode restricts types to those compatible across all Fory implementations. Use it for multi-language systems: - -```python -import pyfory - -f = pyfory.Fory(xlang=True, ref=True) - -# Only supports cross-language compatible types -f.register(MyDataClass, name="com.example.MyDataClass") - -# Data can be read by Java, Go, Rust, etc. -data = f.serialize(MyDataClass(field1="value", field2=42)) -``` +See [Python Configuration](https://fory.apache.org/docs/object-serialization/python/configuration) +for the current `Fory` and `ThreadSafeFory` constructors, mode comparison, registration lifecycle, +configuration options, and root methods. ## Advanced Features From 00b99bace0c7c94b05ad913e45634f31e9aa6ab6 Mon Sep 17 00:00:00 2001 From: chaokunyang Date: Sat, 29 Aug 2026 11:01:29 +0800 Subject: [PATCH 084/168] perf(cpp): preserve byte root call shape --- .agents/languages/cpp.md | 8 +++++--- cpp/fory/serialization/fory.h | 9 +++------ 2 files changed, 8 insertions(+), 9 deletions(-) diff --git a/.agents/languages/cpp.md b/.agents/languages/cpp.md index 6c516e3c69..74d9152ed1 100644 --- a/.agents/languages/cpp.md +++ b/.agents/languages/cpp.md @@ -18,9 +18,11 @@ Load this file when changing `cpp/`, Cython build plumbing, or C++ xlang behavio resource amplification, publish reference or cache state that survives root cleanup, or return success past the required safepoint. Do not add per-field checks, cursor rollback, or tests that pin the first detection point solely to make an error earlier or more precise. -- Every public `Fory` and `ThreadSafeFory` root overload freezes facade registration as its first - action, before constructing stream wrappers, accessing stream buffers, validating arguments, or - acquiring pooled instances. `BaseFory` and the source `TypeResolver` keep separate owner-local +- Every public `Fory` and `ThreadSafeFory` root overload must enter a root owner whose first action + freezes facade registration, before constructing stream wrappers, accessing stream buffers, + validating arguments, or acquiring pooled instances. Overloads for the same byte input shape + share that one owner instead of duplicating the hot-path gate. `BaseFory` and the source + `TypeResolver` keep separate owner-local freeze gates so direct resolver registration cannot bypass the facade gate. Both reject before mutation; do not collapse these gates or describe permanent registry freeze as finalization. - Keep `TypeResolver::check_registration()` as the single out-of-line owner of the frozen and diff --git a/cpp/fory/serialization/fory.h b/cpp/fory/serialization/fory.h index 520d465f6b..5932b39be9 100644 --- a/cpp/fory/serialization/fory.h +++ b/cpp/fory/serialization/fory.h @@ -664,9 +664,6 @@ class Fory : public BaseFory { /// @return Deserialized object, or error. template Result deserialize(const uint8_t *data, size_t size) { - if (FORY_PREDICT_FALSE(!finalized_)) { - ensure_finalized(); - } return deserialize_bytes(data, size); } @@ -677,9 +674,6 @@ class Fory : public BaseFory { /// @return Deserialized object, or error. template Result deserialize(const std::vector &data) { - if (FORY_PREDICT_FALSE(!finalized_)) { - ensure_finalized(); - } return deserialize_bytes(data.data(), data.size()); } @@ -848,6 +842,9 @@ class Fory : public BaseFory { template Result deserialize_bytes(const uint8_t *data, size_t size) { + if (FORY_PREDICT_FALSE(!finalized_)) { + ensure_finalized(); + } if (data == nullptr) { return Unexpected(Error::invalid("Data pointer is null")); } From 59e300f6975759cf97412522ed634981423b3ead Mon Sep 17 00:00:00 2001 From: chaokunyang Date: Sat, 29 Aug 2026 11:01:37 +0800 Subject: [PATCH 085/168] perf(go): make root freeze idempotent --- go/fory/fory.go | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/go/fory/fory.go b/go/fory/fory.go index eb22be7ac4..23222b1192 100644 --- a/go/fory/fory.go +++ b/go/fory/fory.go @@ -308,9 +308,7 @@ func (f *Fory) checkRegistrationOpen() error { } func (f *Fory) beginRoot() { - if !f.registryFrozen { - f.registryFrozen = true - } + f.registryFrozen = true } // RegisterStruct registers a struct type with a numeric ID for cross-language serialization. From 8cdda40392b46bd63e1b2b60beb7f57453345b7b Mon Sep 17 00:00:00 2001 From: chaokunyang Date: Sat, 29 Aug 2026 11:01:44 +0800 Subject: [PATCH 086/168] docs: distinguish native resolution from registration --- .agents/languages/java.md | 5 +++++ .agents/languages/python.md | 9 +++++---- AGENTS.md | 7 +++++++ 3 files changed, 17 insertions(+), 4 deletions(-) diff --git a/.agents/languages/java.md b/.agents/languages/java.md index 22607ff6dc..cbd910011c 100644 --- a/.agents/languages/java.md +++ b/.agents/languages/java.md @@ -115,6 +115,11 @@ Load this file when changing anything under `java/` or when Java drives a cross- no-op return, callback invocation, or publication. Serializer completion methods used by lazy, JIT, and generated serializers are internal resolver-owned operations, remain valid after registration freezes, and must not be treated or repurposed as registration APIs. +- Registration freeze does not disable native runtime type resolution. When class registration is + not required, native roots may discover an unregistered runtime class and materialize its + resolver-owned `TypeInfo`, descriptor, serializer, or JIT cache entry after freeze. This runtime + cache materialization must not create or change an explicit class, serializer, ID, name, or + policy registration. - For GraalVM, use `fory codegen` to generate serializers when building native images. Do not add reflection configuration except for JDK `proxy`. - In Java native mode (`xlang=false`), only `Types.BOOL` through `Types.STRING` share type IDs with xlang mode. Other native-mode type IDs differ. - Choose one serializer ownership location per logical Java type family. Add native/xlang serializer variants only when the wire format or constructor contract truly differs. diff --git a/.agents/languages/python.md b/.agents/languages/python.md index 5efcb8ce04..ee060c6e0d 100644 --- a/.agents/languages/python.md +++ b/.agents/languages/python.md @@ -40,10 +40,11 @@ Load this file when changing `python/`, Cython serialization, or Python xlang be the provided child resolver and normalized declared type; singleton serializers cannot be shared across children. Instance-specific serializer configuration belongs in `fory_factory`, which creates and configures each child. -- Registry freeze prohibits type and serializer publication after the first root; it does not - prohibit policy-authorized resolution of module-global classes or callables during a non-strict - native read when that resolution does not mutate registry state. Do not describe these two - operations as one kind of late discovery. +- Registry freeze prohibits explicit type and serializer registration after the first root; it + does not prohibit policy-authorized native runtime type resolution. Non-strict native roots may + resolve module-global classes or callables and materialize resolver-owned type information or + serializer cache entries without creating or changing an explicit type, serializer, ID, name, or + policy registration. Do not describe these operations as late registration. - In non-strict native mode, public unqualified `register_type` for a built-in native carrier uses the same reserved type identity as pre-root discovery. Ordinary application classes and dataclasses retain their struct registration identity. Configure both through public registration; diff --git a/AGENTS.md b/AGENTS.md index 94ef9c3b14..67fb80a8d5 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -208,6 +208,13 @@ This is the entry point for AI guidance in Apache Fory. Read this file first, th serializer rebinding, metadata rebuilding, or other late-registration machinery. Registration-order finalization before the first root operation remains registration-owned and must not create a runtime invalidation path. + Registry freeze does not make native runtime type resolution immutable. When + registration is not required, Java and Python native modes may still discover + an unregistered runtime type and materialize resolver-owned type information, + descriptors, serializers, or JIT code while processing a root. Lazy, JIT, and + generated serializer completion for an existing binding is likewise allowed + after freeze. These runtime cache operations must not create or change an + explicit type, serializer, ID, name, or policy registration. If serializer construction, factory execution, or another application callback can reenter a root, complete that callback before publishing the entry it prepares, then recheck the authoritative per-instance freeze owner immediately From 956210e4bd57bc70c4571910ba89321e4ab062a7 Mon Sep 17 00:00:00 2001 From: chaokunyang Date: Sun, 30 Aug 2026 02:45:05 +0800 Subject: [PATCH 087/168] refactor(csharp): simplify registry freeze lifecycle --- csharp/src/Fory/Fory.cs | 106 +--- csharp/src/Fory/ThreadSafeFory.cs | 31 +- csharp/src/Fory/TypeResolver.cs | 158 +----- .../tests/Fory.Tests/RuntimeEdgeCaseTests.cs | 461 ------------------ 4 files changed, 47 insertions(+), 709 deletions(-) diff --git a/csharp/src/Fory/Fory.cs b/csharp/src/Fory/Fory.cs index e5d8222cba..9afc47c6cd 100644 --- a/csharp/src/Fory/Fory.cs +++ b/csharp/src/Fory/Fory.cs @@ -73,13 +73,7 @@ public static ForyBuilder Builder() public Fory Register(uint typeId) { EnsureRegistrationOpen(); - if (_typeResolver.CheckRegistration(typeof(T), typeId)) - { - return this; - } - - TypeInfo typeInfo = PrepareRegistration(typeof(T)); - _typeResolver.Register(typeof(T), typeId, typeInfo); + _typeResolver.Register(typeof(T), typeId); return this; } @@ -95,13 +89,7 @@ public Fory Register(string name) { EnsureRegistrationOpen(); (string namespaceName, string typeName) = TypeResolver.SplitTypeName(name); - if (_typeResolver.CheckRegistration(typeof(T), namespaceName, typeName)) - { - return this; - } - - TypeInfo typeInfo = PrepareRegistration(typeof(T)); - _typeResolver.Register(typeof(T), namespaceName, typeName, typeInfo); + _typeResolver.Register(typeof(T), namespaceName, typeName); return this; } @@ -117,14 +105,7 @@ public Fory Register(string name) public Fory Register(string typeNamespace, string typeName) { EnsureRegistrationOpen(); - TypeResolver.ValidateSplitTypeName(typeNamespace, typeName); - if (_typeResolver.CheckRegistration(typeof(T), typeNamespace, typeName)) - { - return this; - } - - TypeInfo typeInfo = PrepareRegistration(typeof(T)); - _typeResolver.Register(typeof(T), typeNamespace, typeName, typeInfo); + _typeResolver.Register(typeof(T), typeNamespace, typeName); return this; } @@ -141,12 +122,7 @@ public Fory Register(uint typeId) where TSerializer : Serializer, new() { EnsureRegistrationOpen(); - if (_typeResolver.CheckRegistration(typeof(T), typeId, typeof(TSerializer))) - { - return this; - } - - TypeInfo typeInfo = PrepareRegistration(); + TypeInfo typeInfo = _typeResolver.RegisterSerializer(); _typeResolver.Register(typeof(T), typeId, typeInfo); return this; } @@ -165,16 +141,7 @@ public Fory Register(string name) { EnsureRegistrationOpen(); (string namespaceName, string typeName) = TypeResolver.SplitTypeName(name); - if (_typeResolver.CheckRegistration( - typeof(T), - namespaceName, - typeName, - typeof(TSerializer))) - { - return this; - } - - TypeInfo typeInfo = PrepareRegistration(); + TypeInfo typeInfo = _typeResolver.RegisterSerializer(); _typeResolver.Register(typeof(T), namespaceName, typeName, typeInfo); return this; } @@ -194,16 +161,7 @@ public Fory Register(string typeNamespace, string typeName) { EnsureRegistrationOpen(); TypeResolver.ValidateSplitTypeName(typeNamespace, typeName); - if (_typeResolver.CheckRegistration( - typeof(T), - typeNamespace, - typeName, - typeof(TSerializer))) - { - return this; - } - - TypeInfo typeInfo = PrepareRegistration(); + TypeInfo typeInfo = _typeResolver.RegisterSerializer(); _typeResolver.Register(typeof(T), typeNamespace, typeName, typeInfo); return this; } @@ -216,7 +174,7 @@ public Fory Register(string typeNamespace, string typeName) /// Serialized bytes. public byte[] Serialize(in T value) { - FreezeRegistry(); + _registryFrozen = true; ByteWriter writer = _writeContext.Writer; writer.Reset(); // A previous failed root may leave references behind. Reset before serializer lookup, @@ -252,10 +210,9 @@ public void Serialize(IBufferWriter output, in T value) /// Thrown when trailing bytes remain after decoding. public T Deserialize(ReadOnlySpan payload) { - FreezeRegistry(); ByteReader reader = _readContext.Reader; reader.Reset(payload); - T value = DeserializeFromReaderCore(reader); + T value = DeserializeFromReader(reader); if (reader.Remaining != 0) { _readContext.ResetAfterFailure(); @@ -274,10 +231,9 @@ public T Deserialize(ReadOnlySpan payload) /// Thrown when trailing bytes remain after decoding. public T Deserialize(byte[] payload) { - FreezeRegistry(); ByteReader reader = _readContext.Reader; reader.Reset(payload); - T value = DeserializeFromReaderCore(reader); + T value = DeserializeFromReader(reader); if (reader.Remaining != 0) { _readContext.ResetAfterFailure(); @@ -326,13 +282,7 @@ private static void ThrowInvalidRootHeader(byte bitmap) => [MethodImpl(MethodImplOptions.AggressiveInlining)] internal T DeserializeFromReader(ByteReader reader) { - FreezeRegistry(); - return DeserializeFromReaderCore(reader); - } - - [MethodImpl(MethodImplOptions.AggressiveInlining)] - private T DeserializeFromReaderCore(ByteReader reader) - { + _registryFrozen = true; ReadContext readContext = _readContext; readContext.ResetFor(reader); readContext._remainingGraphMemoryBytes = Config.MaxGraphMemoryBytes; @@ -363,23 +313,6 @@ private T DeserializeFromReaderCore(ByteReader reader) } } - [MethodImpl(MethodImplOptions.AggressiveInlining)] - private void FreezeRegistry() - { - // Generated descriptors and serializers become operation-visible at the first root, so - // failed roots freeze registration just as successful roots do. - if (!_registryFrozen) - { - FreezeRegistrySlow(); - } - } - - [MethodImpl(MethodImplOptions.NoInlining)] - private void FreezeRegistrySlow() - { - _registryFrozen = true; - } - [MethodImpl(MethodImplOptions.AggressiveInlining)] private void EnsureRegistrationOpen() { @@ -389,25 +322,6 @@ private void EnsureRegistrationOpen() } } - private TypeInfo PrepareRegistration(Type type) - { - TypeInfo typeInfo = _typeResolver.PrepareRegistration(type); - // Serializer factories can call application code that starts a root. Recheck before - // the resolver publishes any type or serializer state. - EnsureRegistrationOpen(); - return typeInfo; - } - - private TypeInfo PrepareRegistration() - where TSerializer : Serializer, new() - { - TypeInfo typeInfo = _typeResolver.PrepareRegistration(); - // Serializer construction can call application code that starts a root. Recheck before - // the resolver publishes any type or serializer state. - EnsureRegistrationOpen(); - return typeInfo; - } - [MethodImpl(MethodImplOptions.NoInlining)] private static void ThrowRegistryFrozen() => throw new InvalidOperationException( diff --git a/csharp/src/Fory/ThreadSafeFory.cs b/csharp/src/Fory/ThreadSafeFory.cs index 0674bb0c06..65319e06ab 100644 --- a/csharp/src/Fory/ThreadSafeFory.cs +++ b/csharp/src/Fory/ThreadSafeFory.cs @@ -29,14 +29,13 @@ public sealed class ThreadSafeFory : IDisposable private readonly object _registrationLock = new(); private readonly List> _registrations = []; private readonly ThreadLocal _threadLocalFory; - private Fory? _registrationFory; private int _registryFrozen; private bool _disposed; internal ThreadSafeFory(Config config) { _config = config; - _threadLocalFory = new ThreadLocal(CreatePerThreadFory); + _threadLocalFory = new ThreadLocal(CreatePerThreadFory, trackAllValues: true); } ///

@@ -45,7 +44,7 @@ internal ThreadSafeFory(Config config) public Config Config => _config; /// - /// Registers a user type by numeric type identifier. + /// Registers a user type by numeric type identifier for all current and future thread-local runtimes. /// /// Type to register. /// Numeric type identifier used on the wire. @@ -59,7 +58,7 @@ public ThreadSafeFory Register(uint typeId) } /// - /// Registers a user type by name. + /// Registers a user type by name for all current and future thread-local runtimes. /// /// Type to register. /// Name used on the wire. A dotted name is split at the last dot. @@ -68,12 +67,13 @@ public ThreadSafeFory Register(uint typeId) /// Registration has closed because a root operation was attempted. public ThreadSafeFory Register(string name) { + _ = TypeResolver.SplitTypeName(name); ApplyRegistration(fory => fory.Register(name)); return this; } /// - /// Registers a user type by namespace and name. + /// Registers a user type by namespace and name for all current and future thread-local runtimes. /// /// Type to register. /// Namespace used on the wire. @@ -83,12 +83,13 @@ public ThreadSafeFory Register(string name) /// Registration has closed because a root operation was attempted. public ThreadSafeFory Register(string typeNamespace, string typeName) { + TypeResolver.ValidateSplitTypeName(typeNamespace, typeName); ApplyRegistration(fory => fory.Register(typeNamespace, typeName)); return this; } /// - /// Registers a user type by numeric type identifier with a custom serializer. + /// Registers a user type by numeric type identifier with a custom serializer for all thread-local runtimes. /// /// Type to register. /// Serializer implementation used for . @@ -104,7 +105,7 @@ public ThreadSafeFory Register(uint typeId) } /// - /// Registers a user type by name with a custom serializer. + /// Registers a user type by name with a custom serializer for all thread-local runtimes. /// /// Type to register. /// Serializer implementation used for . @@ -115,12 +116,13 @@ public ThreadSafeFory Register(uint typeId) public ThreadSafeFory Register(string name) where TSerializer : Serializer, new() { + _ = TypeResolver.SplitTypeName(name); ApplyRegistration(fory => fory.Register(name)); return this; } /// - /// Registers a user type by namespace and name with a custom serializer. + /// Registers a user type by namespace and name with a custom serializer for all thread-local runtimes. /// /// Type to register. /// Serializer implementation used for . @@ -132,6 +134,7 @@ public ThreadSafeFory Register(string name) public ThreadSafeFory Register(string typeNamespace, string typeName) where TSerializer : Serializer, new() { + TypeResolver.ValidateSplitTypeName(typeNamespace, typeName); ApplyRegistration(fory => fory.Register(typeNamespace, typeName)); return this; } @@ -186,7 +189,6 @@ public void Dispose() _threadLocalFory.Dispose(); _registrations.Clear(); - _registrationFory = null; _disposed = true; } } @@ -229,15 +231,11 @@ private void ApplyRegistration(Action registration) ThrowRegistryFrozen(); } - registration(_registrationFory ??= new Fory(_config)); - - ThrowIfDisposed(); - if (_registryFrozen != 0) + _registrations.Add(registration); + foreach (Fory fory in _threadLocalFory.Values) { - ThrowRegistryFrozen(); + registration(fory); } - - _registrations.Add(registration); } } @@ -260,7 +258,6 @@ private void FreezeRegistry() ThrowIfDisposed(); if (_registryFrozen == 0) { - _registrationFory = null; Volatile.Write(ref _registryFrozen, 1); } } diff --git a/csharp/src/Fory/TypeResolver.cs b/csharp/src/Fory/TypeResolver.cs index 6e036dbe5a..9eb0cc9da1 100644 --- a/csharp/src/Fory/TypeResolver.cs +++ b/csharp/src/Fory/TypeResolver.cs @@ -204,7 +204,7 @@ public Serializer GetSerializer() public TypeInfo GetTypeInfo(Type type) { - return GetOrCreateTypeInfo(type); + return GetOrCreateTypeInfo(type, null); } public TypeInfo GetTypeInfo() @@ -453,15 +453,23 @@ internal IReadOnlyList TypeMetaFields(TypeInfo typeInfo, bool return typeInfo.TypeMetaFields(trackRef); } - private TypeInfo GetOrCreateTypeInfo(Type type) + private TypeInfo GetOrCreateTypeInfo(Type type, TypeInfo? explicitTypeInfo) { ulong typeKey = TypeMapKey.Get(type); if (_typeInfos.TryGetValue(typeKey, out TypeInfo? existing)) { - return existing; + if (explicitTypeInfo is null || ReferenceEquals(existing, explicitTypeInfo)) + { + return existing; + } + + if (existing.IsRegistered) + { + throw new InvalidDataException($"cannot override serializer for registered type {type}"); + } } - TypeInfo typeInfo = CreateBindingCore(type); + TypeInfo typeInfo = explicitTypeInfo ?? CreateBindingCore(type); if (typeInfo.Type != type) { throw new InvalidDataException($"serializer type mismatch for {type}, got {typeInfo.Type}"); @@ -477,74 +485,22 @@ private TypeInfo GetOrCreateTypeInfo(Type type) return typeInfo; } - internal TypeInfo PrepareRegistration(Type type) - { - if (_typeInfos.TryGetValue(TypeMapKey.Get(type), out TypeInfo? existing)) - { - return existing; - } - - // Registration factories can run application code. Return the binding without publishing - // it so the facade can recheck its lifecycle boundary first. - TypeInfo typeInfo = CreateBindingCore(type); - if (typeInfo.Type != type) - { - throw new InvalidDataException($"serializer type mismatch for {type}, got {typeInfo.Type}"); - } - - return typeInfo; - } - - internal TypeInfo PrepareRegistration() + internal TypeInfo RegisterSerializer() where TSerializer : Serializer, new() { - return TypeInfo.Create(typeof(T), new TSerializer()); - } - - internal bool CheckRegistration(Type type, uint id, Type? serializerType = null) - { - if (_byUserTypeId.TryGetValue(id, out TypeInfo? wireOwner) && wireOwner.Type != type) - { - throw new InvalidDataException($"type ID {id} is already registered for {wireOwner.Type}"); - } - - if (!_typeInfos.TryGetValue(TypeMapKey.Get(type), out TypeInfo? typeInfo) || !typeInfo.IsRegistered) - { - return false; - } - - if (!typeInfo.RegisterByName && typeInfo.UserTypeId == id) - { - if (serializerType is not null && typeInfo.SerializerType != serializerType) - { - throw new InvalidDataException( - $"type {type} is already registered with serializer {typeInfo.SerializerType}"); - } - - return true; - } - - throw new InvalidDataException($"type {type} is already registered with a different wire identity"); + TypeInfo typeInfo = TypeInfo.Create(typeof(T), new TSerializer()); + RegisterSerializer(typeof(T), typeInfo); + return typeInfo; } - internal void Register(Type type, uint id) + internal void RegisterSerializer(Type type, TypeInfo typeInfo) { - if (CheckRegistration(type, id)) - { - return; - } - - Register(type, id, PrepareRegistration(type)); + GetOrCreateTypeInfo(type, typeInfo); } - internal void Register(Type type, uint id, TypeInfo typeInfo) + internal void Register(Type type, uint id, TypeInfo? explicitTypeInfo = null) { - if (CheckRegistration(type, id, typeInfo.SerializerType)) - { - return; - } - - typeInfo = PrepareRegistration(type, typeInfo).WithTypeIdRegistration(id); + TypeInfo typeInfo = GetOrCreateTypeInfo(type, explicitTypeInfo).WithTypeIdRegistration(id); _typeInfos.Set(TypeMapKey.Get(type), typeInfo); _byUserTypeId[id] = typeInfo; InvalidateFinalizedVersion(); @@ -584,86 +540,18 @@ internal static void ValidateSplitTypeName(string namespaceName, string typeName } } - internal bool CheckRegistration( - Type type, - string namespaceName, - string typeName, - Type? serializerType = null) - { - if (_byTypeName.TryGetValue((namespaceName, typeName), out TypeInfo? wireOwner) && wireOwner.Type != type) - { - throw new InvalidDataException( - $"type name {namespaceName}.{typeName} is already registered for {wireOwner.Type}"); - } - - if (!_typeInfos.TryGetValue(TypeMapKey.Get(type), out TypeInfo? typeInfo) || !typeInfo.IsRegistered) - { - return false; - } - - if (typeInfo.RegisterByName && - typeInfo.NamespaceName?.Value == namespaceName && - typeInfo.TypeName?.Value == typeName) - { - if (serializerType is not null && typeInfo.SerializerType != serializerType) - { - throw new InvalidDataException( - $"type {type} is already registered with serializer {typeInfo.SerializerType}"); - } - - return true; - } - - throw new InvalidDataException($"type {type} is already registered with a different wire identity"); - } - - internal void Register(Type type, string namespaceName, string typeName) - { - ValidateSplitTypeName(namespaceName, typeName); - if (CheckRegistration(type, namespaceName, typeName)) - { - return; - } - - Register(type, namespaceName, typeName, PrepareRegistration(type)); - } - - internal void Register(Type type, string namespaceName, string typeName, TypeInfo typeInfo) + internal void Register(Type type, string namespaceName, string typeName, TypeInfo? explicitTypeInfo = null) { ValidateSplitTypeName(namespaceName, typeName); - if (CheckRegistration(type, namespaceName, typeName, typeInfo.SerializerType)) - { - return; - } - + TypeInfo typeInfo = GetOrCreateTypeInfo(type, explicitTypeInfo); MetaString namespaceMeta = MetaStringEncoder.Namespace.Encode(namespaceName, TypeMetaEncodings.NamespaceMetaStringEncodings); MetaString typeNameMeta = MetaStringEncoder.TypeName.Encode(typeName, TypeMetaEncodings.TypeNameMetaStringEncodings); - typeInfo = PrepareRegistration(type, typeInfo).WithTypeNameRegistration(namespaceMeta, typeNameMeta); + typeInfo = typeInfo.WithTypeNameRegistration(namespaceMeta, typeNameMeta); _typeInfos.Set(TypeMapKey.Get(type), typeInfo); _byTypeName[(namespaceName, typeName)] = typeInfo; InvalidateFinalizedVersion(); } - private TypeInfo PrepareRegistration(Type type, TypeInfo typeInfo) - { - if (typeInfo.Type != type) - { - throw new InvalidDataException($"serializer type mismatch for {type}, got {typeInfo.Type}"); - } - - if (!_typeInfos.TryGetValue(TypeMapKey.Get(type), out TypeInfo? existing) || ReferenceEquals(existing, typeInfo)) - { - return typeInfo; - } - - if (existing.IsRegistered) - { - throw new InvalidDataException($"cannot override serializer for registered type {type}"); - } - - return typeInfo.WithRegistrationFrom(existing); - } - /// /// Returns a finalized semantic resolver version used by generated/static caches. /// The version is computed lazily and changes whenever bindings/registrations change. diff --git a/csharp/tests/Fory.Tests/RuntimeEdgeCaseTests.cs b/csharp/tests/Fory.Tests/RuntimeEdgeCaseTests.cs index e64d082525..4ae1a26d04 100644 --- a/csharp/tests/Fory.Tests/RuntimeEdgeCaseTests.cs +++ b/csharp/tests/Fory.Tests/RuntimeEdgeCaseTests.cs @@ -87,33 +87,8 @@ public sealed class FrozenPayload public sealed class FrozenPayloadSerializer : Serializer { public static int Constructions; - public static Action? ConstructionAction; public FrozenPayloadSerializer() - { - Interlocked.Increment(ref Constructions); - ConstructionAction?.Invoke(); - } - - public override FrozenPayload DefaultValue => null!; - - public override void WriteData(WriteContext context, in FrozenPayload value, bool hasGenerics) - { - _ = hasGenerics; - context.Writer.WriteVarInt32(value.Value); - } - - public override FrozenPayload ReadData(ReadContext context) - { - return new FrozenPayload { Value = context.Reader.ReadVarInt32() }; - } -} - -public sealed class AlternateFrozenSerializer : Serializer -{ - public static int Constructions; - - public AlternateFrozenSerializer() { Interlocked.Increment(ref Constructions); } @@ -132,35 +107,6 @@ public override FrozenPayload ReadData(ReadContext context) } } -public enum GeneratedFrozenValue -{ - Zero, - One, -} - -public sealed class GeneratedFrozenSerializer : Serializer -{ - public static Action? ConstructionAction; - - public GeneratedFrozenSerializer() - { - ConstructionAction?.Invoke(); - } - - public override GeneratedFrozenValue DefaultValue => GeneratedFrozenValue.Zero; - - public override void WriteData(WriteContext context, in GeneratedFrozenValue value, bool hasGenerics) - { - _ = hasGenerics; - context.Writer.WriteVarInt32((int)value); - } - - public override GeneratedFrozenValue ReadData(ReadContext context) - { - return (GeneratedFrozenValue)context.Reader.ReadVarInt32(); - } -} - public enum LookupFailureValue { Zero, @@ -191,35 +137,6 @@ public override LookupFailureValue ReadData(ReadContext context) } } -public enum AtomicRegistrationValue -{ - Zero, - One, -} - -public sealed class AtomicRegistrationSerializer : Serializer -{ - public static Action? ConstructionAction; - - public AtomicRegistrationSerializer() - { - ConstructionAction?.Invoke(); - } - - public override AtomicRegistrationValue DefaultValue => AtomicRegistrationValue.Zero; - - public override void WriteData(WriteContext context, in AtomicRegistrationValue value, bool hasGenerics) - { - _ = hasGenerics; - context.Writer.WriteVarInt32((int)value); - } - - public override AtomicRegistrationValue ReadData(ReadContext context) - { - return (AtomicRegistrationValue)context.Reader.ReadVarInt32(); - } -} - [ForyStruct] public sealed class FailingWritePayload { @@ -951,371 +868,6 @@ public void FrozenRegistryRejectsBeforeMutation() Assert.Equal(0, FrozenPayloadSerializer.Constructions); } - [Fact] - public void DirectIdIdentityOwners() - { - ForyRuntime fory = ForyRuntime.Builder().Build(); - FrozenPayloadSerializer.Constructions = 0; - AlternateFrozenSerializer.Constructions = 0; - int atomicConstructions = 0; - AtomicRegistrationSerializer.ConstructionAction = () => atomicConstructions++; - - try - { - fory.Register(740); - fory.Register(740); - Assert.Equal(1, FrozenPayloadSerializer.Constructions); - Assert.ThrowsAny( - () => fory.Register(740)); - Assert.Equal(0, AlternateFrozenSerializer.Constructions); - - Assert.ThrowsAny( - () => fory.Register(740)); - Assert.Equal(0, atomicConstructions); - Assert.ThrowsAny( - () => fory.Register(741)); - Assert.Equal(1, FrozenPayloadSerializer.Constructions); - - fory.Register(742); - Assert.Equal(1, atomicConstructions); - FrozenPayload value = new() { Value = 11 }; - Assert.Equal(value.Value, fory.Deserialize(fory.Serialize(value)).Value); - Assert.Equal( - AtomicRegistrationValue.One, - fory.Deserialize(fory.Serialize(AtomicRegistrationValue.One))); - } - finally - { - AtomicRegistrationSerializer.ConstructionAction = null; - } - } - - [Fact] - public void DirectNameIdentityOwners() - { - ForyRuntime fory = ForyRuntime.Builder().Build(); - FrozenPayloadSerializer.Constructions = 0; - AlternateFrozenSerializer.Constructions = 0; - int atomicConstructions = 0; - AtomicRegistrationSerializer.ConstructionAction = () => atomicConstructions++; - - try - { - fory.Register("identity.direct"); - fory.Register("identity", "direct"); - Assert.Equal(1, FrozenPayloadSerializer.Constructions); - Assert.ThrowsAny( - () => fory.Register("identity.direct")); - Assert.Equal(0, AlternateFrozenSerializer.Constructions); - - Assert.ThrowsAny( - () => fory.Register("identity.direct")); - Assert.Equal(0, atomicConstructions); - Assert.ThrowsAny( - () => fory.Register("identity.changed")); - Assert.Equal(1, FrozenPayloadSerializer.Constructions); - - fory.Register("identity.atomic"); - Assert.Equal(1, atomicConstructions); - FrozenPayload value = new() { Value = 12 }; - Assert.Equal(value.Value, fory.Deserialize(fory.Serialize(value)).Value); - Assert.Equal( - AtomicRegistrationValue.One, - fory.Deserialize(fory.Serialize(AtomicRegistrationValue.One))); - } - finally - { - AtomicRegistrationSerializer.ConstructionAction = null; - } - } - - [Fact] - public void ThreadSafeIdIdentityOwners() - { - using ThreadSafeFory fory = ForyRuntime.Builder().BuildThreadSafe(); - FrozenPayloadSerializer.Constructions = 0; - AlternateFrozenSerializer.Constructions = 0; - int atomicConstructions = 0; - AtomicRegistrationSerializer.ConstructionAction = () => atomicConstructions++; - - try - { - fory.Register(743); - fory.Register(743); - Assert.Equal(1, FrozenPayloadSerializer.Constructions); - Assert.ThrowsAny( - () => fory.Register(743)); - Assert.Equal(0, AlternateFrozenSerializer.Constructions); - - Assert.ThrowsAny( - () => fory.Register(743)); - Assert.Equal(0, atomicConstructions); - Assert.ThrowsAny( - () => fory.Register(744)); - Assert.Equal(1, FrozenPayloadSerializer.Constructions); - - fory.Register(745); - Assert.Equal(1, atomicConstructions); - FrozenPayload value = new() { Value = 13 }; - Assert.Equal(value.Value, fory.Deserialize(fory.Serialize(value)).Value); - Assert.Equal( - AtomicRegistrationValue.One, - fory.Deserialize(fory.Serialize(AtomicRegistrationValue.One))); - Assert.Equal(2, FrozenPayloadSerializer.Constructions); - Assert.Equal(2, atomicConstructions); - } - finally - { - AtomicRegistrationSerializer.ConstructionAction = null; - } - } - - [Fact] - public void ThreadSafeNameIdentityOwners() - { - using ThreadSafeFory fory = ForyRuntime.Builder().BuildThreadSafe(); - FrozenPayloadSerializer.Constructions = 0; - AlternateFrozenSerializer.Constructions = 0; - int atomicConstructions = 0; - AtomicRegistrationSerializer.ConstructionAction = () => atomicConstructions++; - - try - { - fory.Register("identity.thread_safe"); - fory.Register("identity", "thread_safe"); - Assert.Equal(1, FrozenPayloadSerializer.Constructions); - Assert.ThrowsAny( - () => fory.Register("identity.thread_safe")); - Assert.Equal(0, AlternateFrozenSerializer.Constructions); - - Assert.ThrowsAny( - () => fory.Register("identity.thread_safe")); - Assert.Equal(0, atomicConstructions); - Assert.ThrowsAny( - () => fory.Register("identity.changed")); - Assert.Equal(1, FrozenPayloadSerializer.Constructions); - - fory.Register("identity.atomic"); - Assert.Equal(1, atomicConstructions); - FrozenPayload value = new() { Value = 14 }; - Assert.Equal(value.Value, fory.Deserialize(fory.Serialize(value)).Value); - Assert.Equal( - AtomicRegistrationValue.One, - fory.Deserialize(fory.Serialize(AtomicRegistrationValue.One))); - Assert.Equal(2, FrozenPayloadSerializer.Constructions); - Assert.Equal(2, atomicConstructions); - } - finally - { - AtomicRegistrationSerializer.ConstructionAction = null; - } - } - - [Fact] - public void DirectReentryKeepsOwner() - { - ForyRuntime fory = ForyRuntime.Builder().Build(); - bool callbackStarted = false; - FrozenPayloadSerializer.ConstructionAction = () => - { - callbackStarted = true; - fory.Register(746); - }; - - try - { - Assert.ThrowsAny( - () => fory.Register(746)); - } - finally - { - FrozenPayloadSerializer.ConstructionAction = null; - } - - Assert.True(callbackStarted); - fory.Register(747); - TimeEnvelope first = new() { Date = new DateOnly(2026, 8, 29) }; - Assert.Equal(first.Date, fory.Deserialize(fory.Serialize(first)).Date); - FrozenPayload second = new() { Value = 15 }; - Assert.Equal(second.Value, fory.Deserialize(fory.Serialize(second)).Value); - } - - [Fact] - public void ThreadSafeReentryKeepsOwner() - { - using ThreadSafeFory fory = ForyRuntime.Builder().BuildThreadSafe(); - bool callbackStarted = false; - FrozenPayloadSerializer.ConstructionAction = () => - { - callbackStarted = true; - fory.Register(748); - }; - - try - { - Assert.ThrowsAny( - () => fory.Register(748)); - } - finally - { - FrozenPayloadSerializer.ConstructionAction = null; - } - - Assert.True(callbackStarted); - fory.Register(749); - TimeEnvelope first = new() { Date = new DateOnly(2026, 8, 29) }; - Assert.Equal(first.Date, fory.Deserialize(fory.Serialize(first)).Date); - FrozenPayload second = new() { Value = 16 }; - Assert.Equal(second.Value, fory.Deserialize(fory.Serialize(second)).Value); - } - - [Fact] - public void ReentrantRegistrationFreezes() - { - ForyRuntime fory = ForyRuntime.Builder().Build(); - FrozenPayloadSerializer.ConstructionAction = () => _ = fory.Serialize(1); - - try - { - Assert.Throws( - () => fory.Register(721)); - Assert.Throws(() => fory.Register(722)); - } - finally - { - FrozenPayloadSerializer.ConstructionAction = null; - } - } - - [Fact] - public void GeneratedReentryFreezes() - { - TypeResolver.RegisterGenerated(); - ForyRuntime fory = ForyRuntime.Builder().Build(); - GeneratedFrozenSerializer.ConstructionAction = () => _ = fory.Serialize(1); - - try - { - Assert.Throws(() => fory.Register(725)); - Assert.Throws(() => fory.Register(726)); - } - finally - { - GeneratedFrozenSerializer.ConstructionAction = null; - } - - Assert.Throws( - () => fory.Serialize(GeneratedFrozenValue.One)); - } - - [Fact] - public void ThreadSafeReentryFreezes() - { - using ThreadSafeFory fory = ForyRuntime.Builder().BuildThreadSafe(); - FrozenPayloadSerializer.ConstructionAction = () => _ = fory.Serialize(1); - - try - { - Assert.Throws( - () => fory.Register(723)); - Assert.Throws(() => fory.Register(724)); - } - finally - { - FrozenPayloadSerializer.ConstructionAction = null; - } - - Assert.Throws(() => - Task.Run(() => fory.Serialize(new FrozenPayload { Value = 1 })) - .GetAwaiter() - .GetResult()); - } - - [Fact] - public void ThreadSafeDisposeReentryStops() - { - ThreadSafeFory fory = ForyRuntime.Builder().BuildThreadSafe(); - FrozenPayloadSerializer.ConstructionAction = fory.Dispose; - - try - { - Assert.Throws( - () => fory.Register(727)); - } - finally - { - FrozenPayloadSerializer.ConstructionAction = null; - fory.Dispose(); - } - } - - [Fact] - public void ThreadSafeFailedReentryClears() - { - using ThreadSafeFory fory = ForyRuntime.Builder().BuildThreadSafe(); - fory.Register(728); - FrozenPayloadSerializer.ConstructionAction = - () => _ = fory.Deserialize(Array.Empty()); - - try - { - Assert.ThrowsAny( - () => fory.Register(729)); - Assert.Null(RegistrationForyFor(fory)); - Assert.Throws(() => fory.Register(730)); - } - finally - { - FrozenPayloadSerializer.ConstructionAction = null; - } - } - - [Fact] - public void ThreadSafeFailureDoesNotPublish() - { - TypeResolver.RegisterGenerated(); - using ThreadSafeFory fory = ForyRuntime.Builder().BuildThreadSafe(); - fory.Register(734); - int constructions = 0; - bool nestedFailed = false; - bool stageRetained = false; - AtomicRegistrationSerializer.ConstructionAction = () => constructions++; - FrozenPayloadSerializer.ConstructionAction = () => - { - ForyRuntime? stage = RegistrationForyFor(fory); - try - { - fory.Register(new string('a', 32_767)); - } - catch - { - nestedFailed = true; - } - stageRetained = ReferenceEquals(stage, RegistrationForyFor(fory)); - }; - - try - { - fory.Register(735); - Assert.True(nestedFailed); - Assert.True(stageRetained); - Assert.Equal(1, constructions); - fory.Register(737); - Assert.Equal(2, constructions); - } - finally - { - FrozenPayloadSerializer.ConstructionAction = null; - AtomicRegistrationSerializer.ConstructionAction = null; - } - - FrozenPayload value = new() { Value = 1 }; - Assert.Equal(value.Value, fory.Deserialize(fory.Serialize(value)).Value); - Assert.Equal( - AtomicRegistrationValue.One, - fory.Deserialize(fory.Serialize(AtomicRegistrationValue.One))); - } - [Fact] public void FailedRootFreezesRegistry() { @@ -1384,10 +936,6 @@ public void ThreadSafeFailedRootFreezesRegistry() Assert.ThrowsAny(() => fory.Deserialize(Array.Empty())); Assert.Throws(() => fory.Register(715)); - FrozenPayloadSerializer.Constructions = 0; - Assert.Throws( - () => fory.Register(string.Empty)); - Assert.Equal(0, FrozenPayloadSerializer.Constructions); } [Fact] @@ -1540,15 +1088,6 @@ private static WriteContext WriteContextFor(ForyRuntime fory) return Assert.IsType(field.GetValue(fory)); } - private static ForyRuntime? RegistrationForyFor(ThreadSafeFory fory) - { - System.Reflection.FieldInfo? field = typeof(ThreadSafeFory).GetField( - "_registrationFory", - System.Reflection.BindingFlags.Instance | System.Reflection.BindingFlags.NonPublic); - Assert.NotNull(field); - return field.GetValue(fory) as ForyRuntime; - } - [Fact] public void DeserializeFromReaderReadsFrames() { From b2381623572371c16a38f7fdefe9638fd63c6b1a Mon Sep 17 00:00:00 2001 From: chaokunyang Date: Sun, 30 Aug 2026 02:45:11 +0800 Subject: [PATCH 088/168] refactor(cpp): keep registry freeze single-owned --- cpp/fory/serialization/fory.h | 186 ++++++---------- cpp/fory/serialization/serialization_test.cc | 220 ------------------- cpp/fory/serialization/type_resolver.cc | 29 +-- cpp/fory/serialization/type_resolver.h | 62 +++--- 4 files changed, 98 insertions(+), 399 deletions(-) diff --git a/cpp/fory/serialization/fory.h b/cpp/fory/serialization/fory.h index 5932b39be9..c18a73c09b 100644 --- a/cpp/fory/serialization/fory.h +++ b/cpp/fory/serialization/fory.h @@ -192,9 +192,6 @@ class ForyBuilder { bool compatible_set_ = false; std::shared_ptr type_resolver_; - /// Helper to get or create type resolver and finalize it - std::shared_ptr get_finalized_resolver(); - friend class Fory; friend class ThreadSafeFory; }; @@ -297,9 +294,7 @@ class BaseFory { /// fory.register_struct(1); /// ``` template Result register_struct(uint32_t type_id) { - return register_type([this, type_id]() { - return type_resolver_->template register_by_id(type_id); - }); + return type_resolver_->template register_by_id(type_id); } /// Register a struct type with namespace and type name. @@ -320,9 +315,7 @@ class BaseFory { template Result register_struct(const std::string &ns, const std::string &type_name) { - return register_type([this, &ns, &type_name]() { - return type_resolver_->template register_by_name(ns, type_name); - }); + return type_resolver_->template register_by_name(ns, type_name); } /// Register a struct type with a name. @@ -362,9 +355,7 @@ class BaseFory { /// fory.register_enum(1); /// ``` template Result register_enum(uint32_t type_id) { - return register_type([this, type_id]() { - return type_resolver_->template register_by_id(type_id); - }); + return type_resolver_->template register_by_id(type_id); } /// Register an enum type with namespace and type name. @@ -385,9 +376,7 @@ class BaseFory { template Result register_enum(const std::string &ns, const std::string &type_name) { - return register_type([this, &ns, &type_name]() { - return type_resolver_->template register_by_name(ns, type_name); - }); + return type_resolver_->template register_by_name(ns, type_name); } /// Register an enum type with a name. @@ -417,9 +406,7 @@ class BaseFory { /// @param type_id Unique numeric identifier for this union type. /// @return Success or error if registration fails. template Result register_union(uint32_t type_id) { - return register_type([this, type_id]() { - return type_resolver_->template register_union_by_id(type_id); - }); + return type_resolver_->template register_union_by_id(type_id); } /// Register a union type with namespace and type name. @@ -432,9 +419,7 @@ class BaseFory { template Result register_union(const std::string &ns, const std::string &type_name) { - return register_type([this, &ns, &type_name]() { - return type_resolver_->template register_union_by_name(ns, type_name); - }); + return type_resolver_->template register_union_by_name(ns, type_name); } /// Register a union type with a name. @@ -458,9 +443,7 @@ class BaseFory { /// @return Success or error if registration fails. template Result register_extension_type(uint32_t type_id) { - return register_type([this, type_id]() { - return type_resolver_->template register_ext_type_by_id(type_id); - }); + return type_resolver_->template register_ext_type_by_id(type_id); } /// Register an extension type with namespace and type name. @@ -473,10 +456,7 @@ class BaseFory { template Result register_extension_type(const std::string &ns, const std::string &type_name) { - return register_type([this, &ns, &type_name]() { - return type_resolver_->template register_ext_type_by_name(ns, - type_name); - }); + return type_resolver_->template register_ext_type_by_name(ns, type_name); } /// Register an extension type with a name. @@ -507,23 +487,7 @@ class BaseFory { return std::make_pair(std::move(ns), std::move(type_name)); } - template - Result register_type(RegisterFn &&fn) { - std::lock_guard lock(registration_mutex_); - if (FORY_PREDICT_FALSE(registration_frozen_)) { - return Unexpected(Error::invalid( - "Cannot register types after first serialize/deserialize call")); - } - return std::forward(fn)(); - } - protected: - Result, Error> finalize_type_resolver() const { - std::lock_guard lock(registration_mutex_); - registration_frozen_ = true; - return type_resolver_->build_final_type_resolver(); - } - /// Protected constructor - only derived classes can instantiate. explicit BaseFory(const Config &config, std::shared_ptr resolver) @@ -539,8 +503,6 @@ class BaseFory { Config config_; std::shared_ptr type_resolver_; - mutable std::mutex registration_mutex_; - mutable bool registration_frozen_{false}; }; // ============================================================================ @@ -572,8 +534,8 @@ class Fory : public BaseFory { /// @return Vector containing serialized bytes, or error. template Result, Error> serialize(const T &obj) { - if (FORY_PREDICT_FALSE(!finalized_)) { - ensure_finalized(); + if (FORY_PREDICT_FALSE(!contexts_initialized_)) { + ensure_contexts_initialized(); } WriteContextGuard guard(*write_ctx_); Buffer &buffer = write_ctx_->buffer(); @@ -593,8 +555,8 @@ class Fory : public BaseFory { /// @return Number of bytes written, or error. template Result serialize(OutputStream &output_stream, const T &obj) { - if (FORY_PREDICT_FALSE(!finalized_)) { - ensure_finalized(); + if (FORY_PREDICT_FALSE(!contexts_initialized_)) { + ensure_contexts_initialized(); } return serialize_stream(output_stream, obj); } @@ -607,8 +569,8 @@ class Fory : public BaseFory { /// @return Number of bytes written, or error. template Result serialize(std::ostream &ostream, const T &obj) { - if (FORY_PREDICT_FALSE(!finalized_)) { - ensure_finalized(); + if (FORY_PREDICT_FALSE(!contexts_initialized_)) { + ensure_contexts_initialized(); } StdOutputStream output_stream(ostream); return serialize_stream(output_stream, obj); @@ -623,8 +585,8 @@ class Fory : public BaseFory { template FORY_ALWAYS_INLINE Result serialize_to(Buffer &buffer, const T &obj) { - if (FORY_PREDICT_FALSE(!finalized_)) { - ensure_finalized(); + if (FORY_PREDICT_FALSE(!contexts_initialized_)) { + ensure_contexts_initialized(); } return serialize_buffer(buffer, obj); } @@ -642,8 +604,8 @@ class Fory : public BaseFory { template Result serialize_to(std::vector &output, const T &obj) { - if (FORY_PREDICT_FALSE(!finalized_)) { - ensure_finalized(); + if (FORY_PREDICT_FALSE(!contexts_initialized_)) { + ensure_contexts_initialized(); } // Wrap the output vector in a Buffer for zero-copy serialization // writer_index starts at output.size() for appending @@ -687,8 +649,8 @@ class Fory : public BaseFory { /// @param buffer Buffer to read from. Its reader_index will be updated. /// @return Deserialized object, or error. template Result deserialize(Buffer &buffer) { - if (FORY_PREDICT_FALSE(!finalized_)) { - ensure_finalized(); + if (FORY_PREDICT_FALSE(!contexts_initialized_)) { + ensure_contexts_initialized(); } return deserialize_buffer(buffer); } @@ -703,8 +665,8 @@ class Fory : public BaseFory { /// @return Deserialized object, or error. template Result deserialize(InputStream &input_stream) { - if (FORY_PREDICT_FALSE(!finalized_)) { - ensure_finalized(); + if (FORY_PREDICT_FALSE(!contexts_initialized_)) { + ensure_contexts_initialized(); } return deserialize_stream(input_stream); } @@ -715,8 +677,8 @@ class Fory : public BaseFory { /// @param stream Input stream wrapper to read from. /// @return Deserialized object, or error. template Result deserialize(StdInputStream &stream) { - if (FORY_PREDICT_FALSE(!finalized_)) { - ensure_finalized(); + if (FORY_PREDICT_FALSE(!contexts_initialized_)) { + ensure_contexts_initialized(); } return deserialize_stream(stream); } @@ -738,41 +700,34 @@ class Fory : public BaseFory { ReadContext &read_context() { return *read_ctx_; } private: - /// Constructor for ForyBuilder - resolver will be finalized lazily. + /// Constructor for ForyBuilder - operation contexts are initialized lazily. explicit Fory(const Config &config, std::shared_ptr resolver) - : BaseFory(config, std::move(resolver)), finalized_(false), + : BaseFory(config, std::move(resolver)), contexts_initialized_(false), precomputed_header_(compute_header(config.xlang)) {} - /// Constructor for ThreadSafeFory pool - resolver is already finalized. - struct PreFinalized {}; + /// Constructor for ThreadSafeFory pool - resolver metadata is ready. + struct PreparedResolver {}; explicit Fory(const Config &config, std::shared_ptr resolver, - PreFinalized) - : BaseFory(config, std::move(resolver)), finalized_(true), + PreparedResolver) + : BaseFory(config, std::move(resolver)), contexts_initialized_(true), precomputed_header_(compute_header(config.xlang)) { - // The facade only retains finalized registration metadata. Runtime caches - // belong to the context clones, which stay distinct per pooled Fory. - // Sharing the published facade resolver avoids a third deep clone on every - // pool miss and must not be replaced with another finalization pass. - registration_frozen_ = true; write_ctx_.emplace(config_, type_resolver_->clone()); read_ctx_.emplace(config_, type_resolver_->clone()); } - /// Finalize the type resolver on first use. - void ensure_finalized() { - if (!finalized_) { - auto final_result = finalize_type_resolver(); + /// Initialize operation contexts from the registered type metadata. + void ensure_contexts_initialized() { + if (!contexts_initialized_) { + auto final_result = type_resolver_->build_final_type_resolver(); FORY_CHECK(final_result.ok()) << "Failed to build finalized TypeResolver: " << final_result.error().to_string(); - // Replace with finalized resolver - auto finalized_resolver = std::move(final_result).value(); + auto prepared_resolver = std::move(final_result).value(); // Create contexts with cloned resolvers - write_ctx_.emplace(config_, finalized_resolver->clone()); - read_ctx_.emplace(config_, finalized_resolver->clone()); - // Store finalized resolver - type_resolver_ = std::move(finalized_resolver); - finalized_ = true; + write_ctx_.emplace(config_, prepared_resolver->clone()); + read_ctx_.emplace(config_, prepared_resolver->clone()); + type_resolver_ = std::move(prepared_resolver); + contexts_initialized_ = true; } } @@ -842,8 +797,8 @@ class Fory : public BaseFory { template Result deserialize_bytes(const uint8_t *data, size_t size) { - if (FORY_PREDICT_FALSE(!finalized_)) { - ensure_finalized(); + if (FORY_PREDICT_FALSE(!contexts_initialized_)) { + ensure_contexts_initialized(); } if (data == nullptr) { return Unexpected(Error::invalid("Data pointer is null")); @@ -956,7 +911,7 @@ class Fory : public BaseFory { return type_info; } - bool finalized_; + bool contexts_initialized_; uint8_t precomputed_header_; std::optional write_ctx_; std::optional read_ctx_; @@ -997,28 +952,28 @@ class ThreadSafeFory : public BaseFory { public: template Result, Error> serialize(const T &obj) { - ensure_finalized(); + ensure_resolver_initialized(); auto fory_handle = fory_pool_.acquire(); return fory_handle->serialize(obj); } template Result serialize(OutputStream &output_stream, const T &obj) { - ensure_finalized(); + ensure_resolver_initialized(); auto fory_handle = fory_pool_.acquire(); return fory_handle->serialize(output_stream, obj); } template Result serialize(std::ostream &ostream, const T &obj) { - ensure_finalized(); + ensure_resolver_initialized(); auto fory_handle = fory_pool_.acquire(); return fory_handle->serialize(ostream, obj); } template Result serialize_to(Buffer &buffer, const T &obj) { - ensure_finalized(); + ensure_resolver_initialized(); auto fory_handle = fory_pool_.acquire(); return fory_handle->serialize_to(buffer, obj); } @@ -1026,34 +981,34 @@ class ThreadSafeFory : public BaseFory { template Result serialize_to(std::vector &output, const T &obj) { - ensure_finalized(); + ensure_resolver_initialized(); auto fory_handle = fory_pool_.acquire(); return fory_handle->serialize_to(output, obj); } template Result deserialize(const uint8_t *data, size_t size) { - ensure_finalized(); + ensure_resolver_initialized(); auto fory_handle = fory_pool_.acquire(); return fory_handle->template deserialize(data, size); } template Result deserialize(const std::vector &data) { - ensure_finalized(); + ensure_resolver_initialized(); auto fory_handle = fory_pool_.acquire(); return fory_handle->template deserialize(data.data(), data.size()); } template Result deserialize(InputStream &input_stream) { - ensure_finalized(); + ensure_resolver_initialized(); auto fory_handle = fory_pool_.acquire(); return fory_handle->template deserialize(input_stream); } template Result deserialize(StdInputStream &stream) { - ensure_finalized(); + ensure_resolver_initialized(); auto fory_handle = fory_pool_.acquire(); return fory_handle->template deserialize(stream); } @@ -1061,27 +1016,26 @@ class ThreadSafeFory : public BaseFory { private: explicit ThreadSafeFory(const Config &config, std::shared_ptr resolver) - : BaseFory(config, std::move(resolver)), finalized_resolver_(), - finalized_once_flag_(), fory_pool_([this]() { - // Every public root finalizes before pool acquisition, so a pool miss - // can share the facade resolver published by that root. The pooled - // Fory constructor owns its context clones. + : BaseFory(config, std::move(resolver)), shared_resolver_(), + resolver_once_flag_(), fory_pool_([this]() { + // Every public root prepares the resolver before pool acquisition. + // The pooled Fory constructor owns its context clones. return std::unique_ptr( - new Fory(config_, finalized_resolver_, Fory::PreFinalized{})); + new Fory(config_, shared_resolver_, Fory::PreparedResolver{})); }) {} - void ensure_finalized() const { - std::call_once(finalized_once_flag_, [this]() { - auto final_result = finalize_type_resolver(); + void ensure_resolver_initialized() const { + std::call_once(resolver_once_flag_, [this]() { + auto final_result = type_resolver_->build_final_type_resolver(); FORY_CHECK(final_result.ok()) << "Failed to build finalized TypeResolver: " << final_result.error().to_string(); - finalized_resolver_ = std::move(final_result).value(); + shared_resolver_ = std::move(final_result).value(); }); } - mutable std::shared_ptr finalized_resolver_; - mutable std::once_flag finalized_once_flag_; + mutable std::shared_ptr shared_resolver_; + mutable std::once_flag resolver_once_flag_; util::Pool fory_pool_; friend class ForyBuilder; @@ -1091,23 +1045,13 @@ class ThreadSafeFory : public BaseFory { // ForyBuilder Implementation // ============================================================================ -inline std::shared_ptr ForyBuilder::get_finalized_resolver() { - if (!type_resolver_) { - type_resolver_ = std::make_shared(); - } - type_resolver_->apply_config(normalized_config()); - auto final_result = type_resolver_->build_final_type_resolver(); - FORY_CHECK(final_result.ok()) << "Failed to build finalized TypeResolver: " - << final_result.error().to_string(); - return std::move(final_result).value(); -} - inline Fory ForyBuilder::build() { if (!type_resolver_) { type_resolver_ = std::make_shared(); } type_resolver_->apply_config(normalized_config()); - // Don't finalize yet - allow type registration, finalize on first use + // Allow type registration until the first root operation initializes its + // contexts. return Fory(config_, type_resolver_); } @@ -1116,7 +1060,7 @@ inline ThreadSafeFory ForyBuilder::build_thread_safe() { type_resolver_ = std::make_shared(); } type_resolver_->apply_config(normalized_config()); - // ThreadSafeFory builds finalized resolver lazily + // ThreadSafeFory prepares shared resolver metadata on its first root. return ThreadSafeFory(config_, type_resolver_); } diff --git a/cpp/fory/serialization/serialization_test.cc b/cpp/fory/serialization/serialization_test.cc index e2e4605be2..5f5ab88a29 100644 --- a/cpp/fory/serialization/serialization_test.cc +++ b/cpp/fory/serialization/serialization_test.cc @@ -35,7 +35,6 @@ #include #include #include -#include #include #include #include @@ -78,16 +77,6 @@ struct NestedStruct { FORY_STRUCT(NestedStruct, point, label); }; -struct UnregisteredNested { - int32_t value; - FORY_STRUCT(UnregisteredNested, value); -}; - -struct MissingNestedHolder { - UnregisteredNested nested; - FORY_STRUCT(MissingNestedHolder, nested); -}; - enum class Color { RED, GREEN, BLUE }; enum class SignedScopedStatus : int32_t { NEG = -3, ZERO = 0, LARGE = 42 }; FORY_ENUM(SignedScopedStatus, NEG, ZERO, LARGE); @@ -193,22 +182,6 @@ inline std::vector buffer_bytes(Buffer &buffer) { buffer.data() + buffer.writer_index()); } -template -void expect_numeric_owner(TypeResolver &resolver, uint32_t user_type_id, - const TypeInfo *owner) { - auto by_type = resolver.get_type_info(); - ASSERT_TRUE(by_type.ok()); - EXPECT_EQ(by_type.value(), owner); - - auto by_id = resolver.get_user_type_info_by_id(owner->type_id, user_type_id); - ASSERT_TRUE(by_id.ok()); - EXPECT_EQ(by_id.value(), owner); - - auto by_runtime = resolver.get_type_info(std::type_index(typeid(T))); - ASSERT_TRUE(by_runtime.ok()); - EXPECT_EQ(by_runtime.value(), owner); -} - class RegistryProbeInputStream final : public InputStream { public: explicit RegistryProbeInputStream(Fory &fory) : fory_(fory) {} @@ -1444,107 +1417,6 @@ TEST(SerializationTest, RegistrationByNameFailureDoesNotLeakTypeInfo) { EXPECT_EQ(dotted_type_name.error().code(), ErrorCode::Invalid); } -TEST(SerializationTest, TypeIdentityConflictsAreAtomic) { - using IdentityUnion = std::variant; - using IdentityRoot = std::tuple<::SimpleStruct, ::SignedScopedStatus, - ::IdLimitExt, IdentityUnion>; - - auto fory = - Fory::builder().xlang(true).compatible(false).track_ref(false).build(); - TypeResolver &resolver = fory.type_resolver(); - - ASSERT_TRUE(fory.register_struct<::SimpleStruct>(1).ok()); - ASSERT_TRUE(fory.register_enum<::SignedScopedStatus>(2).ok()); - ASSERT_TRUE(fory.register_extension_type<::IdLimitExt>(3).ok()); - ASSERT_TRUE(fory.register_union(4).ok()); - - auto struct_info = resolver.get_type_info<::SimpleStruct>(); - auto enum_info = resolver.get_type_info<::SignedScopedStatus>(); - auto ext_info = resolver.get_type_info<::IdLimitExt>(); - auto union_info = resolver.get_type_info(); - ASSERT_TRUE(struct_info.ok()); - ASSERT_TRUE(enum_info.ok()); - ASSERT_TRUE(ext_info.ok()); - ASSERT_TRUE(union_info.ok()); - - const TypeInfo *struct_owner = struct_info.value(); - const TypeInfo *enum_owner = enum_info.value(); - const TypeInfo *ext_owner = ext_info.value(); - const TypeInfo *union_owner = union_info.value(); - - EXPECT_FALSE(fory.register_struct<::SimpleStruct>(1).ok()); - EXPECT_FALSE(fory.register_struct<::SimpleStruct>("conflict", "Struct").ok()); - EXPECT_FALSE( - fory.register_enum<::SignedScopedStatus>("conflict", "Enum").ok()); - EXPECT_FALSE( - fory.register_extension_type<::IdLimitExt>("conflict", "Ext").ok()); - EXPECT_FALSE(fory.register_union("conflict", "Union").ok()); - - expect_numeric_owner<::SimpleStruct>(resolver, 1, struct_owner); - expect_numeric_owner<::SignedScopedStatus>(resolver, 2, enum_owner); - expect_numeric_owner<::IdLimitExt>(resolver, 3, ext_owner); - expect_numeric_owner(resolver, 4, union_owner); - - EXPECT_FALSE(resolver.get_type_info_by_name("conflict", "Struct").ok()); - EXPECT_FALSE(resolver.get_type_info_by_name("conflict", "Enum").ok()); - EXPECT_FALSE(resolver.get_type_info_by_name("conflict", "Ext").ok()); - EXPECT_FALSE(resolver.get_type_info_by_name("conflict", "Union").ok()); - - IdentityRoot original{::SimpleStruct{7, 9}, ::SignedScopedStatus::LARGE, - ::IdLimitExt{42}, IdentityUnion{std::string("value")}}; - auto bytes = fory.serialize(original); - ASSERT_TRUE(bytes.ok()) << bytes.error().to_string(); - auto decoded = fory.deserialize(bytes.value()); - ASSERT_TRUE(decoded.ok()) << decoded.error().to_string(); - EXPECT_EQ(decoded.value(), original); -} - -TEST(SerializationTest, NumericIdentityConflictsAreAtomic) { - using IdentityUnion = std::variant; - - auto fory = - Fory::builder().xlang(true).compatible(false).track_ref(false).build(); - TypeResolver &resolver = fory.type_resolver(); - - ASSERT_TRUE(fory.register_struct<::SimpleStruct>(1).ok()); - auto struct_info = resolver.get_type_info<::SimpleStruct>(); - ASSERT_TRUE(struct_info.ok()); - const TypeInfo *struct_owner = struct_info.value(); - - EXPECT_FALSE(fory.register_enum<::SignedScopedStatus>(1).ok()); - EXPECT_FALSE(fory.register_extension_type<::IdLimitExt>(1).ok()); - EXPECT_FALSE(fory.register_union(1).ok()); - - expect_numeric_owner<::SimpleStruct>(resolver, 1, struct_owner); - EXPECT_FALSE(resolver.get_type_info<::SignedScopedStatus>().ok()); - EXPECT_FALSE(resolver.get_type_info<::IdLimitExt>().ok()); - EXPECT_FALSE(resolver.get_type_info().ok()); - EXPECT_FALSE( - resolver.get_user_type_info_by_id(static_cast(TypeId::ENUM), 1) - .ok()); - EXPECT_FALSE( - resolver.get_user_type_info_by_id(static_cast(TypeId::EXT), 1) - .ok()); - EXPECT_FALSE(resolver - .get_user_type_info_by_id( - static_cast(TypeId::TYPED_UNION), 1) - .ok()); - EXPECT_FALSE( - resolver.get_type_info(std::type_index(typeid(::SignedScopedStatus))) - .ok()); - EXPECT_FALSE( - resolver.get_type_info(std::type_index(typeid(::IdLimitExt))).ok()); - EXPECT_FALSE( - resolver.get_type_info(std::type_index(typeid(IdentityUnion))).ok()); - - ::SimpleStruct original{7, 9}; - auto bytes = fory.serialize(original); - ASSERT_TRUE(bytes.ok()) << bytes.error().to_string(); - auto decoded = fory.deserialize<::SimpleStruct>(bytes.value()); - ASSERT_TRUE(decoded.ok()) << decoded.error().to_string(); - EXPECT_EQ(decoded.value(), original); -} - static std::vector make_remote_type_meta(const std::string &type_name, const std::string &field) { std::vector fields; @@ -2654,83 +2526,6 @@ TEST(SerializationTest, ConfigurationBuilder) { // Thread Safety Tests // ============================================================================ -static void expect_finalized_source(TypeResolver &resolver) { - auto source_info = resolver.get_type_info<::SimpleStruct>(); - ASSERT_TRUE(source_info.ok()) << source_info.error().to_string(); - ASSERT_NE(source_info.value()->type_meta, nullptr); - ASSERT_FALSE(source_info.value()->type_def.empty()); - - std::vector source_type_def = source_info.value()->type_def; - Buffer source_bytes(source_type_def); - auto parsed_source = TypeMeta::from_bytes(source_bytes, nullptr); - ASSERT_TRUE(parsed_source.ok()) << parsed_source.error().to_string(); - EXPECT_EQ(source_bytes.remaining_size(), 0u); - EXPECT_EQ(parsed_source.value()->field_infos.size(), 2u); - - auto cloned = resolver.clone(); - auto cloned_info = cloned->get_type_info<::SimpleStruct>(); - ASSERT_TRUE(cloned_info.ok()) << cloned_info.error().to_string(); - ASSERT_NE(cloned_info.value()->type_meta, nullptr); - EXPECT_EQ(cloned_info.value()->type_def, source_info.value()->type_def); - EXPECT_EQ(cloned_info.value()->type_meta->field_infos.size(), 2u); - - auto rebuilt = resolver.build_final_type_resolver(); - ASSERT_TRUE(rebuilt.ok()) << rebuilt.error().to_string(); - auto rebuilt_info = rebuilt.value()->get_type_info<::SimpleStruct>(); - ASSERT_TRUE(rebuilt_info.ok()) << rebuilt_info.error().to_string(); - ASSERT_NE(rebuilt_info.value()->type_meta, nullptr); - EXPECT_EQ(rebuilt_info.value()->type_def, source_info.value()->type_def); -} - -TEST(SerializationTest, SourceResolverFinalizes) { - auto source_resolver = std::make_shared(); - auto fory = Fory::builder() - .xlang(true) - .compatible(false) - .track_ref(false) - .type_resolver(source_resolver) - .build(); - ASSERT_TRUE(fory.register_struct<::SimpleStruct>(1).ok()); - auto pending = source_resolver->get_type_info<::SimpleStruct>(); - ASSERT_TRUE(pending.ok()) << pending.error().to_string(); - EXPECT_EQ(pending.value()->type_meta, nullptr); - - auto bytes = fory.serialize(::SimpleStruct{1, 2}); - ASSERT_TRUE(bytes.ok()) << bytes.error().to_string(); - expect_finalized_source(*source_resolver); -} - -TEST(SerializationTest, FinalizationFailureIsAtomic) { - auto source_resolver = std::make_shared(); - auto fory = Fory::builder() - .xlang(true) - .compatible(false) - .track_ref(false) - .type_resolver(source_resolver) - .build(); - ASSERT_TRUE(fory.register_struct<::SimpleStruct>(1).ok()); - ASSERT_TRUE(fory.register_struct<::MissingNestedHolder>(2).ok()); - - auto simple_info = source_resolver->get_type_info<::SimpleStruct>(); - auto holder_info = source_resolver->get_type_info<::MissingNestedHolder>(); - ASSERT_TRUE(simple_info.ok()); - ASSERT_TRUE(holder_info.ok()); - ASSERT_EQ(simple_info.value()->type_meta, nullptr); - ASSERT_TRUE(simple_info.value()->type_def.empty()); - ASSERT_EQ(holder_info.value()->type_meta, nullptr); - ASSERT_TRUE(holder_info.value()->type_def.empty()); - - auto final_resolver = source_resolver->build_final_type_resolver(); - ASSERT_FALSE(final_resolver.ok()); - - EXPECT_EQ(simple_info.value()->type_meta, nullptr); - EXPECT_TRUE(simple_info.value()->type_def.empty()); - EXPECT_EQ(holder_info.value()->type_meta, nullptr); - EXPECT_TRUE(holder_info.value()->type_def.empty()); - EXPECT_FALSE(fory.register_struct<::UnregisteredNested>(3).ok()); - EXPECT_FALSE(source_resolver->get_type_info<::UnregisteredNested>().ok()); -} - TEST(SerializationTest, DirectFailedRootFreezes) { auto source_resolver = std::make_shared(); auto fory = Fory::builder() @@ -2761,21 +2556,6 @@ TEST(SerializationTest, DirectFailedRootFreezes) { .ok()); } -TEST(SerializationTest, ThreadSafeSourceFinalizes) { - auto source_resolver = std::make_shared(); - auto fory = Fory::builder() - .xlang(true) - .compatible(false) - .track_ref(false) - .type_resolver(source_resolver) - .build_thread_safe(); - ASSERT_TRUE(fory.register_struct<::SimpleStruct>(1).ok()); - - auto bytes = fory.serialize(::SimpleStruct{1, 2}); - ASSERT_TRUE(bytes.ok()) << bytes.error().to_string(); - expect_finalized_source(*source_resolver); -} - TEST(SerializationTest, ThreadSafeForyMultiThread) { auto fory = Fory::builder() .xlang(true) diff --git a/cpp/fory/serialization/type_resolver.cc b/cpp/fory/serialization/type_resolver.cc index b98acbc611..cb8d8cdcab 100644 --- a/cpp/fory/serialization/type_resolver.cc +++ b/cpp/fory/serialization/type_resolver.cc @@ -1775,11 +1775,7 @@ Result TypeResolver::check_registration() { Result, Error> TypeResolver::build_final_type_resolver() { - std::lock_guard lock(*registration_mutex_); - // Freeze the source before building so even failed finalization permanently - // rejects later registration. Holding the registration mutex makes first use - // linearizable with direct registration helpers. ThreadSafeFory retains this - // source resolver after publishing finalized pool owners. + std::lock_guard lock(registration_mutex_); registry_frozen_ = true; auto final_resolver = std::make_unique(); @@ -1863,29 +1859,6 @@ TypeResolver::build_final_type_resolver() { // Clear partial_type_infos in the final resolver since they're all completed final_resolver->partial_type_infos_.clear(); - // ThreadSafeFory retains the source resolver after publishing the finalized - // clone. Prepare every metadata update before mutating the source so failed - // finalization cannot leave it partially completed. - struct FinalizedPartial { - TypeInfo *source; - std::vector type_def; - std::unique_ptr type_meta; - }; - std::vector finalized_partials; - for (const auto &[key, source_ptr] : partial_type_infos_) { - (void)key; - TypeInfo *completed_ptr = remap_type_info(source_ptr); - FORY_CHECK(completed_ptr->type_meta != nullptr); - finalized_partials.push_back( - {source_ptr, completed_ptr->type_def, - std::make_unique(*completed_ptr->type_meta)}); - } - for (auto &partial : finalized_partials) { - partial.source->type_def = std::move(partial.type_def); - partial.source->type_meta = std::move(partial.type_meta); - } - partial_type_infos_.clear(); - return final_resolver; } diff --git a/cpp/fory/serialization/type_resolver.h b/cpp/fory/serialization/type_resolver.h index f3a93a9341..4e0cc1ede0 100644 --- a/cpp/fory/serialization/type_resolver.h +++ b/cpp/fory/serialization/type_resolver.h @@ -1373,8 +1373,7 @@ class TypeResolver { /// 3. Builds complete TypeMeta and serializes it to bytes /// 4. Returns a new TypeResolver with all type infos fully initialized /// - /// Calling this method permanently freezes registration on the source - /// resolver before construction starts, including when construction fails. + /// Registration is permanently frozen before metadata construction starts. /// /// @return A new TypeResolver with all type infos fully initialized and ready /// for use. @@ -1509,6 +1508,7 @@ class TypeResolver { static std::string make_name_key(const std::string &ns, const std::string &name); + static uint64_t make_user_type_key(uint32_t type_id, uint32_t user_type_id); /// Register a TypeInfo, taking ownership and storing in primary storage. /// Returns pointer to the stored TypeInfo (owned by TypeResolver). @@ -1531,9 +1531,7 @@ class TypeResolver { std::thread::id registration_thread_id_; bool registry_frozen_; - // Keep registration-only synchronization out of the hot resolver footprint. - std::unique_ptr registration_mutex_{ - std::make_unique()}; + std::mutex registration_mutex_; // Primary storage - owns all TypeInfo objects std::vector> type_infos_; @@ -1543,7 +1541,7 @@ class TypeResolver { // FlatIntMap is optimized for integer keys with minimal overhead util::U64PtrMap type_info_by_ctid_{256}; util::U32PtrMap type_info_by_id_{256}; - util::U32PtrMap user_type_info_by_id_{256}; + util::U64PtrMap user_type_info_by_id_{256}; fory::flat_hash_map type_info_by_name_; util::U64PtrMap partial_type_infos_{256}; @@ -1751,7 +1749,7 @@ get_type_info_with_resolver(TypeResolver &resolver) { } template Result TypeResolver::register_any_type() { - std::lock_guard lock(*registration_mutex_); + std::lock_guard lock(registration_mutex_); FORY_RETURN_IF_ERROR(check_registration()); using ChronoTimestamp = std::chrono::time_point; @@ -1790,7 +1788,7 @@ template Result TypeResolver::register_any_type() { template Result TypeResolver::register_by_id(uint32_t type_id) { - std::lock_guard lock(*registration_mutex_); + std::lock_guard lock(registration_mutex_); FORY_RETURN_IF_ERROR(check_registration()); if (type_id == kInvalidUserTypeId) { return Unexpected(Error::invalid( @@ -1846,7 +1844,7 @@ template Result TypeResolver::register_by_name(const std::string &ns, const std::string &type_name) { - std::lock_guard lock(*registration_mutex_); + std::lock_guard lock(registration_mutex_); FORY_RETURN_IF_ERROR(check_registration()); if (type_name.empty()) { return Unexpected( @@ -1899,7 +1897,7 @@ TypeResolver::register_by_name(const std::string &ns, template Result TypeResolver::register_ext_type_by_id(uint32_t type_id) { - std::lock_guard lock(*registration_mutex_); + std::lock_guard lock(registration_mutex_); FORY_RETURN_IF_ERROR(check_registration()); if (type_id == kInvalidUserTypeId) { return Unexpected(Error::invalid("type_id must be in range [0, 0xfffffffe] " @@ -1924,7 +1922,7 @@ template Result TypeResolver::register_ext_type_by_name(const std::string &ns, const std::string &type_name) { - std::lock_guard lock(*registration_mutex_); + std::lock_guard lock(registration_mutex_); FORY_RETURN_IF_ERROR(check_registration()); if (type_name.empty()) { return Unexpected(Error::invalid( @@ -1949,7 +1947,7 @@ TypeResolver::register_ext_type_by_name(const std::string &ns, template Result TypeResolver::register_union_by_id(uint32_t type_id) { - std::lock_guard lock(*registration_mutex_); + std::lock_guard lock(registration_mutex_); FORY_RETURN_IF_ERROR(check_registration()); if (type_id == kInvalidUserTypeId) { return Unexpected(Error::invalid( @@ -1973,7 +1971,7 @@ template Result TypeResolver::register_union_by_name(const std::string &ns, const std::string &type_name) { - std::lock_guard lock(*registration_mutex_); + std::lock_guard lock(registration_mutex_); FORY_RETURN_IF_ERROR(check_registration()); if (type_name.empty()) { return Unexpected(Error::invalid( @@ -2352,6 +2350,12 @@ inline std::string TypeResolver::make_name_key(const std::string &ns, return key; } +inline uint64_t TypeResolver::make_user_type_key(uint32_t type_id, + uint32_t user_type_id) { + return (static_cast(type_id) << 32) | + static_cast(user_type_id); +} + inline Result TypeResolver::register_type_internal(uint64_t ctid, std::unique_ptr info) { @@ -2360,16 +2364,13 @@ TypeResolver::register_type_internal(uint64_t ctid, Error::invalid("TypeInfo or harness is invalid during registration")); } - // Validate both directions of the C++ type-to-wire identity before mutating - // resolver state so failed registration leaves every owner index unchanged. + // Validate all uniqueness constraints before mutating resolver state so + // failed registration leaves no partial entries behind. TypeInfo *raw_ptr = info.get(); - TypeInfo *existing_type = type_info_by_ctid_.get_or_default(ctid, nullptr); - if (existing_type != nullptr) { - return Unexpected(Error::invalid("C++ type already registered")); - } const bool is_internal = ::fory::is_internal_type(raw_ptr->type_id); - const bool has_user_type_id = + const bool has_user_type_key = !raw_ptr->register_by_name && raw_ptr->user_type_id != kInvalidUserTypeId; + uint64_t user_type_key = 0; std::string name_key; if (is_internal) { @@ -2379,14 +2380,14 @@ TypeResolver::register_type_internal(uint64_t ctid, return Unexpected(Error::invalid("Type id already registered: " + std::to_string(raw_ptr->type_id))); } - } else if (has_user_type_id) { - // Numeric user IDs share one registry namespace across all user type - // families. TypeInfo retains the family for validation during lookup. + } else if (has_user_type_key) { + user_type_key = make_user_type_key(raw_ptr->type_id, raw_ptr->user_type_id); TypeInfo *existing = - user_type_info_by_id_.get_or_default(raw_ptr->user_type_id, nullptr); + user_type_info_by_id_.get_or_default(user_type_key, nullptr); if (existing != nullptr) { - return Unexpected(Error::invalid("User type id already registered: " + - std::to_string(raw_ptr->user_type_id))); + return Unexpected(Error::invalid( + "Type id already registered: " + std::to_string(raw_ptr->type_id) + + "/" + std::to_string(raw_ptr->user_type_id))); } } @@ -2407,8 +2408,8 @@ TypeResolver::register_type_internal(uint64_t ctid, if (is_internal) { type_info_by_id_.put(stored_ptr->type_id, stored_ptr); - } else if (has_user_type_id) { - user_type_info_by_id_.put(stored_ptr->user_type_id, stored_ptr); + } else if (has_user_type_key) { + user_type_info_by_id_.put(user_type_key, stored_ptr); } if (stored_ptr->register_by_name) { @@ -2438,8 +2439,9 @@ TypeResolver::get_type_info_by_id(uint32_t type_id) const { inline Result TypeResolver::get_user_type_info_by_id(uint32_t type_id, uint32_t user_type_id) const { - TypeInfo *info = user_type_info_by_id_.get_or_default(user_type_id, nullptr); - if (info != nullptr && info->type_id == type_id) { + uint64_t key = make_user_type_key(type_id, user_type_id); + TypeInfo *info = user_type_info_by_id_.get_or_default(key, nullptr); + if (info != nullptr) { return info; } return Unexpected(Error::type_error( From 285f6089db86585f2e70bd35f4dc3aad7ce6c112 Mon Sep 17 00:00:00 2001 From: chaokunyang Date: Sun, 30 Aug 2026 02:45:49 +0800 Subject: [PATCH 089/168] refactor(javascript): remove registration transactions --- javascript/packages/core/lib/fory.ts | 26 +- javascript/packages/core/lib/gen/builder.ts | 9 +- javascript/packages/core/lib/gen/index.ts | 560 ++-------------- javascript/packages/core/lib/gen/map.ts | 2 +- javascript/packages/core/lib/gen/router.ts | 20 +- .../packages/core/lib/gen/serializer.ts | 43 +- javascript/packages/core/lib/gen/struct.ts | 46 +- javascript/packages/core/lib/type.ts | 2 - javascript/packages/core/lib/typeInfo.ts | 90 +-- javascript/packages/core/lib/typeResolver.ts | 246 ++++--- javascript/test/fory.test.ts | 627 +----------------- javascript/test/map.test.ts | 41 -- javascript/test/union.test.ts | 26 - 13 files changed, 278 insertions(+), 1460 deletions(-) diff --git a/javascript/packages/core/lib/fory.ts b/javascript/packages/core/lib/fory.ts index d62293d8e2..cf0f33ff2a 100644 --- a/javascript/packages/core/lib/fory.ts +++ b/javascript/packages/core/lib/fory.ts @@ -47,6 +47,7 @@ export default class Fory { readonly writeContext: WriteContext; readonly readContext: ReadContext; private readonly rootSerializers = new WeakMap PlatformBuffer>(); + private registrationFrozen = false; private readonly rootDeserializers = new WeakMap any>(); @@ -147,17 +148,26 @@ export default class Fory { deserialize(bytes: Uint8Array): InstanceType | null; }; register(constructor: any, customSerializer?: CustomSerializer) { - // Root codegen captures resolver state in generated closures and checked metadata caches. - // Freezing permanently at the first attempt keeps every later root on that same registry. - this.typeResolver.ensureRegistrationOpen(); + if (this.registrationFrozen) { + throw new Error("types and serializers must be registered before the first root operation"); + } let serializer: Serializer; if (constructor.prototype?.[ForyTypeInfoSymbol]) { const typeInfo: TypeInfo = (constructor.prototype[ForyTypeInfoSymbol] as WithForyClsInfo) .structTypeInfo; - serializer = new Gen(this.typeResolver, customSerializer).generateSerializer(typeInfo); + typeInfo.freeze(); + serializer = new Gen(this.typeResolver, { + creator: constructor, + customSerializer, + }).generateSerializer(typeInfo); + this.typeResolver.registerSerializer(typeInfo, serializer); } else { const typeInfo = constructor; - serializer = new Gen(this.typeResolver, customSerializer).generateSerializer(typeInfo); + typeInfo.freeze(); + serializer = new Gen(this.typeResolver, { + customSerializer, + }).generateSerializer(typeInfo); + this.typeResolver.registerSerializer(typeInfo, serializer); } return { serializer, @@ -167,7 +177,7 @@ export default class Fory { } deserialize(bytes: Uint8Array, serializer: Serializer = this.anySerializer): T | null { - this.typeResolver.freezeRegistration(); + this.registrationFrozen = true; this.readContext.reset(bytes); const reader = this.readContext.reader; const bitmap = reader.readUint8(); @@ -197,7 +207,7 @@ export default class Fory { const writer = writeContext.writer; const rootHeader = ConfigFlags.isCrossLanguageFlag; rootSerializer = (data: any) => { - this.typeResolver.freezeRegistration(); + this.registrationFrozen = true; // The entry reset releases state from the previous root before this context is reused. writeContext.reset(); writer.writeUint8(rootHeader); @@ -221,7 +231,7 @@ export default class Fory { : this.anySerializer; const rootHeader = ConfigFlags.isCrossLanguageFlag; rootDeserializer = (bytes: Uint8Array) => { - this.typeResolver.freezeRegistration(); + this.registrationFrozen = true; readContext.reset(bytes); const bitmap = reader.readUint8(); if (bitmap !== rootHeader) { diff --git a/javascript/packages/core/lib/gen/builder.ts b/javascript/packages/core/lib/gen/builder.ts index a89110693b..6fc84efe01 100644 --- a/javascript/packages/core/lib/gen/builder.ts +++ b/javascript/packages/core/lib/gen/builder.ts @@ -20,11 +20,6 @@ import { Scope } from "./scope"; import TypeResolver from "../typeResolver"; -export type SerializerLookup = Pick< - TypeResolver, - "getSerializerByTypeInfo" | "getSerializerById" | "getSerializerByName" ->; - export class BinaryReaderBuilder { constructor(private holder: string) {} @@ -431,19 +426,19 @@ export class CodecBuilder { constructor( scope: Scope, readonly resolver: TypeResolver, - readonly serializerLookup: SerializerLookup = resolver, ) { const writeContext = scope.declareByName("writeContext", "typeResolver.writeContext"); const readContext = scope.declareByName("readContext", "typeResolver.readContext"); const br = scope.declareByName("br", "readContext.reader"); const bw = scope.declareByName("bw", "writeContext.writer"); + const cr = scope.declareByName("cr", "typeResolver"); const rw = scope.declareByName("rw", "writeContext.refWriter"); const rr = scope.declareByName("rr", "readContext.refReader"); const mw = scope.declareByName("mw", "writeContext.metaStringWriter"); scope.declareByName("mr", "readContext.metaStringReader"); this.reader = new BinaryReaderBuilder(br); this.writer = new BinaryWriterBuilder(bw); - this.typeResolver = new TypeResolverBuilder("serializerLookup"); + this.typeResolver = new TypeResolverBuilder(cr); this.referenceResolver = new ReferenceResolverBuilder(rr, rw); this.typeMetaResolver = new TypeMetaContextBuilder(writeContext, readContext); this.metaStringResolver = new MetaStringContextBuilder(writeContext, readContext, mw); diff --git a/javascript/packages/core/lib/gen/index.ts b/javascript/packages/core/lib/gen/index.ts index 93724ee62e..9e62446722 100644 --- a/javascript/packages/core/lib/gen/index.ts +++ b/javascript/packages/core/lib/gen/index.ts @@ -17,10 +17,10 @@ * under the License. */ -import { CustomSerializer, TypeId, Serializer } from "../type"; -import { sealTypeInfo, TypeInfo } from "../typeInfo"; +import { TypeId, Serializer } from "../type"; +import { TypeInfo } from "../typeInfo"; import { CodegenRegistry } from "./router"; -import { CodecBuilder, SerializerLookup } from "./builder"; +import { CodecBuilder } from "./builder"; import { Scope } from "./scope"; import { CompatibleScalarConverter } from "../compatible/scalar"; import "./array"; @@ -50,102 +50,24 @@ CodegenRegistry.registerExternal(CompatibleScalarConverter); type SerializerFactoryBuilder = () => ( typeResolver: TypeResolver, - serializerLookup: SerializerLookup, external: unknown, typeInfo: TypeInfo, - options: object | undefined, + options: { [key: string]: unknown }, localTypeMeta: TypeMeta | undefined, localTypeMetaSymbol: symbol, checkedTypeMetaSerializerSymbol: symbol, checkedTypeMetaWireTypeIdSymbol: symbol, ) => Serializer; -type SerializerFactory = ReturnType; - -interface GeneratedFactory { - create: SerializerFactory; - localTypeMeta: TypeMeta | undefined; - fixedSize: number; - readDataAlwaysAdvances: boolean; -} - -const uninitializedSerializer: Serializer = { - _initialized: false, - fixedSize: 0, - getTypeInfo: () => { - throw new Error("serializer is not initialized"); - }, - getTypeId: () => { - throw new Error("serializer is not initialized"); - }, - getUserTypeId: () => { - throw new Error("serializer is not initialized"); - }, - needToWriteRef: () => { - throw new Error("serializer is not initialized"); - }, - getHash: () => { - throw new Error("serializer is not initialized"); - }, - write: (value: any) => { - void value; - throw new Error("serializer is not initialized"); - }, - writeRef: (value: any) => { - void value; - throw new Error("serializer is not initialized"); - }, - writeNoRef: (value: any) => { - void value; - throw new Error("serializer is not initialized"); - }, - writeRefOrNull: (value: any) => { - void value; - throw new Error("serializer is not initialized"); - }, - writeTypeInfo: (value: any) => { - void value; - throw new Error("serializer is not initialized"); - }, - read: (fromRef: boolean) => { - void fromRef; - throw new Error("serializer is not initialized"); - }, - readRef: () => { - throw new Error("serializer is not initialized"); - }, - readRefWithoutTypeInfo: () => { - throw new Error("serializer is not initialized"); - }, - readNoRef: (fromRef: boolean) => { - void fromRef; - throw new Error("serializer is not initialized"); - }, - readTypeInfo: () => { - throw new Error("serializer is not initialized"); - }, - readDataAlwaysAdvances: false, -}; - -interface GeneratedRegistration { - typeInfo: TypeInfo; - serializer: Serializer; - preparing: boolean; - factory?: GeneratedFactory; -} - export class Gen { static external = CodegenRegistry.getExternal(); constructor( private typeResolver: TypeResolver, - private rootCustomSerializer?: CustomSerializer, + private regOptions: { [key: string]: any } = {}, ) {} - private generateFactory( - typeInfo: TypeInfo, - serializerLookup: SerializerLookup, - ): GeneratedFactory { + private generate(typeInfo: TypeInfo): Serializer { const InnerGeneratorClass = CodegenRegistry.get(typeInfo.typeId); if (!InnerGeneratorClass) { throw new Error(`${typeInfo.typeId} generator not exists`); @@ -153,352 +75,50 @@ export class Gen { const scope = new Scope(); const generator = new InnerGeneratorClass( typeInfo, - new CodecBuilder(scope, this.typeResolver, serializerLookup), + new CodecBuilder(scope, this.typeResolver), scope, ); - const generated = generator.toSerializer(); + const funcString = generator.toSerializer(); let factoryBuilder: SerializerFactoryBuilder; if (this.typeResolver.config && this.typeResolver.config.hooks) { const afterCodeGenerated = this.typeResolver.config.hooks.afterCodeGenerated; if (typeof afterCodeGenerated === "function") { - factoryBuilder = new Function( - afterCodeGenerated(generated.source), - ) as SerializerFactoryBuilder; + factoryBuilder = new Function(afterCodeGenerated(funcString)) as SerializerFactoryBuilder; } else { - factoryBuilder = new Function(generated.source) as SerializerFactoryBuilder; + factoryBuilder = new Function(funcString) as SerializerFactoryBuilder; } } else { - factoryBuilder = new Function(generated.source) as SerializerFactoryBuilder; + factoryBuilder = new Function(funcString) as SerializerFactoryBuilder; } - return { - create: factoryBuilder(), - localTypeMeta: generated.localTypeMeta, - fixedSize: generated.fixedSize, - readDataAlwaysAdvances: generated.readDataAlwaysAdvances, - }; - } - - private createSerializer( - typeInfo: TypeInfo, - serializerLookup: SerializerLookup, - factory: GeneratedFactory, - ): Serializer { - const options = TypeId.extType(typeInfo.typeId) - ? { ...typeInfo.options, customSerializer: this.rootCustomSerializer } - : typeInfo.options; - return factory.create( + return factoryBuilder()( this.typeResolver, - serializerLookup, Gen.external, typeInfo, - options, - factory.localTypeMeta, + this.regOptions, + generator.getLocalTypeMeta(), localTypeMetaSymbol, checkedTypeMetaSerializerSymbol, checkedTypeMetaWireTypeIdSymbol, ); } + private register(typeInfo: TypeInfo, serializer?: Serializer) { + this.typeResolver.registerSerializer(typeInfo, serializer); + } + private isRegistered(typeInfo: TypeInfo) { return !!this.typeResolver.getSerializerByTypeInfo(typeInfo); } - private isFullyGenerated(typeInfo: TypeInfo, registrations: GeneratedRegistration[]) { - const ser = this.getGeneratedSerializer(typeInfo, registrations); + private isFullyGenerated(typeInfo: TypeInfo) { + const ser = this.typeResolver.getSerializerByTypeInfo(typeInfo); return ser && ser._initialized; } - private sameRegistration(left: TypeInfo, right: TypeInfo) { - const leftTypeId = this.typeResolver.computeTypeId(left); - const rightTypeId = this.typeResolver.computeTypeId(right); - if (TypeId.isNamedType(leftTypeId) && TypeId.isNamedType(rightTypeId)) { - return left.named === right.named; - } - if ( - TypeId.needsUserTypeId(leftTypeId) && - TypeId.needsUserTypeId(rightTypeId) && - left.userTypeId !== -1 && - right.userTypeId !== -1 - ) { - return left.userTypeId === right.userTypeId; - } - if (TypeId.userDefinedType(leftTypeId) || TypeId.userDefinedType(rightTypeId)) { - return left === right; - } - return leftTypeId === rightTypeId; - } - - private sameTypeFamily(left: TypeInfo, right: TypeInfo) { - const leftTypeId = left.typeId; - const rightTypeId = right.typeId; - if (TypeId.structType(leftTypeId) && TypeId.structType(rightTypeId)) { - return true; - } - if (TypeId.enumType(leftTypeId) && TypeId.enumType(rightTypeId)) { - return true; - } - if (TypeId.extType(leftTypeId) && TypeId.extType(rightTypeId)) { - return true; - } - const leftUnion = - leftTypeId === TypeId.UNION || - leftTypeId === TypeId.TYPED_UNION || - leftTypeId === TypeId.NAMED_UNION; - const rightUnion = - rightTypeId === TypeId.UNION || - rightTypeId === TypeId.TYPED_UNION || - rightTypeId === TypeId.NAMED_UNION; - return leftUnion && rightUnion; - } - - private hasCompleteDefinition(typeInfo: TypeInfo) { - const options = typeInfo.options; - if (TypeId.structType(typeInfo.typeId)) { - return options?.props !== undefined; - } - if (TypeId.enumType(typeInfo.typeId)) { - return options?.enumProps !== undefined; - } - if (TypeId.extType(typeInfo.typeId)) { - return options?.props !== undefined || options?.creator !== undefined; - } - return ( - (typeInfo.typeId === TypeId.UNION || - typeInfo.typeId === TypeId.TYPED_UNION || - typeInfo.typeId === TypeId.NAMED_UNION) && - options?.cases !== undefined - ); - } - - private hasRegistryIdentity(typeInfo: TypeInfo) { - const typeId = this.typeResolver.computeTypeId(typeInfo); - if (!TypeId.userDefinedType(typeId) || TypeId.isNamedType(typeId)) { - return true; - } - if (TypeId.needsUserTypeId(typeId) && typeInfo.userTypeId !== -1) { - return true; - } - // A complete anonymous schema belongs to this generation graph. Only the definition-free - // generic serializer can be the canonical owner of a raw user-defined wire type ID. - return !this.hasCompleteDefinition(typeInfo); - } - - private sameDefinition(left: TypeInfo, right: TypeInfo) { - if (left === right) { - return true; - } - const leftOptions = left.options!; - const rightOptions = right.options!; - return ( - this.sameTypeFamily(left, right) && - left.named === right.named && - left.namespace === right.namespace && - left.typeName === right.typeName && - left.userTypeId === right.userTypeId && - left.evolving === right.evolving && - leftOptions.props === rightOptions.props && - leftOptions.enumProps === rightOptions.enumProps && - leftOptions.cases === rightOptions.cases && - leftOptions.fieldEntries === rightOptions.fieldEntries && - leftOptions.preserveFieldOrder === rightOptions.preserveFieldOrder && - leftOptions.withConstructor === rightOptions.withConstructor && - leftOptions.creator === rightOptions.creator - ); - } - - private checkTypeFamily(owner: TypeInfo, typeInfo: TypeInfo) { - if ( - TypeId.userDefinedType(typeInfo.typeId) && - (!TypeId.userDefinedType(owner.typeId) || !this.sameTypeFamily(owner, typeInfo)) - ) { - throw new Error("conflicting type families for the same registry identity"); - } - } - - private checkDefinitionOwner(owner: TypeInfo, typeInfo: TypeInfo) { - if (!TypeId.userDefinedType(typeInfo.typeId)) { - return; - } - this.checkTypeFamily(owner, typeInfo); - if ( - this.hasCompleteDefinition(owner) && - this.hasCompleteDefinition(typeInfo) && - !this.sameDefinition(owner, typeInfo) - ) { - throw new Error("conflicting complete definitions for the same registry identity"); - } - } - - private findRegistration(typeInfo: TypeInfo, registrations: GeneratedRegistration[]) { - const registration = registrations.find((entry) => - this.sameRegistration(entry.typeInfo, typeInfo), - ); - if (registration !== undefined) { - this.checkDefinitionOwner(registration.typeInfo, typeInfo); - } - return registration; - } - - private addRegistration(typeInfo: TypeInfo, registrations: GeneratedRegistration[]) { - const owner = { ...uninitializedSerializer }; - const entry: GeneratedRegistration = { - typeInfo, - serializer: owner, - preparing: false, - }; - owner.getTypeInfo = () => entry.typeInfo; - registrations.push(entry); - return entry; - } - - private getGeneratedSerializer(typeInfo: TypeInfo, registrations: GeneratedRegistration[]) { - const published = this.hasRegistryIdentity(typeInfo) - ? this.typeResolver.getSerializerByTypeInfo(typeInfo) - : undefined; - if (published !== undefined) { - this.checkDefinitionOwner(published.getTypeInfo(), typeInfo); - return published; - } - return this.findRegistration(typeInfo, registrations)?.serializer; - } - - private getCapturedSerializerById( - registrations: GeneratedRegistration[], - id: number, - userTypeId?: number, - ) { - const published = this.typeResolver.getSerializerById(id, userTypeId); - if (published !== undefined) { - return published; - } - if (id === TypeId.TYPED_UNION && (userTypeId === undefined || userTypeId === -1)) { - throw new Error("anonymous union serializer requires its TypeInfo owner"); - } - const entry = registrations.find((candidate) => { - const typeId = this.typeResolver.computeTypeId(candidate.typeInfo); - if ( - TypeId.needsUserTypeId(id) && - TypeId.needsUserTypeId(typeId) && - userTypeId !== undefined && - userTypeId !== -1 - ) { - return candidate.typeInfo.userTypeId === userTypeId; - } - return typeId === id; - }); - return entry?.serializer as Serializer; - } - - private getCapturedSerializerByName( - registrations: GeneratedRegistration[], - name: number | string, - ) { - const published = this.typeResolver.getSerializerByName(name); - if (published !== undefined) { - return published; - } - const entry = registrations.find( - (candidate) => - typeof name === "string" && - TypeId.isNamedType(this.typeResolver.computeTypeId(candidate.typeInfo)) && - candidate.typeInfo.named === name, - ); - return entry?.serializer; - } - - private prepareRegistration( - typeInfo: TypeInfo, - children: TypeInfo[], - registrations: GeneratedRegistration[], - factories: GeneratedRegistration[], - serializerLookup: SerializerLookup, - ) { - let entry = this.findRegistration(typeInfo, registrations); - if (entry?.serializer._initialized || entry?.preparing) { - return; - } - if (entry === undefined) { - entry = this.addRegistration(typeInfo, registrations); - } else { - entry.typeInfo = typeInfo; - } - entry.preparing = true; - try { - for (const child of children) { - this.traversalContainer(child, registrations, factories, serializerLookup); - } - entry.factory = this.generateFactory(typeInfo, serializerLookup); - // This local owner is still unreachable by the resolver. Expose only the completed static - // facts to later code generation; the final pass installs its runtime methods after hooks. - entry.serializer.fixedSize = entry.factory.fixedSize; - entry.serializer.readDataAlwaysAdvances = entry.factory.readDataAlwaysAdvances; - entry.serializer._initialized = true; - factories.push(entry); - } finally { - entry.preparing = false; - } - } - - private seedDefinitions(root: TypeInfo, registrations: GeneratedRegistration[]) { - const pending = [root]; - const seen = new Set(); - while (pending.length > 0) { - const typeInfo = pending.pop()!; - if (seen.has(typeInfo)) { - continue; - } - seen.add(typeInfo); - const options = typeInfo.options; - if ( - !TypeId.extType(typeInfo.typeId) && - this.hasCompleteDefinition(typeInfo) && - !this.getGeneratedSerializer(typeInfo, registrations)?._initialized - ) { - const registration = this.findRegistration(typeInfo, registrations); - if (registration === undefined) { - this.addRegistration(typeInfo, registrations); - } - } - if (options === undefined) { - continue; - } - if (options.props !== undefined) { - pending.push(...Object.values(options.props)); - } - if (options.cases !== undefined) { - pending.push(...Object.values(options.cases)); - } - if (options.fieldEntries !== undefined) { - for (const entry of options.fieldEntries) { - pending.push(entry.typeInfo); - } - } - if (options.inner !== undefined) { - pending.push(options.inner); - } - if (options.key !== undefined) { - pending.push(options.key); - } - if (options.value !== undefined) { - pending.push(options.value); - } - } - for (const typeInfo of seen) { - if (TypeId.userDefinedType(typeInfo.typeId)) { - this.findRegistration(typeInfo, registrations); - } - } - } - - private traversalContainer( - typeInfo: TypeInfo, - registrations: GeneratedRegistration[], - factories: GeneratedRegistration[], - serializerLookup: SerializerLookup, - ) { + private traversalContainer(typeInfo: TypeInfo) { if (TypeId.userDefinedType(typeInfo.typeId)) { - if (this.isFullyGenerated(typeInfo, registrations)) { + if (this.isFullyGenerated(typeInfo)) { return; } const options = typeInfo.options; @@ -506,137 +126,59 @@ export class Gen { typeInfo.typeId === TypeId.UNION || typeInfo.typeId === TypeId.TYPED_UNION || typeInfo.typeId === TypeId.NAMED_UNION; - // Extension generation belongs only to an explicit root registration. Check it before the - // generic props path so a decorated nested extension cannot create a second local owner. - if (TypeId.extType(typeInfo.typeId)) { - if (this.findRegistration(typeInfo, registrations) === undefined) { - throw new Error("nested extension serializer must be registered before use"); - } - this.prepareRegistration( - typeInfo, - Object.values(options?.props ?? {}), - registrations, - factories, - serializerLookup, - ); + if (unionType && options?.cases && Object.keys(options.cases).length > 0) { + this.register(typeInfo); + Object.values(options.cases).forEach((x) => { + this.traversalContainer(x); + }); + this.register(typeInfo, this.generate(typeInfo)); return; - } else if (unionType && options?.cases && Object.keys(options.cases).length > 0) { - this.prepareRegistration( - typeInfo, - Object.values(options.cases), - registrations, - factories, - serializerLookup, - ); - return; - } else if (options?.props !== undefined) { - this.prepareRegistration( - typeInfo, - Object.values(options.props), - registrations, - factories, - serializerLookup, - ); + } else if (options?.props && Object.keys(options.props).length > 0) { + this.register(typeInfo); + Object.values(options.props).forEach((x) => { + this.traversalContainer(x); + }); + this.register(typeInfo, this.generate(typeInfo)); } else if (!this.isRegistered(typeInfo) && TypeId.structType(typeInfo.typeId)) { - if (this.findRegistration(typeInfo, registrations) === undefined) { - throw new Error("nested struct schema must be registered or defined before use"); - } + // Forward reference to a struct type not yet fully defined — register a + // placeholder so that serializer factories can capture the object + // reference. The placeholder will be filled in via Object.assign + // when the real serializer is generated later. + this.register(typeInfo); } else if (TypeId.enumType(typeInfo.typeId) && !this.isRegistered(typeInfo)) { - this.prepareRegistration(typeInfo, [], registrations, factories, serializerLookup); + this.register(typeInfo, this.generate(typeInfo)); } } if (typeInfo.typeId === TypeId.LIST) { - this.traversalContainer(typeInfo.options!.inner!, registrations, factories, serializerLookup); + this.traversalContainer(typeInfo.options!.inner!); } if (typeInfo.typeId === TypeId.SET) { - this.traversalContainer(typeInfo.options!.key!, registrations, factories, serializerLookup); + this.traversalContainer(typeInfo.options!.key!); } if (typeInfo.typeId === TypeId.MAP) { if (!typeInfo.options?.key || !typeInfo.options?.value) { throw new Error("map type must have key and value"); } - this.traversalContainer(typeInfo.options!.key!, registrations, factories, serializerLookup); - this.traversalContainer(typeInfo.options!.value!, registrations, factories, serializerLookup); + this.traversalContainer(typeInfo.options!.key!); + this.traversalContainer(typeInfo.options!.value!); } if (typeInfo.options?.cases) { Object.values(typeInfo.options.cases).forEach((caseTypeInfo) => { - this.traversalContainer(caseTypeInfo, registrations, factories, serializerLookup); + this.traversalContainer(caseTypeInfo); }); } } reGenerateSerializer(typeInfo: TypeInfo) { - const factory = this.generateFactory(typeInfo, this.typeResolver); - return this.createSerializer(typeInfo, this.typeResolver, factory); + return this.generate(typeInfo); } generateSerializer(typeInfo: TypeInfo) { - this.typeResolver.ensureRegistrationOpen(); - sealTypeInfo(typeInfo); - // TypeInfo freezing may invoke application-owned proxy traps. A root entered there closes the - // resolver before code generation or publication can continue. - this.typeResolver.ensureRegistrationOpen(); - const registrations: GeneratedRegistration[] = []; - const factories: GeneratedRegistration[] = []; - // Generator-time TypeInfo queries see initialized local serializers for codegen decisions. - // Factory-init ID/name queries instead return the stable owner captured by runtime closures. - const serializerLookup: SerializerLookup = { - getSerializerByTypeInfo: (fieldType) => this.getGeneratedSerializer(fieldType, registrations), - getSerializerById: (id, userTypeId) => - this.getCapturedSerializerById(registrations, id, userTypeId), - getSerializerByName: (name) => this.getCapturedSerializerByName(registrations, name), - }; - this.seedDefinitions(typeInfo, registrations); - if ( - !TypeId.structType(typeInfo.typeId) && - !this.typeResolver.getSerializerByTypeInfo(typeInfo)?._initialized && - this.findRegistration(typeInfo, registrations) === undefined - ) { - this.addRegistration(typeInfo, registrations); - } - this.traversalContainer(typeInfo, registrations, factories, serializerLookup); - const publishedRoot = this.typeResolver.getSerializerByTypeInfo(typeInfo); - if (!publishedRoot?._initialized) { - let registration = this.findRegistration(typeInfo, registrations); - if (registration === undefined) { - registration = this.addRegistration(typeInfo, registrations); - } - if (!registration.serializer._initialized) { - this.prepareRegistration(typeInfo, [], registrations, factories, serializerLookup); - } + this.traversalContainer(typeInfo); + const serializer = this.typeResolver.getSerializerByTypeInfo(typeInfo); + if (serializer?._initialized) { + return serializer; } - - // Hooks may publish an owner after earlier code generation used an equivalent local schema. - // Reconcile every identity before invoking any factory so fixed captures use the final owner. - for (const registration of registrations) { - if (!this.hasRegistryIdentity(registration.typeInfo)) { - continue; - } - const published = this.typeResolver.getSerializerByTypeInfo(registration.typeInfo); - if (published !== undefined) { - this.checkDefinitionOwner(published.getTypeInfo(), registration.typeInfo); - } - } - for (const registration of factories) { - const published = this.hasRegistryIdentity(registration.typeInfo) - ? this.typeResolver.getSerializerByTypeInfo(registration.typeInfo) - : undefined; - if (published !== undefined) { - continue; - } - Object.assign( - registration.serializer, - this.createSerializer(registration.typeInfo, serializerLookup, registration.factory!), - ); - } - const serializer = this.getGeneratedSerializer(typeInfo, registrations)!; - this.typeResolver.commitGeneratedSerializers( - registrations.filter( - (registration) => - this.hasRegistryIdentity(registration.typeInfo) && - this.typeResolver.getSerializerByTypeInfo(registration.typeInfo) === undefined, - ), - ); - return serializer; + return this.reGenerateSerializer(typeInfo); } } diff --git a/javascript/packages/core/lib/gen/map.ts b/javascript/packages/core/lib/gen/map.ts index 6df1f99a10..3e18823541 100644 --- a/javascript/packages/core/lib/gen/map.ts +++ b/javascript/packages/core/lib/gen/map.ts @@ -393,7 +393,7 @@ export class MapSerializerGenerator extends BaseSerializerGenerator { private useDeclaredType(typeInfo: TypeInfo) { const readWriteTypeInfo = - this.builder.serializerLookup.getSerializerByTypeInfo(typeInfo)?.getTypeInfo() ?? typeInfo; + this.builder.resolver.getSerializerByTypeInfo(typeInfo)?.getTypeInfo() ?? typeInfo; // Evolving structs need per-chunk TypeInfo so a compatible reader can discard a removed map // field. A fixed-schema serializer deliberately keeps the declared form: evolving=false is its // same-schema size and speed opt-out, even when the field declaration is only a placeholder. diff --git a/javascript/packages/core/lib/gen/router.ts b/javascript/packages/core/lib/gen/router.ts index 9eefe243e4..5492348aac 100644 --- a/javascript/packages/core/lib/gen/router.ts +++ b/javascript/packages/core/lib/gen/router.ts @@ -17,7 +17,6 @@ * under the License. */ -import { TypeId } from "../type"; import { TypeInfo } from "../typeInfo"; import { SerializerGenerator } from "./serializer"; import { CodecBuilder } from "./builder"; @@ -49,26 +48,11 @@ export class CodegenRegistry { } static newGeneratorByTypeInfo(typeInfo: TypeInfo, builder: CodecBuilder, scope: Scope) { - let generatorTypeInfo = typeInfo; - if (TypeId.userDefinedType(typeInfo.typeId)) { - const ownerTypeInfo = builder.serializerLookup - .getSerializerByTypeInfo(typeInfo) - ?.getTypeInfo(); - if (ownerTypeInfo !== undefined && ownerTypeInfo !== typeInfo) { - // Schema comes from the authoritative serializer owner. Field occurrence modifiers remain - // local to the containing schema and must not be replaced with the owner's modifiers. - generatorTypeInfo = ownerTypeInfo.clone(); - generatorTypeInfo.nullable = typeInfo.nullable; - generatorTypeInfo.trackingRef = typeInfo.trackingRef; - generatorTypeInfo.id = typeInfo.id; - generatorTypeInfo.dynamic = typeInfo.dynamic; - } - } - const constructor = CodegenRegistry.get(generatorTypeInfo.typeId); + const constructor = CodegenRegistry.get(typeInfo.typeId); if (!constructor) { throw new Error("type not registered"); } - return new constructor(generatorTypeInfo, builder, scope); + return new constructor(typeInfo, builder, scope); } static get(typeId: number) { diff --git a/javascript/packages/core/lib/gen/serializer.ts b/javascript/packages/core/lib/gen/serializer.ts index aa621bf38b..c2a05dd542 100644 --- a/javascript/packages/core/lib/gen/serializer.ts +++ b/javascript/packages/core/lib/gen/serializer.ts @@ -40,12 +40,7 @@ export interface SerializerGenerator { write(accessor: string): string; writeEmbed(): any; - toSerializer(): { - source: string; - localTypeMeta: TypeMeta | undefined; - fixedSize: number; - readDataAlwaysAdvances: boolean; - }; + toSerializer(): string; getFixedSize(): number; needToWriteRef(): boolean; @@ -308,7 +303,6 @@ export abstract class BaseSerializerGenerator implements SerializerGenerator { this.scope.assertNameNotDuplicate("write"); this.scope.assertNameNotDuplicate("writeInner"); this.scope.assertNameNotDuplicate("typeResolver"); - this.scope.assertNameNotDuplicate("serializerLookup"); this.scope.assertNameNotDuplicate("external"); this.scope.assertNameNotDuplicate("options"); this.scope.assertNameNotDuplicate("typeInfo"); @@ -337,14 +331,12 @@ export abstract class BaseSerializerGenerator implements SerializerGenerator { ? "" : `[localTypeMetaSymbol]: localTypeMeta, [checkedTypeMetaWireTypeIdSymbol]: localTypeMeta.getTypeId(),`; - const hash = this.getHash(); - const typeMetaBytes = this.getTypeMetaBytes(); const declare = ` const getHash = () => { - return ${hash}; + return ${this.getHash()}; } const getTypeMetaBytes = () => { - return ${typeMetaBytes}; + return ${this.getTypeMetaBytes()}; } const write = (v) => { ${this.write("v")} @@ -377,26 +369,19 @@ export abstract class BaseSerializerGenerator implements SerializerGenerator { ${this.readTypeInfo()} }; `; - const scope = this.scope.generate(); - const fixedSize = this.getFixedSize(); - const needToWriteRef = this.needToWriteRef(); - const typeId = this.getTypeId(); - const userTypeId = this.getUserTypeId(); - const readDataAlwaysAdvances = this.readDataAlwaysAdvances(); // Append read-only capability metadata so existing writer properties keep // their object-layout order on serialization hot paths. - return { - source: ` - return function (typeResolver, serializerLookup, external, typeInfo, options${localTypeMetaParams}) { - ${scope} + return ` + return function (typeResolver, external, typeInfo, options${localTypeMetaParams}) { + ${this.scope.generate()} ${serializerDeclaration} ${declare} ${serializerAssignment} { _initialized: true, - fixedSize: ${fixedSize}, - needToWriteRef: () => ${needToWriteRef}, - getTypeId: () => ${typeId}, - getUserTypeId: () => ${userTypeId}, + fixedSize: ${this.getFixedSize()}, + needToWriteRef: () => ${this.needToWriteRef()}, + getTypeId: () => ${this.getTypeId()}, + getUserTypeId: () => ${this.getUserTypeId()}, getTypeInfo: () => typeInfo, getHash, getTypeMetaBytes, @@ -412,15 +397,11 @@ export abstract class BaseSerializerGenerator implements SerializerGenerator { readRefWithoutTypeInfo, readNoRef, readTypeInfo, - readDataAlwaysAdvances: ${readDataAlwaysAdvances}, + readDataAlwaysAdvances: ${this.readDataAlwaysAdvances()}, ${localTypeMetaProperty} }; ${serializerReturn} } - `, - localTypeMeta, - fixedSize, - readDataAlwaysAdvances, - }; + `; } } diff --git a/javascript/packages/core/lib/gen/struct.ts b/javascript/packages/core/lib/gen/struct.ts index 307a72189a..d9ffae0d3f 100644 --- a/javascript/packages/core/lib/gen/struct.ts +++ b/javascript/packages/core/lib/gen/struct.ts @@ -587,12 +587,6 @@ class StructSerializerGenerator extends BaseSerializerGenerator { return JS_STRUCT_OWNER_BYTES + this.sortedProps.length * REFERENCE_BYTES; } - private serializerCaptureExpr(): string { - return TypeId.isNamedType(this.typeInfo.typeId) - ? this.builder.typeResolver.getSerializerByName(this.typeInfo.named!) - : this.builder.typeResolver.getSerializerById(this.typeInfo.typeId, this.typeInfo.userTypeId); - } - readDataAlwaysAdvances(): boolean { if (!this.builder.resolver.isCompatible()) { return true; @@ -609,8 +603,7 @@ class StructSerializerGenerator extends BaseSerializerGenerator { // recursive placeholder remains unknown and selects the guarded loop; // do not recursively walk the schema graph here. if ( - this.builder.serializerLookup.getSerializerByTypeInfo(typeInfo)?.readDataAlwaysAdvances === - true + this.builder.resolver.getSerializerByTypeInfo(typeInfo)?.readDataAlwaysAdvances === true ) { return true; } @@ -796,11 +789,11 @@ class StructSerializerGenerator extends BaseSerializerGenerator { continue; } } - const innerGenerator = CodegenRegistry.newGeneratorByTypeInfo( - current.typeInfo, - this.builder, - this.scope, - ); + const InnerGeneratorClass = CodegenRegistry.get(current.typeInfo.typeId); + if (!InnerGeneratorClass) { + throw new Error(`${current.typeInfo.typeId} generator not exists`); + } + const innerGenerator = new InnerGeneratorClass(current.typeInfo, this.builder, this.scope); const fieldAccessor = `${accessor}${CodecBuilder.safePropAccessor(current.key)}`; fieldWrites.push( this.writeField(current.key, current.typeInfo, fieldAccessor, innerGenerator.writeEmbed()), @@ -952,11 +945,11 @@ class StructSerializerGenerator extends BaseSerializerGenerator { ${this.maybeReference(result, refState)} ${this.sortedProps .map(({ key, typeInfo }) => { - const innerGenerator = CodegenRegistry.newGeneratorByTypeInfo( - typeInfo, - this.builder, - this.scope, - ); + const InnerGeneratorClass = CodegenRegistry.get(typeInfo.typeId); + if (!InnerGeneratorClass) { + throw new Error(`${typeInfo.typeId} generator not exists`); + } + const innerGenerator = new InnerGeneratorClass(typeInfo, this.builder, this.scope); return ` ${this.readField(key, typeInfo, (expr) => this.readFieldAssign(result, key, expr), innerGenerator.readEmbed())} `; @@ -1237,11 +1230,11 @@ class StructSerializerGenerator extends BaseSerializerGenerator { // Hoist the serializer lookup into a scope-level const, evaluated once during // factory init. Self-recursive structs may still point at a placeholder, so // only the fully generated serializer path can hoist derived values below. - const hoisted = this.scope.declare("ser", this.serializerCaptureExpr()); + const hoisted = this.scope.declare("ser", this.serializerExpr); const scope = this.scope; const builder = this.builder; const internalTypeId = this.getInternalTypeId(); - const serializer = builder.serializerLookup.getSerializerByTypeInfo(this.typeInfo); + const serializer = builder.resolver.getSerializerByTypeInfo(this.typeInfo); const canInlineCompatibleTypeInfo = internalTypeId === TypeId.COMPATIBLE_STRUCT || internalTypeId === TypeId.NAMED_COMPATIBLE_STRUCT || @@ -1355,7 +1348,7 @@ class StructSerializerGenerator extends BaseSerializerGenerator { writeEmbed() { // Hoist the serializer lookup — safe because writeEmbed() is used by // the parent struct whose factory runs after child serializers exist. - const hoisted = this.scope.declare("ser", this.serializerCaptureExpr()); + const hoisted = this.scope.declare("ser", this.serializerExpr); const scope = this.scope; return new Proxy( {}, @@ -1448,16 +1441,11 @@ class StructSerializerGenerator extends BaseSerializerGenerator { let fixedSize = 8; if (options!.props) { Object.values(options!.props).forEach((x) => { - const serializer = this.builder.serializerLookup.getSerializerByTypeInfo(x); - if (TypeId.userDefinedType(x.typeId) && serializer !== undefined) { - fixedSize += serializer.fixedSize; - } else { - const propGenerator = CodegenRegistry.newGeneratorByTypeInfo(x, this.builder, this.scope); - fixedSize += propGenerator.getFixedSize(); - } + const propGenerator = new (CodegenRegistry.get(x.typeId)!)(x, this.builder, this.scope); + fixedSize += propGenerator.getFixedSize(); }); } else { - fixedSize += this.builder.serializerLookup.getSerializerByTypeInfo(typeInfo)!.fixedSize; + fixedSize += this.builder.resolver.getSerializerByName(typeInfo.named!)!.fixedSize; } return fixedSize; } diff --git a/javascript/packages/core/lib/type.ts b/javascript/packages/core/lib/type.ts index 73f0ab34c6..3f2ecc998a 100644 --- a/javascript/packages/core/lib/type.ts +++ b/javascript/packages/core/lib/type.ts @@ -156,8 +156,6 @@ export const TypeId = { TypeId.NAMED_COMPATIBLE_STRUCT, TypeId.EXT, TypeId.NAMED_EXT, - TypeId.TYPED_UNION, - TypeId.NAMED_UNION, ].includes(id as any); }, structType(id: number) { diff --git a/javascript/packages/core/lib/typeInfo.ts b/javascript/packages/core/lib/typeInfo.ts index 8e8ad2d336..809d7efcbc 100644 --- a/javascript/packages/core/lib/typeInfo.ts +++ b/javascript/packages/core/lib/typeInfo.ts @@ -26,20 +26,6 @@ import { Decimal } from "./types/decimal"; const targetFields = new WeakMap any, { [key: string]: TypeInfo }>(); export const MAX_FIELD_ID = (1 << 29) - 1; -const sealedSchemaFields = { - options: { writable: false, configurable: false }, - named: { writable: false, configurable: false }, - namespace: { writable: false, configurable: false }, - typeName: { writable: false, configurable: false }, - userTypeId: { writable: false, configurable: false }, - evolving: { writable: false, configurable: false }, - _typeId: { writable: false, configurable: false }, - nullable: { writable: false, configurable: false }, - trackingRef: { writable: false, configurable: false }, - id: { writable: false, configurable: false }, - dynamic: { writable: false, configurable: false }, -}; - export function checkFieldId(fieldId: number) { if (Number.isFinite(fieldId) && fieldId < 0) { throw new Error("field id must be non-negative"); @@ -163,6 +149,26 @@ export class TypeInfo extends ExtensibleFunction { }); } + public freeze() { + Object.defineProperties(this, { + named: { writable: false, configurable: false }, + namespace: { writable: false, configurable: false }, + typeName: { writable: false, configurable: false }, + userTypeId: { writable: false, configurable: false }, + evolving: { writable: false, configurable: false }, + options: { writable: false, configurable: false }, + _typeId: { writable: false, configurable: false }, + nullable: { writable: false, configurable: false }, + }); + Object.freeze(this.options); + if (this.options?.props) { + Object.freeze(this.options!.props); + } + if (this.options?.enumProps) { + Object.freeze(this.options!.enumProps); + } + } + public constructor(typeId: number, userTypeId = -1) { super(function (target: any, key?: string | { name?: string }) { if (key === undefined) { @@ -327,7 +333,7 @@ export class TypeInfo extends ExtensibleFunction { } const typeInfo = new TypeInfo(finalTypeId, userTypeId); typeInfo.options = { - props, + props: props || {}, withConstructor, }; typeInfo.evolving = evolving; @@ -395,60 +401,6 @@ export class TypeInfo extends ExtensibleFunction { } } -/** @internal */ -export function sealTypeInfo(root: TypeInfo) { - const pending = [root]; - const seen = new Set(); - while (pending.length > 0) { - const typeInfo = pending.pop()!; - if (seen.has(typeInfo)) { - continue; - } - seen.add(typeInfo); - - // Lock the options pointer before any schema read. A proxy trap may replace the pointer while - // it is being locked, but later traps cannot replace the final value code generation observes. - Object.defineProperties(typeInfo, sealedSchemaFields); - const options = typeInfo.options; - if (options === undefined) { - continue; - } - Object.freeze(options); - - const props = options.props; - if (props !== undefined) { - Object.freeze(props); - pending.push(...Object.values(props)); - } - const cases = options.cases; - if (cases !== undefined) { - Object.freeze(cases); - pending.push(...Object.values(cases)); - } - const fieldEntries = options.fieldEntries; - if (fieldEntries !== undefined) { - Object.freeze(fieldEntries); - for (const entry of fieldEntries) { - Object.freeze(entry); - pending.push(entry.typeInfo); - } - } - if (options.inner !== undefined) { - pending.push(options.inner); - } - if (options.key !== undefined) { - pending.push(options.key); - } - if (options.value !== undefined) { - pending.push(options.value); - } - if (options.enumProps !== undefined) { - Object.freeze(options.enumProps); - } - } - // dynamicTypeId is operation-local writer state and remains mutable across roots. -} - export enum Dynamic { TRUE = "TRUE", FALSE = "FALSE", diff --git a/javascript/packages/core/lib/typeResolver.ts b/javascript/packages/core/lib/typeResolver.ts index 7db8d75293..30d27d54c0 100644 --- a/javascript/packages/core/lib/typeResolver.ts +++ b/javascript/packages/core/lib/typeResolver.ts @@ -35,11 +35,68 @@ import { BoolArray } from "./types/boolArray"; import { isFloat16Array } from "./types/float16"; import { getUnknownTypeMeta, UnknownStructSerializer } from "./unknownStruct"; +const uninitSerialize = { + _initialized: false, + fixedSize: 0, + getTypeInfo: () => { + throw new Error("uninitSerialize"); + }, + getTypeId: () => { + throw new Error("uninitSerialize"); + }, + getUserTypeId: () => { + throw new Error("uninitSerialize"); + }, + needToWriteRef: () => { + throw new Error("uninitSerialize"); + }, + getHash: () => { + throw new Error("uninitSerialize"); + }, + write: (v: any) => { + void v; + throw new Error("uninitSerialize"); + }, + writeRef: (v: any) => { + void v; + throw new Error("uninitSerialize"); + }, + writeNoRef: (v: any) => { + void v; + throw new Error("uninitSerialize"); + }, + writeRefOrNull: (v: any) => { + void v; + throw new Error("uninitSerialize"); + }, + writeTypeInfo: (v: any) => { + void v; + throw new Error("uninitSerialize"); + }, + read: (fromRef: boolean) => { + void fromRef; + throw new Error("uninitSerialize"); + }, + readRef: () => { + throw new Error("uninitSerialize"); + }, + readRefWithoutTypeInfo: () => { + throw new Error("uninitSerialize"); + }, + readNoRef: (fromRef: boolean) => { + void fromRef; + throw new Error("uninitSerialize"); + }, + readTypeInfo: () => { + throw new Error("uninitSerialize"); + }, + readDataAlwaysAdvances: false, +}; + export default class TypeResolver { readonly trackingRef: boolean; private internalSerializer: Serializer[] = new Array(300); private customSerializer: Map = new Map(); - private registrationFrozen = false; private writeContext!: WriteContext; private readContext!: ReadContext; @@ -134,53 +191,53 @@ export default class TypeResolver { } private initInternalSerializer() { - const generateInternalSerializer = (typeInfo: TypeInfo) => { - return new Gen(this).generateSerializer(typeInfo); + const registerSerializer = (typeInfo: TypeInfo) => { + return this.registerSerializer(typeInfo, new Gen(this).generateSerializer(typeInfo)); }; - generateInternalSerializer(Type.string()); - generateInternalSerializer(new TypeInfo(TypeId.ENUM)); - generateInternalSerializer(new TypeInfo(TypeId.NAMED_ENUM)); - generateInternalSerializer(Type.any()); - generateInternalSerializer(Type.list(Type.any())); - generateInternalSerializer(Type.map(Type.any(), Type.any())); - generateInternalSerializer(Type.bool()); - generateInternalSerializer(Type.int8()); - generateInternalSerializer(Type.int16()); - generateInternalSerializer(Type.int32({ encoding: "fixed" })); - generateInternalSerializer(Type.int32()); - generateInternalSerializer(Type.uint32({ encoding: "fixed" })); - generateInternalSerializer(Type.uint64({ encoding: "fixed" })); - generateInternalSerializer(Type.int64({ encoding: "fixed" })); - generateInternalSerializer(Type.int64()); - generateInternalSerializer(Type.uint8()); - generateInternalSerializer(Type.uint16()); - generateInternalSerializer(Type.uint32()); - generateInternalSerializer(Type.uint64()); - generateInternalSerializer(Type.uint64({ encoding: "tagged" })); - generateInternalSerializer(Type.int64({ encoding: "tagged" })); - generateInternalSerializer(Type.float16()); - generateInternalSerializer(Type.bfloat16()); - generateInternalSerializer(Type.float32()); - generateInternalSerializer(Type.float64()); - generateInternalSerializer(Type.timestamp()); - generateInternalSerializer(Type.duration()); - generateInternalSerializer(Type.date()); - generateInternalSerializer(Type.decimal()); - generateInternalSerializer(Type.set(Type.any())); - generateInternalSerializer(Type.binary()); - generateInternalSerializer(Type.boolArray()); - generateInternalSerializer(Type.uint8Array()); - generateInternalSerializer(Type.int8Array()); - generateInternalSerializer(Type.uint16Array()); - generateInternalSerializer(Type.int16Array()); - generateInternalSerializer(Type.uint32Array()); - generateInternalSerializer(Type.int32Array()); - generateInternalSerializer(Type.uint64Array()); - generateInternalSerializer(Type.int64Array()); - generateInternalSerializer(Type.float16Array()); - generateInternalSerializer(Type.bfloat16Array()); - generateInternalSerializer(Type.float32Array()); - generateInternalSerializer(Type.float64Array()); + registerSerializer(Type.string()); + registerSerializer(new TypeInfo(TypeId.ENUM)); + registerSerializer(new TypeInfo(TypeId.NAMED_ENUM)); + registerSerializer(Type.any()); + registerSerializer(Type.list(Type.any())); + registerSerializer(Type.map(Type.any(), Type.any())); + registerSerializer(Type.bool()); + registerSerializer(Type.int8()); + registerSerializer(Type.int16()); + registerSerializer(Type.int32({ encoding: "fixed" })); + registerSerializer(Type.int32()); + registerSerializer(Type.uint32({ encoding: "fixed" })); + registerSerializer(Type.uint64({ encoding: "fixed" })); + registerSerializer(Type.int64({ encoding: "fixed" })); + registerSerializer(Type.int64()); + registerSerializer(Type.uint8()); + registerSerializer(Type.uint16()); + registerSerializer(Type.uint32()); + registerSerializer(Type.uint64()); + registerSerializer(Type.uint64({ encoding: "tagged" })); + registerSerializer(Type.int64({ encoding: "tagged" })); + registerSerializer(Type.float16()); + registerSerializer(Type.bfloat16()); + registerSerializer(Type.float32()); + registerSerializer(Type.float64()); + registerSerializer(Type.timestamp()); + registerSerializer(Type.duration()); + registerSerializer(Type.date()); + registerSerializer(Type.decimal()); + registerSerializer(Type.set(Type.any())); + registerSerializer(Type.binary()); + registerSerializer(Type.boolArray()); + registerSerializer(Type.uint8Array()); + registerSerializer(Type.int8Array()); + registerSerializer(Type.uint16Array()); + registerSerializer(Type.int16Array()); + registerSerializer(Type.uint32Array()); + registerSerializer(Type.int32Array()); + registerSerializer(Type.uint64Array()); + registerSerializer(Type.int64Array()); + registerSerializer(Type.float16Array()); + registerSerializer(Type.bfloat16Array()); + registerSerializer(Type.float32Array()); + registerSerializer(Type.float64Array()); this.float64Serializer = this.getSerializerById(TypeId.FLOAT64); this.float32Serializer = this.getSerializerById(TypeId.FLOAT32); @@ -213,64 +270,59 @@ export default class TypeResolver { this.initInternalSerializer(); } - /** @internal */ - freezeRegistration() { - if (!this.registrationFrozen) { - this.registrationFrozen = true; - } - } - - /** @internal */ - ensureRegistrationOpen() { - if (this.registrationFrozen) { - throw new Error("types and serializers must be registered before the first root operation"); - } - } - - /** @internal */ - commitGeneratedSerializers(entries: readonly { typeInfo: TypeInfo; serializer: Serializer }[]) { - this.ensureRegistrationOpen(); - const publications = entries.map((entry) => { - if (!entry.serializer._initialized) { - throw new Error("generated serializer graph is incomplete"); - } - const typeId = this.computeTypeId(entry.typeInfo); - let internalTypeId: number | undefined; - let customTypeKey: number | string | undefined; - if (TypeId.isNamedType(typeId)) { - customTypeKey = entry.typeInfo.named!; - } else if (TypeId.needsUserTypeId(typeId) && entry.typeInfo.userTypeId !== -1) { - customTypeKey = this.makeUserTypeKey(entry.typeInfo.userTypeId); - } else if (typeId <= 0xff) { - internalTypeId = typeId; - } else { - customTypeKey = typeId; + registerSerializer(typeInfo: TypeInfo, serializer: Serializer = uninitSerialize) { + const typeId = this.computeTypeId(typeInfo); + if (!TypeId.isNamedType(typeId)) { + if (TypeId.needsUserTypeId(typeId) && typeInfo.userTypeId !== -1) { + const key = this.makeUserTypeKey(typeInfo.userTypeId); + if (this.customSerializer.has(key)) { + Object.assign(this.customSerializer.get(key)!, serializer); + } else { + this.customSerializer.set(key, { ...serializer }); + } + return this.customSerializer.get(key); } - const existingSerializer = - internalTypeId === undefined - ? this.customSerializer.get(customTypeKey!) - : this.internalSerializer[internalTypeId]; - return { - entry, - internalTypeId, - customTypeKey, - existingSerializer, - }; - }); - for (const publication of publications) { - if (publication.existingSerializer !== undefined) { - continue; + if (typeId <= 0xff) { + if (this.internalSerializer[typeId]) { + Object.assign(this.internalSerializer[typeId], serializer); + } else { + this.internalSerializer[typeId] = { ...serializer }; + } + return this.internalSerializer[typeId]; } - if (publication.internalTypeId !== undefined) { - this.internalSerializer[publication.internalTypeId] = publication.entry.serializer; + if (this.customSerializer.has(typeId)) { + Object.assign(this.customSerializer.get(typeId)!, serializer); } else { - this.customSerializer.set(publication.customTypeKey!, publication.entry.serializer); + this.customSerializer.set(typeId, { ...serializer }); } + return this.customSerializer.get(typeId); } + + const name = typeInfo.named!; + if (this.customSerializer.has(name)) { + Object.assign(this.customSerializer.get(name)!, serializer); + } else { + this.customSerializer.set(name, { ...serializer }); + } + return this.customSerializer.get(name); } generateReadSerializer(typeInfo: TypeInfo) { - return new Gen(this).reGenerateSerializer(typeInfo); + return new Gen(this, { creator: typeInfo.options?.creator }).reGenerateSerializer(typeInfo); + } + + regenerateReadSerializer(typeInfo: TypeInfo) { + const serializer = this.generateReadSerializer(typeInfo); + return this.registerSerializer(typeInfo, { + readDataAlwaysAdvances: serializer.readDataAlwaysAdvances, + getHash: serializer.getHash, + getTypeInfo: serializer.getTypeInfo, + read: serializer.read, + readNoRef: serializer.readNoRef, + readRef: serializer.readRef, + readTypeInfo: serializer.readTypeInfo, + readRefWithoutTypeInfo: serializer.readRefWithoutTypeInfo, + } as any)!; } getSerializerByTypeInfo(typeInfo: TypeInfo) { diff --git a/javascript/test/fory.test.ts b/javascript/test/fory.test.ts index 8f3a0d4796..e1ce9f2721 100644 --- a/javascript/test/fory.test.ts +++ b/javascript/test/fory.test.ts @@ -20,7 +20,6 @@ import Fory, { TypeInfo, Type } from "../packages/core/index"; import { describe, expect, test } from "@jest/globals"; import { fromUint8Array } from "../packages/core/lib/platformBuffer"; -import { TypeId } from "../packages/core/lib/type"; describe("fory", () => { test("defaults to compatible mode unless explicitly set", () => { @@ -87,7 +86,6 @@ describe("fory", () => { test.each(["serialize", "deserialize"] as const)("freezes on failed %s", (operation) => { const fory = new Fory({ compatible: false }); - fory.register(Type.struct(8101, {})); if (operation === "serialize") { expect(() => fory.serialize(Symbol("unsupported"))).toThrow(); @@ -98,632 +96,17 @@ describe("fory", () => { expect(() => fory.register(Type.struct(8102, {}))).toThrow(); }); - test("keeps rejected schema mutable", () => { + test.each(["serialize", "deserialize"] as const)("freezes after %s", (operation) => { const fory = new Fory({ compatible: false }); - const typeInfo = Type.struct(8106, {}); - fory.serialize(1); - - expect(() => fory.register(typeInfo)).toThrow(); - typeInfo.setNullable(true); - expect(typeInfo.nullable).toBe(true); - }); - - test("keeps callback failure local", () => { - let reenterRoot = false; - let fory: Fory; - fory = new Fory({ - compatible: false, - hooks: { - afterCodeGenerated(code) { - if (reenterRoot) { - reenterRoot = false; - fory.serialize(1); - } - return code; - }, - }, - }); - const typeResolver = fory.typeResolver as any; - const internalBefore = Array.from(typeResolver.internalSerializer); - const customBefore = Array.from(typeResolver.customSerializer.entries()); - const childType = Type.struct(8109, { - value: Type.int32(), - }); - const rootType = Type.struct(8110, { - child: childType, - }); - - reenterRoot = true; - expect(() => fory.register(rootType)).toThrow(); - - expect(Array.from(typeResolver.internalSerializer)).toEqual(internalBefore); - expect(Array.from(typeResolver.customSerializer.entries())).toEqual(customBefore); - expect(typeResolver.getSerializerById(TypeId.STRUCT, childType.userTypeId)).toBeUndefined(); - expect(typeResolver.getSerializerById(TypeId.STRUCT, rootType.userTypeId)).toBeUndefined(); - expect(() => rootType.setNullable(true)).toThrow(); - }); - - test("keeps factory failure local", () => { - let failFactory = false; - const fory = new Fory({ - compatible: false, - hooks: { - afterCodeGenerated(code) { - if (!failFactory) { - return code; - } - return code.replace( - /return function \(typeResolver, serializerLookup, external, typeInfo, options([^)]*)\) \{/, - (signature) => `${signature}\nthrow new Error("factory failure");`, - ); - }, - }, - }); - const typeResolver = fory.typeResolver as any; - const internalBefore = Array.from(typeResolver.internalSerializer); - const customBefore = Array.from(typeResolver.customSerializer.entries()); - const childType = Type.struct(8111, { - value: Type.int32(), - }); - const rootType = Type.struct(8112, { - child: childType, - }); - - failFactory = true; - expect(() => fory.register(rootType)).toThrow(); - - expect(Array.from(typeResolver.internalSerializer)).toEqual(internalBefore); - expect(Array.from(typeResolver.customSerializer.entries())).toEqual(customBefore); - expect(typeResolver.getSerializerById(TypeId.STRUCT, childType.userTypeId)).toBeUndefined(); - expect(typeResolver.getSerializerById(TypeId.STRUCT, rootType.userTypeId)).toBeUndefined(); - expect(() => rootType.setNullable(true)).toThrow(); - }); - - test("rejects unresolved schema", () => { - const fory = new Fory({ compatible: false }); - const typeResolver = fory.typeResolver as any; - const internalBefore = Array.from(typeResolver.internalSerializer); - const customBefore = Array.from(typeResolver.customSerializer.entries()); - const forwardType = Type.struct(8113); - const parentType = Type.struct(8114, { child: forwardType }); - - expect(() => fory.register(parentType)).toThrow(); - - expect(Array.from(typeResolver.internalSerializer)).toEqual(internalBefore); - expect(Array.from(typeResolver.customSerializer.entries())).toEqual(customBefore); - expect(typeResolver.getSerializerById(TypeId.STRUCT, forwardType.userTypeId)).toBeUndefined(); - expect(typeResolver.getSerializerById(TypeId.STRUCT, parentType.userTypeId)).toBeUndefined(); - - fory.register( - Type.struct(8113, { - value: Type.int32(), - }), - ); - const parent = fory.register( - Type.struct(8114, { - child: Type.struct(8113), - }), - ); - const value = { child: { value: 7 } }; - expect(parent.deserialize(parent.serialize(value))).toEqual(value); - }); - - test("rejects nested extension", () => { - const extensionType = Type.ext(8152); - @extensionType - class NestedExtension { - @Type.int32() - value = 0; - } - const fory = new Fory({ compatible: false }); - const root = Type.struct(8151, { value: extensionType }); - - expect(() => fory.register(root)).toThrow(); - expect(fory.typeResolver.getSerializerById(TypeId.STRUCT, 8151)).toBeUndefined(); - expect(fory.typeResolver.getSerializerById(TypeId.EXT, 8152)).toBeUndefined(); - }); - - test("uses decorated constructors", () => { - const childType = Type.struct(8153); - @childType - class Child { - @Type.int32() - value = 0; - } - const parentType = Type.struct(8154, { child: childType }); - @parentType - class Parent { - child = new Child(); - } - const registered = new Fory({ compatible: false }).register(Parent); - const value = new Parent(); - value.child.value = 7; - - const result = registered.deserialize(registered.serialize(value)); - expect(result).toBeInstanceOf(Parent); - expect(result!.child).toBeInstanceOf(Child); - expect(result!.child.value).toBe(7); - }); - - test("registers empty roots", () => { - const registered = new Fory({ compatible: false }).register(Type.struct(8122, {})); - - expect(registered.deserialize(registered.serialize({}))).toEqual({}); - }); - - test("registers self recursion", () => { - const nodeType = Type.struct(8123, { - value: Type.int32(), - next: Type.struct(8123).setNullable(true).setTrackingRef(true), - }); - const registered = new Fory({ compatible: false, ref: true }).register(nodeType); - const value: any = { value: 7 }; - value.next = value; - - const result: any = registered.deserialize(registered.serialize(value)); - expect(result.value).toBe(7); - expect(result.next).toBe(result); - }); - - test("registers mutual recursion", () => { - const rightType = Type.struct(8125, { - value: Type.string(), - left: Type.struct(8124).setNullable(true), - }); - const leftType = Type.struct(8124, { - value: Type.int32(), - right: rightType, - }); - const registered = new Fory({ compatible: false }).register(leftType); - const value = { value: 7, right: { value: "right", left: null } }; - - expect(registered.deserialize(registered.serialize(value))).toEqual(value); - }); - - test.each(["userTypeId", "name", "options"] as const)("rejects root %s mutation", (change) => { - let mutateDescriptor: (() => void) | undefined; - const fory = new Fory({ - compatible: false, - hooks: { - afterCodeGenerated(code) { - const mutate = mutateDescriptor; - mutateDescriptor = undefined; - mutate?.(); - return code; - }, - }, - }); - const typeInfo = - change === "name" - ? Type.struct("stable.Root", { value: Type.int32() }) - : Type.struct(8115, { value: Type.int32() }); - const typeResolver = fory.typeResolver as any; - const internalBefore = Array.from(typeResolver.internalSerializer); - const customBefore = Array.from(typeResolver.customSerializer.entries()); - if (change === "userTypeId") { - mutateDescriptor = () => { - typeInfo.userTypeId = 9115; - }; - } else if (change === "name") { - mutateDescriptor = () => { - typeInfo.named = "changed$Root"; - }; - } else { - mutateDescriptor = () => { - typeInfo.options!.props!.extra = Type.string(); - }; - } - - expect(() => fory.register(typeInfo)).toThrow(); - - expect(Array.from(typeResolver.internalSerializer)).toEqual(internalBefore); - expect(Array.from(typeResolver.customSerializer.entries())).toEqual(customBefore); - expect(() => typeInfo.setNullable(true)).toThrow(); - if (change === "options") { - expect(() => { - typeInfo.options!.props!.afterFailure = Type.bool(); - }).toThrow(); - } - }); - - test("rejects nested mutation", () => { - let mutateDescriptor: (() => void) | undefined; - const fory = new Fory({ - compatible: false, - hooks: { - afterCodeGenerated(code) { - const mutate = mutateDescriptor; - mutateDescriptor = undefined; - mutate?.(); - return code; - }, - }, - }); - const childType = Type.struct(8116, { value: Type.int32() }); - const rootType = Type.struct(8117, { child: childType }); - const typeResolver = fory.typeResolver as any; - const internalBefore = Array.from(typeResolver.internalSerializer); - const customBefore = Array.from(typeResolver.customSerializer.entries()); - mutateDescriptor = () => { - childType.options!.props!.extra = Type.string(); - }; - - expect(() => fory.register(rootType)).toThrow(); - - expect(Array.from(typeResolver.internalSerializer)).toEqual(internalBefore); - expect(Array.from(typeResolver.customSerializer.entries())).toEqual(customBefore); - expect(typeResolver.getSerializerById(TypeId.STRUCT, childType.userTypeId)).toBeUndefined(); - expect(typeResolver.getSerializerById(TypeId.STRUCT, rootType.userTypeId)).toBeUndefined(); - expect(() => rootType.setNullable(true)).toThrow(); - expect(() => childType.setNullable(true)).toThrow(); - }); - - test("resolves later definitions", () => { - const itemType = Type.struct(8118, { value: Type.int32() }); - const registered = new Fory({ compatible: false }).register( - Type.struct(8119, { - first: Type.struct(8118), - definition: itemType, - }), - ); - const value = { - first: { value: 1 }, - definition: { value: 2 }, - }; - - expect(registered.deserialize(registered.serialize(value as any))).toEqual(value); - }); - - test("rejects conflicting definitions", () => { - const identities: (number | { namespace: string; typeName: string })[] = [ - 8127, - { namespace: "test", typeName: "Conflict" }, - ]; - for (const identity of identities) { - const definitionPairs = [ - [ - Type.struct(identity, { firstValue: Type.int32() }), - Type.struct(identity, { secondValue: Type.string() }), - ], - [Type.enum(identity, { FIRST: 1 }), Type.enum(identity, { FIRST: 1 })], - [Type.union(identity, { 1: Type.int32() }), Type.union(identity, { 1: Type.int32() })], - ]; - for (const [first, second] of definitionPairs) { - for (const reversed of [false, true]) { - let generated = 0; - const fory = new Fory({ - compatible: false, - hooks: { - afterCodeGenerated(code) { - generated++; - return code; - }, - }, - }); - generated = 0; - const props = reversed ? { second, first } : { first, second }; - - expect(() => fory.register(Type.struct(8128, props))).toThrow(); - expect(generated).toBe(0); - expect(fory.typeResolver.getSerializerById(TypeId.STRUCT, 8128)).toBeUndefined(); - } - } - } - }); - - test("rejects mixed type families", () => { - const identities: (number | { namespace: string; typeName: string })[] = [ - 8129, - { namespace: "test", typeName: "Mixed" }, - ]; - for (const identity of identities) { - const types = [ - Type.struct(identity, { value: Type.int32() }), - Type.enum(identity, { VALUE: 1 }), - Type.ext(identity), - Type.union(identity, { 1: Type.string() }), - ]; - for (let left = 0; left < types.length; left++) { - for (let right = left + 1; right < types.length; right++) { - for (const reversed of [false, true]) { - let generated = 0; - const fory = new Fory({ - compatible: false, - hooks: { - afterCodeGenerated(code) { - generated++; - return code; - }, - }, - }); - generated = 0; - const first = types[reversed ? right : left]; - const second = types[reversed ? left : right]; - - expect(() => fory.register(Type.struct(8130, { first, second }))).toThrow(); - expect(generated).toBe(0); - expect(fory.typeResolver.getSerializerById(TypeId.STRUCT, 8130)).toBeUndefined(); - } - } - } - } - }); - - test("reuses shared definitions", () => { - const struct = Type.struct(8131, { value: Type.int32() }); - const enumType = Type.enum(8132, { VALUE: 1 }); - const union = Type.union(8133, { 1: Type.string() }); - const registered = new Fory({ compatible: false }).register( - Type.struct(8134, { - firstStruct: struct, - secondStruct: struct.clone(), - firstEnum: enumType, - secondEnum: enumType.clone(), - firstUnion: union, - secondUnion: union.clone(), - }), - ); - const value = { - firstStruct: { value: 1 }, - secondStruct: { value: 2 }, - firstEnum: 1, - secondEnum: 1, - firstUnion: { case: 1, value: "first" }, - secondUnion: { case: 1, value: "second" }, - }; - - expect(registered.deserialize(registered.serialize(value as any))).toEqual(value); - }); - - test("rejects reentrant family conflict", () => { - let publishConflict = false; - let fory: Fory; - fory = new Fory({ - compatible: false, - hooks: { - afterCodeGenerated(code) { - if (publishConflict) { - publishConflict = false; - fory.register(Type.enum(8135, { VALUE: 1 })); - } - return code; - }, - }, - }); - publishConflict = true; - - expect(() => - fory.register( - Type.struct(8136, { - value: Type.struct(8135, { value: Type.int32() }), - }), - ), - ).toThrow("conflicting type families"); - expect(fory.typeResolver.getSerializerById(TypeId.ENUM, 8135)).toBeDefined(); - expect(fory.typeResolver.getSerializerById(TypeId.STRUCT, 8136)).toBeUndefined(); - }); - - test("rejects published schema conflicts", () => { - const sharedProps = { value: Type.int32() }; - const definitionPairs: [TypeInfo, TypeInfo][] = [ - [Type.struct(8137, { first: Type.int32() }), Type.struct(8137, { second: Type.string() })], - [ - Type.struct({ typeId: 8138, evolving: false }, sharedProps), - Type.struct({ typeId: 8138, evolving: true }, sharedProps), - ], - [Type.enum(8139, { FIRST: 1 }), Type.enum(8139, { SECOND: 2 })], - [Type.union(8140, { 1: Type.int32() }), Type.union(8140, { 2: Type.string() })], - [ - Type.struct("test.PublishedOwner", { first: Type.int32() }), - Type.struct("test.PublishedOwner", { second: Type.string() }), - ], - ]; - - for (const [first, second] of definitionPairs) { - let generated = 0; - const fory = new Fory({ - compatible: false, - hooks: { - afterCodeGenerated(code) { - generated++; - return code; - }, - }, - }); - const registered = fory.register(first); - generated = 0; - - expect(() => fory.register(second)).toThrow("conflicting complete definitions"); - expect(generated).toBe(0); - expect(fory.typeResolver.getSerializerByTypeInfo(first)).toBe(registered.serializer); - } - }); - - test("keeps extension owner", () => { - class FirstExtension { - value = 0; - } - class SecondExtension { - value = 0; - } - Type.ext(8144)(FirstExtension); - Type.ext(8144)(SecondExtension); - const customSerializer = { - write(writeContext: any, value: FirstExtension | SecondExtension) { - writeContext.writeVarInt32(value.value); - }, - read(readContext: any, value: FirstExtension | SecondExtension) { - value.value = readContext.readVarInt32(); - }, - }; - const fory = new Fory({ compatible: false }); - const extension = fory.register(FirstExtension, customSerializer); - const wrapper = fory.register(Type.struct(8145, { value: Type.ext(8144) })); - - expect(() => fory.register(SecondExtension, customSerializer)).toThrow( - "conflicting complete definitions", - ); - expect(fory.typeResolver.getSerializerById(TypeId.EXT, 8144)).toBe(extension.serializer); - const value = new FirstExtension(); - value.value = 7; - expect(wrapper.serializer).toBeDefined(); - expect(extension.deserialize(extension.serialize(value))).toEqual(value); - }); - - test("uses published schema owners", () => { - const fory = new Fory({ compatible: false }); - fory.register(Type.enum(8146, { VALUE: 7 })); - fory.register(Type.union(8147, { 1: Type.string() })); - const registered = fory.register( - Type.struct(8148, { - enumValue: Type.enum(8146), - unionValue: Type.union(8147), - }), - ); - const value = { enumValue: 7, unionValue: { case: 1, value: "value" } }; - - expect(registered.deserialize(registered.serialize(value))).toEqual(value); - }); - - test("keeps open enum and union", () => { - const fory = new Fory({ compatible: false }); - const enumType = fory.register(Type.enum(8149)); - const unionType = fory.register(Type.union(8150)); - const unionValue = { case: 1, value: "value" }; - - expect(enumType.deserialize(enumType.serialize(7))).toBe(7); - expect(unionType.deserialize(unionType.serialize(unionValue))).toEqual(unionValue); - }); - - test("rejects reentrant schema conflict", () => { - let publishConflict = false; - let reentrant: ReturnType; - let fory: Fory; - fory = new Fory({ - compatible: false, - hooks: { - afterCodeGenerated(code) { - if (publishConflict) { - publishConflict = false; - reentrant = fory.register(Type.struct(8141, { inner: Type.string() })); - } - return code; - }, - }, - }); - publishConflict = true; - - expect(() => - fory.register( - Type.struct(8142, { - trigger: Type.struct(8143, { value: Type.int32() }), - value: Type.struct(8141, { outer: Type.int32() }), - }), - ), - ).toThrow("conflicting complete definitions"); - expect(fory.typeResolver.getSerializerById(TypeId.STRUCT, 8141)).toBe(reentrant!.serializer); - expect(fory.typeResolver.getSerializerById(TypeId.STRUCT, 8142)).toBeUndefined(); - }); - - test("uses a late reentrant owner", () => { - const child = Type.struct(8155, { value: Type.int32() }); - const early = Type.struct(8156, { child }); - const trigger = Type.struct(8157, { value: Type.string() }); - const root = Type.struct(8158, { early, trigger }); - let hookCount = 0; - let generateReentrant = false; - let reentrant: ReturnType; - let fory: Fory; - fory = new Fory({ - compatible: false, - hooks: { - afterCodeGenerated(code) { - hookCount++; - if (generateReentrant && hookCount === 3) { - generateReentrant = false; - reentrant = fory.register(child); - } - return code; - }, - }, - }); - hookCount = 0; - generateReentrant = true; - - const registered = fory.register(root); - expect(hookCount).toBe(5); - expect(fory.typeResolver.getSerializerById(TypeId.STRUCT, 8155)).toBe(reentrant!.serializer); - expect(fory.typeResolver.getSerializerById(TypeId.STRUCT, 8158)).toBe(registered.serializer); - - let winnerReads = 0; - const winnerRead = reentrant!.serializer.read; - reentrant!.serializer.read = (fromRef) => { - winnerReads++; - return winnerRead(fromRef); - }; - const value = { - early: { child: { value: 7 } }, - trigger: { value: "trigger" }, - }; - - expect(registered.deserialize(registered.serialize(value))).toEqual(value); - expect(winnerReads).toBe(1); - }); - - test("seals replaced schema options", () => { - const original = Type.struct(8126, { oldValue: Type.int32() }); - const replacement: { - props: Record; - withConstructor: boolean; - } = { - props: { value: Type.string() }, - withConstructor: false, - }; - let replaceOptions = true; - const typeInfo = new Proxy(original, { - defineProperty(target, property, descriptor) { - if (property === "options" && replaceOptions) { - replaceOptions = false; - target.options = replacement; - } - return Reflect.defineProperty(target, property, descriptor); - }, - }); - const registered = new Fory({ compatible: false }).register(typeInfo); - const value = { value: "sealed" }; - - expect(registered.deserialize(registered.serialize(value as any))).toEqual(value); - expect(() => { - replacement.props.extra = Type.bool(); - }).toThrow(); - }); - - test("seals recursive schemas", () => { - const left = Type.struct(8120, {}); - const right = Type.struct(8121, { left }); - left.options!.props!.right = right; - - new Fory({ compatible: false }).register(left); - - expect(() => left.setNullable(true)).toThrow(); - expect(() => right.setTrackingRef(true)).toThrow(); - left.dynamicTypeId = 7; - expect(left.dynamicTypeId).toBe(7); - }); - - test.each(["serialize", "deserialize"] as const)("freezes after successful %s", (operation) => { - const typeInfo = Type.struct(8103, {}); - const source = new Fory({ compatible: false }).register(typeInfo.clone()); - const fory = new Fory({ compatible: false }); - const registered = fory.register(typeInfo); if (operation === "serialize") { - registered.serialize({}); + fory.serialize(1); } else { - registered.deserialize(source.serialize({})); + const bytes = new Fory({ compatible: false }).serialize(1); + fory.deserialize(bytes); } - expect(() => fory.register(Type.struct(8104, {}))).toThrow(); + expect(() => fory.register(Type.struct(8103, {}))).toThrow(); }); function testTypeInfo(typeinfo: TypeInfo, input: any, expected?: any) { diff --git a/javascript/test/map.test.ts b/javascript/test/map.test.ts index 59e4c0ddd8..ab0fa20cfe 100644 --- a/javascript/test/map.test.ts +++ b/javascript/test/map.test.ts @@ -186,47 +186,6 @@ describe("map", () => { expect((native.header >> 3) & 0b100).toBe(0b100); }); - test("uses reentrant map owner", () => { - let registerSameKey = false; - let reentrant: ReturnType; - let fory: Fory; - fory = new Fory({ - compatible: true, - ref: true, - hooks: { - afterCodeGenerated(code) { - if (registerSameKey) { - registerSameKey = false; - reentrant = fory.register( - Type.struct({ typeId: 350, evolving: true }, { innerValue: Type.string() }), - ); - } - return code; - }, - }, - }); - registerSameKey = true; - const registered = fory.register( - Type.struct(351, { - trigger: Type.struct(352, { value: Type.int32() }).setId(3), - outer: Type.struct(350).setId(2), - values: Type.map(Type.struct(350), Type.struct(350)).setId(1), - }), - ); - const value = { - trigger: { value: 1 }, - outer: { innerValue: "outer" }, - values: new Map([[{ innerValue: "key" }, { innerValue: "value" }]]), - }; - const bytes = registered.serialize(value as any); - const { header } = structMapHeader(fory, bytes, true, 351); - - expect(fory.typeResolver.getSerializerById(TypeId.STRUCT, 350)).toBe(reentrant!.serializer); - expect(header & 0b100).toBe(0); - expect((header >> 3) & 0b100).toBe(0); - expect(registered.deserialize(bytes)).toEqual(value); - }); - test("rejects invalid runtime chunks", () => { const fory = new Fory({ compatible: false, ref: true }); const MapAnySerializer = CodegenRegistry.getExternal().MapAnySerializer; diff --git a/javascript/test/union.test.ts b/javascript/test/union.test.ts index cf46630041..a8b4c11df4 100644 --- a/javascript/test/union.test.ts +++ b/javascript/test/union.test.ts @@ -18,7 +18,6 @@ */ import Fory, { Type } from "../packages/core/index"; -import { TypeId } from "../packages/core/lib/type"; import { describe, expect, test } from "@jest/globals"; describe("union", () => { @@ -220,29 +219,4 @@ describe("union", () => { expect(result.value).toBe(result); expect(readContext.getReadRef(0)).toBe(result); }); - - test("keeps anonymous union owners", () => { - const fory = new Fory({ compatible: false, ref: true }); - const first = fory.register(Type.union({ 1: Type.string() })); - const second = fory.register(Type.union({ 2: Type.int32() })); - expect((fory as any).typeResolver.getSerializerById(TypeId.TYPED_UNION)).toBeUndefined(); - const open = fory.register(Type.union()); - const combined = fory.register( - Type.struct(702, { - first: Type.union({ 1: Type.string() }), - second: Type.union({ 2: Type.int32() }), - }), - ); - const firstValue = { case: 1, value: "first" }; - const secondValue = { case: 2, value: 42 }; - const openValue = { case: 3, value: "open" }; - const combinedValue = { first: firstValue, second: secondValue }; - - expect(first.serializer).not.toBe(second.serializer); - expect((fory as any).typeResolver.getSerializerById(TypeId.TYPED_UNION)).toBe(open.serializer); - expect(first.deserialize(first.serialize(firstValue))).toEqual(firstValue); - expect(second.deserialize(second.serialize(secondValue))).toEqual(secondValue); - expect(open.deserialize(open.serialize(openValue))).toEqual(openValue); - expect(combined.deserialize(combined.serialize(combinedValue))).toEqual(combinedValue); - }); }); From efc24eae80f64bd82041caa34c2d4539102e0ec3 Mon Sep 17 00:00:00 2001 From: chaokunyang Date: Sun, 30 Aug 2026 02:47:49 +0800 Subject: [PATCH 090/168] docs(csharp): clarify thread-safe registry ownership --- csharp/src/Fory/ThreadSafeFory.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/csharp/src/Fory/ThreadSafeFory.cs b/csharp/src/Fory/ThreadSafeFory.cs index 65319e06ab..89fddf80cf 100644 --- a/csharp/src/Fory/ThreadSafeFory.cs +++ b/csharp/src/Fory/ThreadSafeFory.cs @@ -243,7 +243,7 @@ private void ApplyRegistration(Action registration) private void BeginRoot() { // Freeze before Current can create a per-thread runtime so roots and registrations - // linearize against one boundary and every runtime replays the same immutable log. + // linearize against one boundary and every runtime gets the same configuration. if (Volatile.Read(ref _registryFrozen) == 0) { FreezeRegistry(); From 5ecc19ce1a6f39a9eb14a83ed3d8cf7b785935b4 Mon Sep 17 00:00:00 2001 From: chaokunyang Date: Sun, 30 Aug 2026 02:48:03 +0800 Subject: [PATCH 091/168] refactor(cpp): derive context readiness from owners --- cpp/fory/serialization/fory.h | 27 +++++++++++++-------------- 1 file changed, 13 insertions(+), 14 deletions(-) diff --git a/cpp/fory/serialization/fory.h b/cpp/fory/serialization/fory.h index c18a73c09b..300391b50d 100644 --- a/cpp/fory/serialization/fory.h +++ b/cpp/fory/serialization/fory.h @@ -534,7 +534,7 @@ class Fory : public BaseFory { /// @return Vector containing serialized bytes, or error. template Result, Error> serialize(const T &obj) { - if (FORY_PREDICT_FALSE(!contexts_initialized_)) { + if (FORY_PREDICT_FALSE(!write_ctx_.has_value())) { ensure_contexts_initialized(); } WriteContextGuard guard(*write_ctx_); @@ -555,7 +555,7 @@ class Fory : public BaseFory { /// @return Number of bytes written, or error. template Result serialize(OutputStream &output_stream, const T &obj) { - if (FORY_PREDICT_FALSE(!contexts_initialized_)) { + if (FORY_PREDICT_FALSE(!write_ctx_.has_value())) { ensure_contexts_initialized(); } return serialize_stream(output_stream, obj); @@ -569,7 +569,7 @@ class Fory : public BaseFory { /// @return Number of bytes written, or error. template Result serialize(std::ostream &ostream, const T &obj) { - if (FORY_PREDICT_FALSE(!contexts_initialized_)) { + if (FORY_PREDICT_FALSE(!write_ctx_.has_value())) { ensure_contexts_initialized(); } StdOutputStream output_stream(ostream); @@ -585,7 +585,7 @@ class Fory : public BaseFory { template FORY_ALWAYS_INLINE Result serialize_to(Buffer &buffer, const T &obj) { - if (FORY_PREDICT_FALSE(!contexts_initialized_)) { + if (FORY_PREDICT_FALSE(!write_ctx_.has_value())) { ensure_contexts_initialized(); } return serialize_buffer(buffer, obj); @@ -604,7 +604,7 @@ class Fory : public BaseFory { template Result serialize_to(std::vector &output, const T &obj) { - if (FORY_PREDICT_FALSE(!contexts_initialized_)) { + if (FORY_PREDICT_FALSE(!write_ctx_.has_value())) { ensure_contexts_initialized(); } // Wrap the output vector in a Buffer for zero-copy serialization @@ -649,7 +649,7 @@ class Fory : public BaseFory { /// @param buffer Buffer to read from. Its reader_index will be updated. /// @return Deserialized object, or error. template Result deserialize(Buffer &buffer) { - if (FORY_PREDICT_FALSE(!contexts_initialized_)) { + if (FORY_PREDICT_FALSE(!write_ctx_.has_value())) { ensure_contexts_initialized(); } return deserialize_buffer(buffer); @@ -665,7 +665,7 @@ class Fory : public BaseFory { /// @return Deserialized object, or error. template Result deserialize(InputStream &input_stream) { - if (FORY_PREDICT_FALSE(!contexts_initialized_)) { + if (FORY_PREDICT_FALSE(!write_ctx_.has_value())) { ensure_contexts_initialized(); } return deserialize_stream(input_stream); @@ -677,7 +677,7 @@ class Fory : public BaseFory { /// @param stream Input stream wrapper to read from. /// @return Deserialized object, or error. template Result deserialize(StdInputStream &stream) { - if (FORY_PREDICT_FALSE(!contexts_initialized_)) { + if (FORY_PREDICT_FALSE(!write_ctx_.has_value())) { ensure_contexts_initialized(); } return deserialize_stream(stream); @@ -702,14 +702,14 @@ class Fory : public BaseFory { private: /// Constructor for ForyBuilder - operation contexts are initialized lazily. explicit Fory(const Config &config, std::shared_ptr resolver) - : BaseFory(config, std::move(resolver)), contexts_initialized_(false), + : BaseFory(config, std::move(resolver)), precomputed_header_(compute_header(config.xlang)) {} /// Constructor for ThreadSafeFory pool - resolver metadata is ready. struct PreparedResolver {}; explicit Fory(const Config &config, std::shared_ptr resolver, PreparedResolver) - : BaseFory(config, std::move(resolver)), contexts_initialized_(true), + : BaseFory(config, std::move(resolver)), precomputed_header_(compute_header(config.xlang)) { write_ctx_.emplace(config_, type_resolver_->clone()); read_ctx_.emplace(config_, type_resolver_->clone()); @@ -717,7 +717,8 @@ class Fory : public BaseFory { /// Initialize operation contexts from the registered type metadata. void ensure_contexts_initialized() { - if (!contexts_initialized_) { + if (!write_ctx_.has_value()) { + FORY_CHECK(!read_ctx_.has_value()); auto final_result = type_resolver_->build_final_type_resolver(); FORY_CHECK(final_result.ok()) << "Failed to build finalized TypeResolver: " @@ -727,7 +728,6 @@ class Fory : public BaseFory { write_ctx_.emplace(config_, prepared_resolver->clone()); read_ctx_.emplace(config_, prepared_resolver->clone()); type_resolver_ = std::move(prepared_resolver); - contexts_initialized_ = true; } } @@ -797,7 +797,7 @@ class Fory : public BaseFory { template Result deserialize_bytes(const uint8_t *data, size_t size) { - if (FORY_PREDICT_FALSE(!contexts_initialized_)) { + if (FORY_PREDICT_FALSE(!write_ctx_.has_value())) { ensure_contexts_initialized(); } if (data == nullptr) { @@ -911,7 +911,6 @@ class Fory : public BaseFory { return type_info; } - bool contexts_initialized_; uint8_t precomputed_header_; std::optional write_ctx_; std::optional read_ctx_; From 9d4635303bb9566302f11770e5d022364c5e51ba Mon Sep 17 00:00:00 2001 From: chaokunyang Date: Sun, 30 Aug 2026 02:48:52 +0800 Subject: [PATCH 092/168] refactor(go): remove thread-safe registry replay --- .agents/languages/go.md | 23 +- docs/object-serialization/go/configuration.md | 17 +- docs/object-serialization/go/native.md | 11 +- docs/object-serialization/go/thread-safety.md | 101 ++-- .../go/type-registration.md | 6 +- go/fory/fory.go | 100 +++- go/fory/registry_freeze_lifecycle_test.go | 458 +++--------------- go/fory/stream.go | 4 +- go/fory/threadsafe/fory.go | 145 +----- go/fory/threadsafe/fory_test.go | 14 +- .../registry_freeze_lifecycle_test.go | 275 ----------- go/fory/type_resolver.go | 280 ++++------- 12 files changed, 335 insertions(+), 1099 deletions(-) delete mode 100644 go/fory/threadsafe/registry_freeze_lifecycle_test.go diff --git a/.agents/languages/go.md b/.agents/languages/go.md index 98d1efce8b..64ae88efbc 100644 --- a/.agents/languages/go.md +++ b/.agents/languages/go.md @@ -7,22 +7,13 @@ Load this file when changing `go/fory/` or Go xlang behavior. - Run Go commands from within `go/fory/`. - Changes under `go/` must pass formatting and tests. - The Go implementation focuses on fast serializers. -- A Go `Fory` instance owns permanent registry freeze at the start of its first root, including a - failed root. Exported resolver registration entries recheck that facade-owned state before - mutation. `threadsafe.Fory` owns the cross-pool boundary with one frozen state, one prepared - validation instance, and one log of successful named-struct registrations. Failed registrations - are failure-atomic and are not logged, so they leave the prepared instance intact; pool misses - after freeze replay the immutable successful log. Its custom factory runs without the - registration mutex because application code may reenter a root; after the factory returns, - registration rechecks the frozen state before publishing prepared or replay state. Numeric IDs - and registered names are bidirectional identities: each identity owns one registered Go value - type, and passing that type's pointer form refers to the same registration. Facade and exported - resolver registration entries normalize values and `reflect.Type` inputs to that non-pointer - value owner before type validation or resolver mutation; only resolver publication creates its - single pointer companion. Resolver registration must validate both directions before changing - serializers or identity indexes. Resolver duplicate diagnostics identify application serializers - by concrete type only; they must not invoke application string or format methods while - registration holds the lifecycle mutex. +- A Go `Fory` instance has one authoritative registry-frozen flag. The first root serialization or + deserialization sets it before codec work and leaves it set after failure. Explicit registration + checks that flag before mutation. Do not add registry finalization or failure states, alternate + identity/preflight machinery, or input normalization beyond the existing registration behavior. + `threadsafe.Fory` has no registration API or facade registry: configure every pooled child in the + factory passed to `NewWithFactory` before returning it. Do not add prepared runtimes, replay logs, + facade registry state, or callback-reentry lifecycle machinery. - Go `ReadContext` intentionally defers codec errors to existing `HasError` or `CheckError` boundaries. After an error, work may continue only while it remains panic- and bounds-safe and cannot cause disproportionate work or allocation, publish state that survives root cleanup, or diff --git a/docs/object-serialization/go/configuration.md b/docs/object-serialization/go/configuration.md index df738a4d8b..006244a916 100644 --- a/docs/object-serialization/go/configuration.md +++ b/docs/object-serialization/go/configuration.md @@ -386,13 +386,16 @@ type Request struct { Payload string } -f := threadsafe.New( - fory.WithXlang(true), - fory.WithMaxDepth(30), -) -if err := f.RegisterStructByName(Request{}, "example.Request"); err != nil { - panic(err) -} +f := threadsafe.NewWithFactory(func() *fory.Fory { + inner := fory.New( + fory.WithXlang(true), + fory.WithMaxDepth(30), + ) + if err := inner.RegisterStructByName(Request{}, "example.Request"); err != nil { + panic(err) + } + return inner +}) // Process requests concurrently for req := range requests { diff --git a/docs/object-serialization/go/native.md b/docs/object-serialization/go/native.md index d226807d78..1fb6b8e9f1 100644 --- a/docs/object-serialization/go/native.md +++ b/docs/object-serialization/go/native.md @@ -79,10 +79,13 @@ import ( "github.com/apache/fory/go/fory/threadsafe" ) -f := threadsafe.New(fory.WithXlang(false), fory.WithTrackRef(true)) -if err := f.RegisterStructByName(Order{}, "example.Order"); err != nil { - panic(err) -} +f := threadsafe.NewWithFactory(func() *fory.Fory { + inner := fory.New(fory.WithXlang(false), fory.WithTrackRef(true)) + if err := inner.RegisterStructByName(Order{}, "example.Order"); err != nil { + panic(err) + } + return inner +}) ``` ## Schema Evolution diff --git a/docs/object-serialization/go/thread-safety.md b/docs/object-serialization/go/thread-safety.md index 3a82823b64..36bad27a6e 100644 --- a/docs/object-serialization/go/thread-safety.md +++ b/docs/object-serialization/go/thread-safety.md @@ -87,42 +87,44 @@ err = threadsafe.Unmarshal(data, &target) ## Type Registration -Register every type before the first serialization or deserialization attempt. Starting a root -operation permanently freezes registration on that Fory instance, including when the operation -fails: +Each pooled `Fory` instance owns its registry. Configure every instance in `NewWithFactory` before +returning it to the pool: ```go -f := threadsafe.New() - -// Register types BEFORE concurrent access -if err := f.RegisterStructByName(User{}, "example.User"); err != nil { - panic(err) -} -if err := f.RegisterStructByName(Order{}, "example.Order"); err != nil { - panic(err) -} +f := threadsafe.NewWithFactory(func() *fory.Fory { + inner := fory.New(fory.WithXlang(true)) + if err := inner.RegisterStructByName(User{}, "example.User"); err != nil { + panic(err) + } + if err := inner.RegisterStructByName(Order{}, "example.Order"); err != nil { + panic(err) + } + return inner +}) -// Now safe to use concurrently go func() { - f.Serialize(&User{ID: 1}) + _, _ = f.Serialize(&User{ID: 1}) }() ``` -### Thread-Safe Registration +The factory is the sole registration path for the thread-safe wrapper. The wrapper has no registry +of its own, and a registration applied to one pooled instance would not configure future instances. +Every factory invocation must return an instance with the same configuration and registrations. -The thread-safe wrapper exposes named struct registration and serializes it against the first root -operation: +For a directly owned `Fory`, register types on that instance before its first root operation. +Starting serialization or deserialization permanently freezes that instance's registry, including +when the root fails: ```go -f := threadsafe.New() -if err := f.RegisterStructByName(User{}, "example.User"); err != nil { +inner := fory.New(fory.WithXlang(true)) +if err := inner.RegisterStructByName(User{}, "example.User"); err != nil { panic(err) } + +_, _ = inner.Serialize(&User{ID: 1}) ``` -If registration races with the first root, one operation wins the boundary. When the root wins, -the registration call returns `fory.ErrRegistryFrozen` without changing the registry. Register all -types during startup so the application does not depend on race ordering. +Later registration on `inner` returns `fory.ErrRegistryFrozen` without changing its registry. ## Zero-Copy Considerations @@ -180,10 +182,13 @@ func BenchmarkNonThreadSafe(b *testing.B) { } func BenchmarkThreadSafe(b *testing.B) { - f := threadsafe.New() - if err := f.RegisterStructByName(User{}, "example.User"); err != nil { - b.Fatal(err) - } + f := threadsafe.NewWithFactory(func() *fory.Fory { + inner := fory.New(fory.WithXlang(true)) + if err := inner.RegisterStructByName(User{}, "example.User"); err != nil { + panic(err) + } + return inner + }) user := &User{ID: 1, Name: "Alice"} for i := 0; i < b.N; i++ { @@ -224,14 +229,13 @@ for i := 0; i < numWorkers; i++ { For dynamic goroutine count or simplicity: ```go -// Single shared instance -var f = threadsafe.New() - -func init() { - if err := f.RegisterStructByName(User{}, "example.User"); err != nil { +var f = threadsafe.NewWithFactory(func() *fory.Fory { + inner := fory.New(fory.WithXlang(true)) + if err := inner.RegisterStructByName(User{}, "example.User"); err != nil { panic(err) } -} + return inner +}) func handleRequest(user *User) []byte { // Safe from any goroutine @@ -243,13 +247,13 @@ func handleRequest(user *User) []byte { ### HTTP Handler Example ```go -var fory = threadsafe.New() - -func init() { - if err := fory.RegisterStructByName(Response{}, "example.Response"); err != nil { +var serializer = threadsafe.NewWithFactory(func() *fory.Fory { + inner := fory.New(fory.WithXlang(true)) + if err := inner.RegisterStructByName(Response{}, "example.Response"); err != nil { panic(err) } -} + return inner +}) func handler(w http.ResponseWriter, r *http.Request) { response := &Response{ @@ -258,7 +262,7 @@ func handler(w http.ResponseWriter, r *http.Request) { } // Safe: threadsafe.Fory handles concurrency - data, err := fory.Serialize(response) + data, err := serializer.Serialize(response) if err != nil { http.Error(w, err.Error(), 500) return @@ -312,26 +316,21 @@ f := threadsafe.New() data, _ := f.Serialize(value1) // Already copied ``` -### Registering Types Concurrently +### Registering Only One Pooled Instance ```go -// The root may freeze the registry first. -go func() { - if err := f.RegisterStructByName(TypeA{}, "example.TypeA"); err != nil { - panic(err) - } -}() -go func() { - _, _ = f.Serialize(value) -}() +// WRONG: a configured instance cannot be installed into threadsafe.New. +inner := fory.New(fory.WithXlang(true)) +_ = inner.RegisterStructByName(TypeA{}, "example.TypeA") +f := threadsafe.New(fory.WithXlang(true)) ``` -If serialization wins, registration returns `fory.ErrRegistryFrozen`. Register all types before -starting concurrent roots. +`f` creates different pooled instances, so the registration on `inner` has no effect. Configure the +registration inside `NewWithFactory` so every pooled instance receives it. ## Best Practices -1. **Register types at startup**: Before any concurrent operations +1. **Configure registrations in the factory**: Every pooled instance must receive the same setup 2. **Clone data if keeping references**: With non-thread-safe instance 3. **Use per-worker instances for hot paths**: Eliminates pool contention 4. **Profile before optimizing**: Thread-safe overhead may be negligible diff --git a/docs/object-serialization/go/type-registration.md b/docs/object-serialization/go/type-registration.md index 70188fe466..ed4577d05c 100644 --- a/docs/object-serialization/go/type-registration.md +++ b/docs/object-serialization/go/type-registration.md @@ -126,9 +126,9 @@ f1.RegisterStruct(User{}, 1) f2.RegisterStruct(User{}, 1) ``` -Within one Fory instance, each numeric ID or registered name identifies one registered Go value -type. Passing a pointer value for that type uses the same registration. A conflicting ID, name, or -type registration returns an error without replacing the first registration. +The thread-safe wrapper creates multiple `Fory` instances. Configure registrations in +`threadsafe.NewWithFactory` so every pooled instance receives the same registry before use; the +wrapper does not expose registration methods. ## Registration Timing diff --git a/go/fory/fory.go b/go/fory/fory.go index 23222b1192..369b55bca1 100644 --- a/go/fory/fory.go +++ b/go/fory/fory.go @@ -307,10 +307,6 @@ func (f *Fory) checkRegistrationOpen() error { return nil } -func (f *Fory) beginRoot() { - f.registryFrozen = true -} - // RegisterStruct registers a struct type with a numeric ID for cross-language serialization. // This is compatible with Java's fory.register(Class, int) method. // type_ can be either a reflect.Type or an instance of the type @@ -325,7 +321,15 @@ func (f *Fory) RegisterStruct(type_ any, typeID uint32) error { if err := validateUserTypeID(typeID); err != nil { return err } - t := valueRegistrationType(type_) + var t reflect.Type + if rt, ok := type_.(reflect.Type); ok { + t = rt + } else { + t = reflect.TypeOf(type_) + if t.Kind() == reflect.Ptr { + t = t.Elem() + } + } // Only struct types are supported via RegisterStruct // For enums, use RegisterEnum @@ -356,7 +360,15 @@ func (f *Fory) RegisterUnion(type_ any, typeID uint32, serializer Serializer) er if err := validateUserTypeID(typeID); err != nil { return err } - t := valueRegistrationType(type_) + var t reflect.Type + if rt, ok := type_.(reflect.Type); ok { + t = rt + } else { + t = reflect.TypeOf(type_) + if t.Kind() == reflect.Ptr { + t = t.Elem() + } + } if t.Kind() != reflect.Struct { return fmt.Errorf("RegisterUnion only supports struct types; got: %v", t.Kind()) } @@ -376,7 +388,15 @@ func (f *Fory) RegisterUnionByName(type_ any, name string, serializer Serializer if serializer == nil { return fmt.Errorf("RegisterUnionByName requires a non-nil serializer") } - t := valueRegistrationType(type_) + var t reflect.Type + if rt, ok := type_.(reflect.Type); ok { + t = rt + } else { + t = reflect.TypeOf(type_) + if t.Kind() == reflect.Ptr { + t = t.Elem() + } + } if t.Kind() != reflect.Struct { return fmt.Errorf("RegisterUnionByName only supports struct types; got: %v", t.Kind()) } @@ -397,7 +417,15 @@ func (f *Fory) RegisterStructByName(type_ any, name string) error { if err := f.checkRegistrationOpen(); err != nil { return err } - t := valueRegistrationType(type_) + var t reflect.Type + if rt, ok := type_.(reflect.Type); ok { + t = rt + } else { + t = reflect.TypeOf(type_) + if t.Kind() == reflect.Ptr { + t = t.Elem() + } + } if t.Kind() != reflect.Struct { return fmt.Errorf("RegisterStructByName only supports struct types; for enum types use RegisterEnumByName. Got: %v", t.Kind()) } @@ -422,7 +450,15 @@ func (f *Fory) RegisterEnum(type_ any, typeID uint32) error { if err := validateUserTypeID(typeID); err != nil { return err } - t := valueRegistrationType(type_) + var t reflect.Type + if rt, ok := type_.(reflect.Type); ok { + t = rt + } else { + t = reflect.TypeOf(type_) + if t.Kind() == reflect.Ptr { + t = t.Elem() + } + } // Verify it's a numeric type (Go enums are int-based) switch t.Kind() { @@ -446,7 +482,15 @@ func (f *Fory) RegisterEnumByName(type_ any, name string) error { if err := f.checkRegistrationOpen(); err != nil { return err } - t := valueRegistrationType(type_) + var t reflect.Type + if rt, ok := type_.(reflect.Type); ok { + t = rt + } else { + t = reflect.TypeOf(type_) + if t.Kind() == reflect.Ptr { + t = t.Elem() + } + } // Verify it's a numeric type (Go enums are int-based) switch t.Kind() { @@ -476,7 +520,15 @@ func (f *Fory) RegisterExtension(type_ any, typeID uint32, serializer ExtensionS if err := validateUserTypeID(typeID); err != nil { return err } - t := valueRegistrationType(type_) + var t reflect.Type + if rt, ok := type_.(reflect.Type); ok { + t = rt + } else { + t = reflect.TypeOf(type_) + if t.Kind() == reflect.Ptr { + t = t.Elem() + } + } return f.typeResolver.RegisterExtension(t, typeID, serializer) } @@ -507,7 +559,15 @@ func (f *Fory) RegisterExtensionByName(type_ any, name string, serializer Extens if err := f.checkRegistrationOpen(); err != nil { return err } - t := valueRegistrationType(type_) + var t reflect.Type + if rt, ok := type_.(reflect.Type); ok { + t = rt + } else { + t = reflect.TypeOf(type_) + if t.Kind() == reflect.Ptr { + t = t.Elem() + } + } namespace, typeName, err := splitRegisteredName(name) if err != nil { return err @@ -540,7 +600,7 @@ func (f *Fory) Reset() { // // For thread-safe usage, use threadsafe.Fory which copies the data internally. func (f *Fory) Serialize(value any) ([]byte, error) { - f.beginRoot() + f.registryFrozen = true defer f.resetWriteState() if !validateRootDecimal(f.writeCtx.Err(), value) { return nil, f.writeCtx.TakeError() @@ -576,7 +636,7 @@ func (f *Fory) rootRefMode() RefMode { // Deserialize deserializes data directly into the provided target value. // The target must be a pointer to the value to deserialize into. func (f *Fory) Deserialize(data []byte, v any) error { - f.beginRoot() + f.registryFrozen = true defer f.resetReadState() f.readCtx.SetData(data) target := reflect.ValueOf(v).Elem() @@ -616,7 +676,7 @@ func (f *Fory) resetWriteState() { // This is useful when you need to write multiple serialized values to the same buffer. // Returns error if serialization fails. func (f *Fory) SerializeTo(buf *ByteBuffer, value any) error { - f.beginRoot() + f.registryFrozen = true origBuffer := f.writeCtx.buffer defer func() { // Restore the owned buffer before reset so a serializer panic cannot reset or retain the @@ -670,7 +730,7 @@ func (f *Fory) SerializeTo(buf *ByteBuffer, value any) error { // The buffer's reader index is advanced as data is read. // This is useful when reading multiple serialized values from the same buffer. func (f *Fory) DeserializeFrom(buf *ByteBuffer, v any) error { - f.beginRoot() + f.registryFrozen = true // Reset contexts for each independent serialized object // Temporarily swap buffer origBuffer := f.readCtx.buffer @@ -723,7 +783,7 @@ func (f *Fory) Unmarshal(data []byte, v any) error { // If callback is provided, it will be called for each BufferObject during serialization. // Return true from callback to write in-band, false for out-of-band. func (f *Fory) SerializeWithCallback(buffer *ByteBuffer, v any, callback func(BufferObject) bool) error { - f.beginRoot() + f.registryFrozen = true buf := f.writeCtx.buffer defer func() { // Reset internal state but NOT the buffer - caller manages buffer state @@ -762,7 +822,7 @@ func (f *Fory) SerializeWithCallback(buffer *ByteBuffer, v any, callback func(Bu // DeserializeWithCallbackBuffers deserializes from buffer into the provided value (for streaming/cross-language use). // The third parameter is optional external buffers for out-of-band data (can be nil). func (f *Fory) DeserializeWithCallbackBuffers(buffer *ByteBuffer, v any, buffers []*ByteBuffer) error { - f.beginRoot() + f.registryFrozen = true // Use the caller buffer only for this root; later stream roots reuse the // original internal buffer. origBuffer := f.readCtx.buffer @@ -897,7 +957,7 @@ func readHeaderSlow(ctx *ReadContext, bitmap byte) { // // For thread-safe usage, use threadsafe.Serialize which copies the data internally. func Serialize[T any](f *Fory, value T) ([]byte, error) { - f.beginRoot() + f.registryFrozen = true defer f.resetWriteState() v := any(value) if !validateRootDecimal(f.writeCtx.Err(), v) { @@ -1054,7 +1114,7 @@ func Serialize[T any](f *Fory, value T) ([]byte, error) { // For structs, it reads directly into the struct fields. // Note: Fory instance is NOT thread-safe. Use ThreadSafeFory for concurrent use. func Deserialize[T any](f *Fory, data []byte, target *T) error { - f.beginRoot() + f.registryFrozen = true // Generic roots share the same reusable read and metadata owners as the // method API, so both entry and every exit must start from a root-clean state. f.resetReadState() diff --git a/go/fory/registry_freeze_lifecycle_test.go b/go/fory/registry_freeze_lifecycle_test.go index 68c1e851c5..37d0997197 100644 --- a/go/fory/registry_freeze_lifecycle_test.go +++ b/go/fory/registry_freeze_lifecycle_test.go @@ -20,7 +20,6 @@ package fory import ( "bytes" "reflect" - "strings" "testing" "github.com/stretchr/testify/require" @@ -34,50 +33,6 @@ type registryFreezeUnion struct{} type registryFreezeEnum int32 -type registryIdentityEnum int32 - -type pointerRegistrationStruct struct { - Value int32 -} - -type pointerRegistrationEnum int32 - -type pointerRegistrationUnion struct { - caseID uint32 - value any -} - -func (pointerRegistrationUnion) ForyUnionMarker() {} - -func (u pointerRegistrationUnion) ForyUnionGet() (uint32, any) { - return u.caseID, u.value -} - -func (u *pointerRegistrationUnion) ForyUnionSet(caseID uint32, value any) { - u.caseID = caseID - u.value = value -} - -type pointerRegistrationExtension struct { - Value int32 -} - -type pointerExtensionSerializer struct{} - -func (pointerExtensionSerializer) WriteData(ctx *WriteContext, value reflect.Value) { - if value.Kind() == reflect.Ptr { - value = value.Elem() - } - ctx.Buffer().WriteInt32(int32(value.FieldByName("Value").Int())) -} - -func (pointerExtensionSerializer) ReadData(ctx *ReadContext, value reflect.Value) { - if value.Kind() == reflect.Ptr { - value = value.Elem() - } - value.FieldByName("Value").SetInt(int64(ctx.Buffer().ReadInt32(ctx.Err()))) -} - type registryFreezeExtension struct { Value int32 } @@ -91,60 +46,34 @@ func (registryPanicSerializer) WriteData(ctx *WriteContext, _ reflect.Value) { func (registryPanicSerializer) ReadData(*ReadContext, reflect.Value) {} -type registryFreezeSnapshot struct { - serializers int - typeNames int - typeIDs int - userTypeIDs int - types int - namespacedTypes int - namedTypes int - typeDefs int - definitionIDs int - typePointers int - unionTypes int - typeIDCounter uint32 - dynamicWriteIndex uint32 -} - -func takeRegistryFreezeSnapshot(r *TypeResolver) registryFreezeSnapshot { - return registryFreezeSnapshot{ - serializers: len(r.typeToSerializers), - typeNames: len(r.typeToTypeInfo), - typeIDs: len(r.typeIDToTypeInfo), - userTypeIDs: len(r.userTypeIdToTypeInfo), - types: len(r.typesInfo), - namespacedTypes: len(r.nsTypeToTypeInfo), - namedTypes: len(r.namedTypeToTypeInfo), - typeDefs: len(r.typeToTypeDef), - definitionIDs: len(r.defIdToTypeDef), - typePointers: len(r.typePointerCache), - unionTypes: len(r.unionTypeCache), - typeIDCounter: r.typeIDCounter, - dynamicWriteIndex: r.dynamicWriteStringID, - } -} - func TestRegistryFreezeRegistrations(t *testing.T) { f := New(WithXlang(false), WithCompatible(false)) _, err := f.Serialize(int32(1)) require.NoError(t, err) f.Reset() - before := takeRegistryFreezeSnapshot(f.typeResolver) attempts := []struct { name string call func() error }{ {"struct ID", func() error { return f.RegisterStruct(registryFreezeStruct{}, 7101) }}, - {"struct name", func() error { return f.RegisterStructByName(registryFreezeStruct{}, "test.RegistryFreezeStruct") }}, + {"struct name", func() error { + return f.RegisterStructByName(registryFreezeStruct{}, "test.RegistryFreezeStruct") + }}, {"union ID", func() error { return f.RegisterUnion(registryFreezeUnion{}, 7102, nil) }}, - {"union name", func() error { return f.RegisterUnionByName(registryFreezeUnion{}, "test.RegistryFreezeUnion", nil) }}, + {"union name", func() error { + return f.RegisterUnionByName(registryFreezeUnion{}, "test.RegistryFreezeUnion", nil) + }}, {"enum ID", func() error { return f.RegisterEnum(registryFreezeEnum(0), 7103) }}, - {"enum name", func() error { return f.RegisterEnumByName(registryFreezeEnum(0), "test.RegistryFreezeEnum") }}, - {"extension ID", func() error { return f.RegisterExtension(registryFreezeExtension{}, 7104, nil) }}, + {"enum name", func() error { + return f.RegisterEnumByName(registryFreezeEnum(0), "test.RegistryFreezeEnum") + }}, + {"extension ID", func() error { + return f.RegisterExtension(registryFreezeExtension{}, 7104, nil) + }}, {"extension name", func() error { - return f.RegisterExtensionByName(registryFreezeExtension{}, "test.RegistryFreezeExtension", nil) + return f.RegisterExtensionByName( + registryFreezeExtension{}, "test.RegistryFreezeExtension", nil) }}, } for _, attempt := range attempts { @@ -152,307 +81,44 @@ func TestRegistryFreezeRegistrations(t *testing.T) { require.ErrorIs(t, attempt.call(), ErrRegistryFrozen) }) } - type_ := reflect.TypeOf(registryFreezeStruct{}) - require.ErrorIs(t, - f.typeResolver.RegisterStruct(type_, f.typeResolver.structTypeID(type_, false), 7105), - ErrRegistryFrozen) - require.Equal(t, before, takeRegistryFreezeSnapshot(f.typeResolver)) -} -func TestNamedEncoderPreflight(t *testing.T) { - f := New(WithXlang(false), WithCompatible(false)) - before := takeRegistryFreezeSnapshot(f.typeResolver) - overlong := strings.Repeat("a", 32_768) - attempts := []struct { - name string - wireName string - }{ - {"namespace", overlong + ".RegistryFreezeStruct"}, - {"type name", overlong}, + structType := reflect.TypeOf(registryFreezeStruct{}) + resolverAttempts := []func() error{ + func() error { + return f.typeResolver.RegisterStruct( + structType, f.typeResolver.structTypeID(structType, false), 7105) + }, + func() error { + return f.typeResolver.RegisterUnion( + reflect.TypeOf(registryFreezeUnion{}), 7106, nil) + }, + func() error { + return f.typeResolver.RegisterEnum( + reflect.TypeOf(registryFreezeEnum(0)), 7107) + }, + func() error { + return f.typeResolver.RegisterExtension( + reflect.TypeOf(registryFreezeExtension{}), 7108, nil) + }, } - for _, attempt := range attempts { - t.Run(attempt.name, func(t *testing.T) { - err := f.RegisterStructByName(registryFreezeStruct{}, attempt.wireName) - require.Error(t, err) - require.Equal(t, before, takeRegistryFreezeSnapshot(f.typeResolver)) - }) + for _, attempt := range resolverAttempts { + require.ErrorIs(t, attempt(), ErrRegistryFrozen) } - - require.NoError(t, - f.RegisterStructByName(registryFreezeStruct{}, "test.RegistryFreezeStruct")) } -func TestNamedRegistryIdentity(t *testing.T) { +func TestFrozenRegistryAllowsLazySerializer(t *testing.T) { f := New(WithXlang(false), WithCompatible(false)) - const name = "test.RegistryIdentity" - require.NoError(t, f.RegisterStructByName(registryFreezeStruct{}, name)) - - nameKey := namedTypeKey{"test", "RegistryIdentity"} - owner := f.typeResolver.namedTypeToTypeInfo[nameKey] - require.NotNil(t, owner) - hashKey := nsTypeKey{owner.PkgPathBytes.Hashcode, owner.NameBytes.Hashcode} - hashOwner := f.typeResolver.nsTypeToTypeInfo[hashKey] - require.NotNil(t, hashOwner) - before := takeRegistryFreezeSnapshot(f.typeResolver) - attempts := []struct { - name string - call func() error - }{ - {"same name struct", func() error { - return f.RegisterStructByName(registryFreezeUnion{}, name) - }}, - {"same name enum", func() error { - return f.RegisterEnumByName(registryFreezeEnum(0), name) - }}, - {"same name union", func() error { - return f.RegisterUnionByName(registryFreezeUnion{}, name, NewUnionSerializer( - UnionCase{ID: 0, Type: reflect.TypeOf(int32(0)), TypeID: INT32})) - }}, - {"same name extension", func() error { - return f.RegisterExtensionByName( - registryFreezeExtension{}, name, registryPanicSerializer{}) - }}, - {"same type name", func() error { - return f.RegisterStructByName(®istryFreezeStruct{}, "test.OtherIdentity") - }}, - {"same type ID", func() error { - return f.RegisterStruct(®istryFreezeStruct{}, 7110) - }}, - } - for _, attempt := range attempts { - t.Run(attempt.name, func(t *testing.T) { - require.Error(t, attempt.call()) - require.Equal(t, before, takeRegistryFreezeSnapshot(f.typeResolver)) - require.Same(t, owner, f.typeResolver.namedTypeToTypeInfo[nameKey]) - require.Same(t, hashOwner, f.typeResolver.nsTypeToTypeInfo[hashKey]) - }) - } - - want := registryFreezeStruct{Value: 7} - data, err := f.Serialize(&want) + _, err := f.Serialize(int32(1)) require.NoError(t, err) - var got registryFreezeStruct - require.NoError(t, f.Deserialize(data, &got)) - require.Equal(t, want, got) -} - -func TestNumericRegistryIdentity(t *testing.T) { - f := New(WithXlang(false), WithCompatible(false)) - const typeID = 7111 - require.NoError(t, f.RegisterEnum(registryFreezeEnum(0), typeID)) - - owner := f.typeResolver.userTypeIdToTypeInfo[typeID] - require.NotNil(t, owner) - before := takeRegistryFreezeSnapshot(f.typeResolver) - attempts := []struct { - name string - call func() error - }{ - {"same type new ID", func() error { - value := registryFreezeEnum(0) - return f.RegisterEnum(&value, typeID+1) - }}, - {"same ID new type", func() error { - return f.RegisterEnum(registryIdentityEnum(0), typeID) - }}, - {"same ID struct", func() error { - return f.RegisterStruct(registryFreezeStruct{}, typeID) - }}, - {"same ID union", func() error { - return f.RegisterUnion(registryFreezeUnion{}, typeID, NewUnionSerializer( - UnionCase{ID: 0, Type: reflect.TypeOf(int32(0)), TypeID: INT32})) - }}, - {"same ID extension", func() error { - return f.RegisterExtension( - registryFreezeExtension{}, typeID, registryPanicSerializer{}) - }}, - {"same type name", func() error { - value := registryFreezeEnum(0) - return f.RegisterEnumByName(&value, "test.RegistryIdentityEnum") - }}, - } - for _, attempt := range attempts { - t.Run(attempt.name, func(t *testing.T) { - require.Error(t, attempt.call()) - require.Equal(t, before, takeRegistryFreezeSnapshot(f.typeResolver)) - require.Same(t, owner, f.typeResolver.userTypeIdToTypeInfo[typeID]) - require.Same(t, owner, f.typeResolver.typesInfo[reflect.TypeOf(registryFreezeEnum(0))]) - }) - } - want := registryFreezeEnum(7) + want := map[int32]string{7: "value"} data, err := f.Serialize(want) require.NoError(t, err) - var got registryFreezeEnum - require.NoError(t, f.Deserialize(data, &got)) - require.Equal(t, want, got) -} - -func requireRegisteredRoundTrip[T any](t *testing.T, f *Fory, want T) { - t.Helper() - data, err := f.Serialize(&want) - require.NoError(t, err) - var got T + var got map[int32]string require.NoError(t, f.Deserialize(data, &got)) require.Equal(t, want, got) } -func requireValueRegistrationOwner( - t *testing.T, - f *Fory, - valueType reflect.Type, - owner *TypeInfo, -) { - t.Helper() - pointerType := reflect.PointerTo(valueType) - doublePointerType := reflect.PointerTo(pointerType) - require.NotNil(t, owner) - require.Equal(t, valueType, owner.Type) - valueInfo := f.typeResolver.typesInfo[valueType] - pointerInfo := f.typeResolver.typesInfo[pointerType] - require.NotNil(t, valueInfo) - require.NotNil(t, pointerInfo) - require.Equal(t, valueInfo.TypeID, pointerInfo.TypeID) - require.Equal(t, valueInfo.UserTypeID, pointerInfo.UserTypeID) - require.Contains(t, f.typeResolver.typeToSerializers, valueType) - require.NotContains(t, f.typeResolver.typeToSerializers, doublePointerType) - require.NotContains(t, f.typeResolver.typeToTypeInfo, doublePointerType) - require.NotContains(t, f.typeResolver.typesInfo, doublePointerType) - require.NotContains(t, f.typeResolver.typeToTypeDef, doublePointerType) - require.NotContains(t, f.typeResolver.unionTypeCache, doublePointerType) - require.NotContains(t, f.typeResolver.typePointerCache, typePointer(doublePointerType)) -} - -func TestPointerReflectTypeRegistration(t *testing.T) { - type registrationFamily struct { - name string - valueType reflect.Type - pointerType reflect.Type - userTypeID uint32 - wireName string - registerID func(*Fory, reflect.Type) error - registerName func(*Fory, reflect.Type) error - registerResolver func(*Fory, reflect.Type) error - roundTrip func(*testing.T, *Fory) - } - unionSerializer := func() *UnionSerializer { - return NewUnionSerializer( - UnionCase{ID: 0, Type: reflect.TypeOf(int32(0)), TypeID: INT32}) - } - families := []registrationFamily{ - { - name: "struct", - valueType: reflect.TypeOf(pointerRegistrationStruct{}), - pointerType: reflect.TypeOf((*pointerRegistrationStruct)(nil)), - userTypeID: 7120, - wireName: "test.PointerRegistrationStruct", - registerID: func(f *Fory, type_ reflect.Type) error { - return f.RegisterStruct(type_, 7120) - }, - registerName: func(f *Fory, type_ reflect.Type) error { - return f.RegisterStructByName(type_, "test.PointerRegistrationStruct") - }, - registerResolver: func(f *Fory, type_ reflect.Type) error { - return f.GetTypeResolver().RegisterStruct(type_, STRUCT, 7120) - }, - roundTrip: func(t *testing.T, f *Fory) { - requireRegisteredRoundTrip(t, f, pointerRegistrationStruct{Value: 7}) - }, - }, - { - name: "enum", - valueType: reflect.TypeOf(pointerRegistrationEnum(0)), - pointerType: reflect.TypeOf((*pointerRegistrationEnum)(nil)), - userTypeID: 7121, - wireName: "test.PointerRegistrationEnum", - registerID: func(f *Fory, type_ reflect.Type) error { - return f.RegisterEnum(type_, 7121) - }, - registerName: func(f *Fory, type_ reflect.Type) error { - return f.RegisterEnumByName(type_, "test.PointerRegistrationEnum") - }, - registerResolver: func(f *Fory, type_ reflect.Type) error { - return f.GetTypeResolver().RegisterEnum(type_, 7121) - }, - roundTrip: func(t *testing.T, f *Fory) { - requireRegisteredRoundTrip(t, f, pointerRegistrationEnum(7)) - }, - }, - { - name: "union", - valueType: reflect.TypeOf(pointerRegistrationUnion{}), - pointerType: reflect.TypeOf((*pointerRegistrationUnion)(nil)), - userTypeID: 7122, - wireName: "test.PointerRegistrationUnion", - registerID: func(f *Fory, type_ reflect.Type) error { - return f.RegisterUnion(type_, 7122, unionSerializer()) - }, - registerName: func(f *Fory, type_ reflect.Type) error { - return f.RegisterUnionByName( - type_, "test.PointerRegistrationUnion", unionSerializer()) - }, - registerResolver: func(f *Fory, type_ reflect.Type) error { - return f.GetTypeResolver().RegisterUnion(type_, 7122, unionSerializer()) - }, - roundTrip: func(t *testing.T, f *Fory) { - requireRegisteredRoundTrip(t, f, pointerRegistrationUnion{ - caseID: 0, - value: int32(7), - }) - }, - }, - { - name: "extension", - valueType: reflect.TypeOf(pointerRegistrationExtension{}), - pointerType: reflect.TypeOf((*pointerRegistrationExtension)(nil)), - userTypeID: 7123, - wireName: "test.PointerRegistrationExtension", - registerID: func(f *Fory, type_ reflect.Type) error { - return f.RegisterExtension(type_, 7123, pointerExtensionSerializer{}) - }, - registerName: func(f *Fory, type_ reflect.Type) error { - return f.RegisterExtensionByName( - type_, "test.PointerRegistrationExtension", pointerExtensionSerializer{}) - }, - registerResolver: func(f *Fory, type_ reflect.Type) error { - return f.GetTypeResolver().RegisterExtension( - type_, 7123, pointerExtensionSerializer{}) - }, - roundTrip: func(t *testing.T, f *Fory) { - requireRegisteredRoundTrip(t, f, pointerRegistrationExtension{Value: 7}) - }, - }, - } - - for _, family := range families { - pointerType := family.pointerType - require.Equal(t, family.valueType, pointerType.Elem()) - t.Run(family.name+" facade ID", func(t *testing.T) { - f := New(WithXlang(true), WithCompatible(false)) - require.NoError(t, family.registerID(f, pointerType)) - family.roundTrip(t, f) - requireValueRegistrationOwner( - t, f, family.valueType, f.typeResolver.userTypeIdToTypeInfo[family.userTypeID]) - }) - t.Run(family.name+" facade name", func(t *testing.T) { - f := New(WithXlang(true), WithCompatible(false)) - require.NoError(t, family.registerName(f, pointerType)) - family.roundTrip(t, f) - namespace, typeName, err := splitRegisteredName(family.wireName) - require.NoError(t, err) - requireValueRegistrationOwner(t, f, family.valueType, - f.typeResolver.namedTypeToTypeInfo[namedTypeKey{namespace, typeName}]) - }) - t.Run(family.name+" resolver", func(t *testing.T) { - f := New(WithXlang(true), WithCompatible(false)) - require.NoError(t, family.registerResolver(f, pointerType)) - family.roundTrip(t, f) - requireValueRegistrationOwner( - t, f, family.valueType, f.typeResolver.userTypeIdToTypeInfo[family.userTypeID]) - }) - } -} - func TestRegistryFreezeRoots(t *testing.T) { badDecimal := Decimal{Scale: maxDecimalScale + 1} tests := []struct { @@ -462,8 +128,12 @@ func TestRegistryFreezeRoots(t *testing.T) { {"Serialize", func(f *Fory) error { _, err := f.Serialize(badDecimal); return err }}, {"Deserialize", func(f *Fory) error { return f.Deserialize(nil, new(int32)) }}, {"SerializeTo", func(f *Fory) error { return f.SerializeTo(NewByteBuffer(nil), badDecimal) }}, - {"DeserializeFrom", func(f *Fory) error { return f.DeserializeFrom(NewByteBuffer(nil), new(int32)) }}, - {"SerializeWithCallback", func(f *Fory) error { return f.SerializeWithCallback(NewByteBuffer(nil), badDecimal, nil) }}, + {"DeserializeFrom", func(f *Fory) error { + return f.DeserializeFrom(NewByteBuffer(nil), new(int32)) + }}, + {"SerializeWithCallback", func(f *Fory) error { + return f.SerializeWithCallback(NewByteBuffer(nil), badDecimal, nil) + }}, {"DeserializeWithCallbackBuffers", func(f *Fory) error { return f.DeserializeWithCallbackBuffers(NewByteBuffer(nil), nil, nil) }}, @@ -472,20 +142,21 @@ func TestRegistryFreezeRoots(t *testing.T) { {"DeserializeFromStream", func(f *Fory) error { return f.DeserializeFromStream(NewInputStream(bytes.NewReader(nil)), new(int32)) }}, - {"DeserializeFromReader", func(f *Fory) error { return f.DeserializeFromReader(bytes.NewReader(nil), new(int32)) }}, + {"DeserializeFromReader", func(f *Fory) error { + return f.DeserializeFromReader(bytes.NewReader(nil), new(int32)) + }}, } for _, test := range tests { t.Run(test.name, func(t *testing.T) { f := New(WithXlang(false), WithCompatible(false)) require.Error(t, test.root(f)) - require.ErrorIs(t, - f.RegisterStructByName(registryFreezeStruct{}, "test.RegistryFreezeRoot"), - ErrRegistryFrozen) + require.ErrorIs(t, f.RegisterStructByName( + registryFreezeStruct{}, "test.RegistryFreezeRoot"), ErrRegistryFrozen) }) } } -func TestBorrowedBufferPanicRestore(t *testing.T) { +func TestBorrowedBufferPanicCleanup(t *testing.T) { writer := New(WithXlang(false), WithCompatible(false)) data, err := writer.Serialize(int32(7)) require.NoError(t, err) @@ -499,26 +170,9 @@ func TestBorrowedBufferPanicRestore(t *testing.T) { var value int32 require.NoError(t, f.Deserialize(data, &value)) require.Equal(t, int32(7), value) - require.ErrorIs(t, - f.RegisterStructByName(registryFreezeStruct{}, "test.RegistryFreezePanic"), - ErrRegistryFrozen) -} - -func TestCallbackBufferRestoreOrder(t *testing.T) { - f := New(WithXlang(false), WithCompatible(false)) - owned := f.readCtx.buffer - borrowed := NewByteBuffer(nil) - // Force root cleanup to panic so restoring the owned buffer after Reset - // cannot accidentally satisfy this test. - f.readCtx.refReader = nil - - require.Panics(t, func() { - _ = f.DeserializeWithCallbackBuffers(borrowed, nil, nil) - }) - require.Same(t, owned, f.readCtx.buffer) } -func TestStreamBufferPanicRestore(t *testing.T) { +func TestStreamBufferPanicCleanup(t *testing.T) { f := New(WithXlang(false), WithCompatible(false)) owned := f.readCtx.buffer stream := NewInputStream(bytes.NewReader(nil)) @@ -529,9 +183,10 @@ func TestStreamBufferPanicRestore(t *testing.T) { require.Same(t, owned, f.readCtx.buffer) } -func TestSerializeToPanicRestore(t *testing.T) { +func TestSerializePanicCleanup(t *testing.T) { f := New(WithXlang(false), WithCompatible(false)) - require.NoError(t, f.RegisterExtension(registryFreezeExtension{}, 7104, registryPanicSerializer{})) + require.NoError(t, f.RegisterExtension( + registryFreezeExtension{}, 7104, registryPanicSerializer{})) owned := f.writeCtx.buffer borrowed := NewByteBuffer(nil) @@ -539,7 +194,6 @@ func TestSerializeToPanicRestore(t *testing.T) { _ = f.SerializeTo(borrowed, ®istryFreezeExtension{Value: 1}) }) require.Same(t, owned, f.writeCtx.buffer) - require.NotZero(t, borrowed.WriterIndex()) data, err := f.Serialize(int32(7)) require.NoError(t, err) @@ -548,12 +202,12 @@ func TestSerializeToPanicRestore(t *testing.T) { func TestCallbackPanicCleanup(t *testing.T) { f := New(WithXlang(false), WithCompatible(false)) - require.NoError(t, f.RegisterExtension(registryFreezeExtension{}, 7104, registryPanicSerializer{})) + require.NoError(t, f.RegisterExtension( + registryFreezeExtension{}, 7104, registryPanicSerializer{})) require.Panics(t, func() { _ = f.SerializeWithCallback( NewByteBuffer(nil), ®istryFreezeExtension{Value: 1}, nil) }) - require.NoError(t, f.SerializeWithCallback(NewByteBuffer(nil), int32(7), nil)) } diff --git a/go/fory/stream.go b/go/fory/stream.go index 0f304ba818..5579ff7c52 100644 --- a/go/fory/stream.go +++ b/go/fory/stream.go @@ -96,7 +96,7 @@ func (is *InputStream) Shrink() { // DeserializeFromStream reads the next object from the stream into the provided value. // It preserves the stream buffer while clearing root-scoped read metadata between calls. func (f *Fory) DeserializeFromStream(is *InputStream, v any) error { - f.beginRoot() + f.registryFrozen = true origBuffer := f.readCtx.buffer f.readCtx.buffer = is.buffer defer func() { @@ -127,7 +127,7 @@ func (f *Fory) DeserializeFromStream(is *InputStream, v any) error { // each call, discarding any prefetched data and type metadata. // For sequential multi-object reads on the same stream, use NewInputStream instead. func (f *Fory) DeserializeFromReader(r io.Reader, v any) error { - f.beginRoot() + f.registryFrozen = true defer f.resetReadState() // Always reset to enforce stateless semantics. f.readCtx.buffer.ResetWithReader(r, 0) diff --git a/go/fory/threadsafe/fory.go b/go/fory/threadsafe/fory.go index f6dc0d741b..4cd3fb2bbd 100644 --- a/go/fory/threadsafe/fory.go +++ b/go/fory/threadsafe/fory.go @@ -19,29 +19,15 @@ package threadsafe import ( - "fmt" - "reflect" "sync" - "sync/atomic" "github.com/apache/fory/go/fory" ) -type structRegistration struct { - typ reflect.Type - name string -} - // Fory is a thread-safe wrapper around fory.Fory using sync.Pool. // It provides the same API as fory.Fory but is safe for concurrent use. -// Registration must finish before its first root operation. type Fory struct { - pool sync.Pool - registrationMu sync.Mutex - registryFrozen atomic.Bool - factory func() *fory.Fory - registrations []structRegistration - prepared *fory.Fory + pool sync.Pool } // New creates a new thread-safe Fory instance. @@ -51,60 +37,26 @@ func New(opts ...fory.Option) *Fory { }) } -// NewWithFactory creates a new thread-safe Fory instance using a custom factory. +// NewWithFactory creates a thread-safe Fory whose factory configures each pooled instance. func NewWithFactory(factory func() *fory.Fory) *Fory { if factory == nil { panic("threadsafe.NewWithFactory requires a non-nil factory") } - return &Fory{factory: factory} -} - -func (f *Fory) createInner() *fory.Fory { - inner := f.factory() - if inner == nil { - panic("threadsafe.NewWithFactory factory returned nil") - } - return inner -} - -func (f *Fory) applyRegistrations(inner *fory.Fory) error { - for _, registration := range f.registrations { - if err := inner.RegisterStructByName(registration.typ, registration.name); err != nil { - return fmt.Errorf("apply registration %q to new Fory instance: %w", registration.name, err) - } - } - return nil -} - -func (f *Fory) newInner() (*fory.Fory, error) { - inner := f.createInner() - // Registry freeze is published before pool misses reach this path, so the - // registration log is immutable and needs no root hot-path lock. - if err := f.applyRegistrations(inner); err != nil { - return nil, err + f := &Fory{} + f.pool = sync.Pool{ + New: func() any { + inner := factory() + if inner == nil { + panic("threadsafe.NewWithFactory factory returned nil") + } + return inner + }, } - return inner, nil + return f } -func (f *Fory) acquire() (*fory.Fory, error) { - if !f.registryFrozen.Load() { - f.registrationMu.Lock() - if !f.registryFrozen.Load() { - f.registryFrozen.Store(true) - inner := f.prepared - f.prepared = nil - f.registrationMu.Unlock() - if inner != nil { - return inner, nil - } - return f.newInner() - } - f.registrationMu.Unlock() - } - if pooled := f.pool.Get(); pooled != nil { - return pooled.(*fory.Fory), nil - } - return f.newInner() +func (f *Fory) acquire() *fory.Fory { + return f.pool.Get().(*fory.Fory) } func (f *Fory) release(inner *fory.Fory) { @@ -118,10 +70,7 @@ func (f *Fory) release(inner *fory.Fory) { // Serialize serializes a value using a pooled Fory instance func (f *Fory) Serialize(v any) ([]byte, error) { - inner, err := f.acquire() - if err != nil { - return nil, err - } + inner := f.acquire() data, err := inner.Serialize(v) if err != nil { f.release(inner) @@ -136,63 +85,11 @@ func (f *Fory) Serialize(v any) ([]byte, error) { // Deserialize deserializes data into the provided value using a pooled Fory instance func (f *Fory) Deserialize(data []byte, v any) error { - inner, err := f.acquire() - if err != nil { - return err - } + inner := f.acquire() defer f.release(inner) return inner.Deserialize(data, v) } -// RegisterStructByName registers a struct type by name before the first root operation. -func (f *Fory) RegisterStructByName(type_ any, name string) error { - f.registrationMu.Lock() - if f.registryFrozen.Load() { - f.registrationMu.Unlock() - return fory.ErrRegistryFrozen - } - if f.prepared != nil { - defer f.registrationMu.Unlock() - return f.registerPrepared(type_, name) - } - f.registrationMu.Unlock() - - // The factory is application code and may reenter a root. Never hold the - // registration mutex across it, and recheck freeze before publishing its result. - inner := f.createInner() - - f.registrationMu.Lock() - defer f.registrationMu.Unlock() - if f.registryFrozen.Load() { - return fory.ErrRegistryFrozen - } - if f.prepared == nil { - if err := f.applyRegistrations(inner); err != nil { - return err - } - f.prepared = inner - } - return f.registerPrepared(type_, name) -} - -func (f *Fory) registerPrepared(type_ any, name string) error { - registration := structRegistration{name: name} - if err := f.prepared.RegisterStructByName(type_, name); err != nil { - // Direct registration errors leave the prepared resolver unchanged. - return err - } - if registeredType, ok := type_.(reflect.Type); ok { - registration.typ = registeredType - } else { - registration.typ = reflect.TypeOf(type_) - if registration.typ.Kind() == reflect.Ptr { - registration.typ = registration.typ.Elem() - } - } - f.registrations = append(f.registrations, registration) - return nil -} - // ============================================================================ // Generic package-level functions // ============================================================================ @@ -200,10 +97,7 @@ func (f *Fory) registerPrepared(type_ any, name string) error { // Serialize serializes a value with type T inferred, thread-safe. // Takes pointer to avoid interface heap allocation and struct copy. func Serialize[T any](f *Fory, value *T) ([]byte, error) { - inner, err := f.acquire() - if err != nil { - return nil, err - } + inner := f.acquire() data, err := fory.Serialize(inner, value) if err != nil { f.release(inner) @@ -219,10 +113,7 @@ func Serialize[T any](f *Fory, value *T) ([]byte, error) { // Deserialize deserializes data directly into the provided target, thread-safe. // Takes pointer to avoid interface heap allocation and enable direct writes. func Deserialize[T any](f *Fory, data []byte, target *T) error { - inner, err := f.acquire() - if err != nil { - return err - } + inner := f.acquire() defer f.release(inner) return fory.Deserialize(inner, data, target) } diff --git a/go/fory/threadsafe/fory_test.go b/go/fory/threadsafe/fory_test.go index acfd15212f..b813de02c3 100644 --- a/go/fory/threadsafe/fory_test.go +++ b/go/fory/threadsafe/fory_test.go @@ -140,12 +140,22 @@ func TestDeserialize(t *testing.T) { }) t.Run("Slice", func(t *testing.T) { - f := New(fory.WithXlang(false), fory.WithRefTracking(true), fory.WithCompatible(false)) // Serialize a struct containing the slice since *[]T is not supported type SliceWrapper struct { Items []int32 } - require.NoError(t, f.RegisterStructByName(SliceWrapper{}, "threadsafe.SliceWrapper")) + f := NewWithFactory(func() *fory.Fory { + inner := fory.New( + fory.WithXlang(false), + fory.WithRefTracking(true), + fory.WithCompatible(false), + ) + if err := inner.RegisterStructByName( + SliceWrapper{}, "threadsafe.SliceWrapper"); err != nil { + panic(err) + } + return inner + }) original := SliceWrapper{Items: []int32{1, 2, 3, 4, 5}} data, err := Serialize(f, &original) require.NoError(t, err) diff --git a/go/fory/threadsafe/registry_freeze_lifecycle_test.go b/go/fory/threadsafe/registry_freeze_lifecycle_test.go deleted file mode 100644 index 7d58d67d33..0000000000 --- a/go/fory/threadsafe/registry_freeze_lifecycle_test.go +++ /dev/null @@ -1,275 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -package threadsafe - -import ( - "reflect" - "sync" - "sync/atomic" - "testing" - "time" - - "github.com/apache/fory/go/fory" - "github.com/stretchr/testify/require" -) - -type registryFreezePooled struct { - Value int32 -} - -type registryFreezeRace struct { - Value int32 -} - -type registryIdentityPooled struct { - Value int32 -} - -type registryInvalidPooled struct { - First int32 `fory:"id=0"` - Second int32 `fory:"id=0"` -} - -type reentrantStringSerializer struct { - f *Fory - called chan struct{} -} - -func (s *reentrantStringSerializer) String() string { - select { - case s.called <- struct{}{}: - default: - } - _, _ = s.f.Serialize(int32(1)) - return "reentrant serializer" -} - -func (*reentrantStringSerializer) Write( - *fory.WriteContext, fory.RefMode, bool, bool, reflect.Value, -) { -} - -func (*reentrantStringSerializer) WriteData(*fory.WriteContext, reflect.Value) {} - -func (*reentrantStringSerializer) Read( - *fory.ReadContext, fory.RefMode, bool, bool, reflect.Value, -) { -} - -func (*reentrantStringSerializer) ReadData(*fory.ReadContext, reflect.Value) {} - -func (*reentrantStringSerializer) ReadWithTypeInfo( - *fory.ReadContext, fory.RefMode, *fory.TypeInfo, reflect.Value, -) { -} - -func TestDuplicateSerializerFormatting(t *testing.T) { - called := make(chan struct{}, 1) - serializer := &reentrantStringSerializer{called: called} - var f *Fory - var prepared *fory.Fory - f = NewWithFactory(func() *fory.Fory { - prepared = fory.New(fory.WithXlang(false), fory.WithCompatible(false)) - if err := prepared.RegisterUnionByName( - registryFreezePooled{}, "test.DuplicateSerializer", serializer, - ); err != nil { - panic(err) - } - return prepared - }) - serializer.f = f - - result := make(chan error, 1) - go func() { - result <- f.RegisterStructByName( - registryFreezePooled{}, "test.DuplicateSerializer") - }() - timer := time.NewTimer(2 * time.Second) - defer timer.Stop() - select { - case err := <-result: - require.Error(t, err) - case <-timer.C: - t.Fatal("duplicate registration deadlocked while formatting the serializer") - } - select { - case <-called: - t.Fatal("duplicate registration formatted the application serializer") - default: - } - require.False(t, f.registryFrozen.Load()) - require.Empty(t, f.registrations) - require.Same(t, prepared, f.prepared) -} - -func TestFactoryRootReentry(t *testing.T) { - var f *Fory - var factoryEntered atomic.Bool - var factoryUnlocked bool - var rootErr error - f = NewWithFactory(func() *fory.Fory { - if factoryEntered.CompareAndSwap(false, true) { - factoryUnlocked = f.registrationMu.TryLock() - if factoryUnlocked { - f.registrationMu.Unlock() - _, rootErr = f.Serialize(int32(1)) - } - } - return fory.New(fory.WithXlang(false), fory.WithCompatible(false)) - }) - - err := f.RegisterStructByName(registryFreezePooled{}, "test.FactoryRootReentry") - require.True(t, factoryUnlocked) - require.NoError(t, rootErr) - require.ErrorIs(t, err, fory.ErrRegistryFrozen) - require.True(t, f.registryFrozen.Load()) - require.Empty(t, f.registrations) - require.Nil(t, f.prepared) -} - -func TestRegistryFreezePropagation(t *testing.T) { - var factoryCalls atomic.Int32 - f := NewWithFactory(func() *fory.Fory { - factoryCalls.Add(1) - return fory.New(fory.WithXlang(false), fory.WithCompatible(false)) - }) - require.NoError(t, f.RegisterStructByName(registryFreezePooled{}, "test.RegistryFreezePooled")) - - const innerCount = 8 - inners := make([]*fory.Fory, 0, innerCount) - for i := 0; i < innerCount; i++ { - inner, err := f.acquire() - require.NoError(t, err) - inners = append(inners, inner) - value := registryFreezePooled{Value: int32(i)} - data, err := inner.Serialize(&value) - require.NoError(t, err) - var result registryFreezePooled - require.NoError(t, inner.Deserialize(data, &result)) - require.Equal(t, value, result) - } - require.Equal(t, int32(innerCount), factoryCalls.Load()) - require.ErrorIs(t, - f.RegisterStructByName(registryFreezePooled{}, "test.RegistryFreezeLate"), - fory.ErrRegistryFrozen) - for _, inner := range inners { - f.release(inner) - } -} - -func TestPreparedSurvivesConflict(t *testing.T) { - f := New(fory.WithXlang(false), fory.WithCompatible(false)) - const name = "test.PreparedIdentity" - require.NoError(t, f.RegisterStructByName(registryFreezePooled{}, name)) - prepared := f.prepared - require.NotNil(t, prepared) - - require.Error(t, f.RegisterStructByName(registryIdentityPooled{}, name)) - require.Same(t, prepared, f.prepared) - require.Len(t, f.registrations, 1) - require.Error(t, - f.RegisterStructByName(registryFreezePooled{}, "test.OtherPreparedIdentity")) - require.Same(t, prepared, f.prepared) - require.Len(t, f.registrations, 1) - require.Error(t, - f.RegisterStructByName(registryInvalidPooled{}, "test.InvalidPrepared")) - require.Same(t, prepared, f.prepared) - require.Len(t, f.registrations, 1) - - want := registryFreezePooled{Value: 7} - data, err := f.Serialize(&want) - require.NoError(t, err) - var got registryFreezePooled - require.NoError(t, f.Deserialize(data, &got)) - require.Equal(t, want, got) -} - -func TestRegistryFreezeOnFailure(t *testing.T) { - f := New(fory.WithXlang(false), fory.WithCompatible(false)) - _, err := f.Serialize(fory.Decimal{Scale: 10_001}) - require.Error(t, err) - require.ErrorIs(t, - f.RegisterStructByName(registryFreezePooled{}, "test.RegistryFreezeFailure"), - fory.ErrRegistryFrozen) -} - -func TestRegistryFreezeReplayFailure(t *testing.T) { - frozenInner := fory.New(fory.WithXlang(false), fory.WithCompatible(false)) - _, err := frozenInner.Serialize(int32(1)) - require.NoError(t, err) - - var factoryCalls atomic.Int32 - f := NewWithFactory(func() *fory.Fory { - if factoryCalls.Add(1) == 1 { - return fory.New(fory.WithXlang(false), fory.WithCompatible(false)) - } - return frozenInner - }) - require.NoError(t, f.RegisterStructByName(registryFreezePooled{}, "test.RegistryFreezeReplay")) - - prepared, err := f.acquire() - require.NoError(t, err) - defer f.release(prepared) - _, err = f.Serialize(®istryFreezePooled{Value: 1}) - require.ErrorIs(t, err, fory.ErrRegistryFrozen) -} - -func TestRegistryFreezeOnFactoryPanic(t *testing.T) { - f := NewWithFactory(func() *fory.Fory { panic("factory failure") }) - require.Panics(t, func() { - _, _ = f.Serialize(int32(1)) - }) - require.ErrorIs(t, - f.RegisterStructByName(registryFreezePooled{}, "test.RegistryFreezeFactory"), - fory.ErrRegistryFrozen) -} - -func TestRegistryFreezeRace(t *testing.T) { - const iterations = 100 - for i := 0; i < iterations; i++ { - f := New(fory.WithXlang(false), fory.WithCompatible(false)) - start := make(chan struct{}) - registrationResult := make(chan error, 1) - rootResult := make(chan error, 1) - var ready sync.WaitGroup - ready.Add(2) - - go func() { - ready.Done() - <-start - registrationResult <- f.RegisterStructByName(registryFreezeRace{}, "test.RegistryFreezeRace") - }() - go func() { - ready.Done() - <-start - _, err := f.Serialize(®istryFreezeRace{Value: int32(i)}) - rootResult <- err - }() - ready.Wait() - close(start) - - registrationErr := <-registrationResult - rootErr := <-rootResult - if registrationErr == nil { - require.NoError(t, rootErr) - } else { - require.ErrorIs(t, registrationErr, fory.ErrRegistryFrozen) - require.Error(t, rootErr) - } - } -} diff --git a/go/fory/type_resolver.go b/go/fory/type_resolver.go index ac30dec83b..93e22f4797 100644 --- a/go/fory/type_resolver.go +++ b/go/fory/type_resolver.go @@ -121,60 +121,6 @@ func joinRegisteredName(namespace, typeName string) string { return namespace + "." + typeName } -func (r *TypeResolver) validateNamedRegistration(namespace, typeName string) error { - if _, err := r.namespaceEncoder.EncodePackage(namespace); err != nil { - return fmt.Errorf("invalid type namespace: %w", err) - } - if _, err := r.typeNameEncoder.EncodeTypeName(typeName); err != nil { - return fmt.Errorf("invalid type name: %w", err) - } - return nil -} - -func hasWireIdentity(info *TypeInfo) bool { - return info != nil && - (info.UserTypeID != invalidUserTypeID || info.NameBytes != nil) -} - -// A registered Go value type and its pointer form share one wire identity. -// Check both indexes before serializer creation so failure cannot alter registration. -func (r *TypeResolver) checkUserTypeIDOwner( - type_ reflect.Type, - typeID TypeId, - userTypeID uint32, -) (bool, error) { - typeInfo := r.typesInfo[type_] - if idInfo, ok := r.userTypeIdToTypeInfo[userTypeID]; ok { - if idInfo.Type == type_ && TypeId(idInfo.TypeID) == typeID && typeInfo == idInfo { - return true, nil - } - return false, fmt.Errorf( - "wire type ID %d already identifies %s", userTypeID, idInfo.Type) - } - if hasWireIdentity(typeInfo) { - return false, fmt.Errorf("type %s already has a different wire identity", type_) - } - return false, nil -} - -func (r *TypeResolver) checkNamedTypeOwner( - type_ reflect.Type, - namespace string, - typeName string, -) error { - typeInfo := r.typesInfo[type_] - nameKey := namedTypeKey{namespace, typeName} - if nameInfo, ok := r.namedTypeToTypeInfo[nameKey]; ok { - return fmt.Errorf( - "wire name %q already identifies %s", - joinRegisteredName(namespace, typeName), nameInfo.Type) - } - if hasWireIdentity(typeInfo) { - return fmt.Errorf("type %s already has a different wire identity", type_) - } - return nil -} - type TypeInfo struct { Type reflect.Type FullNameBytes []byte @@ -499,7 +445,7 @@ func (r *TypeResolver) initialize() { func (r *TypeResolver) registerSerializer(type_ reflect.Type, typeId TypeId, s Serializer) error { if prev, ok := r.typeToSerializers[type_]; ok { - return fmt.Errorf("type %s already has a serializer of type %T registered", type_, prev) + return fmt.Errorf("type %s already has a serializer %s registered", type_, prev) } r.typeToSerializers[type_] = s // Skip type ID registration for namespaced types, collection types, and primitive array types @@ -516,30 +462,16 @@ func (r *TypeResolver) registerSerializer(type_ reflect.Type, typeId TypeId, s S return nil } -// valueRegistrationType returns the canonical value owner represented by an -// instance or reflect.Type. Registration publishes that value type and creates -// at most one pointer companion from it. -func valueRegistrationType(type_ any) reflect.Type { - registeredType, ok := type_.(reflect.Type) - if !ok { - registeredType = reflect.TypeOf(type_) - } - for registeredType != nil && registeredType.Kind() == reflect.Ptr { - registeredType = registeredType.Elem() - } - return registeredType -} - func validateOptionalFields(type_ reflect.Type) error { if type_ == nil { return nil } + if type_.Kind() == reflect.Ptr { + type_ = type_.Elem() + } if type_.Kind() != reflect.Struct { return nil } - if err := validateForyTags(type_); err != nil { - return err - } for i := 0; i < type_.NumField(); i++ { field := type_.Field(i) if field.PkgPath != "" { @@ -567,48 +499,59 @@ func (r *TypeResolver) RegisterStruct(type_ reflect.Type, typeID TypeId, userTyp if err := r.fory.checkRegistrationOpen(); err != nil { return err } - type_ = valueRegistrationType(type_) - if type_.Kind() != reflect.Struct { - return fmt.Errorf("unsupported type for ID registration: %v (use RegisterEnum for enum types)", type_.Kind()) - } - if err := validateOptionalFields(type_); err != nil { - return err - } - alreadyRegistered, err := r.checkUserTypeIDOwner(type_, typeID, userTypeID) - if err != nil { - return err - } - if alreadyRegistered { - return nil + // Check if already registered + if info, ok := r.userTypeIdToTypeInfo[userTypeID]; ok { + if info.Type == type_ { + return nil + } + return fmt.Errorf("type %s with id %d has been registered", info.Type, userTypeID) } - if prev, ok := r.typeToSerializers[type_]; ok { - return fmt.Errorf("type %s already has a serializer of type %T registered", type_, prev) - } + switch type_.Kind() { + case reflect.Struct: + if err := validateForyTags(type_); err != nil { + return err + } + if err := validateOptionalFields(type_); err != nil { + return err + } + // For struct types, check if serializer already registered + if prev, ok := r.typeToSerializers[type_]; ok { + return fmt.Errorf("type %s already has a serializer %s registered", type_, prev) + } - tag := type_.Name() - serializer := newStructSerializer(type_, tag) - ptrType := reflect.PtrTo(type_) - ptrSerializer, ok := r.typeToSerializers[ptrType] - if !ok { - ptrSerializer = &ptrToValueSerializer{ - valueSerializer: serializer, - valueBytes: serializer.valueBytes, + // Create struct serializer + tag := type_.Name() + serializer := newStructSerializer(type_, tag) + r.typeToSerializers[type_] = serializer + r.typeToTypeInfo[type_] = "@" + tag + + // Create pointer serializer + ptrType := reflect.PtrTo(type_) + ptrSerializer, ok := r.typeToSerializers[ptrType] + if !ok { + ptrSerializer = &ptrToValueSerializer{ + valueSerializer: serializer, + valueBytes: serializer.valueBytes, + } + r.typeToSerializers[ptrType] = ptrSerializer } - } + r.typeToTypeInfo[ptrType] = "*@" + tag - r.typeToSerializers[type_] = serializer - r.typeToTypeInfo[type_] = "@" + tag - r.typeToSerializers[ptrType] = ptrSerializer - r.typeToTypeInfo[ptrType] = "*@" + tag + // Register value type with fullTypeID + _, err := r.registerType(type_, uint32(typeID), userTypeID, "", "", serializer, false) + if err != nil { + return fmt.Errorf("failed to register type by ID: %w", err) + } - if _, err = r.registerType( - type_, uint32(typeID), userTypeID, "", "", serializer, false); err != nil { - return fmt.Errorf("failed to register type by ID: %w", err) - } - if _, err = r.registerType( - ptrType, uint32(typeID), userTypeID, "", "", ptrSerializer, false); err != nil { - return fmt.Errorf("failed to register pointer type by ID: %w", err) + // Register pointer type with same fullTypeID (Java treats value and pointer types the same) + _, err = r.registerType(ptrType, uint32(typeID), userTypeID, "", "", ptrSerializer, false) + if err != nil { + return fmt.Errorf("failed to register pointer type by ID: %w", err) + } + + default: + return fmt.Errorf("unsupported type for ID registration: %v (use RegisterEnum for enum types)", type_.Kind()) } return nil @@ -619,21 +562,17 @@ func (r *TypeResolver) RegisterUnion(type_ reflect.Type, userTypeID uint32, seri if err := r.fory.checkRegistrationOpen(); err != nil { return err } - type_ = valueRegistrationType(type_) if serializer == nil { return fmt.Errorf("RegisterUnion requires a non-nil serializer") } + if info, ok := r.userTypeIdToTypeInfo[userTypeID]; ok { + return fmt.Errorf("type %s with id %d has been registered", info.Type, userTypeID) + } if type_.Kind() != reflect.Struct { return fmt.Errorf("RegisterUnion only supports struct types; got: %v", type_.Kind()) } - if alreadyRegistered, err := r.checkUserTypeIDOwner( - type_, TYPED_UNION, userTypeID); err != nil { - return err - } else if alreadyRegistered { - return fmt.Errorf("type %s with id %d has been registered", type_, userTypeID) - } if prev, ok := r.typeToSerializers[type_]; ok { - return fmt.Errorf("type %s already has a serializer of type %T registered", type_, prev) + return fmt.Errorf("type %s already has a serializer %s registered", type_, prev) } tag := type_.Name() @@ -661,7 +600,12 @@ func (r *TypeResolver) RegisterEnum(type_ reflect.Type, userTypeID uint32) error if err := r.fory.checkRegistrationOpen(); err != nil { return err } - type_ = valueRegistrationType(type_) + // Check if already registered + if info, ok := r.userTypeIdToTypeInfo[userTypeID]; ok { + return fmt.Errorf("type %s with id %d has been registered", info.Type, userTypeID) + } + + // Verify it's a numeric type switch type_.Kind() { case reflect.Int, reflect.Int8, reflect.Int16, reflect.Int32, reflect.Int64, reflect.Uint, reflect.Uint8, reflect.Uint16, reflect.Uint32, reflect.Uint64: @@ -669,14 +613,6 @@ func (r *TypeResolver) RegisterEnum(type_ reflect.Type, userTypeID uint32) error default: return fmt.Errorf("RegisterEnum only supports numeric types; got: %v", type_.Kind()) } - if alreadyRegistered, err := r.checkUserTypeIDOwner(type_, ENUM, userTypeID); err != nil { - return err - } else if alreadyRegistered { - return fmt.Errorf("type %s with id %d has been registered", type_, userTypeID) - } - if prev, ok := r.typeToSerializers[type_]; ok { - return fmt.Errorf("type %s already has a serializer of type %T registered", type_, prev) - } // Create enum serializer serializer := &enumSerializer{type_: type_, typeID: uint32(ENUM)} @@ -703,13 +639,13 @@ func (r *TypeResolver) RegisterEnum(type_ reflect.Type, userTypeID uint32) error } func (r *TypeResolver) registerEnumByName(type_ reflect.Type, namespace, typeName string) error { - type_ = valueRegistrationType(type_) + // Check if already registered + if prev, ok := r.typeToSerializers[type_]; ok { + return fmt.Errorf("type %s already has a serializer %s registered", type_, prev) + } if typeName == "" { return fmt.Errorf("typeName must be non-empty") } - if err := r.validateNamedRegistration(namespace, typeName); err != nil { - return err - } // Verify it's a numeric type switch type_.Kind() { @@ -719,16 +655,12 @@ func (r *TypeResolver) registerEnumByName(type_ reflect.Type, namespace, typeNam default: return fmt.Errorf("RegisterEnumByName only supports numeric types; got: %v", type_.Kind()) } - if err := r.checkNamedTypeOwner(type_, namespace, typeName); err != nil { - return err - } - if prev, ok := r.typeToSerializers[type_]; ok { - return fmt.Errorf("type %s already has a serializer of type %T registered", type_, prev) - } + + // Compute type ID for NAMED_ENUM + typeId := uint32(NAMED_ENUM) // Create enum serializer - typeID := uint32(NAMED_ENUM) - serializer := &enumSerializer{type_: type_, typeID: typeID} + serializer := &enumSerializer{type_: type_, typeID: typeId} tag := joinRegisteredName(namespace, typeName) @@ -736,7 +668,7 @@ func (r *TypeResolver) registerEnumByName(type_ reflect.Type, namespace, typeNam r.typeToTypeInfo[type_] = "@" + tag // Register the type - _, err := r.registerType(type_, typeID, invalidUserTypeID, namespace, typeName, serializer, false) + _, err := r.registerType(type_, typeId, invalidUserTypeID, namespace, typeName, serializer, false) if err != nil { return fmt.Errorf("failed to register enum by name: %w", err) } @@ -745,35 +677,29 @@ func (r *TypeResolver) registerEnumByName(type_ reflect.Type, namespace, typeNam } func (r *TypeResolver) registerStructByName(type_ reflect.Type, namespace, typeName string) error { - type_ = valueRegistrationType(type_) + if prev, ok := r.typeToSerializers[type_]; ok { + return fmt.Errorf("type %s already has a serializer %s registered", type_, prev) + } if typeName == "" { return fmt.Errorf("typeName must be non-empty") } - if err := r.validateNamedRegistration(namespace, typeName); err != nil { - return err - } - if err := validateOptionalFields(type_); err != nil { - return err - } - internalTypeID := r.structTypeID(type_, true) - if err := r.checkNamedTypeOwner(type_, namespace, typeName); err != nil { + if err := validateForyTags(type_); err != nil { return err } - if prev, ok := r.typeToSerializers[type_]; ok { - return fmt.Errorf("type %s already has a serializer of type %T registered", type_, prev) - } tag := joinRegisteredName(namespace, typeName) serializer := newStructSerializer(type_, tag) - ptrType := reflect.PtrTo(type_) - ptrSerializer := &ptrToValueSerializer{valueSerializer: serializer, valueBytes: int(type_.Size())} - r.typeToSerializers[type_] = serializer - // Distinct Go types can have the same display name, so registered wire names - // own the encoded type information. + // multiple struct with same name defined inside function will have same `type_.String()`, but they are + // different types. so we use tag to encode type info. + // tagged type encode as `@$tag`/`*@$tag`. r.typeToTypeInfo[type_] = "@" + tag + + ptrType := reflect.PtrTo(type_) + ptrSerializer := &ptrToValueSerializer{valueSerializer: serializer, valueBytes: int(type_.Size())} r.typeToSerializers[ptrType] = ptrSerializer // use `ptrToValueSerializer` as default deserializer when deserializing data from other languages. r.typeToTypeInfo[ptrType] = "*@" + tag + internalTypeID := r.structTypeID(type_, true) userTypeID := invalidUserTypeID // For structs registered by name, directly register both their value and pointer types. _, err := r.registerType(type_, uint32(internalTypeID), userTypeID, namespace, typeName, nil, false) @@ -793,25 +719,18 @@ func (r *TypeResolver) registerUnionByName( typeName string, serializer Serializer, ) error { - type_ = valueRegistrationType(type_) if serializer == nil { return fmt.Errorf("RegisterUnionByName requires a non-nil serializer") } + if prev, ok := r.typeToSerializers[type_]; ok { + return fmt.Errorf("type %s already has a serializer %s registered", type_, prev) + } if type_.Kind() != reflect.Struct { return fmt.Errorf("RegisterUnionByName only supports struct types; got: %v", type_.Kind()) } if typeName == "" { return fmt.Errorf("typeName must be non-empty") } - if err := r.validateNamedRegistration(namespace, typeName); err != nil { - return err - } - if err := r.checkNamedTypeOwner(type_, namespace, typeName); err != nil { - return err - } - if prev, ok := r.typeToSerializers[type_]; ok { - return fmt.Errorf("type %s already has a serializer of type %T registered", type_, prev) - } tag := joinRegisteredName(namespace, typeName) r.typeToSerializers[type_] = serializer r.typeToTypeInfo[type_] = "@" + tag @@ -839,22 +758,15 @@ func (r *TypeResolver) registerExtensionByName( typeName string, userSerializer ExtensionSerializer, ) error { - type_ = valueRegistrationType(type_) if userSerializer == nil { return fmt.Errorf("serializer cannot be nil for extension type %s", type_) } + if prev, ok := r.typeToSerializers[type_]; ok { + return fmt.Errorf("type %s already has a serializer %s registered", type_, prev) + } if typeName == "" { return fmt.Errorf("typeName must be non-empty") } - if err := r.validateNamedRegistration(namespace, typeName); err != nil { - return err - } - if err := r.checkNamedTypeOwner(type_, namespace, typeName); err != nil { - return err - } - if prev, ok := r.typeToSerializers[type_]; ok { - return fmt.Errorf("type %s already has a serializer of type %T registered", type_, prev) - } tag := joinRegisteredName(namespace, typeName) // Create adapter wrapping the user's ExtensionSerializer @@ -891,20 +803,14 @@ func (r *TypeResolver) RegisterExtension( if err := r.fory.checkRegistrationOpen(); err != nil { return err } - type_ = valueRegistrationType(type_) if userTypeID > maxUserTypeID { return fmt.Errorf("typeID must be in range [0, 0xfffffffe], got %d", userTypeID) } if userSerializer == nil { return fmt.Errorf("serializer cannot be nil for extension type %s", type_) } - if alreadyRegistered, err := r.checkUserTypeIDOwner(type_, EXT, userTypeID); err != nil { - return err - } else if alreadyRegistered { - return fmt.Errorf("type %s with id %d has been registered", type_, userTypeID) - } if prev, ok := r.typeToSerializers[type_]; ok { - return fmt.Errorf("type %s already has a serializer of type %T registered", type_, prev) + return fmt.Errorf("type %s already has a serializer %s registered", type_, prev) } // Create adapter wrapping the user's ExtensionSerializer @@ -1303,18 +1209,12 @@ func (r *TypeResolver) registerType( } } - nsMeta, encodeErr := r.namespaceEncoder.EncodePackage(namespace) - if encodeErr != nil { - return nil, fmt.Errorf("invalid type namespace: %w", encodeErr) - } + nsMeta, _ := r.namespaceEncoder.EncodePackage(namespace) if nsBytes = r.metaStringResolver.GetMetaStrBytes(&nsMeta); nsBytes == nil { panic("failed to encode namespace") } - typeMeta, encodeErr := r.typeNameEncoder.EncodeTypeName(typeName) - if encodeErr != nil { - return nil, fmt.Errorf("invalid type name: %w", encodeErr) - } + typeMeta, _ := r.typeNameEncoder.EncodeTypeName(typeName) if typeBytes = r.metaStringResolver.GetMetaStrBytes(&typeMeta); typeBytes == nil { panic("failed to encode type name") } From c5313b921119e830895d88c46c073ff908c85326 Mon Sep 17 00:00:00 2001 From: chaokunyang Date: Sun, 30 Aug 2026 02:53:19 +0800 Subject: [PATCH 093/168] refactor(swift): freeze registry without finalization state --- .agents/languages/swift.md | 15 +-- swift/Sources/Fory/Fory.swift | 8 +- swift/Sources/Fory/ReadContext.swift | 3 + swift/Sources/Fory/Serializer.swift | 2 +- swift/Sources/Fory/TypeResolver.swift | 61 ++++------ swift/Sources/Fory/WriteContext.swift | 6 + swift/Sources/ForyMacro/ForyObjectMacro.swift | 24 +--- .../ForyTests/CollectionSerializerTests.swift | 2 +- swift/Tests/ForyTests/DecoderStateTests.swift | 2 +- .../ExternalTypeSerializationTests.swift | 6 +- swift/Tests/ForyTests/ForySwiftTests.swift | 106 +++--------------- .../ForyTests/GraphMemoryBudgetTests.swift | 2 +- .../Tests/ForyTests/TypeMetaDepthTests.swift | 8 +- 13 files changed, 66 insertions(+), 179 deletions(-) diff --git a/.agents/languages/swift.md b/.agents/languages/swift.md index 2d2d31ef90..ffdf8f6d46 100644 --- a/.agents/languages/swift.md +++ b/.agents/languages/swift.md @@ -37,13 +37,14 @@ Load this file when changing `swift/` or Swift xlang behavior. ignored declaration fields are budget-only and must not enter target access, construction, metadata, or wire code. Omitted large value storage must be declared explicitly and ignored. -- `@ForyStruct` supports protocol conformances but rejects every superclass during registration - finalization because macros cannot inspect inherited storage. SwiftSyntax represents both in one - inheritance clause, and Swift provides no public superclass query for arbitrary Swift classes; - keep the minimal `_getSuperclass` check finalization-owned and out of root hot paths. -- Swift registration finalization invokes application-owned `StructSerializer.foryFieldsInfo`. - Reject a same-`Fory` root that reenters while finalization is in progress, cache that first - failure, and do not retry partially finalized metadata builders. +- `@ForyStruct` supports protocol conformances but registration rejects every superclass because + macros cannot inspect inherited storage. SwiftSyntax represents both in one inheritance clause, + so keep the minimal `_getSuperclass` check in the existing TypeResolver registration preflight and + out of root hot paths. +- Swift registry lifecycle uses one authoritative frozen flag set by the first root serialization or + deserialization. Do not add finalizing, finalized, or failed states or cache a registration + preparation failure as a second lifecycle state. Registered TypeInfo owns lazy TypeMeta completion + after freeze; do not restore an eager whole-registry metadata pass. - Direct `Any` and `AnyObject` root overloads remain disfavored forwarding facades over `DynamicSerializer` and `DynamicSerializer`, including their Data-buffer forms. Arbitrary protocol roots explicitly select `DynamicSerializer`. Do not add an unconstrained diff --git a/swift/Sources/Fory/Fory.swift b/swift/Sources/Fory/Fory.swift index d243ca9936..eb7b00c6a9 100644 --- a/swift/Sources/Fory/Fory.swift +++ b/swift/Sources/Fory/Fory.swift @@ -363,7 +363,7 @@ public final class Fory { private func serializeRoot( _ body: (WriteContext) throws -> Void ) throws -> Data { - try typeResolver.finishRegistration() + typeResolver.freezeRegistration() let context = writeContext context.buffer.clear() defer { @@ -379,7 +379,7 @@ public final class Fory { to output: inout Data, _ body: (WriteContext) throws -> Void ) throws { - try typeResolver.finishRegistration() + typeResolver.freezeRegistration() let context = writeContext context.buffer.clear() defer { @@ -395,7 +395,7 @@ public final class Fory { data: Data, _ body: (ReadContext) throws -> R ) throws -> R { - try typeResolver.finishRegistration() + typeResolver.freezeRegistration() return try withReusableReadContext(data: data) { context in try readHead(buffer: context.buffer) let value = try body(context) @@ -411,7 +411,7 @@ public final class Fory { from buffer: ByteBuffer, _ body: (ReadContext) throws -> R ) throws -> R { - try typeResolver.finishRegistration() + typeResolver.freezeRegistration() readContext.buffer.swapState(with: buffer) readContext.remainingGraphMemoryBytes = Int(self.config.maxGraphMemoryBytes) readContext.remainingUnbackedContainerItems = self.config.maxUnbackedContainerItems diff --git a/swift/Sources/Fory/ReadContext.swift b/swift/Sources/Fory/ReadContext.swift index 2f59cf7570..a1dd309495 100644 --- a/swift/Sources/Fory/ReadContext.swift +++ b/swift/Sources/Fory/ReadContext.swift @@ -184,6 +184,9 @@ public final class ReadContext { return lastTypeInfo } let info = try typeResolver.requireTypeInfo(for: type) + if compatible { + try info.ensureTypeMeta(resolver: typeResolver) + } lastTypeInfo = info return info } diff --git a/swift/Sources/Fory/Serializer.swift b/swift/Sources/Fory/Serializer.swift index 6e8359f5df..7980f99182 100644 --- a/swift/Sources/Fory/Serializer.swift +++ b/swift/Sources/Fory/Serializer.swift @@ -66,7 +66,7 @@ public protocol StructSerializer: Serializer { static func foryFieldsInfo(trackRef: Bool) -> [TypeMeta.FieldInfo] /// Builds field metadata after all serializer registrations are visible to the resolver. - /// Serialization and deserialization hot paths must use finalized TypeInfo metadata instead. + /// The registered TypeInfo completes this metadata lazily on first use. static func foryFieldsInfo( trackRef: Bool, resolveSerializerTypeId: (Any.Type) throws -> TypeId diff --git a/swift/Sources/Fory/TypeResolver.swift b/swift/Sources/Fory/TypeResolver.swift index 961e70aa3d..b11287cd43 100644 --- a/swift/Sources/Fory/TypeResolver.swift +++ b/swift/Sources/Fory/TypeResolver.swift @@ -271,7 +271,7 @@ public final class TypeInfo: @unchecked Sendable { let evolving: Bool let namespace: MetaString let typeName: MetaString - /// Finalized local metadata. Generated compatible readers use this for local field comparison; + /// Local metadata prepared on demand. Generated compatible readers use it for field comparison; /// remote metadata remains exposed through `compatibleTypeMeta`. public private(set) var typeMeta: TypeMeta? public var compatibleTypeMeta: TypeMeta? { remoteCompatibleTypeMeta ?? typeMeta } @@ -501,8 +501,8 @@ public final class TypeInfo: @unchecked Sendable { } @inline(never) - func finalizeTypeMeta(resolver: TypeResolver) throws { - guard typeDefBytes == nil, let typeMetaFieldsBuilder else { + func ensureTypeMeta(resolver: TypeResolver) throws { + guard let typeMetaFieldsBuilder else { return } let fields = try typeMetaFieldsBuilder(resolver) @@ -516,7 +516,7 @@ public final class TypeInfo: @unchecked Sendable { ) let typeDefBytes = try typeMeta.encode() let typeDefHeaderHash = try encodedTypeDefHeaderHash(typeDefBytes) - self.typeMeta = try TypeMeta( + let resolvedTypeMeta = try TypeMeta( typeID: compatibleWireTypeID.rawValue, userTypeID: registerByName ? nil : userTypeID, namespace: namespace, @@ -525,6 +525,9 @@ public final class TypeInfo: @unchecked Sendable { fields: fields, headerHash: typeDefHeaderHash ) + // Publish only after all fallible work succeeds. A failed lazy build keeps its builder and + // can be retried without exposing partial metadata. + self.typeMeta = resolvedTypeMeta self.typeDefBytes = typeDefBytes self.typeDefHeaderHash = typeDefHeaderHash self.typeDefHasUserTypeFields = encodedTypeDefHasUserTypeFields(fields) @@ -623,20 +626,16 @@ final class TypeResolver { private let trackRef: Bool private var registryFrozen = false - private var registrationFinalized = false private var bySerializerType = UInt64Map(initialCapacity: 64) private var byTargetType = UInt64Map(initialCapacity: 64) private var byUserTypeID = UInt64Map(initialCapacity: 64) private var byTypeName: [TypeNameKey: TypeInfo] = [:] - private var registeredTypeInfos: [TypeInfo] = [] private var builtinTypeInfoByID: [TypeInfo?] = [] // Never key this cache by the complete header: its low 12 framing bits may vary on a hit. private var typeInfoByHeaderHash = UInt64Map(initialCapacity: 64) private var remoteSchemaVersionsByType: [String: Int] = [:] private var totalAcceptedSchemaVersions = 0 - private var registrationFailure: (any Error)? - init(trackRef: Bool = false) { self.trackRef = trackRef seedBuiltinTypeInfos() @@ -834,35 +833,8 @@ final class TypeResolver { } @inline(__always) - func finishRegistration() throws { - if registrationFinalized { - return - } - try finishRegistrationSlow() - } - - @inline(never) - private func finishRegistrationSlow() throws { - // Freezing and finalization are separate states: the first root permanently closes - // registration, while a partial builder failure must never be mistaken for success. - if let registrationFailure { - throw registrationFailure - } - // Application-owned field metadata can invoke another root on this Fory. Once frozen, - // an unfinished registry is already inside finalization and must not rerun its builders. - guard !registryFrozen else { - throw ForyError.invalidData("registration finalization is already in progress") - } + func freezeRegistration() { registryFrozen = true - do { - for typeInfo in registeredTypeInfos { - try typeInfo.finalizeTypeMeta(resolver: self) - } - registrationFinalized = true - } catch { - registrationFailure = error - throw error - } } func register(_ type: T.Type, id: UInt32) throws { @@ -1055,6 +1027,7 @@ final class TypeResolver { if let cached = typeInfoByHeaderHash.value(for: headerHash) { return cached } + try localTypeInfo.ensureTypeMeta(resolver: self) if localTypeInfo.typeDefHeaderHash == headerHash { // A validated 52-bit hash is the complete schema identity. The local metadata bytes // may use different current-frame low bits, so byte equality must not decide ownership. @@ -1062,7 +1035,7 @@ final class TypeResolver { return localTypeInfo } guard let localTypeMeta = localTypeInfo.typeMeta else { - throw ForyError.invalidData("local type metadata for \(localTypeInfo.typeID) is not finalized") + throw ForyError.invalidData("local type metadata for \(localTypeInfo.typeID) is unavailable") } let canonicalTypeMeta = try typeMeta.assigningFieldIDs(from: localTypeMeta) // Failed compatibility checks must not consult or mutate persistent remote accounting. @@ -1142,7 +1115,6 @@ final class TypeResolver { if let typeNameKey { byTypeName[typeNameKey] = typeInfo } - registeredTypeInfos.append(typeInfo) } @inline(never) @@ -1305,9 +1277,16 @@ final class TypeResolver { throw ForyError.invalidData( "structural serializer \(type) must use STRUCT, ENUM, or UNION type identity") } - if !T.isRefType, T.Target.self is AnyObject.Type { - throw ForyError.invalidData( - "value structural serializer \(type) cannot target class type \(T.Target.self)") + if let targetClass = T.Target.self as? AnyClass { + if !T.isRefType { + throw ForyError.invalidData( + "value structural serializer \(type) cannot target class type \(T.Target.self)") + } + if _getSuperclass(targetClass) != nil { + throw ForyError.invalidData( + "@ForyStruct classes cannot inherit from a superclass because macros cannot inspect inherited storage" + ) + } } return } diff --git a/swift/Sources/Fory/WriteContext.swift b/swift/Sources/Fory/WriteContext.swift index 0c4b3d5536..22ca1203ec 100644 --- a/swift/Sources/Fory/WriteContext.swift +++ b/swift/Sources/Fory/WriteContext.swift @@ -139,6 +139,9 @@ public final class WriteContext { return lastTypeInfo } let info = try typeResolver.requireTypeInfo(for: type) + if compatible { + try info.ensureTypeMeta(resolver: typeResolver) + } lastTypeInfo = info return info } @@ -150,6 +153,9 @@ public final class WriteContext { return lastTargetTypeInfo } let info = try typeResolver.requireTypeInfo(forTarget: type) + if compatible { + try info.ensureTypeMeta(resolver: typeResolver) + } lastTargetTypeInfo = info return info } diff --git a/swift/Sources/ForyMacro/ForyObjectMacro.swift b/swift/Sources/ForyMacro/ForyObjectMacro.swift index 4f342e0e57..0b1b230392 100644 --- a/swift/Sources/ForyMacro/ForyObjectMacro.swift +++ b/swift/Sources/ForyMacro/ForyObjectMacro.swift @@ -62,9 +62,6 @@ public struct ForyStructMacro: MemberMacro, ExtensionMacro { objectConfig.targetType ?? declaration.as(ClassDeclSyntax.self)?.name.text ?? "Self" - // SwiftSyntax uses the same inheritance clause for a superclass and protocol - // conformances, so semantic superclass validation must happen after registration. - let needsSuperclassValidation = declaration.as(ClassDeclSyntax.self)?.inheritanceClause != nil let successBodyAttribute = objectConfig.targetType != nil && parsed.fields.contains(where: { @@ -107,8 +104,7 @@ public struct ForyStructMacro: MemberMacro, ExtensionMacro { let compatibleTypeMetaDecl: DeclSyntax = DeclSyntax( stringLiteral: buildCompatibleTypeMetaFieldsDecl( sortedFields: sortedFields, - accessPrefix: accessPrefix, - needsSuperclassValidation: needsSuperclassValidation + accessPrefix: accessPrefix ) ) let defaultDecl: DeclSyntax = DeclSyntax( @@ -2549,26 +2545,11 @@ private func buildSchemaHashDecl(fields: [ParsedField]) throws -> String { private func buildCompatibleTypeMetaFieldsDecl( sortedFields: [ParsedField], - accessPrefix: String, - needsSuperclassValidation: Bool + accessPrefix: String ) -> String { let disabledExpr = compatibleTypeMetaFieldsExpr(sortedFields: sortedFields, trackRefExpression: "false") let enabledExpr = compatibleTypeMetaFieldsExpr(sortedFields: sortedFields, trackRefExpression: "true") let resolvedBody = resolvedTypeMetaFieldsBody(sortedFields: sortedFields) - let superclassValidation: String - if needsSuperclassValidation { - // Swift has no public API for querying an arbitrary Swift class's superclass. - // Keep the underscored query in this registration-finalization check only. - superclassValidation = """ - if _getSuperclass(Self.self) != nil { - throw ForyError.encodingError( - "@ForyStruct classes cannot inherit from a superclass because macros cannot inspect inherited storage" - ) - } - """ - } else { - superclassValidation = "" - } return """ private static let __foryFieldsInfoTrackRefDisabled: [TypeMeta.FieldInfo] = \(disabledExpr) private static let __foryFieldsInfoTrackRefEnabled: [TypeMeta.FieldInfo] = \(enabledExpr) @@ -2581,7 +2562,6 @@ private func buildCompatibleTypeMetaFieldsDecl( trackRef: Bool, resolveSerializerTypeId: (Any.Type) throws -> TypeId ) throws -> [TypeMeta.FieldInfo] { - \(superclassValidation) \(resolvedBody) } """ diff --git a/swift/Tests/ForyTests/CollectionSerializerTests.swift b/swift/Tests/ForyTests/CollectionSerializerTests.swift index 814478b829..d0be7f3aca 100644 --- a/swift/Tests/ForyTests/CollectionSerializerTests.swift +++ b/swift/Tests/ForyTests/CollectionSerializerTests.swift @@ -792,7 +792,7 @@ func generatedReadProgress() throws { let fory = Fory(config: Config(trackRef: false, compatible: true)) try fory.register(AdvancingReadStruct.self, id: 9705) - try fory.typeResolver.finishRegistration() + fory.typeResolver.freezeRegistration() let local = try fory.typeResolver.requireTypeInfo(for: AdvancingReadStruct.self) let emptyMeta = try TypeMeta( typeID: TypeId.compatibleStruct.rawValue, diff --git a/swift/Tests/ForyTests/DecoderStateTests.swift b/swift/Tests/ForyTests/DecoderStateTests.swift index 624374e2dd..211014f4c7 100644 --- a/swift/Tests/ForyTests/DecoderStateTests.swift +++ b/swift/Tests/ForyTests/DecoderStateTests.swift @@ -143,7 +143,7 @@ func remoteSchemaLogicalKeyLimitPersists() throws { let resolver = TypeResolver(config: config) try resolver.register(Person.self, id: 901) try resolver.register(Address.self, id: 902) - try resolver.finishRegistration() + resolver.freezeRegistration() let localTypeInfo = try resolver.requireTypeInfo(for: Person.self) func remoteTypeMeta( diff --git a/swift/Tests/ForyTests/ExternalTypeSerializationTests.swift b/swift/Tests/ForyTests/ExternalTypeSerializationTests.swift index f52401e2af..7c80303c33 100644 --- a/swift/Tests/ForyTests/ExternalTypeSerializationTests.swift +++ b/swift/Tests/ForyTests/ExternalTypeSerializationTests.swift @@ -1183,9 +1183,6 @@ func hiddenCarrierAliasIsRejected() throws { #expect(throws: ForyError.self) { _ = try fory.serialize(HiddenCarrierHolder(users: [])) } - #expect(throws: ForyError.self) { - _ = try fory.serialize(Int32(1)) - } #expect(throws: ForyError.self) { try fory.register(KeySerializer.self, id: 80) } @@ -1194,9 +1191,8 @@ func hiddenCarrierAliasIsRejected() throws { @Test func superclassIsRejected() throws { let fory = Fory() - try fory.register(SuperclassChild.self, id: 133) #expect(throws: ForyError.self) { - _ = try fory.serialize(SuperclassChild()) + try fory.register(SuperclassChild.self, id: 133) } } diff --git a/swift/Tests/ForyTests/ForySwiftTests.swift b/swift/Tests/ForyTests/ForySwiftTests.swift index 68c1639244..40cccd8ef7 100644 --- a/swift/Tests/ForyTests/ForySwiftTests.swift +++ b/swift/Tests/ForyTests/ForySwiftTests.swift @@ -202,47 +202,6 @@ struct LateMetaExt: Serializer, Equatable { } } -private final class RegistrationFinalizationError: Error {} - -private struct FailingRegistrationSerializer: StructSerializer { - typealias Target = Self - - static let failure = RegistrationFinalizationError() - static var staticTypeId: TypeId { .structType } - - static func defaultValue(_: ReadContext) throws -> Self { Self() } - static func writeData(_: Self, _: WriteContext) throws {} - static func readData(_: ReadContext) throws -> Self { Self() } - static func readCompatible(_: ReadContext, typeInfo _: TypeInfo) throws -> Self { Self() } - - static func foryFieldsInfo( - trackRef _: Bool, - resolveSerializerTypeId _: (Any.Type) throws -> TypeId - ) throws -> [TypeMeta.FieldInfo] { - throw failure - } -} - -private struct ReentrantRegistrationSerializer: StructSerializer { - typealias Target = Self - - nonisolated(unsafe) static var fieldsCallback: (() throws -> Void)? - static var staticTypeId: TypeId { .structType } - - static func defaultValue(_: ReadContext) throws -> Self { Self() } - static func writeData(_: Self, _: WriteContext) throws {} - static func readData(_: ReadContext) throws -> Self { Self() } - static func readCompatible(_: ReadContext, typeInfo _: TypeInfo) throws -> Self { Self() } - - static func foryFieldsInfo( - trackRef _: Bool, - resolveSerializerTypeId _: (Any.Type) throws -> TypeId - ) throws -> [TypeMeta.FieldInfo] { - try fieldsCallback?() - return [] - } -} - @ForyStruct struct LateMetaHolder: Equatable { var ext: LateMetaExt @@ -682,7 +641,7 @@ func schemaLimitTracksStructTypesSeparately() throws { let resolver = TypeResolver(config: config) try resolver.register(Person.self, id: 901) try resolver.register(Address.self, id: 902) - try resolver.finishRegistration() + resolver.freezeRegistration() func remoteTypeMeta(userTypeID: UInt32, fieldName: String) throws -> TypeMeta { try TypeMeta( @@ -728,7 +687,7 @@ func nonStructTypeMetaUsesSchemaLimit() throws { let config = Config(maxSchemaVersionsPerType: 1) let resolver = TypeResolver(config: config) try resolver.register(SparseStatus.self, name: "example.SharedEnum") - try resolver.finishRegistration() + resolver.freezeRegistration() let namespace = try MetaStringEncoder.namespace.encode("example") let typeName = try MetaStringEncoder.typeName.encode("SharedEnum") @@ -769,8 +728,9 @@ func localNonStructMetaBypassesLimit() throws { let config = Config(compatible: true, maxSchemaVersionsPerType: 1) let resolver = TypeResolver(config: config) try resolver.register(SparseStatus.self, name: "example.SharedEnum") - try resolver.finishRegistration() + resolver.freezeRegistration() let localTypeInfo = try resolver.requireTypeInfo(for: SparseStatus.self) + try localTypeInfo.ensureTypeMeta(resolver: resolver) let namespace = try MetaStringEncoder.namespace.encode("example") let typeName = try MetaStringEncoder.typeName.encode("SharedEnum") @@ -804,7 +764,7 @@ func localNonStructMetaBypassesLimit() throws { } @Test -func typeMetaUsesFinalRegistration() throws { +func typeMetaUsesAllRegistrations() throws { func holderTypeDefBytes(registerFieldTypeFirst: Bool) throws -> [UInt8] { let resolver = TypeResolver(config: Config(compatible: true)) if registerFieldTypeFirst { @@ -814,8 +774,10 @@ func typeMetaUsesFinalRegistration() throws { try resolver.register(LateMetaHolder.self, name: "example.LateMetaHolder") try resolver.register(LateMetaExt.self, name: "example.LateMetaExt") } - try resolver.finishRegistration() - return try resolver.requireTypeInfo(for: LateMetaHolder.self).typeDefBytes! + resolver.freezeRegistration() + let typeInfo = try resolver.requireTypeInfo(for: LateMetaHolder.self) + try typeInfo.ensureTypeMeta(resolver: resolver) + return typeInfo.typeDefBytes! } let fieldFirst = try holderTypeDefBytes(registerFieldTypeFirst: true) @@ -833,7 +795,7 @@ func failedSchemaDoesNotConsumeLimit() throws { let resolver = TypeResolver(config: config) try resolver.register(Person.self, id: 901) try resolver.register(Address.self, id: 902) - try resolver.finishRegistration() + resolver.freezeRegistration() func remoteTypeMeta(fieldName: String, fieldType: TypeMeta.FieldType) throws -> TypeMeta { try TypeMeta( @@ -894,7 +856,7 @@ func staticTypeRejectsWrongMetaOwner() throws { let resolver = TypeResolver(config: config) try resolver.register(Person.self, id: 901) try resolver.register(Address.self, id: 902) - try resolver.finishRegistration() + resolver.freezeRegistration() let wrongTypeMeta = try TypeMeta( typeID: TypeId.compatibleStruct.rawValue, userTypeID: 901, @@ -934,7 +896,7 @@ func cachedMetaChecksConcreteOwner() throws { let resolver = TypeResolver(config: config) try resolver.register(Person.self, id: 901) try resolver.register(Address.self, id: 902) - try resolver.finishRegistration() + resolver.freezeRegistration() let remote = try TypeMeta( typeID: TypeId.compatibleStruct.rawValue, userTypeID: 901, @@ -982,7 +944,7 @@ func failedStaticMetaDoesNotCount() throws { let resolver = TypeResolver(config: config) try resolver.register(Person.self, id: 901) try resolver.register(Address.self, id: 902) - try resolver.finishRegistration() + resolver.freezeRegistration() func typeMeta(userTypeID: UInt32, fieldName: String) throws -> TypeMeta { try TypeMeta( @@ -1221,48 +1183,6 @@ func registrationIsRejectedAfterFirstTopLevelUse() throws { } } -@Test -func finalizationPreservesFailure() throws { - let fory = Fory() - try fory.register(FailingRegistrationSerializer.self, id: 701) - - for _ in 0..<2 { - do { - _ = try fory.serialize(FailingRegistrationSerializer()) - Issue.record("expected registration finalization failure") - } catch { - #expect( - (error as? RegistrationFinalizationError) - === FailingRegistrationSerializer.failure - ) - } - } -} - -@Test -func reentrantFinalizationIsRejected() throws { - let fory = Fory() - var callbackCount = 0 - ReentrantRegistrationSerializer.fieldsCallback = { - callbackCount += 1 - _ = try fory.serialize(Int32(1)) - } - defer { - ReentrantRegistrationSerializer.fieldsCallback = nil - } - try fory.register(ReentrantRegistrationSerializer.self, id: 702) - - for _ in 0..<2 { - #expect(throws: ForyError.self) { - _ = try fory.serialize(ReentrantRegistrationSerializer()) - } - } - #expect(callbackCount == 1) - #expect(throws: ForyError.self) { - try fory.register(Address.self, id: 703) - } -} - @Test func serializeToAppendsRoots() throws { let fory = Fory() diff --git a/swift/Tests/ForyTests/GraphMemoryBudgetTests.swift b/swift/Tests/ForyTests/GraphMemoryBudgetTests.swift index d7b28cb1f3..b0f0add009 100644 --- a/swift/Tests/ForyTests/GraphMemoryBudgetTests.swift +++ b/swift/Tests/ForyTests/GraphMemoryBudgetTests.swift @@ -716,7 +716,7 @@ func unknownCaseChargesDynamicHeapBox() throws { let config = Config(compatible: false) let resolver = TypeResolver(config: config) try resolver.register(DynamicBoxBudgetV1.self, id: 9821) - try resolver.finishRegistration() + resolver.freezeRegistration() let value = DynamicBoxBudgetV1(first: 1, second: 2, third: 3, fourth: 4) let buffer = ByteBuffer() let writeContext = WriteContext( diff --git a/swift/Tests/ForyTests/TypeMetaDepthTests.swift b/swift/Tests/ForyTests/TypeMetaDepthTests.swift index cc92b3733f..7d4d55b896 100644 --- a/swift/Tests/ForyTests/TypeMetaDepthTests.swift +++ b/swift/Tests/ForyTests/TypeMetaDepthTests.swift @@ -61,7 +61,7 @@ func remoteTypeMetaUsesFixedDepth() throws { let config = Config(compatible: true, maxDepth: 2) let resolver = TypeResolver(config: config) try resolver.register(Address.self, id: 902) - try resolver.finishRegistration() + resolver.freezeRegistration() func context(_ encoded: [UInt8]) -> ReadContext { let buffer = ByteBuffer() @@ -129,7 +129,7 @@ func cachedMetaUsesHeaderHash() throws { let resolver = TypeResolver(config: config) try resolver.register(Person.self, id: 901) try resolver.register(Address.self, id: 902) - try resolver.finishRegistration() + resolver.freezeRegistration() let remote = try TypeMeta( typeID: TypeId.compatibleStruct.rawValue, userTypeID: 901, @@ -189,10 +189,12 @@ func localMetaUsesHeaderHash() throws { let resolver = TypeResolver(config: config) try resolver.register(Person.self, id: 901) try resolver.register(Address.self, id: 902) - try resolver.finishRegistration() + resolver.freezeRegistration() let firstTypeInfo = try resolver.requireTypeInfo(for: Person.self) + try firstTypeInfo.ensureTypeMeta(resolver: resolver) let firstBytes = try #require(firstTypeInfo.typeDefBytes) let localTypeInfo = try resolver.requireTypeInfo(for: Address.self) + try localTypeInfo.ensureTypeMeta(resolver: resolver) let headerHash = try #require(localTypeInfo.typeDefHeaderHash) let currentBody: [UInt8] = [0xD1, 0xD2, 0xD3] let currentHeader = (headerHash << 12) | UInt64(currentBody.count) From 6465d70674bf6f2b25670075ae7badab016b90c3 Mon Sep 17 00:00:00 2001 From: chaokunyang Date: Sun, 30 Aug 2026 02:57:17 +0800 Subject: [PATCH 094/168] refactor(python): remove registry finalization machinery --- python/pyfory/_fory.py | 204 +---- python/pyfory/registry.py | 309 +++---- python/pyfory/serialization.pyx | 30 +- python/pyfory/struct.pxi | 14 +- python/pyfory/struct.py | 11 +- python/pyfory/tests/test_class_serializer.py | 65 +- python/pyfory/tests/test_collection_safety.py | 8 - python/pyfory/tests/test_function.py | 23 +- .../pyfory/tests/test_graph_memory_budget.py | 22 +- python/pyfory/tests/test_method.py | 61 +- python/pyfory/tests/test_pickle_buffer.py | 47 + python/pyfory/tests/test_policy.py | 62 +- python/pyfory/tests/test_reduce_serializer.py | 24 +- python/pyfory/tests/test_ref_tracking.py | 6 +- python/pyfory/tests/test_serializer.py | 810 +----------------- .../pyfory/tests/test_stateful_serializer.py | 10 +- python/pyfory/tests/test_struct.py | 11 +- python/pyfory/tests/test_thread_safe.py | 427 --------- 18 files changed, 307 insertions(+), 1837 deletions(-) diff --git a/python/pyfory/_fory.py b/python/pyfory/_fory.py index a504fed858..d14dd536b7 100644 --- a/python/pyfory/_fory.py +++ b/python/pyfory/_fory.py @@ -16,7 +16,6 @@ # under the License. import os -import threading from abc import ABC, abstractmethod from typing import Iterable, Optional, Union @@ -27,7 +26,6 @@ from pyfory.policy import DEFAULT_POLICY, DeserializationPolicy from pyfory.resolver import NOT_NULL_VALUE_FLAG -from pyfory.type_util import normalize_fory_type from pyfory.types import TypeId DYNAMIC_TYPE_ID = -1 @@ -90,10 +88,9 @@ class Fory: objects and cross-language reference metadata; Python native mode handles the broader Python object graph surface, including circular Python objects. - In Python native mode (xlang=False), Fory can serialize a configured Python - type surface including dataclasses, classes with custom serialization methods, - and local functions/classes. Register application types and native carriers - before the first root operation, which permanently freezes the registry. + In Python native mode (xlang=False), Fory can serialize Python objects including + dataclasses, classes with custom serialization methods, and local functions/classes. + With strict mode disabled, policy-authorized types are discovered lazily. In xlang mode, the default, Fory serializes objects in a format that can be deserialized by other Fory-supported languages (Java, Go, Rust, C++, etc.). @@ -155,11 +152,11 @@ def __init__( Args: xlang: Enable xlang mode. When False, uses - Python native mode supporting configured Python objects (dataclasses, - __reduce__, local functions/classes). Register all application types and - native carriers before the first root operation. When True, uses the xlang wire format - compatible with other Fory languages (Java, Go, Rust, etc), but Python- - specific features like functions and __reduce__ methods are not supported. + Python native mode supporting Python objects such as dataclasses, + __reduce__, and local functions/classes. When True, uses the xlang wire + format compatible with other Fory languages (Java, Go, Rust, etc), but + Python-specific features like functions and __reduce__ methods are not + supported. ref: Enable reference tracking for shared references and Python native-mode circular references. When enabled, duplicate objects are stored once. @@ -169,11 +166,10 @@ def __init__( classes (default: True). Compatible metadata for an unregistered remote Struct uses the fixed data-only UnknownStruct carrier instead of loading or generating the sender-named class. Disabling strict mode authorizes - configured native carriers. Policy-authorized module globals may be - resolved while reading trusted native payloads, but the first root still - freezes type and serializer registration. Dynamic application types can - be insecure if malicious code exists in __new__/__init__/__eq__/__hash__ - methods. + lazy native type discovery. Policy-authorized module globals may be + resolved while reading trusted native payloads. Dynamic application types + can be insecure if malicious code exists in + __new__/__init__/__eq__/__hash__ methods. **WARNING**: Only disable in trusted environments. When disabling strict mode, you should provide a custom `policy` parameter to control which types are allowed. We are not responsible for security risks when this option @@ -431,8 +427,7 @@ def dump(self, obj, stream): the passed object (or a view of it) is unsupported. If your sink needs retention, copy bytes inside ``write``. """ - if not self.type_resolver._registry_finalization_complete: - self.type_resolver._freeze_registry() + self.type_resolver._freeze_registry() try: self.buffer.set_writer_index(0) output_stream = Buffer.wrap_output_stream(stream) @@ -488,8 +483,7 @@ def serialize( >>> print(type(data)) """ - if not self.type_resolver._registry_finalization_complete: - self.type_resolver._freeze_registry() + self.type_resolver._freeze_registry() try: write_buffer = self._serialize( obj, @@ -567,8 +561,7 @@ def deserialize( >>> print(obj) {'key': 'value'} """ - if not self.type_resolver._registry_finalization_complete: - self.type_resolver._freeze_registry() + self.type_resolver._freeze_registry() try: return self._deserialize(buffer, buffers, unsupported_objects) finally: @@ -633,44 +626,6 @@ def reset(self): self.reset_read() -class _Registration: - __slots__ = ( - "cls", - "declared_type", - "name", - "operation", - "serializer", - "type_id", - ) - - def __init__(self, operation, cls, type_id, name, serializer): - self.operation = operation - self.cls = cls - self.declared_type = normalize_fory_type(cls) - self.type_id = int(type_id) if isinstance(type_id, int) and not isinstance(type_id, bool) else type_id - self.name = str(name) if isinstance(name, str) else name - self.serializer = serializer - - def same_request(self, other): - return ( - self.operation == other.operation - and self.declared_type == other.declared_type - and type(self.type_id) is type(other.type_id) - and self.type_id == other.type_id - and type(self.name) is type(other.name) - and self.name == other.name - and self.serializer is other.serializer - ) - - def apply(self, fory): - getattr(fory, self.operation)( - self.cls, - type_id=self.type_id, - name=self.name, - serializer=self.serializer, - ) - - class ThreadSafeFory: """ Thread-safe wrapper for Fory using instance pooling. @@ -682,14 +637,9 @@ class ThreadSafeFory: All type registrations must be performed before the first root serialization or deserialization attempt to ensure consistency across all pooled instances. Registration - remains closed even when that first operation fails. Custom serializer registrations accept - a serializer class or factory so every pooled instance owns a serializer bound to its own - resolver and declared type. A serializer factory cannot reuse one serializer across children; - use ``fory_factory`` for configured serializer instances. + remains closed even when that first operation fails. Args: - fory_factory (Callable): Optional factory that creates and configures each pooled Fory. - When omitted, remaining keyword arguments are forwarded to Fory. xlang (bool): Whether to enable xlang mode. Defaults to True. ref (bool): Whether to enable reference tracking. Defaults to False. strict (bool): Whether to require type registration. Defaults to True. @@ -730,16 +680,12 @@ class ThreadSafeFory: """ def __init__(self, fory_factory=None, **kwargs): + import threading + self._config = kwargs self._fory_factory = fory_factory - self._registrations = [] + self._callbacks = [] self._lock = threading.Lock() - self._registration_lock = threading.RLock() - self._registration_depth = 0 - self._registration_fory = None - self._building_thread = None - # Number of accepted registrations already applied to the child being built. - self._replay_limit = 0 self._pool = [] if fory_factory is not None: self._fory_class = None @@ -749,106 +695,30 @@ def __init__(self, fory_factory=None, **kwargs): self._fory_class = CythonFory else: self._fory_class = Fory - self._root_started = False - - def _build_fory(self): - thread_id = threading.get_ident() - with self._registration_lock: - if self._building_thread == thread_id: - raise RuntimeError("Cannot start a root serialization or deserialization operation while a Fory instance is being built.") - self._building_thread = thread_id - self._replay_limit = 0 - try: - if self._fory_factory is not None: - fory = self._fory_factory() - else: - fory = self._fory_class(**self._config) - for registration in self._registrations: - registration.apply(fory) - self._replay_limit += 1 - return fory - finally: - self._replay_limit = 0 - self._building_thread = None + self._registry_frozen = False def _get_fory(self): - # Check the active builder before the pool: factory callbacks must not evade root-reentry - # rejection by returning an instance to the facade that is still building it. - building_thread = self._building_thread - if building_thread is not None and building_thread == threading.get_ident(): - with self._lock: - self._root_started = True - raise RuntimeError("Cannot start a root serialization or deserialization operation while a Fory instance is being built.") with self._lock: + self._registry_frozen = True if self._pool: return self._pool.pop() - self._root_started = True - # Nested registrations share the staging instance, but a root may reuse it only - # after the outermost registration has published its descriptor. - if self._registration_depth == 0: - fory = self._registration_fory + if self._fory_factory is not None: + fory = self._fory_factory() else: - fory = None - self._registration_fory = None - if fory is not None: - # The validation instance already contains every published registration. + fory = self._fory_class(**self._config) + for callback in self._callbacks: + callback(fory) return fory - # Factories and registrations are application code. Keep them outside the - # non-reentrant pool lock so a callback can enter the same facade root. - return self._build_fory() def _return_fory(self, fory): with self._lock: self._pool.append(fory) - def _register_registration(self, registration): - # The reentrant lock gives nested facade registrations one publication order while the - # pool lock keeps a concurrently starting root atomic with registration publication. - with self._registration_lock: - if self._building_thread == threading.get_ident(): - index = 0 - while index < self._replay_limit: - accepted = self._registrations[index] - if accepted.same_request(registration): - return - index += 1 - raise RuntimeError("A child may replay only a registration already accepted by this ThreadSafeFory.") - with self._lock: - self._check_registration_open() - self._registration_depth += 1 - registration_fory = self._registration_fory - try: - if registration_fory is None: - registration_fory = self._build_fory() - with self._lock: - self._check_registration_open() - self._registration_fory = registration_fory - registration.apply(registration_fory) - with self._lock: - self._check_registration_open() - self._registrations.append(registration) - self._registration_fory = registration_fory - except BaseException: - with self._lock: - if self._registration_depth == 1: - self._registration_fory = None - raise - finally: - with self._lock: - self._registration_depth -= 1 - - def _check_registration_open(self): - if self._root_started: - raise RuntimeError("Cannot register types after the first root serialization or deserialization operation has started.") - - @staticmethod - def _check_serializer_factory(serializer): - if serializer is None: - return - from pyfory.serializer import Serializer - - if isinstance(serializer, Serializer): - raise TypeError("ThreadSafeFory requires a serializer class or factory; use fory_factory to install serializer instances per Fory") + def _register_callback(self, callback): + with self._lock: + if self._registry_frozen: + raise RuntimeError("Cannot register types after the first root serialization or deserialization operation has started.") + self._callbacks.append(callback) def register( self, @@ -858,8 +728,7 @@ def register( name: str = None, serializer=None, ): - self._check_serializer_factory(serializer) - self._register_registration(_Registration("register", cls, type_id, name, serializer)) + self._register_callback(lambda f: f.register(cls, type_id=type_id, name=name, serializer=serializer)) def register_type( self, @@ -869,8 +738,7 @@ def register_type( name: str = None, serializer=None, ): - self._check_serializer_factory(serializer) - self._register_registration(_Registration("register_type", cls, type_id, name, serializer)) + self._register_callback(lambda f: f.register_type(cls, type_id=type_id, name=name, serializer=serializer)) def register_union( self, @@ -880,8 +748,10 @@ def register_union( name: str = None, serializer=None, ): - self._check_serializer_factory(serializer) - self._register_registration(_Registration("register_union", cls, type_id, name, serializer)) + self._register_callback(lambda f: f.register_union(cls, type_id=type_id, name=name, serializer=serializer)) + + def register_serializer(self, cls: type, serializer): + self._register_callback(lambda f: f.register_serializer(cls, serializer)) def serialize( self, diff --git a/python/pyfory/registry.py b/python/pyfory/registry.py index 60672c196b..9a3fe0d042 100644 --- a/python/pyfory/registry.py +++ b/python/pyfory/registry.py @@ -220,17 +220,11 @@ def _construct_serializer(serializer_factory, type_resolver, cls): for nargs, args in ( (2, (type_resolver, cls)), (1, (type_resolver,)), + (0, ()), ): if _accepts_n_positional_args(serializer_factory, nargs): - serializer = serializer_factory(*args) - if not isinstance(serializer, (Serializer, CythonSerializer)): - raise TypeError("Serializer factory must return a supported Serializer carrier") - if serializer.type_resolver is not type_resolver: - raise TypeError("Serializer factory returned a serializer bound to a different resolver") - if normalize_fory_type(serializer.type_) != normalize_fory_type(cls): - raise TypeError("Serializer factory returned a serializer bound to a different type") - return serializer - raise TypeError(f"Unsupported serializer constructor for {serializer_factory!r}; expected `(type_resolver, cls)` or `(type_resolver)`.") + return serializer_factory(*args) + raise TypeError(f"Unsupported serializer constructor for {serializer_factory!r}; expected `(type_resolver, cls)`, `(type_resolver)`, or `()`.") def _split_registration_name(name: str): @@ -378,9 +372,7 @@ class TypeResolver: "meta_share", "_internal_py_serializer_map", "_actual_type_resolver", - "_registry_finalization_complete", "_registry_frozen", - "_registry_finalizing", ) def __init__(self, config, *, shared_registry): @@ -421,41 +413,14 @@ def __init__(self, config, *, shared_registry): self.meta_share = config.meta_share self._internal_py_serializer_map = {} self._actual_type_resolver = self - # Fory exposes this resolver, so the resolver must own the root-use gate; - # facade-only state would leave direct registration methods mutable. Freeze - # is permanent, while completion records only a successful finalization. - self._registry_finalization_complete = False self._registry_frozen = False - self._registry_finalizing = False def _check_registry_mutable(self): if self._registry_frozen: raise RuntimeError("Cannot register types or serializers after the first root operation has started") - def _needs_registration_finalization(self, type_info): - if type_info.serializer is None: - return True - if is_struct_type(type_info.type_id): - from pyfory.struct import DataClassStubSerializer - - if isinstance(type_info.serializer, DataClassStubSerializer): - return True - return self.meta_share and type_info.type_def is None and TypeId.is_type_share_meta(type_info.type_id) - def _freeze_registry(self): - if self._registry_finalization_complete: - return - if self._registry_frozen: - raise RuntimeError("Registry finalization did not complete") self._registry_frozen = True - self._registry_finalizing = True - try: - for type_info in self._types_info.values(): - if self._needs_registration_finalization(type_info): - self._set_type_info(type_info) - self._registry_finalization_complete = True - finally: - self._registry_finalizing = False def _set_actual_resolver(self, type_resolver): # Cython mode injects the compiled companion before initialize() so all @@ -469,7 +434,6 @@ def initialize(self): self._initialize_py() else: self._initialize_xlang() - self._get_nonexist_enum_type_info() def _initialize_py(self): register = functools.partial(self._register_type, internal=True) @@ -642,22 +606,21 @@ def register_union( serializer=None, ): self._check_registry_mutable() - cls = normalize_fory_type(cls) - if cls in self._types_info: - raise TypeError(f"{cls} registered already") namespace, typename = _split_registration_name(name) if serializer is None: raise TypeError("register_union requires a serializer") + if serializer is not None and not isinstance(serializer, Serializer): + serializer = _construct_serializer( + serializer, + self._actual_type_resolver, + cls, + ) + self._check_registry_mutable() if typename is not None and type_id is not None: raise TypeError(f"type name {typename} and id {type_id} should not be set at the same time") - if type_id is None: - if typename is None: - user_type_id = None - type_id = TypeId.TYPED_UNION - else: - user_type_id = NO_USER_TYPE_ID - type_id = TypeId.NAMED_UNION - elif type_id != 0: + if typename is None and type_id is None: + type_id = self._next_type_id() + if type_id not in {0, None}: user_type_id = type_id type_id = TypeId.TYPED_UNION else: @@ -690,10 +653,26 @@ def _register_type( if internal: if type_id is not None and type_id >= 0 and type_id > 0xFF: raise ValueError(f"Internal type id overflow: {type_id}") - if cls in self._types_info: - if type_id is None and typename is None and namespace is None and serializer is None and user_type_id in {None, NO_USER_TYPE_ID}: - return self._types_info[cls] - raise TypeError(f"{cls} registered already") + else: + if user_type_id not in {None, NO_USER_TYPE_ID} and (user_type_id < 0 or user_type_id > 0xFFFFFFFE): + raise ValueError(f"user_type_id must be in range [0, 0xfffffffe], got {user_type_id}") + if serializer is not None and not isinstance(serializer, Serializer): + serializer = _construct_serializer( + serializer, + self._actual_type_resolver, + cls, + ) + if not internal: + self._check_registry_mutable() + if ( + cls in self._types_info + and type_id is None + and typename is None + and namespace is None + and serializer is None + and user_type_id in {None, NO_USER_TYPE_ID} + ): + return self._types_info[cls] if not internal and not self.xlang and not self.strict and type_id is None and typename is None and namespace is None and serializer is None: # Native carriers keep their reserved discovery identity when users # configure them explicitly; application classes retain struct registration. @@ -701,8 +680,12 @@ def _register_type( if typeinfo is not None: return typeinfo n_params = len({typename, type_id, None}) - 1 + if n_params == 0 and typename is None: + type_id = self._next_type_id() if n_params == 2: raise TypeError(f"type name {typename} and id {type_id} should not be set at the same time") + if cls in self._types_info: + raise TypeError(f"{cls} registered already") return self._register_xtype( cls, type_id=type_id, @@ -730,13 +713,10 @@ def _register_xtype( evolving = object_meta.evolving if serializer is None: if issubclass(cls, enum.Enum): + serializer = EnumSerializer(self._actual_type_resolver, cls) if type_id is None: - if typename is None: - type_id = TypeId.ENUM - user_type_id = None - else: - type_id = TypeId.NAMED_ENUM - user_type_id = NO_USER_TYPE_ID + type_id = TypeId.NAMED_ENUM + user_type_id = NO_USER_TYPE_ID else: user_type_id = type_id type_id = TypeId.ENUM @@ -744,34 +724,22 @@ def _register_xtype( serializer = None if self.meta_share and evolving: if type_id is None: - if typename is None: - type_id = TypeId.COMPATIBLE_STRUCT - user_type_id = None - else: - type_id = TypeId.NAMED_COMPATIBLE_STRUCT - user_type_id = NO_USER_TYPE_ID + type_id = TypeId.NAMED_COMPATIBLE_STRUCT + user_type_id = NO_USER_TYPE_ID else: user_type_id = type_id type_id = TypeId.COMPATIBLE_STRUCT else: if type_id is None: - if typename is None: - type_id = TypeId.STRUCT - user_type_id = None - else: - type_id = TypeId.NAMED_STRUCT - user_type_id = NO_USER_TYPE_ID + type_id = TypeId.NAMED_STRUCT + user_type_id = NO_USER_TYPE_ID else: user_type_id = type_id type_id = TypeId.STRUCT elif not internal: if type_id is None: - if typename is None: - type_id = TypeId.EXT - user_type_id = None - else: - type_id = TypeId.NAMED_EXT - user_type_id = NO_USER_TYPE_ID + type_id = TypeId.NAMED_EXT + user_type_id = NO_USER_TYPE_ID else: user_type_id = type_id type_id = TypeId.EXT @@ -797,42 +765,7 @@ def __register_type( serializer: Serializer = None, internal: bool = False, ): - namespace_metastr = None - typename_metastr = None - if typename is not None: - if namespace is None: - splits = typename.rsplit(".", 1) - if len(splits) == 2: - namespace, typename = splits - else: - namespace = "" - else: - namespace = namespace or "" - if not typename: - raise ValueError("type name must not be empty") - if ( - not internal - and needs_user_type_id(type_id) - and user_type_id is not None - and (not isinstance(user_type_id, int) or isinstance(user_type_id, bool) or user_type_id < 0 or user_type_id > 0xFFFFFFFE) - ): - raise ValueError(f"user_type_id must be an integer in range [0, 0xfffffffe], got {user_type_id}") - self._preflight_registration( - cls, - type_id=type_id, - user_type_id=user_type_id, - namespace=namespace, - typename=typename, - ) - if typename is not None: - namespace_metastr = self.namespace_encoder.encode(namespace or "") - typename_metastr = self.typename_encoder.encode(typename) - if serializer is not None and not isinstance(serializer, Serializer): - serializer = _construct_serializer( - serializer, - self._actual_type_resolver, - cls, - ) + dynamic_type = type_id is not None and type_id < 0 # In metashare mode, for struct types, we want to keep serializer=None # so that _set_type_info will be called to create the TypeDef-based serializer # This applies to both types registered by name and by ID @@ -840,34 +773,35 @@ def __register_type( if should_create_serializer: serializer = self._create_serializer(cls) - # Serializer construction can run application code and mutate or freeze this registry. - # Recheck both invariants before allocating an automatic ID or publishing state. - self._check_registry_mutable() - self._preflight_registration( - cls, - type_id=type_id, - user_type_id=user_type_id, - namespace=namespace, - typename=typename, - ) - # Allocate automatic IDs only at the common commit point. Nested registrations therefore - # receive IDs in publication order without reservations or rollback state. - if type_id is None: - type_id = self._next_type_id() - elif not internal and needs_user_type_id(type_id) and user_type_id is None: - user_type_id = self._next_type_id() - dynamic_type = type_id < 0 + if not internal: + self._check_registry_mutable() + if needs_user_type_id(type_id) and user_type_id not in {None, NO_USER_TYPE_ID}: + existing = self._user_type_id_to_type_info.get(user_type_id) + if existing is not None and existing.cls is not cls: + raise TypeError(f"user_type_id {user_type_id} already registered for {existing.cls}") if serializer is not None and type_id in _NO_REF_NUMERIC_TYPE_IDS: serializer.need_to_write_ref = False if typename is None: typeinfo = TypeInfo(cls, type_id, user_type_id, serializer, None, None, dynamic_type) else: - ns_meta_bytes = self.shared_registry.get_encoded_meta_string(namespace_metastr) - type_meta_bytes = self.shared_registry.get_encoded_meta_string(typename_metastr) + if namespace is None: + splits = typename.rsplit(".", 1) + if len(splits) == 2: + namespace, typename = splits + else: + namespace = "" # Use empty string for consistency with lookup + if not typename: + raise ValueError("type name must not be empty") + ns_metastr = self.namespace_encoder.encode(namespace or "") + ns_meta_bytes = self.shared_registry.get_encoded_meta_string(ns_metastr) + type_metastr = self.typename_encoder.encode(typename) + type_meta_bytes = self.shared_registry.get_encoded_meta_string(type_metastr) typeinfo = TypeInfo(cls, type_id, user_type_id, serializer, ns_meta_bytes, type_meta_bytes, dynamic_type) + if typename is not None: self._named_type_to_type_info[(namespace, typename)] = typeinfo self._ns_type_to_type_info[(ns_meta_bytes, type_meta_bytes)] = typeinfo + self._types_info[cls] = typeinfo if type_id is not None and type_id != 0: if needs_user_type_id(type_id) and user_type_id not in {None, NO_USER_TYPE_ID}: if user_type_id not in self._user_type_id_to_type_info or not internal: @@ -876,33 +810,9 @@ def __register_type( elif not TypeId.is_namespaced_type(type_id): if type_id not in self._type_id_to_type_info or not internal: self._type_id_to_type_info[type_id] = typeinfo - self._types_info[cls] = typeinfo self._index_python_type(cls) return typeinfo - def _preflight_registration( - self, - cls, - *, - type_id, - user_type_id, - namespace, - typename, - ): - if cls in self._types_info: - raise TypeError(f"{cls} registered already") - if typename is not None: - existing = self._named_type_to_type_info.get((namespace, typename)) - if existing is not None and existing.cls is not cls: - raise TypeError(f"type name {(namespace, typename)!r} already registered for {existing.cls}") - if needs_user_type_id(type_id) and user_type_id not in { - None, - NO_USER_TYPE_ID, - }: - existing = self._user_type_id_to_type_info.get(user_type_id) - if existing is not None and existing.cls is not cls: - raise TypeError(f"user_type_id {user_type_id} already registered for {existing.cls}") - def _index_python_type(self, cls): if self._python_name_to_type is None or not isinstance(cls, type): return @@ -935,7 +845,7 @@ def register_serializer(self, cls, serializer): prev_user_type_id = typeinfo.user_type_id if needs_user_type_id(prev_type_id) and prev_user_type_id not in {None, NO_USER_TYPE_ID}: self._user_type_id_to_type_info.pop(prev_user_type_id, None) - elif not TypeId.is_namespaced_type(prev_type_id): + else: self._type_id_to_type_info.pop(prev_type_id, None) if typeinfo.serializer is not serializer: if typeinfo.typename_bytes is not None: @@ -944,10 +854,9 @@ def register_serializer(self, cls, serializer): else: typeinfo.type_id = TypeId.EXT typeinfo.serializer = serializer - typeinfo.type_def = None if needs_user_type_id(typeinfo.type_id) and typeinfo.user_type_id not in {None, NO_USER_TYPE_ID}: self._user_type_id_to_type_info[typeinfo.user_type_id] = typeinfo - elif not TypeId.is_namespaced_type(typeinfo.type_id): + else: self._type_id_to_type_info[typeinfo.type_id] = typeinfo def get_serializer(self, cls: type): @@ -965,13 +874,12 @@ def get_type_info(self, cls, create=True): type_info = self._types_info.get(cls) if type_info is not None: if type_info.serializer is None: - self._finalize_type_info(type_info) + self._set_type_info(type_info) return type_info elif not create: return None if self.require_registration and not issubclass(cls, Enum): raise TypeUnregisteredError(f"{cls} not registered") - self._check_registry_mutable() if cls is NonExistEnum: return self._get_nonexist_enum_type_info() logger.info("Type %s not registered", cls) @@ -1011,55 +919,40 @@ def _register_inferred_type(self, cls, native_only=False): namespace=cls.__module__, typename=cls.__qualname__, serializer=serializer, + internal=True, ) - def _finalize_type_info(self, typeinfo): - if not self._registry_finalizing: - self._check_registry_mutable() - return self._set_type_info(typeinfo) - def _set_type_info(self, typeinfo): serializer_type_resolver = self._actual_type_resolver type_id = typeinfo.type_id - previous_serializer = typeinfo.serializer - previous_type_def = typeinfo.type_def - try: - if is_struct_type(type_id): - from pyfory.struct import DataClassSerializer, DataClassStubSerializer - - if typeinfo.serializer is None or isinstance(typeinfo.serializer, DataClassStubSerializer): - # Publish the stub only for recursive construction. If later - # work fails, restore the pre-finalization state so a frozen - # registry cannot retain a replaceable partial descriptor. - typeinfo.serializer = DataClassStubSerializer(serializer_type_resolver, typeinfo.cls) - - if self.meta_share: - type_def = encode_typedef(serializer_type_resolver, typeinfo.cls) - if type_def is not None: - typeinfo.serializer = type_def.create_serializer(serializer_type_resolver) - typeinfo.type_def = type_def - else: - typeinfo.serializer = DataClassSerializer(serializer_type_resolver, typeinfo.cls) - else: - typeinfo.serializer = DataClassSerializer(serializer_type_resolver, typeinfo.cls) - elif self.meta_share and typeinfo.type_def is None and TypeId.is_type_share_meta(type_id): - typeinfo.type_def = encode_typedef(serializer_type_resolver, typeinfo.cls) + if is_struct_type(type_id): + from pyfory.struct import DataClassSerializer, DataClassStubSerializer + + # Set a stub serializer FIRST to break recursion for self-referencing types. + # get_type_info() only calls _set_type_info when serializer is None, + # so setting stub first prevents re-entry for circular type references. + typeinfo.serializer = DataClassStubSerializer(serializer_type_resolver, typeinfo.cls) + + if self.meta_share: + type_def = encode_typedef(serializer_type_resolver, typeinfo.cls) + if type_def is not None: + typeinfo.serializer = type_def.create_serializer(serializer_type_resolver) + typeinfo.type_def = type_def + else: + typeinfo.serializer = DataClassSerializer(serializer_type_resolver, typeinfo.cls) else: - if typeinfo.serializer is None: - typeinfo.serializer = self._create_serializer(typeinfo.cls) - if ( - self.meta_share - and typeinfo.type_def is None - and ( - TypeId.is_namespaced_type(type_id) - or (needs_user_type_id(type_id) and typeinfo.user_type_id is not None and typeinfo.user_type_id != NO_USER_TYPE_ID) - ) - ): - typeinfo.type_def = encode_typedef(serializer_type_resolver, typeinfo.cls) - except BaseException: - typeinfo.serializer = previous_serializer - typeinfo.type_def = previous_type_def - raise + typeinfo.serializer = DataClassSerializer(serializer_type_resolver, typeinfo.cls) + else: + typeinfo.serializer = self._create_serializer(typeinfo.cls) + if ( + self.meta_share + and typeinfo.type_def is None + and ( + TypeId.is_namespaced_type(type_id) + or (needs_user_type_id(type_id) and typeinfo.user_type_id is not None and typeinfo.user_type_id != NO_USER_TYPE_ID) + ) + ): + typeinfo.type_def = encode_typedef(serializer_type_resolver, typeinfo.cls) return typeinfo @@ -1191,7 +1084,6 @@ def _load_metabytes_to_type_info(self, ns_metabytes, type_metabytes): if self.strict: name = ns + "." + typename if ns else typename raise TypeUnregisteredError(f"{name} not registered") - self._check_registry_mutable() cls = load_class(ns + "#" + typename, policy=self.policy) typeinfo = self.get_type_info(cls) self._cache_wire_type_info(ns_metabytes, type_metabytes, typeinfo) @@ -1275,7 +1167,6 @@ def _get_nonexist_enum_type_info(self): typeinfo = self._types_info.get(NonExistEnum) if typeinfo is None: - self._check_registry_mutable() serializer = NonExistEnumSerializer(self._actual_type_resolver) typeinfo = TypeInfo(NonExistEnum, TypeId.ENUM, NO_USER_TYPE_ID, serializer, None, None, False) self._types_info[NonExistEnum] = typeinfo @@ -1311,7 +1202,7 @@ def write_shared_type_meta(self, write_context, typeinfo): buffer.write_var_uint32(index << 1) type_def = typeinfo.type_def if type_def is None: - self._finalize_type_info(typeinfo) + self._set_type_info(typeinfo) type_def = typeinfo.type_def buffer.write_bytes(type_def.encoded) @@ -1494,7 +1385,7 @@ def _read_uncached_type_info(self, buffer, header, expected_typeinfo=None): raise TypeError("Type metadata owner does not match the declared type") if local_type_info is not None: if local_type_info.type_def is None: - self._finalize_type_info(local_type_info) + self._set_type_info(local_type_info) if local_type_info.type_def is not None: local_header = int.from_bytes(local_type_info.type_def.encoded[:8], "little", signed=True) if _typedef_hash_key(local_header) == hash_key: diff --git a/python/pyfory/serialization.pyx b/python/pyfory/serialization.pyx index 4a909a1d75..55720f413b 100644 --- a/python/pyfory/serialization.pyx +++ b/python/pyfory/serialization.pyx @@ -269,12 +269,6 @@ cdef class TypeResolver: cdef flat_hash_map[uint32_t, PyObject *] _c_user_type_id_to_type_info cdef flat_hash_map[uint64_t, PyObject *] _c_types_info cdef flat_hash_map[pair[int64_t, int64_t], PyObject *] _c_meta_hash_to_type_info - # The Python resolver owns registry mutability. Native completion is published only - # after that owner succeeds and every native table is synchronized. Failure is - # retained without its exception so a partial native cache is never retried or used. - cdef bint registry_finalization_complete - cdef bint registry_finalization_failed - def __init__(self, Config config, *, shared_registry): """ Build the Cython resolver and its hot caches. @@ -305,8 +299,6 @@ cdef class TypeResolver: self._ns_type_to_type_info = resolver._ns_type_to_type_info self._local_type_info_by_hash = resolver._local_type_info_by_hash self._meta_shared_type_info = resolver._meta_shared_type_info - self.registry_finalization_complete = False - self.registry_finalization_failed = False for typeinfo in resolver._types_info.values(): self._populate_type_info(typeinfo) @@ -318,18 +310,7 @@ cdef class TypeResolver: self._populate_type_info(typeinfo) cdef inline void _freeze_registry(self): - cdef object typeinfo - if not self.registry_finalization_complete: - if self.registry_finalization_failed: - raise RuntimeError("Registry finalization did not complete") - self.resolver._freeze_registry() - try: - for typeinfo in self.resolver._types_info.values(): - self._populate_type_info(typeinfo) - except BaseException: - self.registry_finalization_failed = True - raise - self.registry_finalization_complete = True + self.resolver._freeze_registry() def register_type( self, @@ -370,10 +351,7 @@ cdef class TypeResolver: cdef uint8_t previous_type_id cdef uint32_t previous_user_type_id self.resolver._check_registry_mutable() - typeinfo = self._types_info.get(normalize_fory_type(cls)) - if typeinfo is None: - self.resolver.register_serializer(cls, serializer) - return + typeinfo = self.resolver.get_type_info(cls) previous_type_id = typeinfo.type_id previous_user_type_id = typeinfo.user_type_id self.resolver.register_serializer(cls, serializer) @@ -590,7 +568,7 @@ cdef class TypeResolver: write_context.write_var_uint32(index << 1) type_def = typeinfo.type_def if type_def is None: - self.resolver._finalize_type_info(typeinfo) + self.resolver._set_type_info(typeinfo) type_def = typeinfo.type_def write_context.write_bytes(type_def.encoded) @@ -733,7 +711,7 @@ cdef class TypeResolver: raise TypeError("Type metadata owner does not match the declared type") if typeinfo is not None: if typeinfo.type_def is None: - self.resolver._finalize_type_info(typeinfo) + self.resolver._set_type_info(typeinfo) if typeinfo.type_def is not None: local_header = Buffer(typeinfo.type_def.encoded).read_int64() if _typedef_hash_key(local_header) == hash_key: diff --git a/python/pyfory/struct.pxi b/python/pyfory/struct.pxi index 76f6df55e0..184ed07b41 100644 --- a/python/pyfory/struct.pxi +++ b/python/pyfory/struct.pxi @@ -632,8 +632,8 @@ cdef class DataClassSerializer(Serializer): @cython.final cdef class DataClassStubSerializer(Serializer): - # Keep a delegate stub so recursive dataclass construction can refer to the - # canonical serializer without re-entering construction. + # Keep a lazy stub so recursive dataclass registration can install the real + # serializer on first use without re-entering construction. cpdef write(self, WriteContext write_context, value): self._replace().write(write_context, value) @@ -641,10 +641,6 @@ cdef class DataClassStubSerializer(Serializer): return self._replace().read(read_context) cpdef object _replace(self): - cdef TypeInfo typeinfo = self.type_resolver.get_type_info(self.type_, create=False) - cdef object serializer = None if typeinfo is None else typeinfo.serializer - # Root-entry finalization must commit the canonical serializer before - # use; this stub must never repair registry state after the freeze. - if serializer is None or isinstance(serializer, DataClassStubSerializer): - raise RuntimeError(f"Serializer finalization incomplete for {self.type_}") - return serializer + cdef TypeInfo typeinfo = self.type_resolver.get_type_info(self.type_) + typeinfo.serializer = DataClassSerializer(self.type_resolver, self.type_) + return typeinfo.serializer diff --git a/python/pyfory/struct.py b/python/pyfory/struct.py index b2256a85f7..4cac1a32d3 100644 --- a/python/pyfory/struct.py +++ b/python/pyfory/struct.py @@ -972,14 +972,9 @@ def read(self, read_context): return self._replace().read(read_context) def _replace(self): - typeinfo = self.type_resolver.get_type_info(self.type_, create=False) - serializer = None if typeinfo is None else typeinfo.serializer - # Recursive serializers may retain this stub while the canonical - # serializer is being built. Root-entry finalization must commit that - # serializer before use; the stub must never repair registry state later. - if serializer is None or isinstance(serializer, DataClassStubSerializer): - raise RuntimeError(f"Serializer finalization incomplete for {self.type_}") - return serializer + typeinfo = self.type_resolver.get_type_info(self.type_) + typeinfo.serializer = DataClassSerializer(self.type_resolver, self.type_) + return typeinfo.serializer basic_types = { diff --git a/python/pyfory/tests/test_class_serializer.py b/python/pyfory/tests/test_class_serializer.py index 52364b28d9..f96095aac8 100644 --- a/python/pyfory/tests/test_class_serializer.py +++ b/python/pyfory/tests/test_class_serializer.py @@ -15,22 +15,8 @@ # specific language governing permissions and limitations # under the License. -from dataclasses import dataclass -import types - from pyfory import Fory - - -def register_class_types(fory, *classes): - for cls in ( - type, - types.FunctionType, - types.MethodType, - staticmethod, - classmethod, - *classes, - ): - fory.register_type(cls) +from dataclasses import dataclass def test_local_class_serialization(): @@ -56,7 +42,6 @@ def __eq__(self, other): # Test basic serialization of the class type itself fory = Fory(xlang=False, ref=True, strict=False, compatible=False) - register_class_types(fory, LocalClass) # Serialize the class type serialized = fory.serialize(LocalClass) @@ -94,7 +79,6 @@ def get_multiplied_value(self): LocalClassWithClosure = create_local_class_with_closure(3) fory = Fory(xlang=False, ref=True, strict=False, compatible=False) - register_class_types(fory, LocalClassWithClosure) # Serialize the class type serialized = fory.serialize(LocalClassWithClosure) @@ -132,7 +116,6 @@ def get_value(self): LocalClass = create_local_class_with_inheritance() fory = Fory(xlang=False, ref=True, strict=False, compatible=False) - register_class_types(fory, LocalClass) # Serialize and deserialize the class serialized = fory.serialize(LocalClass) @@ -148,7 +131,7 @@ def get_value(self): assert instance2.base_method() == "base" -def test_local_class_variables(): +def test_local_class_with_class_variables(): """Test local class with class variables""" def create_class_with_vars(): @@ -174,7 +157,6 @@ def get_info(self): LocalClass = create_class_with_vars() fory = Fory(xlang=False, ref=True, strict=False, compatible=False) - register_class_types(fory, LocalClass) # Create some instances to modify class state LocalClass(1) # This increments the counter @@ -220,7 +202,6 @@ def create_inner(self, inner_val): return self.InnerGlobalClass(inner_val) fory = Fory(xlang=False, ref=True, strict=False, compatible=False) - register_class_types(fory, OuterGlobalClass, OuterGlobalClass.InnerGlobalClass) # Test serializing the outer class serialized_outer = fory.serialize(OuterGlobalClass) @@ -278,10 +259,10 @@ def inner_method(self): return OuterLocalClass + fory = Fory(xlang=False, ref=True, strict=False, compatible=False) + # Create complex local class with nested closures ComplexLocalClass = create_complex_local_scenario(5) - fory = Fory(xlang=False, ref=True, strict=False, compatible=False) - register_class_types(fory, ComplexLocalClass) # Serialize and deserialize serialized = fory.serialize(ComplexLocalClass) @@ -302,10 +283,10 @@ def inner_method(self): assert inner2.inner_method() == 17 # 12 + 5 -def test_local_multiple_inheritance(): +def test_local_class_with_multiple_inheritance(): """Test local class with multiple inheritance""" - def create_local_multiple_inheritance(): + def create_local_class_with_multiple_inheritance(): class MixinA: def method_a(self): return "A" @@ -323,9 +304,9 @@ def combined_method(self): return LocalMultipleInheritanceClass - LocalClass = create_local_multiple_inheritance() fory = Fory(xlang=False, ref=True, strict=False, compatible=False) - register_class_types(fory, LocalClass) + + LocalClass = create_local_class_with_multiple_inheritance() # Serialize and deserialize serialized = fory.serialize(LocalClass) @@ -360,9 +341,29 @@ def h(x): def test_dataclass_serialize(): fory = Fory(xlang=False, ref=True, strict=False, compatible=False) - register_class_types(fory, Person) - assert str(fory.loads(fory.dumps(Person))("Bob", 25)) == str(Person("Bob", 25)) - assert fory.loads(fory.dumps(Person("Bob", 20).f))(10) == 200 - assert fory.loads(fory.dumps(Person.g))(10) == 100 - assert fory.loads(fory.dumps(Person.h))(10) == 100 + # serialize global class + @dataclass + class LocalPerson: + name: str + age: int + + def f(self, x): + return self.age * x + + @classmethod + def g(cls, x): + return 10 * x + + @staticmethod + def h(x): + return 10 * x + + for cls in [LocalPerson, LocalPerson]: + assert str(fory.loads(fory.dumps(cls))("Bob", 25)) == str(cls("Bob", 25)) + # serialize global class instance method + assert fory.loads(fory.dumps(cls("Bob", 20).f))(10) == 200 + # serialize global class class method + assert fory.loads(fory.dumps(cls.g))(10) == 100 + # serialize global class static method + assert fory.loads(fory.dumps(cls.h))(10) == 100 diff --git a/python/pyfory/tests/test_collection_safety.py b/python/pyfory/tests/test_collection_safety.py index dfaee720ee..70ec36e3d7 100644 --- a/python/pyfory/tests/test_collection_safety.py +++ b/python/pyfory/tests/test_collection_safety.py @@ -15,8 +15,6 @@ # specific language governing permissions and limitations # under the License. -import types - import pytest import pyfory @@ -122,8 +120,6 @@ def __reduce__(self): def test_published_list_has_valid_slots(): fory = pyfory.Fory(xlang=False, ref=True, strict=False, compatible=False) - fory.register_type(type) - fory.register_type(ListCapture) outer = [] outer.append(ListCapture(outer)) @@ -134,8 +130,6 @@ def test_published_list_has_valid_slots(): def test_published_list_failure_cleanup(): fory = pyfory.Fory(xlang=False, ref=True, strict=False, compatible=False) - fory.register_type(FailingListCapture) - fory.register_type(types.FunctionType) outer = [] outer.append(FailingListCapture(outer)) data = fory.serialize(outer) @@ -147,8 +141,6 @@ def test_published_list_failure_cleanup(): def test_reentrant_list_clear(): fory = pyfory.Fory(xlang=False, ref=True, strict=False, compatible=False) - fory.register_type(ClearingListCapture) - fory.register_type(types.FunctionType) outer = [] outer.append(ClearingListCapture(outer)) data = fory.serialize(outer) diff --git a/python/pyfory/tests/test_function.py b/python/pyfory/tests/test_function.py index 2cc4d664ae..ebb5de638c 100644 --- a/python/pyfory/tests/test_function.py +++ b/python/pyfory/tests/test_function.py @@ -98,17 +98,12 @@ def test_lambda_functions_serialization(): fory = pyfory.Fory( xlang=False, strict=False, + compatible=False, ) test_input = 5 - # Register the necessary types - fory.register_type(tuple) - fory.register_type(list) - # dict is already registered by default with MapSerializer - # Simple lambda simple_lambda = lambda x: x * 2 # noqa: E731 - fory.register_type(type(simple_lambda)) serialized = fory.serialize(simple_lambda) deserialized = fory.deserialize(serialized) assert simple_lambda(test_input) == deserialized(test_input) @@ -138,10 +133,6 @@ def complex_function(a, b, c=10): return a * b + c # Test regular function - fory.register_type(type(add_one)) - # Registry contents are finalized by the first root operation. - fory.register_type(tuple) - fory.register_type(list) serialized = fory.serialize(add_one) deserialized = fory.deserialize(serialized) assert add_one(test_input) == deserialized(test_input) @@ -162,11 +153,6 @@ def test_nested_functions_serialization(): compatible=False, ) - # Register the necessary types - fory.register_type(tuple) - fory.register_type(list) - # dict is already registered by default with MapSerializer - def outer_function(x): def inner_function(y): return x + y @@ -175,7 +161,6 @@ def inner_function(y): # Create a nested function nested_func = outer_function(10) - fory.register_type(type(nested_func)) serialized = fory.serialize(nested_func) deserialized = fory.deserialize(serialized) @@ -191,11 +176,6 @@ def test_local_class_serialization(): compatible=False, ) - # Register the necessary types - fory.register_type(tuple) - fory.register_type(list) - # dict is already registered by default with MapSerializer - def create_local_class(): from dataclasses import dataclass @@ -207,7 +187,6 @@ class LocalClass: return LocalClass(42, "test") local_obj = create_local_class() - fory.register_type(type(local_obj)) serialized = fory.serialize(local_obj) deserialized = fory.deserialize(serialized) diff --git a/python/pyfory/tests/test_graph_memory_budget.py b/python/pyfory/tests/test_graph_memory_budget.py index 8b42749f0f..5acb047443 100644 --- a/python/pyfory/tests/test_graph_memory_budget.py +++ b/python/pyfory/tests/test_graph_memory_budget.py @@ -19,7 +19,6 @@ import dataclasses import struct import sys -import types from typing import Any, List import pytest @@ -375,19 +374,16 @@ def test_reduce_object_budget(): value = BudgetReduceObject() writer = new_fory(xlang=False) writer.register_type(BudgetReduceObject) - writer.register_type(type) data = writer.serialize(value) reduce_args_budget = tuple_memory(0) + PY_OBJECT_OWNER_BYTES with pytest.raises(ValueError, match="Estimated graph memory budget exceeded"): reader = new_fory(reduce_args_budget - 1, xlang=False) reader.register_type(BudgetReduceObject) - reader.register_type(type) reader.deserialize(data) reader = new_fory(reduce_args_budget, xlang=False) reader.register_type(BudgetReduceObject) - reader.register_type(type) assert isinstance(reader.deserialize(data), BudgetReduceObject) @@ -398,7 +394,6 @@ def local_func(value=5): return value + captured writer = new_fory(xlang=False) - writer.register_type(types.FunctionType) data = writer.serialize(local_func) module_entries = len(sys.modules[local_func.__module__].__dict__) @@ -413,13 +408,9 @@ def local_func(value=5): + map_memory(0) ) with pytest.raises(ValueError, match="Estimated graph memory budget exceeded"): - reader = new_fory(budget - PY_OBJECT_OWNER_BYTES, xlang=False) - reader.register_type(types.FunctionType) - reader.deserialize(data) + new_fory(budget - PY_OBJECT_OWNER_BYTES, xlang=False).deserialize(data) - reader = new_fory(budget, xlang=False) - reader.register_type(types.FunctionType) - restored = reader.deserialize(data) + restored = new_fory(budget, xlang=False).deserialize(data) assert restored() == local_func() @@ -432,20 +423,15 @@ class LocalBudgetClass: cls = make_class() writer = new_fory(xlang=False) - writer.register_type(type) data = writer.serialize(cls) class_attrs = {name: value for name, value in cls.__dict__.items() if name not in SKIP_CLASS_ATTR_NAMES} class_attr_value_budget = sum(collection_memory(len(value)) for value in class_attrs.values() if isinstance(value, tuple)) budget = collection_memory(1) + PY_OBJECT_OWNER_BYTES + map_memory(len(class_attrs)) + class_attr_value_budget with pytest.raises(ValueError, match="Estimated graph memory budget exceeded"): - reader = new_fory(collection_memory(1), xlang=False) - reader.register_type(type) - reader.deserialize(data) + new_fory(collection_memory(1), xlang=False).deserialize(data) - reader = new_fory(budget, xlang=False) - reader.register_type(type) - restored = reader.deserialize(data) + restored = new_fory(budget, xlang=False).deserialize(data) assert restored.__name__ == cls.__name__ assert restored.__bases__ == cls.__bases__ diff --git a/python/pyfory/tests/test_method.py b/python/pyfory/tests/test_method.py index 20933ee8ab..41b21b2962 100644 --- a/python/pyfory/tests/test_method.py +++ b/python/pyfory/tests/test_method.py @@ -15,23 +15,9 @@ # specific language governing permissions and limitations # under the License. -import types - import pyfory -def register_method_types(fory, *classes): - for cls in ( - type, - types.FunctionType, - types.MethodType, - staticmethod, - classmethod, - *classes, - ): - fory.register_type(cls) - - # Global classes for testing global class method serialization class GlobalTestClass: """Global test class for method serialization.""" @@ -100,7 +86,6 @@ def instance_method(self): obj = TestClass(5) method = obj.instance_method - register_method_types(fory, TestClass) # Test serialization/deserialization serialized = fory.serialize(method) @@ -121,7 +106,6 @@ def class_method(cls): return cls.class_var method = TestClass.class_method - register_method_types(fory, TestClass) # Test serialization/deserialization serialized = fory.serialize(method) @@ -140,7 +124,6 @@ def static_method(): return "static_result" method = TestClass.static_method - register_method_types(fory, TestClass) # Test serialization/deserialization serialized = fory.serialize(method) @@ -169,7 +152,6 @@ def subtract(a, b): return a - b obj = TestClass(10) - register_method_types(fory, TestClass) # Test instance method instance_method = obj.add @@ -192,7 +174,7 @@ def subtract(a, b): assert static_method(10, 3) == deserialized(10, 3) assert static_method(10, 3) == 7 - def test_nested_class_method(self): + def test_nested_class_method_serialization(self): """Test serialization of methods from nested classes.""" fory = pyfory.Fory(xlang=False, strict=False, ref=True, compatible=False) @@ -203,7 +185,6 @@ def inner_class_method(cls): return "inner_result" method = OuterClass.InnerClass.inner_class_method - register_method_types(fory, OuterClass, OuterClass.InnerClass) # Test serialization/deserialization serialized = fory.serialize(method) @@ -227,7 +208,6 @@ def g(): return A method = A.f - register_method_types(fory, A) serialized = fory.serialize(method) deserialized = fory.deserialize(serialized) @@ -243,13 +223,29 @@ def g(): assert original_result == deserialized_result +def test_staticmethod_serialization(): + """Standalone test for staticmethod serialization.""" + fory = pyfory.Fory(xlang=False, strict=False, ref=True, compatible=False) + + class A: + @staticmethod + def g(): + return "static_result" + + method = A.g + serialized = fory.serialize(method) + deserialized = fory.deserialize(serialized) + + assert method() == deserialized() + assert method() == "static_result" + + # Global class method tests -def test_global_classmethod(): +def test_global_classmethod_serialization(): """Test serialization of global class methods.""" fory = pyfory.Fory(xlang=False, strict=False, ref=True, compatible=False) method = GlobalTestClass.class_method - register_method_types(fory, GlobalTestClass) serialized = fory.serialize(method) deserialized = fory.deserialize(serialized) @@ -263,7 +259,6 @@ def test_global_classmethod_with_args(): fory = pyfory.Fory(xlang=False, strict=False, ref=True, compatible=False) method = GlobalTestClass.class_method_with_args - register_method_types(fory, GlobalTestClass) serialized = fory.serialize(method) deserialized = fory.deserialize(serialized) @@ -272,12 +267,11 @@ def test_global_classmethod_with_args(): assert deserialized(*args) == "class_global_class_value_arg1_arg2" -def test_global_staticmethod(): +def test_global_staticmethod_serialization(): """Test serialization of global static methods.""" fory = pyfory.Fory(xlang=False, strict=False, ref=True, compatible=False) method = GlobalTestClass.static_method - register_method_types(fory, GlobalTestClass) serialized = fory.serialize(method) deserialized = fory.deserialize(serialized) @@ -290,7 +284,6 @@ def test_global_staticmethod_with_args(): fory = pyfory.Fory(xlang=False, strict=False, ref=True, compatible=False) method = GlobalTestClass.static_method_with_args - register_method_types(fory, GlobalTestClass) serialized = fory.serialize(method) deserialized = fory.deserialize(serialized) @@ -299,13 +292,12 @@ def test_global_staticmethod_with_args(): assert deserialized(*args) == "static_test1_test2" -def test_global_instance_method(): +def test_global_instance_method_serialization(): """Test serialization of global instance methods.""" fory = pyfory.Fory(xlang=False, strict=False, ref=True, compatible=False) obj = GlobalTestClass("test_value") method = obj.instance_method - register_method_types(fory, GlobalTestClass) serialized = fory.serialize(method) deserialized = fory.deserialize(serialized) @@ -320,7 +312,6 @@ def test_multiple_global_classes(): # Test methods from different global classes method1 = GlobalTestClass.class_method method2 = AnotherGlobalClass.another_class_method - register_method_types(fory, GlobalTestClass, AnotherGlobalClass) serialized1 = fory.serialize(method1) serialized2 = fory.serialize(method2) @@ -340,7 +331,6 @@ def test_global_class_inheritance(): # Test inherited class method method = GlobalClassWithInheritance.inherited_class_method - register_method_types(fory, GlobalTestClass, GlobalClassWithInheritance) serialized = fory.serialize(method) deserialized = fory.deserialize(serialized) @@ -356,13 +346,12 @@ def test_global_class_inheritance(): assert deserialized_parent() == "class_inherited_value" # Uses child's class_variable -def test_global_methods_without_refs(): +def test_global_methods_without_ref_tracking(): """Test serialization of global class methods without reference tracking.""" fory = pyfory.Fory(xlang=False, strict=False, ref=False, compatible=False) # Global classes should work even without track_ref method = GlobalTestClass.class_method - register_method_types(fory, GlobalTestClass) serialized = fory.serialize(method) deserialized = fory.deserialize(serialized) @@ -379,7 +368,6 @@ def test_global_method_collection(): GlobalTestClass.static_method, AnotherGlobalClass.another_class_method, ] - register_method_types(fory, GlobalTestClass, AnotherGlobalClass) serialized = fory.serialize(methods) deserialized = fory.deserialize(serialized) @@ -398,14 +386,13 @@ def test_global_method_in_dict(): "static_method": GlobalTestClass.static_method, "another_method": AnotherGlobalClass.another_class_method, } - register_method_types(fory, GlobalTestClass, AnotherGlobalClass) serialized = fory.serialize(method_dict) deserialized = fory.deserialize(serialized) assert len(deserialized) == len(method_dict) - for key, method in method_dict.items(): - assert method() == deserialized[key]() + for key in method_dict: + assert method_dict[key]() == deserialized[key]() if __name__ == "__main__": diff --git a/python/pyfory/tests/test_pickle_buffer.py b/python/pyfory/tests/test_pickle_buffer.py index d571b40a22..a073c084b5 100644 --- a/python/pyfory/tests/test_pickle_buffer.py +++ b/python/pyfory/tests/test_pickle_buffer.py @@ -25,6 +25,11 @@ except ImportError: np = None +try: + import pandas as pd +except ImportError: + pd = None + def test_pickle_buffer_serialization(): fory = Fory(xlang=False, ref=False, strict=False, compatible=False) @@ -55,6 +60,28 @@ def test_numpy_out_of_band_serialization(): np.testing.assert_array_equal(arr, deserialized) +@pytest.mark.skipif(pd is None, reason="Requires pandas") +def test_pandas_out_of_band_serialization(): + fory = Fory(xlang=False, ref=False, strict=False, compatible=False) + + df = pd.DataFrame( + { + "a": np.arange(1000, dtype=np.float64), + "b": np.arange(1000, dtype=np.int64), + "c": ["text"] * 1000, + } + ) + + buffer_objects = [] + serialized = fory.serialize(df, buffer_callback=buffer_objects.append) + + buffers = [o.getbuffer() for o in buffer_objects] + + deserialized = fory.deserialize(serialized, buffers=buffers) + + pd.testing.assert_frame_equal(df, deserialized) + + @pytest.mark.skipif(np is None, reason="Requires numpy") def test_numpy_multiple_arrays_out_of_band(): fory = Fory(xlang=False, ref=True, strict=False, compatible=False) @@ -100,6 +127,26 @@ def test_numpy_with_mixed_types(): np.testing.assert_array_equal(arr, deserialized["array"]) +@pytest.mark.skipif(pd is None or np is None, reason="Requires numpy and pandas") +def test_mixed_numpy_pandas_out_of_band(): + fory = Fory(xlang=False, ref=True, strict=False, compatible=False) + + arr = np.arange(500, dtype=np.float64) + df = pd.DataFrame({"x": np.arange(500, dtype=np.int64), "y": np.arange(500, dtype=np.float32)}) + + data = {"array": arr, "dataframe": df} + + buffer_objects = [] + serialized = fory.serialize(data, buffer_callback=buffer_objects.append) + + buffers = [o.getbuffer() for o in buffer_objects] + + deserialized = fory.deserialize(serialized, buffers=buffers) + + np.testing.assert_array_equal(arr, deserialized["array"]) + pd.testing.assert_frame_equal(df, deserialized["dataframe"]) + + @pytest.mark.skipif(np is None, reason="Requires numpy") def test_selective_out_of_band_serialization(): fory = Fory(xlang=False, ref=True, strict=False, compatible=False) diff --git a/python/pyfory/tests/test_policy.py b/python/pyfory/tests/test_policy.py index 61873348a9..d8312c070f 100644 --- a/python/pyfory/tests/test_policy.py +++ b/python/pyfory/tests/test_policy.py @@ -267,7 +267,6 @@ class UnsafeClass: policy = BlockClassPolicy(blocked_class_names=["UnsafeClass"]) fory = Fory(xlang=False, ref=True, strict=False, policy=policy, compatible=False) - fory.register_type(type) # Serialize and deserialize the class type itself (not an instance) safe_data = fory.serialize(SafeClass) @@ -292,9 +291,6 @@ def __reduce__(self): policy = BlockReduceCallPolicy(blocked_names=["ReducibleClass"]) fory = Fory(xlang=False, ref=True, strict=False, policy=policy, compatible=False) - fory.register_type(type) - fory.register_type(types.FunctionType) - fory.register_type(ReducibleClass) data = fory.serialize(ReducibleClass(42)) with pytest.raises(ValueError, match="ReducibleClass is blocked"): @@ -313,9 +309,6 @@ def __reduce__(self): policy = ReplaceObjectPolicy(replacement_value="REPLACED") fory = Fory(xlang=False, ref=True, strict=False, policy=policy, compatible=False) - fory.register_type(type) - fory.register_type(types.FunctionType) - fory.register_type(ReducibleClass) data = fory.serialize(ReducibleClass(42)) result = fory.deserialize(data) @@ -338,7 +331,6 @@ def __setstate__(self, state): policy = SanitizeStatePolicy() fory = Fory(xlang=False, ref=False, strict=False, policy=policy, compatible=False) - fory.register_type(SecretHolder) data = fory.serialize(SecretHolder("admin", "secret123")) result = fory.deserialize(data) @@ -372,9 +364,6 @@ def __setstate__(self, state): policy = CountingSanitizePolicy() fory = Fory(xlang=False, ref=True, strict=False, policy=policy, compatible=False) - fory.register_type(type) - fory.register_type(types.FunctionType) - fory.register_type(SecretReduceHolder) data = fory.serialize(SecretReduceHolder()) result = fory.deserialize(data) @@ -398,8 +387,6 @@ def intercept_setstate(self, obj, state, **kwargs): policy=BlockSetStatePolicy(), compatible=False, ) - fory.register_type(FalseyState) - fory.register_type(FalseyStatePayload) data = fory.serialize(FalseyStatePayload()) with pytest.raises(ValueError, match="state blocked"): @@ -447,7 +434,6 @@ class LocalClass: policy = BlockClassPolicy(blocked_class_names=["LocalClass"]) fory = Fory(xlang=False, ref=True, strict=False, policy=policy, compatible=False) - fory.register_type(type) # Serialize the local class type data = fory.serialize(LocalCls) @@ -468,9 +454,6 @@ def __reduce__(self): policy = BlockReduceCallPolicy(blocked_names=["ReducibleClass"]) fory = Fory(xlang=False, ref=True, strict=False, policy=policy, compatible=False) - fory.register_type(type) - fory.register_type(types.FunctionType) - fory.register_type(ReducibleClass) data = fory.serialize(ReducibleClass(42)) @@ -517,9 +500,6 @@ def __reduce__(self): policy = MultiHookPolicy() fory = Fory(xlang=False, ref=True, strict=False, policy=policy, compatible=False) - fory.register_type(type) - fory.register_type(types.FunctionType) - fory.register_type(TestClass) data = fory.serialize(TestClass(42)) result = fory.deserialize(data) @@ -549,10 +529,6 @@ def __reduce__(self): policy = BlockReduceCallPolicy(blocked_names=["Inner"]) fory = Fory(xlang=False, ref=True, strict=False, policy=policy, compatible=False) - fory.register_type(type) - fory.register_type(types.FunctionType) - fory.register_type(Inner) - fory.register_type(Outer) data = fory.serialize(Outer(Inner(42))) @@ -585,7 +561,6 @@ def authorize_instantiation(self, cls, **kwargs): policy = BlockInstantiationPolicy() fory = Fory(xlang=False, ref=True, strict=False, policy=policy, compatible=False) - fory.register_type(StatefulPayload) with pytest.raises(ValueError, match="StatefulPayload blocked"): fory.deserialize(fory.serialize(StatefulPayload())) assert policy.authorize_instantiation_calls == 1 @@ -615,9 +590,6 @@ def authorize_instantiation(self, cls, **kwargs): policy = BlockInstantiationPolicy() fory = Fory(xlang=False, ref=True, strict=False, policy=policy, compatible=False) - fory.register_type(type) - fory.register_type(ReducePayload) - fory.register_type(ReduceTarget) with pytest.raises(ValueError, match="ReduceTarget blocked"): fory.deserialize(fory.serialize(ReducePayload())) assert policy.reduce_target_calls == 1 @@ -802,8 +774,6 @@ def validate_method(self, method, is_local, **kwargs): policy = ReturnPolicy() fory = Fory(xlang=False, ref=True, strict=False, policy=policy, compatible=False) - fory.register_type(type) - fory.register_type(types.FunctionType) assert fory.deserialize(fory.serialize(json)) is json assert fory.deserialize(fory.serialize(PolicyGlobalClass)) is PolicyGlobalClass assert fory.deserialize(fory.serialize(policy_global_function)) is policy_global_function @@ -838,11 +808,6 @@ def validate_class(self, cls, is_local, **kwargs): policy=ReturnClassPolicy(), compatible=False, ) - fory.register_type(type) - fory.register_type(types.FunctionType) - fory.register_type(types.MethodType) - fory.register_type(staticmethod) - fory.register_type(classmethod) decoded = fory.deserialize(fory.serialize(make_payload_class())) assert decoded is not SafeClass assert decoded.run() == "payload" @@ -867,7 +832,6 @@ def validate_module(self, module_name, is_local, **kwargs): policy = BlockModulePolicy() fory = Fory(xlang=False, ref=True, strict=False, policy=policy, compatible=False) - fory.register_type(type) with pytest.raises(ValueError, match="subprocess blocked"): fory.deserialize(fory.serialize(subprocess.Popen)) assert policy.validate_module_calls == 1 @@ -892,7 +856,6 @@ def validate_function(self, func, is_local, **kwargs): policy = BlockMethodPolicy() fory = Fory(xlang=False, ref=True, strict=False, policy=policy, compatible=False) - fory.register_type(type(abs)) with pytest.raises(ValueError, match="method blocked"): fory.deserialize(fory.serialize([].append)) @@ -927,9 +890,6 @@ def validate_method(self, method, is_local, **kwargs): policy=BlockMethodPolicy(), compatible=False, ) - fory.register_type(types.FunctionType) - fory.register_type(types.MethodType) - fory.register_type(GuardedMethod) data = fory.serialize(method) GuardedMethod.getattribute_called = False @@ -1307,10 +1267,7 @@ def test_default_global_round_trips(): import time fory = Fory(xlang=False, ref=True, strict=False, compatible=False) - values = (PolicyGlobalClass, policy_global_function, time.time, policy_reduce_global) - for value in values: - fory.register_type(type(value)) - for value in values: + for value in (PolicyGlobalClass, policy_global_function, time.time, policy_reduce_global): assert fory.deserialize(fory.serialize(value)) is value @@ -1440,16 +1397,6 @@ def validate_method(self, method, is_local, **kwargs): writer = Fory(xlang=False, ref=True, strict=False, compatible=False) policy = ClassMethodPolicy() reader = Fory(xlang=False, ref=True, strict=False, policy=policy, compatible=False) - writer.register_type(type) - writer.register_type(types.FunctionType) - writer.register_type(types.MethodType) - writer.register_type(staticmethod) - writer.register_type(classmethod) - reader.register_type(type) - reader.register_type(types.FunctionType) - reader.register_type(types.MethodType) - reader.register_type(staticmethod) - reader.register_type(classmethod) data = writer.serialize(make_local_class()) with pytest.raises(ValueError, match="classmethod blocked"): @@ -1673,7 +1620,6 @@ def validate_module(self, module_name, is_local, **kwargs): policy = BlockModulePolicy() fory = Fory(xlang=False, ref=True, strict=False, policy=policy, compatible=False) - fory.register_type(types.FunctionType) with pytest.raises(ValueError, match="function module blocked"): fory.deserialize(fory.serialize(policy_global_function)) assert policy.validate_module_calls == 1 @@ -1700,7 +1646,6 @@ def validate_module(self, module_name, is_local, **kwargs): policy = BlockModulePolicy() fory = Fory(xlang=False, ref=True, strict=False, policy=policy, compatible=False) - fory.register_type(types.FunctionType) with pytest.raises(ValueError, match="local function module blocked"): fory.deserialize(fory.serialize(local_function)) assert policy.validate_module_calls == 1 @@ -1750,7 +1695,6 @@ def validate_module(self, module_name, is_local, **kwargs): policy = BlockModulePolicy() fory = Fory(xlang=False, ref=True, strict=False, policy=policy, compatible=False) - fory.register_type(type(abs)) with pytest.raises(ValueError, match="time blocked"): fory.deserialize(fory.serialize(time.time)) assert policy.validate_module_calls == 1 @@ -1837,7 +1781,6 @@ def validate_module(self, module_name, is_local, **kwargs): policy = BlockModulePolicy() fory = Fory(xlang=False, ref=True, strict=False, policy=policy, compatible=False) - fory.register_type(GlobalNamePayload) with pytest.raises(ValueError, match="subprocess blocked"): fory.deserialize(fory.serialize(GlobalNamePayload())) assert policy.validate_module_calls == 1 @@ -1869,7 +1812,6 @@ def validate_class(self, cls, is_local, **kwargs): policy = BlockClassPolicy() fory = Fory(xlang=False, ref=True, strict=False, policy=policy, compatible=False) - fory.register_type(GlobalNamePayload) with pytest.raises(ValueError, match="subprocess.Popen blocked"): fory.deserialize(fory.serialize(GlobalNamePayload())) assert policy.validate_module_calls == 1 @@ -1901,7 +1843,6 @@ def validate_function(self, func, is_local, **kwargs): policy = BlockFunctionPolicy() fory = Fory(xlang=False, ref=True, strict=False, policy=policy, compatible=False) - fory.register_type(GlobalNamePayload) with pytest.raises(ValueError, match="eval blocked"): fory.deserialize(fory.serialize(GlobalNamePayload())) assert policy.validate_module_calls == 1 @@ -1936,7 +1877,6 @@ def validate_function(self, func, is_local, **kwargs): policy = MethodPolicy() fory = Fory(xlang=False, ref=True, strict=False, policy=policy, compatible=False) - fory.register_type(GlobalNamePayload) with pytest.raises(ValueError, match="method blocked"): fory.deserialize(fory.serialize(GlobalNamePayload())) assert policy.validate_module_calls == 1 diff --git a/python/pyfory/tests/test_reduce_serializer.py b/python/pyfory/tests/test_reduce_serializer.py index 8b5fea5e7b..3fa13339f5 100644 --- a/python/pyfory/tests/test_reduce_serializer.py +++ b/python/pyfory/tests/test_reduce_serializer.py @@ -26,17 +26,6 @@ from pyfory.serializer import ReduceSerializer -def register_reduce_types(fory, *classes): - for cls in ( - type, - types.BuiltinFunctionType, - type(iter([])), - type(iter({}.items())), - *classes, - ): - fory.register_type(cls) - - _reduce_factory_calls = [] _class_callable_calls = [] _function_factory_calls = [] @@ -306,7 +295,6 @@ def validate_function(self, func, is_local, **kwargs): def test_nonstrict_reduce_global(): _reduce_factory_calls.clear() fory = Fory(xlang=False, ref=True, strict=False, compatible=False) - fory.register_type(NestedGlobalReduce) assert fory.deserialize(fory.serialize(NestedGlobalReduce("outer", "allowed"))) == NestedGlobalReduce("result", "allowed") assert _reduce_factory_calls == ["allowed"] @@ -401,8 +389,6 @@ def validate_function(self, func, is_local, **kwargs): def test_nonstrict_reduce_function(): fory = Fory(xlang=False, ref=True, strict=False, compatible=False) - fory.register_type(types.FunctionType) - fory.register_type(FunctionCallableReduce) _function_factory_calls.clear() value = FunctionCallableReduce("allowed") @@ -471,7 +457,6 @@ def test_basic_reduce_object(): fory = Fory(xlang=False, ref=True, strict=False, compatible=False) obj = BasicReduceObject(42, 3) - register_reduce_types(fory, BasicReduceObject) # Verify ReduceSerializer is used serializer = fory.type_resolver.get_serializer(BasicReduceObject) @@ -491,7 +476,6 @@ def test_reduce_with_state_object(): fory = Fory(xlang=False, ref=True, strict=False, compatible=False) obj = ReduceWithStateObject("test", {"key": "value"}) - register_reduce_types(fory, ReduceWithStateObject) # Verify ReduceSerializer is used serializer = fory.type_resolver.get_serializer(ReduceWithStateObject) @@ -512,7 +496,6 @@ def test_reduce_ex_object(): fory = Fory(xlang=False, ref=True, strict=False, compatible=False) obj = ReduceExObject(5, 7) - register_reduce_types(fory, ReduceExObject) # Verify ReduceSerializer is used serializer = fory.type_resolver.get_serializer(ReduceExObject) @@ -533,7 +516,7 @@ def test_reduce_with_list_items(): fory = Fory(xlang=False, ref=True, strict=False, compatible=False) obj = ReduceWithListItems([1, 2, 3, 4]) - register_reduce_types(fory, ReduceWithListItems) + fory.register_type(type(iter([]))) # Verify ReduceSerializer is used serializer = fory.type_resolver.get_serializer(ReduceWithListItems) @@ -553,7 +536,7 @@ def test_reduce_with_dict_items(): fory = Fory(xlang=False, ref=True, strict=False, compatible=False) obj = ReduceWithDictItems({"a": 1, "b": 2}) - register_reduce_types(fory, ReduceWithDictItems) + fory.register_type(type(iter({}.items()))) # Verify ReduceSerializer is used serializer = fory.type_resolver.get_serializer(ReduceWithDictItems) @@ -573,7 +556,6 @@ def test_reduce_precedes_stateful(): fory = Fory(xlang=False, ref=True, strict=False, compatible=False) obj = BothReduceAndStateful(100) - register_reduce_types(fory, BothReduceAndStateful) # Verify ReduceSerializer is used, not StatefulSerializer serializer = fory.type_resolver.get_serializer(BothReduceAndStateful) @@ -596,7 +578,6 @@ def test_reference_tracking(): obj1 = BasicReduceObject(42) obj2 = BasicReduceObject(42) container = [obj1, obj1, obj2] # obj1 appears twice - register_reduce_types(fory, BasicReduceObject) serialized = fory.serialize(container) deserialized = fory.deserialize(serialized) @@ -616,7 +597,6 @@ def test_nested_reduce_objects(): inner = BasicReduceObject(10, 2) outer = ReduceWithStateObject("outer", {"inner": inner}) - register_reduce_types(fory, BasicReduceObject, ReduceWithStateObject) serialized = fory.serialize(outer) deserialized = fory.deserialize(serialized) diff --git a/python/pyfory/tests/test_ref_tracking.py b/python/pyfory/tests/test_ref_tracking.py index 3ac1265dba..f5dd689b98 100644 --- a/python/pyfory/tests/test_ref_tracking.py +++ b/python/pyfory/tests/test_ref_tracking.py @@ -120,10 +120,9 @@ def test_collection_tuple_shared_reference_python_mode(): assert restored[2][0] is restored[0] -def test_set_element_outer_alias(): +def test_collection_set_element_alias_with_outer_reference_python_mode(): fory = pyfory.Fory(xlang=False, ref=True, strict=False, compatible=False) token = HashKey("shared-key") - fory.register_type(HashKey) payload = [{token}, token] restored = _roundtrip(fory, payload) @@ -153,10 +152,9 @@ def test_map_self_cycle_and_shared_submap_python_mode(): assert restored["self"] is restored -def test_map_key_outer_alias(): +def test_map_key_alias_with_outer_reference_python_mode(): fory = pyfory.Fory(xlang=False, ref=True, strict=False, compatible=False) key = HashKey("k") - fory.register_type(HashKey) payload = [{key: "value"}, key] restored = _roundtrip(fory, payload) diff --git a/python/pyfory/tests/test_serializer.py b/python/pyfory/tests/test_serializer.py index 8d54b35ed2..099796a841 100644 --- a/python/pyfory/tests/test_serializer.py +++ b/python/pyfory/tests/test_serializer.py @@ -22,7 +22,6 @@ import io import os import pickle -import types import weakref from collections.abc import MutableSequence from enum import Enum, IntEnum @@ -36,7 +35,6 @@ import pytest import pyfory -import pyfory.registry as registry_module from pyfory.serialization import Buffer, _bfloat16_from_bits, _bfloat16_to_bits, _float16_from_bits, _float16_to_bits from pyfory import Fory, EnumSerializer from pyfory.serializer import ( @@ -48,7 +46,6 @@ Numpy1DArraySerializer, ) from pyfory.types import TypeId -from pyfory.union import UnionSerializer from pyfory.utils import lazy_import pa = lazy_import("pyarrow") @@ -688,8 +685,6 @@ def test_ref_cleanup(): fory = Fory(xlang=False, ref=True, strict=False, compatible=False) o1 = RefTestClass1() o2 = RefTestClass2(f1=o1) - fory.register_type(RefTestClass1) - fory.register_type(RefTestClass2) pickle.loads(pickle.dumps(o2)) ref1 = weakref.ref(o1) ref2 = weakref.ref(o2) @@ -815,79 +810,6 @@ class FrozenParent: child: FrozenChild -@dataclass -class BrokenFinalization: - value: int - - -@dataclass -class FrozenRecursive: - child: Optional["FrozenRecursive"] = None - - -class FrozenMetadataEnum(Enum): - VALUE = 1 - - -@dataclass -class FrozenExt: - value: int - - -@dataclass -class FrozenSecondExt: - value: int - - -class FrozenExtSerializer(pyfory.Serializer): - def write(self, write_context, value): - write_context.write_int32(value.value) - - def read(self, read_context): - return self.type_(read_context.read_int32()) - - -class FrozenUnion: - def __init__(self, case_id, value): - self._case_id = case_id - self._value = value - - def case_id(self): - return self._case_id - - @staticmethod - def _from_case_id(case_id, value): - return FrozenUnion(case_id, value) - - def __eq__(self, other): - return isinstance(other, FrozenUnion) and (self._case_id, self._value) == (other._case_id, other._value) - - -def registration_state(fory): - resolver = fory.type_resolver - maps = tuple( - (name, dict(getattr(resolver, name))) - for name in ( - "_types_info", - "_type_id_to_type_info", - "_user_type_id_to_type_info", - "_ns_type_to_type_info", - "_named_type_to_type_info", - "_local_type_info_by_hash", - "_meta_shared_type_info", - ) - if hasattr(resolver, name) - ) - used_type_ids = getattr(resolver, "_used_user_type_ids", None) - return ( - maps, - None if used_type_ids is None else set(used_type_ids), - getattr(resolver, "_type_id_counter", None), - dict(resolver.shared_registry._metastr_to_bytes), - dict(resolver.shared_registry._encoded_metastrings), - ) - - def test_register_py_serializer(): fory = Fory(xlang=False, ref=True, strict=False, compatible=False) @@ -896,12 +818,12 @@ def write(self, write_context, value): write_context.write_int32(value.f1) def read(self, read_context): - return self.type_(read_context.read_int32()) + a = A() + a.f1 = read_context.read_int32() + return a - fory.register_type(RegisterClass, serializer=Serializer(fory.type_resolver, RegisterClass)) - value = fory.deserialize(fory.serialize(RegisterClass(100))) - assert isinstance(value, RegisterClass) - assert value.f1 == 100 + fory.register_type(A, serializer=Serializer(fory.type_resolver, RegisterClass)) + assert fory.deserialize(fory.serialize(RegisterClass(100))).f1 == 100 @pytest.mark.parametrize("registration", ["id", "name"]) @@ -1035,99 +957,35 @@ def test_registry_freezes_at_root(root): assert fory.type_resolver.get_type_info(FrozenRegistration).type_id == TypeId.STRUCT -@pytest.mark.parametrize("registration", ["type", "union"]) -def test_reentrant_freeze(registration): +def test_factory_root_freeze(): fory = Fory(xlang=True, compatible=False) - before = registration_state(fory) - constructions = 0 - target = FrozenExt if registration == "type" else FrozenUnion - def serializer_factory(type_resolver, cls): - nonlocal constructions - constructions += 1 + def factory(type_resolver, cls): fory.serialize(None) - if registration == "type": - return FrozenExtSerializer(type_resolver, cls) - return UnionSerializer(type_resolver, cls, {0: str}) + return BarSerializer(type_resolver, cls) with pytest.raises(RuntimeError): - if registration == "type": - fory.register_type(target, type_id=725, serializer=serializer_factory) - else: - fory.register_union(target, type_id=725, serializer=serializer_factory) - - assert constructions == 1 - assert fory.type_resolver.get_type_info(target, create=False) is None - assert registration_state(fory) == before - with pytest.raises(RuntimeError): - fory.register_type(RejectedRegistration, type_id=726) - + fory.register_type(FrozenRegistration, serializer=factory) + assert fory.type_resolver.get_type_info(FrozenRegistration, create=False) is None -def test_frozen_serializer_lookup(monkeypatch): - fory = Fory(xlang=False, strict=False, compatible=False) - fory.serialize(None) - serializer_count = 0 - serializer_type = registry_module._DefaultPolicyObjectSerializer - - class CountingSerializer(serializer_type): - def __init__(self, type_resolver, cls): - nonlocal serializer_count - serializer_count += 1 - super().__init__(type_resolver, cls) - - monkeypatch.setattr( - registry_module, - "_DefaultPolicyObjectSerializer", - CountingSerializer, - ) - registry_sizes = tuple( - len(getattr(fory.type_resolver, attr)) - for attr in ( - "_types_info", - "_type_id_to_type_info", - "_user_type_id_to_type_info", - "_ns_type_to_type_info", - ) - ) - with pytest.raises(Exception): - fory.type_resolver.register_serializer(RejectedRegistration, object()) - with pytest.raises(Exception): - fory.type_resolver.get_type_info(RejectedRegistration) - - assert serializer_count == 0 - assert registry_sizes == tuple( - len(getattr(fory.type_resolver, attr)) - for attr in ( - "_types_info", - "_type_id_to_type_info", - "_user_type_id_to_type_info", - "_ns_type_to_type_info", - ) - ) - assert fory.type_resolver.get_type_info(RejectedRegistration, create=False) is None - - -def test_frozen_named_lookup(monkeypatch): - writer = Fory(xlang=True, compatible=False) - writer.register_type(FrozenRegistration, name="test.FrozenWireType") - data = writer.serialize(FrozenRegistration()) - - reader = Fory(xlang=True, strict=False, compatible=False) - loaded = False +def test_id_conflict_no_mutation(): + fory = Fory(xlang=True, compatible=False) + fory.register_type(FrozenRegistration, type_id=701) + resolver = fory.type_resolver + types_info = dict(resolver._types_info) + id_info = dict(resolver._user_type_id_to_type_info) + name_info = dict(resolver._ns_type_to_type_info) - def reject_load(*_args, **_kwargs): - nonlocal loaded - loaded = True - raise AssertionError("frozen named lookup must not load a class") + with pytest.raises(TypeError, match="user_type_id 701"): + fory.register_type(RejectedRegistration, type_id=701) - monkeypatch.setattr(registry_module, "load_class", reject_load) - with pytest.raises(Exception): - reader.deserialize(data) - assert not loaded + assert resolver._types_info == types_info + assert resolver._user_type_id_to_type_info == id_info + assert resolver._ns_type_to_type_info == name_info -def test_registered_types_finalize(): +def test_registered_types_build_lazily(): fory = Fory(xlang=True, compatible=True) parent_info = fory.register_type(FrozenParent, name="test.FrozenParent") child_info = fory.register_type(FrozenChild, name="test.FrozenChild") @@ -1144,613 +1002,33 @@ def test_registered_types_finalize(): assert fory.deserialize(data) == value -def test_lazy_dataclass_finalizes(): +def test_lazy_dataclass_serializer(): from pyfory.struct import DataClassStubSerializer fory = Fory(xlang=False, strict=False, compatible=False) - type_info = fory.register_type(BrokenFinalization) + type_info = fory.register_type(FrozenChild) assert isinstance(type_info.serializer, DataClassStubSerializer) - value = BrokenFinalization(7) + value = FrozenChild(7) data = fory.serialize(value) assert not isinstance(type_info.serializer, DataClassStubSerializer) assert fory.deserialize(data) == value -def test_recursive_serializer_stable(): - fory = Fory(xlang=True, compatible=True, ref=True) - type_info = fory.register_type(FrozenRecursive, name="test.FrozenRecursive") - value = FrozenRecursive(FrozenRecursive()) - - first = fory.serialize(value) - serializer = type_info.serializer - maps = tuple( - dict(getattr(fory.type_resolver, attr)) - for attr in ( - "_types_info", - "_type_id_to_type_info", - "_user_type_id_to_type_info", - "_ns_type_to_type_info", - ) - ) - - assert fory.deserialize(first) == value - assert fory.deserialize(fory.serialize(value)) == value - assert type_info.serializer is serializer - assert maps == tuple( - dict(getattr(fory.type_resolver, attr)) - for attr in ( - "_types_info", - "_type_id_to_type_info", - "_user_type_id_to_type_info", - "_ns_type_to_type_info", - ) - ) - - -def test_named_metadata_finalizes(): - enum_fory = Fory(xlang=True, compatible=True) - enum_info = enum_fory.register_type(FrozenMetadataEnum, name="test.FrozenMetadataEnum") - enum_serializer = enum_info.serializer - enum_data = enum_fory.serialize(FrozenMetadataEnum.VALUE) - assert enum_fory.deserialize(enum_data) is FrozenMetadataEnum.VALUE - assert enum_info.serializer is enum_serializer - - ext_fory = Fory(xlang=True, compatible=True) - ext_serializer = FrozenExtSerializer(ext_fory.type_resolver, FrozenExt) - ext_info = ext_fory.register_type(FrozenExt, name="test.FrozenExt", serializer=ext_serializer) - ext_data = ext_fory.serialize(FrozenExt(7)) - assert ext_fory.deserialize(ext_data) == FrozenExt(7) - assert ext_info.serializer is ext_serializer - - union_fory = Fory(xlang=True, compatible=True) - union_serializer = UnionSerializer(union_fory.type_resolver, FrozenUnion, {0: str}) - union_info = union_fory.register_union( - FrozenUnion, - name="test.FrozenUnion", - serializer=union_serializer, - ) - union_data = union_fory.serialize(FrozenUnion(0, "value")) - assert union_fory.deserialize(union_data) == FrozenUnion(0, "value") - assert union_info.serializer is union_serializer - - for type_info in (enum_info, ext_info, union_info): - assert type_info.type_def is not None - - -def test_pre_root_serializer_rebind(): - fory = Fory(xlang=True, compatible=True) - type_info = fory.register_type(FrozenExt, name="test.ReboundExt") - fory.type_resolver.get_serializer(FrozenExt) - old_header = Buffer(type_info.type_def.encoded).read_int64() - id_map = dict(fory.type_resolver._type_id_to_type_info) - - serializer = FrozenExtSerializer(fory.type_resolver, FrozenExt) - fory.register_serializer(FrozenExt, serializer) - assert type_info.type_def is None - assert fory.type_resolver._type_id_to_type_info == id_map - - data = fory.serialize(FrozenExt(9)) - assert fory.deserialize(data) == FrozenExt(9) - assert type_info.serializer is serializer - assert type_info.type_id == TypeId.NAMED_EXT - new_header = Buffer(type_info.type_def.encoded).read_int64() - assert new_header != old_header - - -def test_rebind_skips_default_build(monkeypatch): - fory = Fory(xlang=True, compatible=True) - type_info = fory.register_type(BrokenFinalization, name="test.ReboundPending") - serializer = FrozenExtSerializer(fory.type_resolver, BrokenFinalization) - - def reject_default_build(*_args): - raise AssertionError("custom serializer registration must not build the default serializer") - - monkeypatch.setattr(registry_module, "encode_typedef", reject_default_build) - fory.register_serializer(BrokenFinalization, serializer) - - assert type_info.serializer is serializer - assert type_info.type_def is None - - -def test_named_serializer_id_map(): - fory = Fory(xlang=True, compatible=True) - first = fory.register_type(FrozenExt, name="test.NamedFirst") - second = fory.register_type(FrozenSecondExt, name="test.NamedSecond") - for cls in (FrozenExt, FrozenSecondExt): - fory.type_resolver.get_serializer(cls) - id_map = dict(fory.type_resolver._type_id_to_type_info) - wire_map = dict(fory.type_resolver._ns_type_to_type_info) - named_infos = ( - fory.type_resolver.get_type_info_by_name("test", "NamedFirst"), - fory.type_resolver.get_type_info_by_name("test", "NamedSecond"), - ) - - first_serializer = FrozenExtSerializer(fory.type_resolver, FrozenExt) - second_serializer = FrozenExtSerializer(fory.type_resolver, FrozenSecondExt) - fory.register_serializer(FrozenExt, first_serializer) - fory.register_serializer(FrozenSecondExt, second_serializer) - - assert fory.type_resolver._type_id_to_type_info == id_map - assert TypeId.NAMED_EXT not in fory.type_resolver._type_id_to_type_info - assert fory.type_resolver._ns_type_to_type_info == wire_map - assert named_infos == ( - fory.type_resolver.get_type_info_by_name("test", "NamedFirst"), - fory.type_resolver.get_type_info_by_name("test", "NamedSecond"), - ) - assert fory.deserialize(fory.serialize(FrozenExt(3))) == FrozenExt(3) - assert first.serializer is first_serializer - assert second.serializer is second_serializer - - -def test_failed_finalization_freezes(monkeypatch): - writer = Fory(xlang=True, compatible=True) - writer.register_type( - FrozenExt, - name="test.FinalizedRoot", - serializer=FrozenExtSerializer(writer.type_resolver, FrozenExt), - ) - data = writer.serialize(FrozenExt(7)) - - fory = Fory(xlang=True, compatible=True) - read_calls = 0 - - class CountingSerializer(FrozenExtSerializer): - def read(self, read_context): - nonlocal read_calls - read_calls += 1 - return super().read(read_context) - - fory.register_type( - FrozenExt, - name="test.FinalizedRoot", - serializer=CountingSerializer(fory.type_resolver, FrozenExt), - ) - type_info = fory.register_type( - BrokenFinalization, - name="test.BrokenFinalization", - ) - pending_info = fory.register_type( - FrozenChild, - name="test.PendingFinalization", - ) - encode_type_def = registry_module.encode_typedef - finalization_calls = 0 - - class FinalizationAbort(BaseException): - pass - - def fail_finalization(resolver, cls): - nonlocal finalization_calls - if cls is type_info.cls: - finalization_calls += 1 - assert type_info.serializer is not None - raise FinalizationAbort - return encode_type_def(resolver, cls) - - monkeypatch.setattr( - registry_module, - "encode_typedef", - fail_finalization, - ) - - with pytest.raises(FinalizationAbort): - fory.deserialize(data) - assert read_calls == 0 - assert finalization_calls == 1 - assert type_info.serializer is None - assert type_info.type_def is None - assert pending_info.serializer is None - assert pending_info.type_def is None - - with pytest.raises(RuntimeError): - fory.deserialize(data) - assert read_calls == 0 - assert finalization_calls == 1 - assert pending_info.serializer is None - assert pending_info.type_def is None - - with pytest.raises(Exception): - fory.type_resolver.get_type_info(type_info.cls) - assert type_info.serializer is None - assert type_info.type_def is None - with pytest.raises(Exception): - fory.register_type(RejectedRegistration, name="test.Rejected") - assert fory.type_resolver.get_type_info(RejectedRegistration, create=False) is None - - -def test_reentrant_finalization_freezes(monkeypatch): - writer = Fory(xlang=True, compatible=True) - writer.register_type( - FrozenExt, - name="test.ReentrantRoot", - serializer=FrozenExtSerializer(writer.type_resolver, FrozenExt), - ) - data = writer.serialize(FrozenExt(7)) - - fory = Fory(xlang=True, compatible=True) - read_calls = 0 - - class CountingSerializer(FrozenExtSerializer): - def read(self, read_context): - nonlocal read_calls - read_calls += 1 - return super().read(read_context) - - fory.register_type( - FrozenExt, - name="test.ReentrantRoot", - serializer=CountingSerializer(fory.type_resolver, FrozenExt), - ) - type_info = fory.register_type( - BrokenFinalization, - name="test.ReentrantFinalization", - ) - encode_type_def = registry_module.encode_typedef - finalization_calls = 0 - - def reenter_root(resolver, cls): - nonlocal finalization_calls - if cls is type_info.cls: - finalization_calls += 1 - fory.deserialize(data) - return encode_type_def(resolver, cls) - - monkeypatch.setattr( - registry_module, - "encode_typedef", - reenter_root, - ) - - with pytest.raises(RuntimeError): - fory.deserialize(data) - assert read_calls == 0 - assert finalization_calls == 1 - assert type_info.serializer is None - assert type_info.type_def is None - - with pytest.raises(RuntimeError): - fory.deserialize(data) - assert read_calls == 0 - assert finalization_calls == 1 - with pytest.raises(Exception): - fory.register_type(RejectedRegistration, name="test.Rejected") - assert fory.type_resolver.get_type_info(RejectedRegistration, create=False) is None - - -@pytest.mark.skipif( - not pyfory.ENABLE_FORY_CYTHON_SERIALIZATION, - reason="Requires the Cython resolver cache", -) -def test_native_finalization_failure(): - write_calls = 0 - - class CountingSerializer(FrozenExtSerializer): - def write(self, write_context, value): - nonlocal write_calls - write_calls += 1 - super().write(write_context, value) - - fory = Fory(xlang=True, compatible=False) - type_info = fory.register_type( - FrozenExt, - name="test.NativeFinalization", - serializer=CountingSerializer(fory.type_resolver, FrozenExt), - ) - hash_reads = 0 - - class NativeSyncAbort(BaseException): - pass - - class BrokenMetaString: - @property - def hashcode(self): - nonlocal hash_reads - hash_reads += 1 - raise NativeSyncAbort - - type_info.namespace_bytes = BrokenMetaString() - - with pytest.raises(NativeSyncAbort): - fory.serialize(FrozenExt(7)) - assert hash_reads == 1 - assert write_calls == 0 - - with pytest.raises(RuntimeError): - fory.serialize(FrozenExt(7)) - assert hash_reads == 1 - assert write_calls == 0 - with pytest.raises(Exception): - fory.register_type(RejectedRegistration, name="test.Rejected") - assert fory.type_resolver.get_type_info(RejectedRegistration, create=False) is None - - -def test_native_carrier_registration(): - writer = Fory(xlang=False, strict=False, compatible=False) - reader = Fory(xlang=False, strict=False, compatible=False) - discovered = Fory(xlang=False, strict=False, compatible=False) - - writer_info = writer.register_type(types.FunctionType) - reader_info = reader.register_type(types.FunctionType) - discovered_info = discovered.type_resolver.get_type_info(types.FunctionType) - - assert writer_info.type_id == reader_info.type_id == discovered_info.type_id - value = lambda number: number + 1 # noqa: E731 - assert reader.deserialize(writer.serialize(value))(4) == 5 - - -def test_native_application_type(): - class PlainValue: - pass - - @dataclass - class DataValue: - value: int - - fory = Fory(xlang=False, strict=False, compatible=False) - plain_info = fory.register_type(PlainValue) - data_info = fory.register_type(DataValue) - - assert plain_info.type_id == TypeId.STRUCT - assert data_info.type_id == TypeId.STRUCT - - -def test_registration_conflicts(): - class First: - pass - - class Second: - pass - - class DifferentKind(Enum): - VALUE = 1 - - fory = Fory(xlang=True, compatible=False) - first = fory.register_type(First, name="SameName") - with pytest.raises(Exception): - fory.register_type(Second, name=".SameName") - with pytest.raises(Exception): - fory.register_type(DifferentKind, name="SameName") - assert fory.type_resolver.get_type_info_by_name("", "SameName") is first - assert fory.type_resolver.get_type_info(Second, create=False) is None - assert fory.type_resolver.get_type_info(DifferentKind, create=False) is None - - numeric = Fory(xlang=True, compatible=False) - first = numeric.register_type(First, type_id=703) - with pytest.raises(Exception): - numeric.register_type(Second, type_id=703) - assert ( - numeric.type_resolver.get_type_info_by_id( - TypeId.STRUCT, - user_type_id=703, - ) - is first - ) - assert numeric.type_resolver.get_type_info(Second, create=False) is None - - for registration in ("name", "id"): - union_fory = Fory(xlang=True, compatible=False) - serializer = UnionSerializer(union_fory.type_resolver, FrozenUnion, {0: str}) - if registration == "name": - union_fory.register_union( - FrozenUnion, - name="test.FirstUnion", - serializer=serializer, - ) - - def duplicate(): - union_fory.register_union( - FrozenUnion, - name="test.SecondUnion", - serializer=serializer, - ) - - else: - union_fory.register_union(FrozenUnion, type_id=704, serializer=serializer) - - def duplicate(): - union_fory.register_union( - FrozenUnion, - type_id=705, - serializer=serializer, - ) - - state = ( - dict(union_fory.type_resolver._types_info), - dict(union_fory.type_resolver._ns_type_to_type_info), - dict(union_fory.type_resolver._user_type_id_to_type_info), - ) - with pytest.raises(Exception): - duplicate() - assert state == ( - dict(union_fory.type_resolver._types_info), - dict(union_fory.type_resolver._ns_type_to_type_info), - dict(union_fory.type_resolver._user_type_id_to_type_info), - ) - - -@pytest.mark.parametrize("registration", ["type", "union"]) -@pytest.mark.parametrize("type_id", [-1, 0xFFFFFFFF, 0x100000000, 701.5, "701", True]) -def test_registration_id_range(registration, type_id): - fory = Fory(xlang=True, compatible=False) - before = registration_state(fory) - if registration == "type": - - def register(): - fory.register_type(RejectedRegistration, type_id=type_id) - - else: - serializer = UnionSerializer(fory.type_resolver, FrozenUnion, {0: str}) - - def register(): - fory.register_union( - FrozenUnion, - type_id=type_id, - serializer=serializer, - ) - - with pytest.raises(Exception): - register() - assert registration_state(fory) == before - - -@pytest.mark.parametrize( - "name", - [ - pytest.param(f"{'n' * 32768}.Value", id="namespace"), - pytest.param(f"scope.{'V' * 32768}", id="typename"), - ], -) -def test_registration_name_preflight(name): - class NamedValue: - pass - - serializer_count = 0 - - class CountingSerializer(pyfory.Serializer): - def __init__(self, type_resolver, cls): - nonlocal serializer_count - serializer_count += 1 - super().__init__(type_resolver, cls) - - fory = Fory(xlang=True, compatible=False) - before = registration_state(fory) - with pytest.raises(Exception): - fory.register_type(NamedValue, name=name, serializer=CountingSerializer) - assert serializer_count == 0 - assert registration_state(fory) == before - - -@pytest.mark.parametrize("registration", ["type", "union"]) -def test_failed_registration_keeps_id(registration): - class FailedValue: - pass - - class NextValue: - pass - - class BrokenSerializer: - def __init__(self, *_args): - raise ValueError("serializer construction failed") - - fory = Fory(xlang=True, compatible=False) - before = registration_state(fory) - with pytest.raises(Exception): - if registration == "type": - fory.register_type(FailedValue, serializer=BrokenSerializer) - else: - fory.register_union(FailedValue, serializer=BrokenSerializer) - assert registration_state(fory) == before - - actual = fory.register_type(NextValue) - expected_fory = Fory(xlang=True, compatible=False) - expected = expected_fory.register_type(NextValue) - assert actual.user_type_id == expected.user_type_id - - -@pytest.mark.parametrize("registration", ["type", "union"]) -def test_nested_registration_ids(registration): - fory = Fory(xlang=True, compatible=False) - nested_info = None - target = FrozenExt if registration == "type" else FrozenUnion - - def serializer_factory(type_resolver, cls): - nonlocal nested_info - nested_info = fory.register_type(FrozenSecondExt) - if registration == "type": - return FrozenExtSerializer(type_resolver, cls) - return UnionSerializer(type_resolver, cls, {0: str}) - - if registration == "type": - type_info = fory.register_type(target, serializer=serializer_factory) - else: - type_info = fory.register_union(target, serializer=serializer_factory) - - assert nested_info.user_type_id + 1 == type_info.user_type_id - - -@pytest.mark.parametrize("conflict", ["class", "id", "name"]) -def test_nested_registration_conflict(conflict): - fory = Fory(xlang=True, compatible=False) - resolver = fory.type_resolver - nested_info = None - nested_cls = FrozenExt if conflict == "class" else FrozenSecondExt - outer_args = {} - nested_args = {} - if conflict == "class": - nested_args["type_id"] = 735 - elif conflict == "id": - outer_args["type_id"] = 735 - nested_args["type_id"] = 735 - else: - outer_args["name"] = "test.NestedConflict" - nested_args["name"] = "test.NestedConflict" - - def serializer_factory(type_resolver, cls): - nonlocal nested_info - serializer = FrozenExtSerializer(type_resolver, nested_cls) - nested_info = fory.register_type( - nested_cls, - serializer=serializer, - **nested_args, - ) - return FrozenExtSerializer(type_resolver, cls) - - with pytest.raises(TypeError): - fory.register_type( - FrozenExt, - serializer=serializer_factory, - **outer_args, - ) - - assert resolver.get_type_info(nested_cls, create=False) is nested_info - assert resolver._types_info[nested_cls] is nested_info - if conflict == "class": - assert resolver._user_type_id_to_type_info[735] is nested_info - - class NextValue: - pass - - actual = fory.register_type(NextValue) - expected_fory = Fory(xlang=True, compatible=False) - expected = expected_fory.register_type(NextValue) - assert actual.user_type_id == expected.user_type_id - else: - assert resolver.get_type_info(FrozenExt, create=False) is None - if conflict == "id": - assert resolver._user_type_id_to_type_info[735] is nested_info - else: - assert resolver.get_type_info_by_name("test", "NestedConflict") is nested_info - key = (nested_info.namespace_bytes, nested_info.typename_bytes) - assert resolver._ns_type_to_type_info[key] is nested_info - - -def test_duplicate_type_keeps_id(): - class FirstValue: - pass - - class NextValue: - pass - - serializer_count = 0 - - class CountingSerializer(pyfory.Serializer): - def __init__(self, type_resolver, cls): - nonlocal serializer_count - serializer_count += 1 - super().__init__(type_resolver, cls) +def test_np_types(): + fory = Fory(xlang=False, ref=True, strict=False, compatible=False) + o1 = [1, True, np.dtype(np.int32)] + data1 = fory.serialize(o1) + new_o1 = fory.deserialize(data1) + assert o1 == new_o1 - fory = Fory(xlang=True, compatible=False) - first = fory.register_type(FirstValue) - before = registration_state(fory) - with pytest.raises(Exception): - fory.register_type(FirstValue, serializer=CountingSerializer) - assert serializer_count == 0 - assert registration_state(fory) == before - next_value = fory.register_type(NextValue) - assert next_value.user_type_id == first.user_type_id + 1 +def test_pandas_dataframe(): + fory = Fory(xlang=False, ref=True, strict=False, compatible=False) + df = pd.DataFrame({"a": list(range(10))}) + df2 = fory.deserialize(fory.serialize(df)) + assert df2.equals(df) def test_unsupported_callback(): @@ -1765,7 +1043,6 @@ def f2(x): return x + x obj1 = [1, True, f1, f2, {1: 2}] - fory.register_type(type(f1)) unsupported_objects = [] binary1 = fory.serialize(obj1, unsupported_callback=unsupported_objects.append) # Functions are now properly supported, so unsupported_objects should be empty @@ -1821,7 +1098,6 @@ class SparseIntEnum(IntEnum): def test_enum(): fory = Fory(xlang=False, ref=True, compatible=False) - fory.register_type(EnumClass) assert ser_de(fory, EnumClass.E1) == EnumClass.E1 assert ser_de(fory, EnumClass.E2) == EnumClass.E2 assert ser_de(fory, EnumClass.E3) == EnumClass.E3 @@ -1838,7 +1114,6 @@ def test_xlang_enum_uses_sparse_integer_values(): def test_duplicate_serialize(): fory = Fory(xlang=False, ref=True, compatible=False) - fory.register_type(EnumClass) assert ser_de(fory, EnumClass.E1) == EnumClass.E1 assert ser_de(fory, EnumClass.E2) == EnumClass.E2 assert ser_de(fory, EnumClass.E4) == EnumClass.E4 @@ -1852,7 +1127,7 @@ def test_pandas_range_index(): fory.register_type(pd.RangeIndex, serializer=pyfory.serializer.PandasRangeIndexSerializer(fory.type_resolver)) index = pd.RangeIndex(1, 100, 2, name="a") new_index = ser_de(fory, index) - pd.testing.assert_index_equal(new_index, index) + pd.testing.assert_index_equal(new_index, new_index) @dataclass(unsafe_hash=True) @@ -1874,7 +1149,6 @@ def test_py_serialize_dataclass(track_ref): strict=False, compatible=False, ) - fory.register_type(PyDataClass1) obj1 = PyDataClass1(f1=1, f2=-2.0, f3="abc", f4=True, f5="xyz", f6=[1, 2], f7={"k1": "v1"}) assert ser_de(fory, obj1) == obj1 obj2 = PyDataClass1(f1=None, f2=-2.0, f3="abc", f4=None, f5="xyz", f6=None, f7=None) @@ -1935,7 +1209,6 @@ def test_function(track_ref): strict=False, compatible=False, ) - fory.register_type(types.FunctionType) c = fory.deserialize(fory.serialize(lambda x: x * 2)) assert c(2) == 4 @@ -1945,6 +1218,10 @@ def func(x): c = fory.deserialize(fory.serialize(func)) assert c(2) == 4 + df = pd.DataFrame({"a": list(range(10))}) + df_sum = fory.deserialize(fory.serialize(df.sum)) + assert df_sum().equals(df.sum()) + @dataclass(unsafe_hash=True) class MapFields: @@ -2000,8 +1277,6 @@ def __eq__(self, other): map_fields_object.dict_with_custom_obj = dict_with_custom_obj map_fields_object.single_key_dict = single_key_dict - fory.register_type(MapFields) - fory.register_type(CustomClass) serialized = fory.serialize(map_fields_object) deserialized = fory.deserialize(serialized) @@ -2055,7 +1330,6 @@ def test_py_serialize_object(track_ref): def test_py_serialize_empty_object(track_ref): fory = Fory(xlang=False, ref=track_ref, strict=False, compatible=False) obj = object() - fory.register_type(object) result = ser_de(fory, obj) assert type(result) is object diff --git a/python/pyfory/tests/test_stateful_serializer.py b/python/pyfory/tests/test_stateful_serializer.py index 1cb9d4c95d..c0cd6f859d 100644 --- a/python/pyfory/tests/test_stateful_serializer.py +++ b/python/pyfory/tests/test_stateful_serializer.py @@ -149,7 +149,6 @@ def test_basic_stateful_object(): fory = Fory(xlang=False, ref=True, strict=False, compatible=False) obj = BasicStatefulObject(42, "original_secret") - fory.register_type(BasicStatefulObject) serialized = fory.serialize(obj) deserialized = fory.deserialize(serialized) @@ -169,7 +168,6 @@ def test_immutable_with_getnewargs_ex(): fory = Fory(xlang=False, ref=True, strict=False, compatible=False) obj = ImmutableWithArgsEx(10, 20, "test") - fory.register_type(ImmutableWithArgsEx) # Simulate the state that would be set by __setstate__ for comparison obj._extra = "some_extra" @@ -193,7 +191,6 @@ def test_immutable_with_getnewargs(): fory = Fory(xlang=False, ref=True, strict=False, compatible=False) obj = ImmutableWithArgs(100, 200) - fory.register_type(ImmutableWithArgs) # Simulate the state that would be set by __setstate__ for comparison obj._metadata = "old_style" @@ -216,7 +213,6 @@ def test_stateful_only_object(): fory = Fory(xlang=False, ref=True, strict=False, compatible=False) obj = StatefulOnlyObject("test_data") - fory.register_type(StatefulOnlyObject) # Simulate the state that would be set by __setstate__ for comparison obj.processed = "restored_test_data" @@ -238,7 +234,6 @@ def test_complex_state_object(): fory = Fory(xlang=False, ref=True, strict=False, compatible=False) obj = ComplexStateObject("test", [1, 2, 3, {"nested": "value"}]) - fory.register_type(ComplexStateObject) # Simulate the state that would be set by __setstate__ for comparison obj.extra_info = {"serialized_at": "test_time"} @@ -262,7 +257,6 @@ def test_reference_tracking(): fory = Fory(xlang=False, ref=True, strict=False, compatible=False) obj = BasicStatefulObject(42) - fory.register_type(BasicStatefulObject) # Create a list with the same object referenced twice container = [obj, obj, {"ref": obj}] @@ -281,8 +275,6 @@ def test_nested_stateful_objects(): inner = BasicStatefulObject(10) outer = ComplexStateObject("outer", [inner, BasicStatefulObject(20)]) - fory.register_type(BasicStatefulObject) - fory.register_type(ComplexStateObject) serialized = fory.serialize(outer) deserialized = fory.deserialize(serialized) @@ -296,7 +288,7 @@ def test_nested_stateful_objects(): assert deserialized.items[1].value == 20 -def test_registered_stateful_roundtrip(): +def test_cross_language_compatibility(): """Test that StatefulSerializer works with type registration""" fory = Fory(xlang=False, ref=True, strict=True, compatible=False) diff --git a/python/pyfory/tests/test_struct.py b/python/pyfory/tests/test_struct.py index 8bcd34a943..fc49cd8058 100644 --- a/python/pyfory/tests/test_struct.py +++ b/python/pyfory/tests/test_struct.py @@ -663,7 +663,6 @@ def test_inheritance(): print(type_hints) assert type_hints.keys() == {"f1", "f2", "f3"} fory = Fory(xlang=False, ref=True, strict=False, compatible=False) - fory.register_type(ChildClass1) obj = ChildClass1(f1="a", f2=-10, f3={"a": -10.0, "b": 1 / 3}) assert ser_de(fory, obj) == obj assert type(fory.type_resolver.get_serializer(ChildClass1)) is pyfory.DataClassSerializer @@ -815,7 +814,6 @@ class TemporalNumberClass: ) def test_bool_field_coercion(value, expected): fory = Fory(xlang=False, ref=True, strict=False, compatible=False) - fory.register_type(BoolCoercionObject) result = ser_de(fory, BoolCoercionObject(value)) assert result.b is expected @@ -823,7 +821,6 @@ def test_bool_field_coercion(value, expected): def test_bool_field_coercion_numpy_bool(): np = pytest.importorskip("numpy") fory = Fory(xlang=False, ref=True, strict=False, compatible=False) - fory.register_type(BoolCoercionObject) result_true = ser_de(fory, BoolCoercionObject(np.bool_(True))) assert result_true.b is True @@ -918,9 +915,8 @@ def test_data_class_serializer_xlang(): @pytest.mark.parametrize("track_ref", [False, True]) -def test_dataclass_typed_tuple(track_ref): +def test_dataclass_with_typed_tuple_field(track_ref): fory = Fory(xlang=False, ref=track_ref, strict=False, compatible=False) - fory.register_type(TupleFieldObject) obj = TupleFieldObject(bar=("a", 1)) assert ser_de(fory, obj) == obj @@ -1135,8 +1131,6 @@ def test_optional_fields(xlang, compatible): fory = Fory(xlang=xlang, ref=True, compatible=compatible, strict=False) if xlang: fory.register_type(OptionalFieldsObject, name="example.OptionalFieldsObject") - else: - fory.register_type(OptionalFieldsObject) obj_with_none = OptionalFieldsObject(f1=None, f2=None, f3=None, f4=42, f5="test") result = ser_de(fory, obj_with_none) @@ -1177,9 +1171,6 @@ def test_nested_optional_fields(xlang, compatible): if xlang: fory.register_type(ComplexObject, name="example.ComplexObject") fory.register_type(NestedOptionalObject, name="example.NestedOptionalObject") - else: - fory.register_type(ComplexObject) - fory.register_type(NestedOptionalObject) obj_with_none = NestedOptionalObject(f1=None, f2=None, f3="test") result = ser_de(fory, obj_with_none) diff --git a/python/pyfory/tests/test_thread_safe.py b/python/pyfory/tests/test_thread_safe.py index 29c1bbce0f..d7111283cd 100644 --- a/python/pyfory/tests/test_thread_safe.py +++ b/python/pyfory/tests/test_thread_safe.py @@ -20,7 +20,6 @@ import pytest -import pyfory from pyfory import ThreadSafeFory @@ -192,429 +191,3 @@ def test_thread_safe_fory_register_after_use(): with pytest.raises(RuntimeError): fory.register(Address) - - -def test_invalid_registration(): - failed_constructions = 0 - valid_constructions = 0 - - class BrokenSerializer: - def __init__(self, *_args): - nonlocal failed_constructions - failed_constructions += 1 - raise ValueError("serializer construction failed") - - class AddressSerializer(pyfory.Serializer): - def write(self, write_context, value): - write_context.write_string(value.city) - write_context.write_string(value.country) - - def read(self, read_context): - return Address(read_context.read_string(), read_context.read_string()) - - def serializer_factory(type_resolver, cls): - nonlocal valid_constructions - valid_constructions += 1 - return AddressSerializer(type_resolver, cls) - - fory = ThreadSafeFory(xlang=False, compatible=False) - with pytest.raises(ValueError): - fory.register_type(Address, serializer=BrokenSerializer) - assert failed_constructions == 1 - - fory.register_type(Address, serializer=serializer_factory) - address = Address(city="Oslo", country="Norway") - assert fory.deserialize(fory.serialize(address)) == address - assert failed_constructions == 1 - assert valid_constructions == 1 - - -def test_zero_arg_serializer_rejected(): - class AddressSerializer(pyfory.Serializer): - def write(self, write_context, value): - write_context.write_string(value.city) - write_context.write_string(value.country) - - def read(self, read_context): - return Address(read_context.read_string(), read_context.read_string()) - - children = [] - constructions = 0 - - def fory_factory(): - child = pyfory.Fory(xlang=False, compatible=False) - children.append(child) - return child - - def serializer_factory(): - nonlocal constructions - constructions += 1 - child = children[-1] - return AddressSerializer(child.type_resolver, Address) - - fory = ThreadSafeFory(fory_factory=fory_factory) - with pytest.raises(TypeError): - fory.register_type(Address, serializer=serializer_factory) - - assert constructions == 0 - assert len(children) == 1 - assert children[0].type_resolver.get_type_info(Address, create=False) is None - - fory.register_type(Address, serializer=AddressSerializer) - value = Address(city="Oslo", country="Norway") - assert fory.deserialize(fory.serialize(value)) == value - - -@pytest.mark.parametrize("method", ["register", "register_type", "register_union"]) -def test_serializer_instance_rejected(method): - class AddressSerializer(pyfory.Serializer): - def write(self, write_context, value): - write_context.write_string(value.city) - write_context.write_string(value.country) - - def read(self, read_context): - return Address(read_context.read_string(), read_context.read_string()) - - owner = pyfory.Fory(xlang=False, compatible=False) - serializer = AddressSerializer(owner.type_resolver, Address) - builds = 0 - - def fory_factory(): - nonlocal builds - builds += 1 - return pyfory.Fory(xlang=False, compatible=False) - - fory = ThreadSafeFory(fory_factory=fory_factory) - - with pytest.raises(TypeError): - getattr(fory, method)(Address, serializer=serializer) - - assert fory._registrations == [] - assert fory._registration_fory is None - assert not fory._root_started - assert builds == 0 - - -def test_serializer_factory_per_child(): - class AddressSerializer(pyfory.Serializer): - def write(self, write_context, value): - write_context.write_string(value.city) - write_context.write_string(value.country) - - def read(self, read_context): - return Address(read_context.read_string(), read_context.read_string()) - - serializers = [] - - def serializer_factory(type_resolver, cls): - serializer = AddressSerializer(type_resolver, cls) - serializers.append(serializer) - return serializer - - fory = ThreadSafeFory(xlang=False, compatible=False) - fory.register_type(Address, serializer=serializer_factory) - first = fory._registration_fory - second = fory._build_fory() - - assert len(serializers) == 2 - assert serializers[0] is not serializers[1] - assert serializers[0].type_resolver is first.type_resolver - assert serializers[1].type_resolver is second.type_resolver - address = Address(city="Oslo", country="Norway") - assert second.deserialize(second.serialize(address)) == address - assert fory.deserialize(fory.serialize(address)) == address - - -@pytest.mark.parametrize("result", ["value", "resolver", "type"]) -def test_serializer_factory_result(result): - class AddressSerializer(pyfory.Serializer): - def write(self, write_context, value): - write_context.write_string(value.city) - write_context.write_string(value.country) - - def read(self, read_context): - return Address(read_context.read_string(), read_context.read_string()) - - foreign = pyfory.Fory(xlang=False, compatible=False) - - def serializer_factory(type_resolver, cls): - if result == "value": - return object() - if result == "resolver": - return AddressSerializer(foreign.type_resolver, cls) - return AddressSerializer(type_resolver, Person) - - fory = ThreadSafeFory(xlang=False, compatible=False) - with pytest.raises(TypeError): - fory.register_type(Address, serializer=serializer_factory) - - assert not fory._registrations - assert fory._registration_fory is None - - -def test_singleton_serializer_factory(): - class AddressSerializer(pyfory.Serializer): - def write(self, write_context, value): - write_context.write_string(value.city) - write_context.write_string(value.country) - - def read(self, read_context): - return Address(read_context.read_string(), read_context.read_string()) - - children = [] - singleton = None - - def child_factory(): - child = pyfory.Fory(xlang=False, compatible=False) - children.append(child) - return child - - def serializer_factory(type_resolver, cls): - nonlocal singleton - if singleton is None: - singleton = AddressSerializer(type_resolver, cls) - return singleton - - fory = ThreadSafeFory(fory_factory=child_factory) - fory.register_type(Address, serializer=serializer_factory) - assert singleton.type_resolver is children[0].type_resolver - - with pytest.raises(TypeError): - fory._build_fory() - - assert len(children) == 2 - assert children[1].type_resolver.get_type_info(Address, create=False) is None - - -def test_factory_serializer_owner(): - class AddressSerializer(pyfory.Serializer): - def write(self, write_context, value): - write_context.write_string(value.city) - write_context.write_string(value.country) - - def read(self, read_context): - return Address(read_context.read_string(), read_context.read_string()) - - serializers = [] - - def fory_factory(): - fory = pyfory.Fory(xlang=False, compatible=False) - serializer = AddressSerializer(fory.type_resolver, Address) - fory.register_type(Address, serializer=serializer) - serializers.append(serializer) - return fory - - fory = ThreadSafeFory(fory_factory=fory_factory) - first = fory._build_fory() - second = fory._build_fory() - - assert len(serializers) == 2 - assert serializers[0] is not serializers[1] - assert serializers[0].type_resolver is first.type_resolver - assert serializers[1].type_resolver is second.type_resolver - address = Address(city="Oslo", country="Norway") - assert first.deserialize(first.serialize(address)) == address - assert second.deserialize(second.serialize(address)) == address - - -def test_reentrant_registration(): - class AddressSerializer(pyfory.Serializer): - def write(self, write_context, value): - write_context.write_string(value.city) - write_context.write_string(value.country) - - def read(self, read_context): - return Address(read_context.read_string(), read_context.read_string()) - - fory = ThreadSafeFory(xlang=False, compatible=False) - constructions = 0 - errors = [] - - def serializer_factory(type_resolver, cls): - nonlocal constructions - constructions += 1 - fory.serialize(None) - return AddressSerializer(type_resolver, cls) - - def register(): - try: - fory.register_type(Address, serializer=serializer_factory) - except RuntimeError as exc: - errors.append(exc) - - thread = threading.Thread(target=register, daemon=True) - thread.start() - thread.join(timeout=5) - - assert not thread.is_alive() - assert constructions == 1 - assert len(errors) == 1 - assert isinstance(errors[0], RuntimeError) - assert not fory._registrations - assert fory._registration_fory is None - assert fory.deserialize(fory.serialize(None)) is None - with pytest.raises(RuntimeError): - fory.register_type(Person) - - -def test_nested_registration(): - class AddressSerializer(pyfory.Serializer): - def write(self, write_context, value): - write_context.write_string(value.city) - write_context.write_string(value.country) - - def read(self, read_context): - return Address(read_context.read_string(), read_context.read_string()) - - fory = ThreadSafeFory(xlang=True, compatible=False) - constructions = 0 - errors = [] - - def serializer_factory(type_resolver, cls): - nonlocal constructions - constructions += 1 - fory.register_type(Person) - return AddressSerializer(type_resolver, cls) - - def register(): - try: - fory.register_type(Address, serializer=serializer_factory) - except (RuntimeError, TypeError) as exc: - errors.append(exc) - - thread = threading.Thread(target=register, daemon=True) - thread.start() - thread.join(timeout=5) - - assert not thread.is_alive() - assert not errors - first = fory._registration_fory - second = fory._build_fory() - for child in (first, second): - resolver = child.type_resolver - person_info = resolver.get_type_info(Person, create=False) - address_info = resolver.get_type_info(Address, create=False) - assert person_info.user_type_id + 1 == address_info.user_type_id - address = Address(city="Oslo", country="Norway") - assert second.deserialize(second.serialize(address)) == address - assert fory.deserialize(fory.serialize(address)) == address - assert constructions == 2 - - -@pytest.mark.parametrize("scenario", ["unknown", "different"]) -def test_nested_replay_rejected(scenario): - class AddressSerializer(pyfory.Serializer): - def write(self, write_context, value): - write_context.write_string(value.city) - write_context.write_string(value.country) - - def read(self, read_context): - return Address(read_context.read_string(), read_context.read_string()) - - class Unknown: - pass - - children = [] - replay = False - - def child_factory(): - child = pyfory.Fory(xlang=True, compatible=False) - children.append(child) - return child - - def serializer_factory(type_resolver, cls): - if not replay: - fory.register_type(Person) - elif scenario == "unknown": - fory.register_type(Unknown) - else: - fory.register_type(Person, type_id=101) - return AddressSerializer(type_resolver, cls) - - fory = ThreadSafeFory(fory_factory=child_factory) - fory.register_type(Address, serializer=serializer_factory) - replay = True - - with pytest.raises(RuntimeError): - fory._build_fory() - - child = children[1] - person_info = child.type_resolver.get_type_info(Person, create=False) - assert person_info is not None - assert person_info.user_type_id != 101 - assert child.type_resolver.get_type_info(Address, create=False) is None - assert child.type_resolver.get_type_info(Unknown, create=False) is None - assert fory._replay_limit == 0 - assert fory._building_thread is None - - -def test_factory_root_reentry(): - fory = None - constructions = 0 - - def fory_factory(): - nonlocal constructions - constructions += 1 - fory.serialize(None) - return pyfory.Fory(xlang=False, compatible=False) - - fory = ThreadSafeFory(fory_factory=fory_factory) - with pytest.raises(Exception): - fory.register_type(Person) - - assert constructions == 1 - assert fory._root_started - assert not fory._registrations - assert fory._registration_fory is None - with pytest.raises(RuntimeError): - fory.register_type(Address) - - -def test_build_owner_precedes_pool(): - fory = None - pooled = pyfory.Fory(xlang=False, compatible=False) - - def fory_factory(): - fory._return_fory(pooled) - fory.serialize(None) - return pyfory.Fory(xlang=False, compatible=False) - - fory = ThreadSafeFory(fory_factory=fory_factory) - - with pytest.raises(RuntimeError): - fory.serialize(None) - - assert fory._pool == [pooled] - - -def test_callback_root_reentry(): - class AddressSerializer(pyfory.Serializer): - def write(self, write_context, value): - write_context.write_string(value.city) - write_context.write_string(value.country) - - def read(self, read_context): - return Address(read_context.read_string(), read_context.read_string()) - - fory = pyfory.ThreadSafeFory(xlang=False, compatible=False) - constructions = 0 - reenter_root = False - - def serializer_factory(type_resolver, cls): - nonlocal constructions, reenter_root - constructions += 1 - if reenter_root: - fory.serialize(None) - return AddressSerializer(type_resolver, cls) - - fory.register_type(Address, type_id=100, serializer=serializer_factory) - initial_constructions = constructions - with pytest.raises(TypeError): - fory.register_type(Person, type_id=100) - - reenter_root = True - with pytest.raises(RuntimeError): - fory.serialize(None) - - assert constructions == initial_constructions + 1 - assert fory._root_started From 88adec82adc37857e78e734a2f01020852e93551 Mon Sep 17 00:00:00 2001 From: chaokunyang Date: Sun, 30 Aug 2026 02:59:51 +0800 Subject: [PATCH 095/168] refactor(cpp): clarify context resolver construction --- cpp/fory/serialization/fory.h | 20 ++++----- cpp/fory/serialization/serialization_test.cc | 26 +++++------ cpp/fory/serialization/type_resolver.cc | 45 ++++++++++---------- cpp/fory/serialization/type_resolver.h | 11 +++-- 4 files changed, 51 insertions(+), 51 deletions(-) diff --git a/cpp/fory/serialization/fory.h b/cpp/fory/serialization/fory.h index 300391b50d..12d84e6248 100644 --- a/cpp/fory/serialization/fory.h +++ b/cpp/fory/serialization/fory.h @@ -719,11 +719,11 @@ class Fory : public BaseFory { void ensure_contexts_initialized() { if (!write_ctx_.has_value()) { FORY_CHECK(!read_ctx_.has_value()); - auto final_result = type_resolver_->build_final_type_resolver(); - FORY_CHECK(final_result.ok()) - << "Failed to build finalized TypeResolver: " - << final_result.error().to_string(); - auto prepared_resolver = std::move(final_result).value(); + auto context_result = type_resolver_->build_context_type_resolver(); + FORY_CHECK(context_result.ok()) + << "Failed to build context TypeResolver: " + << context_result.error().to_string(); + auto prepared_resolver = std::move(context_result).value(); // Create contexts with cloned resolvers write_ctx_.emplace(config_, prepared_resolver->clone()); read_ctx_.emplace(config_, prepared_resolver->clone()); @@ -1025,11 +1025,11 @@ class ThreadSafeFory : public BaseFory { void ensure_resolver_initialized() const { std::call_once(resolver_once_flag_, [this]() { - auto final_result = type_resolver_->build_final_type_resolver(); - FORY_CHECK(final_result.ok()) - << "Failed to build finalized TypeResolver: " - << final_result.error().to_string(); - shared_resolver_ = std::move(final_result).value(); + auto context_result = type_resolver_->build_context_type_resolver(); + FORY_CHECK(context_result.ok()) + << "Failed to build context TypeResolver: " + << context_result.error().to_string(); + shared_resolver_ = std::move(context_result).value(); }); } diff --git a/cpp/fory/serialization/serialization_test.cc b/cpp/fory/serialization/serialization_test.cc index 5f5ab88a29..d34b7368e6 100644 --- a/cpp/fory/serialization/serialization_test.cc +++ b/cpp/fory/serialization/serialization_test.cc @@ -1741,8 +1741,8 @@ TEST(SerializationTest, ExpectedLocalTypeMetaStaysRootLocal) { ASSERT_TRUE( fory.register_extension_type("example", "ExpectedExt").ok()); ASSERT_TRUE(fory.register_union("example", "ExpectedUnion").ok()); - auto finalized = fory.serialize(SimpleStruct{}); - ASSERT_TRUE(finalized.ok()) << finalized.error().to_string(); + auto serialized = fory.serialize(SimpleStruct{}); + ASSERT_TRUE(serialized.ok()) << serialized.error().to_string(); ReadContext ctx(fory.config(), fory.type_resolver().clone()); auto struct_info = ctx.type_resolver().get_type_info(); @@ -1801,8 +1801,8 @@ TEST(SerializationTest, LocalTypeMetaPrecedesRemoteCache) { auto fory = Fory::builder().xlang(true).compatible(true).build(); ASSERT_TRUE( fory.register_enum("example", "WarmLocal").ok()); - auto finalized = fory.serialize(SignedScopedStatus::ZERO); - ASSERT_TRUE(finalized.ok()) << finalized.error().to_string(); + auto serialized = fory.serialize(SignedScopedStatus::ZERO); + ASSERT_TRUE(serialized.ok()) << serialized.error().to_string(); auto expected = fory.type_resolver().get_type_info(); ASSERT_TRUE(expected.ok()) << expected.error().to_string(); const std::vector &type_def = expected.value()->type_def; @@ -1846,8 +1846,8 @@ TEST(SerializationTest, StaticTypeMetaChecksOwner) { ASSERT_TRUE(fory.register_enum("example", "EnumB").ok()); ASSERT_TRUE(fory.register_union("example", "UnionA").ok()); ASSERT_TRUE(fory.register_union("example", "UnionB").ok()); - auto finalized = fory.serialize(SimpleStruct{}); - ASSERT_TRUE(finalized.ok()) << finalized.error().to_string(); + auto serialized = fory.serialize(SimpleStruct{}); + ASSERT_TRUE(serialized.ok()) << serialized.error().to_string(); ReadContext ctx(fory.config(), fory.type_resolver().clone()); auto struct_b = ctx.type_resolver().get_type_info(); @@ -1884,8 +1884,8 @@ TEST(SerializationTest, CachedTypeMetaChecksOwner) { ASSERT_TRUE( fory.register_enum("example", "CacheEnumA").ok()); ASSERT_TRUE(fory.register_union("example", "CacheUnionA").ok()); - auto finalized = fory.serialize(SimpleStruct{}); - ASSERT_TRUE(finalized.ok()) << finalized.error().to_string(); + auto serialized = fory.serialize(SimpleStruct{}); + ASSERT_TRUE(serialized.ok()) << serialized.error().to_string(); ReadContext ctx(fory.config(), fory.type_resolver().clone()); expect_cached_owner_mismatch( @@ -1910,8 +1910,8 @@ TEST(SerializationTest, StaticCollectionChecksOwner) { fory.register_struct("example", "CollectionA").ok()); ASSERT_TRUE( fory.register_struct("example", "CollectionB").ok()); - auto finalized = fory.serialize(SimpleStruct{}); - ASSERT_TRUE(finalized.ok()) << finalized.error().to_string(); + auto serialized = fory.serialize(SimpleStruct{}); + ASSERT_TRUE(serialized.ok()) << serialized.error().to_string(); auto first = make_remote_type_meta("CollectionB", "first_remote"); auto second = make_remote_type_meta("CollectionB", "second_remote"); @@ -1950,8 +1950,8 @@ TEST(SerializationTest, StaticMapChecksOwner) { .build(); ASSERT_TRUE(fory.register_struct("example", "MapA").ok()); ASSERT_TRUE(fory.register_struct("example", "MapB").ok()); - auto finalized = fory.serialize(SimpleStruct{}); - ASSERT_TRUE(finalized.ok()) << finalized.error().to_string(); + auto serialized = fory.serialize(SimpleStruct{}); + ASSERT_TRUE(serialized.ok()) << serialized.error().to_string(); auto first = make_remote_type_meta("MapB", "first_remote"); auto second = make_remote_type_meta("MapB", "second_remote"); @@ -2142,7 +2142,7 @@ TEST(SerializationTest, IdExtDoesNotUseTypeMetaLimits) { EXPECT_EQ(decoded.value(), IdLimitExt{42}); } -TEST(SerializationTest, LocalTypeMetaFinalizationIgnoresReceiveBodyLimit) { +TEST(SerializationTest, LocalTypeMetaCompletionIgnoresReceiveBodyLimit) { auto fory = Fory::builder() .xlang(true) .compatible(true) diff --git a/cpp/fory/serialization/type_resolver.cc b/cpp/fory/serialization/type_resolver.cc index cb8d8cdcab..7a4b729eb7 100644 --- a/cpp/fory/serialization/type_resolver.cc +++ b/cpp/fory/serialization/type_resolver.cc @@ -1774,17 +1774,17 @@ Result TypeResolver::check_registration() { } Result, Error> -TypeResolver::build_final_type_resolver() { +TypeResolver::build_context_type_resolver() { std::lock_guard lock(registration_mutex_); registry_frozen_ = true; - auto final_resolver = std::make_unique(); + auto context_resolver = std::make_unique(); // copy configuration - final_resolver->compatible_ = compatible_; - final_resolver->xlang_ = xlang_; - final_resolver->check_struct_version_ = check_struct_version_; - final_resolver->track_ref_ = track_ref_; - final_resolver->registry_frozen_ = true; + context_resolver->compatible_ = compatible_; + context_resolver->xlang_ = xlang_; + context_resolver->check_struct_version_ = check_struct_version_; + context_resolver->track_ref_ = track_ref_; + context_resolver->registry_frozen_ = true; // Build mapping from old pointers to new pointers for rebuilding lookup maps fory::flat_hash_map ptr_map; @@ -1794,7 +1794,7 @@ TypeResolver::build_final_type_resolver() { auto cloned = info->deep_clone(); TypeInfo *new_ptr = cloned.get(); ptr_map[info.get()] = new_ptr; - final_resolver->type_infos_.push_back(std::move(cloned)); + context_resolver->type_infos_.push_back(std::move(cloned)); } auto remap_type_info = [&ptr_map](const TypeInfo *old_ptr) { auto *entry = ptr_map.find(old_ptr); @@ -1804,31 +1804,32 @@ TypeResolver::build_final_type_resolver() { // Rebuild lookup maps with new pointers for (const auto &[key, old_ptr] : type_info_by_ctid_) { - final_resolver->type_info_by_ctid_.put(key, remap_type_info(old_ptr)); + context_resolver->type_info_by_ctid_.put(key, remap_type_info(old_ptr)); } for (const auto &[key, old_ptr] : type_info_by_id_) { - final_resolver->type_info_by_id_.put(key, remap_type_info(old_ptr)); + context_resolver->type_info_by_id_.put(key, remap_type_info(old_ptr)); } for (const auto &[key, old_ptr] : user_type_info_by_id_) { - final_resolver->user_type_info_by_id_.put(key, remap_type_info(old_ptr)); + context_resolver->user_type_info_by_id_.put(key, remap_type_info(old_ptr)); } for (const auto &[key, old_ptr] : type_info_by_name_) { - final_resolver->type_info_by_name_[key] = remap_type_info(old_ptr); + context_resolver->type_info_by_name_[key] = remap_type_info(old_ptr); } for (const auto &[key, old_ptr] : type_info_by_runtime_type_) { - final_resolver->type_info_by_runtime_type_[key] = remap_type_info(old_ptr); + context_resolver->type_info_by_runtime_type_[key] = + remap_type_info(old_ptr); } for (const auto &[key, old_ptr] : partial_type_infos_) { - final_resolver->partial_type_infos_.put(key, remap_type_info(old_ptr)); + context_resolver->partial_type_infos_.put(key, remap_type_info(old_ptr)); } // Process all partial type infos to build complete type metadata for (const auto &[rust_type_id, partial_ptr] : - final_resolver->partial_type_infos_) { + context_resolver->partial_type_infos_) { // Call the harness's sorted_field_infos function to get complete field info FORY_TRY(sorted_fields, - partial_ptr->harness.sorted_field_infos_fn(*final_resolver)); + partial_ptr->harness.sorted_field_infos_fn(*context_resolver)); // Build complete TypeMeta TypeMeta meta = TypeMeta::from_fields( @@ -1848,7 +1849,7 @@ TypeResolver::build_final_type_resolver() { buffer.writer_index(static_cast(partial_ptr->type_def.size())); // This metadata was just generated from local registration state. Remote // receive limits are enforced only on remote metadata parse/cache-miss - // paths, so large trusted local schemas do not fail during finalization. + // paths, so large trusted local schemas do not fail metadata completion. FORY_TRY(parsed_meta, TypeMeta::from_bytes(buffer, nullptr, std::numeric_limits::max(), @@ -1856,10 +1857,10 @@ TypeResolver::build_final_type_resolver() { partial_ptr->type_meta = std::move(parsed_meta); } - // Clear partial_type_infos in the final resolver since they're all completed - final_resolver->partial_type_infos_.clear(); + // The context resolver retains only completed metadata. + context_resolver->partial_type_infos_.clear(); - return final_resolver; + return context_resolver; } std::unique_ptr TypeResolver::clone() const { @@ -1904,8 +1905,8 @@ std::unique_ptr TypeResolver::clone() const { for (const auto &[key, old_ptr] : type_info_by_runtime_type_) { cloned->type_info_by_runtime_type_[key] = remap_type_info(old_ptr); } - // Note: Don't copy partial_type_infos_ - clone should only be used on - // finalized resolvers + // Note: Don't copy partial_type_infos_ - clone is used only after metadata + // completion. return cloned; } diff --git a/cpp/fory/serialization/type_resolver.h b/cpp/fory/serialization/type_resolver.h index 4e0cc1ede0..02b05dcac3 100644 --- a/cpp/fory/serialization/type_resolver.h +++ b/cpp/fory/serialization/type_resolver.h @@ -1360,8 +1360,8 @@ class TypeResolver { template Result register_any_type(); - /// Builds the final TypeResolver by completing all partial type infos - /// created during registration. + /// Builds the TypeResolver used by operation contexts by completing all + /// partial type infos created during registration. /// /// This method processes all types that were registered. During registration, /// types are stored in `partial_type_infos` without their complete @@ -1371,13 +1371,12 @@ class TypeResolver { /// 2. Calls their `sorted_field_infos` function to get complete field /// information /// 3. Builds complete TypeMeta and serializes it to bytes - /// 4. Returns a new TypeResolver with all type infos fully initialized + /// 4. Returns a new TypeResolver with complete metadata /// /// Registration is permanently frozen before metadata construction starts. /// - /// @return A new TypeResolver with all type infos fully initialized and ready - /// for use. - Result, Error> build_final_type_resolver(); + /// @return A TypeResolver ready to create operation contexts. + Result, Error> build_context_type_resolver(); /// Deep clones the TypeResolver for use in a new context. /// From 8cc3296bd875d97c73347e1a3e2dca83d88aa7f0 Mon Sep 17 00:00:00 2001 From: chaokunyang Date: Sun, 30 Aug 2026 03:03:19 +0800 Subject: [PATCH 096/168] refactor(csharp): name version hash cache precisely --- csharp/src/Fory/TypeResolver.cs | 28 ++++++++++----------- csharp/tests/Fory.Tests/ForyRuntimeTests.cs | 2 +- 2 files changed, 15 insertions(+), 15 deletions(-) diff --git a/csharp/src/Fory/TypeResolver.cs b/csharp/src/Fory/TypeResolver.cs index 9eb0cc9da1..af6c8dee30 100644 --- a/csharp/src/Fory/TypeResolver.cs +++ b/csharp/src/Fory/TypeResolver.cs @@ -109,7 +109,7 @@ private static class GenericTypeCache private readonly Dictionary<(string NamespaceName, string TypeName), TypeInfo> _byTypeName = []; private readonly UInt64Map _typeInfos = new(); private ulong _versionHash; - private bool _finalized; + private bool _versionHashReady; /// /// Registers a generated enum or union serializer factory for a runtime target type. @@ -188,7 +188,7 @@ private static UInt64Map CreateTypeMap(params (Type Key, Type Value)[] ent public Serializer GetSerializer() { - if (_finalized) + if (_versionHashReady) { ulong version = _versionHash; GenericTypeCacheEntry? cacheEntry = Volatile.Read(ref GenericTypeCache.Entry); @@ -209,7 +209,7 @@ public TypeInfo GetTypeInfo(Type type) public TypeInfo GetTypeInfo() { - if (_finalized) + if (_versionHashReady) { ulong version = _versionHash; GenericTypeCacheEntry? cacheEntry = Volatile.Read(ref GenericTypeCache.Entry); @@ -220,7 +220,7 @@ public TypeInfo GetTypeInfo() } TypeInfo typeInfo = GetTypeInfo(typeof(T)); - EnsureFinalizedVersion(); + EnsureVersionHash(); Volatile.Write( ref GenericTypeCache.Entry, new GenericTypeCacheEntry(_versionHash, typeInfo)); @@ -481,7 +481,7 @@ private TypeInfo GetOrCreateTypeInfo(Type type, TypeInfo? explicitTypeInfo) } _typeInfos.Set(typeKey, typeInfo); - InvalidateFinalizedVersion(); + InvalidateVersionHash(); return typeInfo; } @@ -503,7 +503,7 @@ internal void Register(Type type, uint id, TypeInfo? explicitTypeInfo = null) TypeInfo typeInfo = GetOrCreateTypeInfo(type, explicitTypeInfo).WithTypeIdRegistration(id); _typeInfos.Set(TypeMapKey.Get(type), typeInfo); _byUserTypeId[id] = typeInfo; - InvalidateFinalizedVersion(); + InvalidateVersionHash(); } internal static (string NamespaceName, string TypeName) SplitTypeName(string name) @@ -549,35 +549,35 @@ internal void Register(Type type, string namespaceName, string typeName, TypeInf typeInfo = typeInfo.WithTypeNameRegistration(namespaceMeta, typeNameMeta); _typeInfos.Set(TypeMapKey.Get(type), typeInfo); _byTypeName[(namespaceName, typeName)] = typeInfo; - InvalidateFinalizedVersion(); + InvalidateVersionHash(); } /// - /// Returns a finalized semantic resolver version used by generated/static caches. + /// Returns the semantic resolver version used by generated/static caches. /// The version is computed lazily and changes whenever bindings/registrations change. /// /// Resolver version token. public ulong VersionHash() { - EnsureFinalizedVersion(); + EnsureVersionHash(); return _versionHash; } - private void InvalidateFinalizedVersion() + private void InvalidateVersionHash() { - _finalized = false; + _versionHashReady = false; _versionHash = 0; } - private void EnsureFinalizedVersion() + private void EnsureVersionHash() { - if (_finalized) + if (_versionHashReady) { return; } _versionHash = ComputeVersionHash(); - _finalized = true; + _versionHashReady = true; } private ulong ComputeVersionHash() diff --git a/csharp/tests/Fory.Tests/ForyRuntimeTests.cs b/csharp/tests/Fory.Tests/ForyRuntimeTests.cs index 630fab26c7..d093e061a3 100644 --- a/csharp/tests/Fory.Tests/ForyRuntimeTests.cs +++ b/csharp/tests/Fory.Tests/ForyRuntimeTests.cs @@ -3093,7 +3093,7 @@ public void TypeResolverVersionHashIncludesUnregisteredTypeBindings() } [Fact] - public void TypeResolverVersionHashIsStableWithinSameFinalizedResolver() + public void VersionHashIsStable() { TypeResolver resolver = new(); _ = resolver.GetTypeInfo>(); From 7f1715a581330472b30fdde3598053bde1d11ca3 Mon Sep 17 00:00:00 2001 From: chaokunyang Date: Sun, 30 Aug 2026 03:08:11 +0800 Subject: [PATCH 097/168] test(go): remove stale facade setup split --- go/fory/threadsafe/fory_test.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/go/fory/threadsafe/fory_test.go b/go/fory/threadsafe/fory_test.go index b813de02c3..a6cdad0272 100644 --- a/go/fory/threadsafe/fory_test.go +++ b/go/fory/threadsafe/fory_test.go @@ -115,8 +115,9 @@ func TestSerializeAny(t *testing.T) { // TestDeserialize tests the Deserialize generic function func TestDeserialize(t *testing.T) { + f := New(fory.WithXlang(false), fory.WithRefTracking(true), fory.WithCompatible(false)) + t.Run("Int32", func(t *testing.T) { - f := New(fory.WithXlang(false), fory.WithRefTracking(true), fory.WithCompatible(false)) val := int32(42) data, err := Serialize(f, &val) require.NoError(t, err) @@ -128,7 +129,6 @@ func TestDeserialize(t *testing.T) { }) t.Run("String", func(t *testing.T) { - f := New(fory.WithXlang(false), fory.WithRefTracking(true), fory.WithCompatible(false)) val := "hello" data, err := Serialize(f, &val) require.NoError(t, err) From e3cb41e2fc5416be883f8a551347f84544448ea2 Mon Sep 17 00:00:00 2001 From: chaokunyang Date: Sun, 30 Aug 2026 03:08:11 +0800 Subject: [PATCH 098/168] test(swift): keep registry lifecycle cases focused --- swift/Sources/ForyMacro/ForyObjectMacro.swift | 10 ++-------- .../ForyTests/ExternalTypeSerializationTests.swift | 14 +++++++++++--- 2 files changed, 13 insertions(+), 11 deletions(-) diff --git a/swift/Sources/ForyMacro/ForyObjectMacro.swift b/swift/Sources/ForyMacro/ForyObjectMacro.swift index 0b1b230392..ee0244ecab 100644 --- a/swift/Sources/ForyMacro/ForyObjectMacro.swift +++ b/swift/Sources/ForyMacro/ForyObjectMacro.swift @@ -102,10 +102,7 @@ public struct ForyStructMacro: MemberMacro, ExtensionMacro { let schemaHashDecl: DeclSyntax = DeclSyntax(stringLiteral: try buildSchemaHashDecl(fields: parsed.fields)) let compatibleTypeMetaDecl: DeclSyntax = DeclSyntax( - stringLiteral: buildCompatibleTypeMetaFieldsDecl( - sortedFields: sortedFields, - accessPrefix: accessPrefix - ) + stringLiteral: buildCompatibleTypeMetaFieldsDecl(sortedFields: sortedFields, accessPrefix: accessPrefix) ) let defaultDecl: DeclSyntax = DeclSyntax( stringLiteral: buildDefaultDecl( @@ -2543,10 +2540,7 @@ private func buildSchemaHashDecl(fields: [ParsedField]) throws -> String { """ } -private func buildCompatibleTypeMetaFieldsDecl( - sortedFields: [ParsedField], - accessPrefix: String -) -> String { +private func buildCompatibleTypeMetaFieldsDecl(sortedFields: [ParsedField], accessPrefix: String) -> String { let disabledExpr = compatibleTypeMetaFieldsExpr(sortedFields: sortedFields, trackRefExpression: "false") let enabledExpr = compatibleTypeMetaFieldsExpr(sortedFields: sortedFields, trackRefExpression: "true") let resolvedBody = resolvedTypeMetaFieldsBody(sortedFields: sortedFields) diff --git a/swift/Tests/ForyTests/ExternalTypeSerializationTests.swift b/swift/Tests/ForyTests/ExternalTypeSerializationTests.swift index 7c80303c33..3605173049 100644 --- a/swift/Tests/ForyTests/ExternalTypeSerializationTests.swift +++ b/swift/Tests/ForyTests/ExternalTypeSerializationTests.swift @@ -1183,9 +1183,6 @@ func hiddenCarrierAliasIsRejected() throws { #expect(throws: ForyError.self) { _ = try fory.serialize(HiddenCarrierHolder(users: [])) } - #expect(throws: ForyError.self) { - try fory.register(KeySerializer.self, id: 80) - } } @Test @@ -1227,3 +1224,14 @@ func registrationFreezesAtFirstRoot() throws { try fory.register(UserSerializer.self, id: 131) } } + +@Test +func failedRootFreezesRegistration() throws { + let fory = Fory() + #expect(throws: ForyError.self) { + let _: Int32 = try fory.deserialize(Data()) + } + #expect(throws: ForyError.self) { + try fory.register(UserSerializer.self, id: 132) + } +} From 6be170ad6b04283375f5774ddef78d6f78cdf1bd Mon Sep 17 00:00:00 2001 From: chaokunyang Date: Sun, 30 Aug 2026 03:10:23 +0800 Subject: [PATCH 099/168] fix(python): recheck registry before explicit publication --- python/pyfory/registry.py | 3 ++ python/pyfory/tests/test_serializer.py | 49 ++++++++++++++++++++++++++ 2 files changed, 52 insertions(+) diff --git a/python/pyfory/registry.py b/python/pyfory/registry.py index 9a3fe0d042..ecb60d51fd 100644 --- a/python/pyfory/registry.py +++ b/python/pyfory/registry.py @@ -843,6 +843,7 @@ def register_serializer(self, cls, serializer): typeinfo = self._types_info[cls] prev_type_id = typeinfo.type_id prev_user_type_id = typeinfo.user_type_id + self._check_registry_mutable() if needs_user_type_id(prev_type_id) and prev_user_type_id not in {None, NO_USER_TYPE_ID}: self._user_type_id_to_type_info.pop(prev_user_type_id, None) else: @@ -887,6 +888,8 @@ def get_type_info(self, cls, create=True): def _register_inferred_type(self, cls, native_only=False): serializer = self._create_serializer(cls) + if native_only: + self._check_registry_mutable() native_registration = self._internal_py_serializer_map.get(type(serializer)) if native_registration is not None: type_id = native_registration[1] diff --git a/python/pyfory/tests/test_serializer.py b/python/pyfory/tests/test_serializer.py index 099796a841..8dc98ede2b 100644 --- a/python/pyfory/tests/test_serializer.py +++ b/python/pyfory/tests/test_serializer.py @@ -969,6 +969,55 @@ def factory(type_resolver, cls): assert fory.type_resolver.get_type_info(FrozenRegistration, create=False) is None +def test_inferred_registration_freeze(): + fory = Fory(xlang=False, strict=False, compatible=False) + armed = False + + class RootDuringMro(type): + def __getattribute__(cls, name): + nonlocal armed + if armed and name == "__mro__": + armed = False + fory.serialize(None) + return super().__getattribute__(name) + + class Reduced(metaclass=RootDuringMro): + def __reduce__(self): + return Reduced, () + + armed = True + with pytest.raises(RuntimeError): + fory.register_type(Reduced) + assert fory.type_resolver.get_type_info(Reduced, create=False) is None + + +def test_serializer_registration_freeze(): + fory = Fory(xlang=False, strict=False, compatible=False) + armed = False + + class RootDuringHash(type): + def __hash__(cls): + nonlocal armed + if armed: + armed = False + fory.serialize(None) + return super().__hash__() + + class Registered(metaclass=RootDuringHash): + pass + + fory.register_type(Registered, name="test.Registered") + typeinfo = fory.type_resolver.get_type_info(Registered) + serializer = typeinfo.serializer + armed = True + with pytest.raises(RuntimeError): + fory.register_serializer( + Registered, + BarSerializer(fory.type_resolver, Registered), + ) + assert typeinfo.serializer is serializer + + def test_id_conflict_no_mutation(): fory = Fory(xlang=True, compatible=False) fory.register_type(FrozenRegistration, type_id=701) From d197185fd3bb6033e495f43ac55060a2175062ee Mon Sep 17 00:00:00 2001 From: chaokunyang Date: Sun, 30 Aug 2026 03:10:50 +0800 Subject: [PATCH 100/168] refactor(javascript): remove stale metadata adapters --- javascript/packages/core/lib/typeResolver.ts | 14 -------------- javascript/packages/core/tsconfig.json | 2 +- javascript/test/decimal.test.ts | 9 ++------- javascript/test/protocol/struct.test.ts | 2 +- javascript/test/rootCleanup.test.ts | 2 +- javascript/test/typemeta.test.ts | 4 ++-- 6 files changed, 7 insertions(+), 26 deletions(-) diff --git a/javascript/packages/core/lib/typeResolver.ts b/javascript/packages/core/lib/typeResolver.ts index 30d27d54c0..b08844138e 100644 --- a/javascript/packages/core/lib/typeResolver.ts +++ b/javascript/packages/core/lib/typeResolver.ts @@ -311,20 +311,6 @@ export default class TypeResolver { return new Gen(this, { creator: typeInfo.options?.creator }).reGenerateSerializer(typeInfo); } - regenerateReadSerializer(typeInfo: TypeInfo) { - const serializer = this.generateReadSerializer(typeInfo); - return this.registerSerializer(typeInfo, { - readDataAlwaysAdvances: serializer.readDataAlwaysAdvances, - getHash: serializer.getHash, - getTypeInfo: serializer.getTypeInfo, - read: serializer.read, - readNoRef: serializer.readNoRef, - readRef: serializer.readRef, - readTypeInfo: serializer.readTypeInfo, - readRefWithoutTypeInfo: serializer.readRefWithoutTypeInfo, - } as any)!; - } - getSerializerByTypeInfo(typeInfo: TypeInfo) { const typeId = this.computeTypeId(typeInfo); if (TypeId.isNamedType(typeId)) { diff --git a/javascript/packages/core/tsconfig.json b/javascript/packages/core/tsconfig.json index f0558acdfe..87cefde930 100644 --- a/javascript/packages/core/tsconfig.json +++ b/javascript/packages/core/tsconfig.json @@ -41,7 +41,7 @@ // "inlineSources": true, /* Include source code in the sourcemaps inside the emitted JavaScript. */ // "emitBOM": true, /* Emit a UTF-8 Byte Order Mark (BOM) in the beginning of output files. */ // "newLine": "crlf", /* Set the newline character for emitting files. */ - "stripInternal": true, /* Disable emitting declarations that have '@internal' in their JSDoc comments. */ + // "stripInternal": true, /* Disable emitting declarations that have '@internal' in their JSDoc comments. */ "noEmitHelpers": true, /* Disable generating custom helper functions like '__extends' in compiled output. */ "noEmitOnError": true, /* Disable emitting files if any type checking errors are reported. */ // "preserveConstEnums": true, /* Disable erasing 'const enum' declarations in generated code. */ diff --git a/javascript/test/decimal.test.ts b/javascript/test/decimal.test.ts index 58177997ad..b6a169cc8e 100644 --- a/javascript/test/decimal.test.ts +++ b/javascript/test/decimal.test.ts @@ -176,14 +176,8 @@ describe("decimal", () => { const roundTrip = fory.deserialize(fory.serialize(value)) as Decimal; expect(roundTrip.equals(value)).toBe(true); } else { - const writer = (fory as any).writeContext.writer; - const bodyBefore = Array.from( - writer.getPlatformBuffer().subarray(bodyOffset, bodyOffset + 5), - ); expect(() => fory.serialize(value)).toThrow(/Decimal scale/); - expect(Array.from(writer.getPlatformBuffer().subarray(bodyOffset, bodyOffset + 5))).toEqual( - bodyBefore, - ); + expect((fory as any).writeContext.writer.writeGetCursor()).toBe(bodyOffset); } const payload = decimalPayload(scale); @@ -216,6 +210,7 @@ describe("decimal", () => { writer.getPlatformBuffer().subarray(bodyOffset, bodyOffset + 5), ); expect(() => fory.serialize(value)).toThrow(/Decimal magnitude/); + expect(writer.writeGetCursor()).toBe(bodyOffset); expect(Array.from(writer.getPlatformBuffer().subarray(bodyOffset, bodyOffset + 5))).toEqual( bodyBefore, ); diff --git a/javascript/test/protocol/struct.test.ts b/javascript/test/protocol/struct.test.ts index 4bbe8db3e6..edaad7ef66 100644 --- a/javascript/test/protocol/struct.test.ts +++ b/javascript/test/protocol/struct.test.ts @@ -71,7 +71,7 @@ describe("protocol", () => { }, ); const { serialize, deserialize } = fory.register(nullableUnspecified); - expect(() => nonNullableSer.serialize({ a: null })).toThrow(); + expect(() => nonNullableSer.serialize({ a: null })).toThrow(/Field "a" is not nullable/); expect(deserialize(serialize({ a: null }))).toEqual({ a: null }); }); diff --git a/javascript/test/rootCleanup.test.ts b/javascript/test/rootCleanup.test.ts index 9c2fe95ed4..fb96e0458f 100644 --- a/javascript/test/rootCleanup.test.ts +++ b/javascript/test/rootCleanup.test.ts @@ -241,7 +241,7 @@ test("releases failed write buffer", () => { expect(writer.getPlatformBuffer().byteLength).toBeLessThan(4 * 1024 * 1024); }); -test("clears failed write refs", () => { +test("clears failed read refs", () => { const fory = new Fory({ compatible: false, ref: true }); const registered = fory.register(Type.struct(7612, {})); const refReader = (fory as any).readContext.refReader; diff --git a/javascript/test/typemeta.test.ts b/javascript/test/typemeta.test.ts index 5f4117f82f..d399d74314 100644 --- a/javascript/test/typemeta.test.ts +++ b/javascript/test/typemeta.test.ts @@ -985,7 +985,7 @@ describe("typemeta", () => { const writerChild = writerFory.register(remoteChild); const writerRoot = writerFory.register( Type.struct(rootId, { - child: remoteChild.clone().setId(1), + child: remoteChild.setId(1), }), ); const remoteTypeMeta = TypeMeta.fromTypeInfo(remoteChild, (writerFory as any).typeResolver); @@ -1000,7 +1000,7 @@ describe("typemeta", () => { const child = fory.register(localChild); const root = fory.register( Type.struct(rootId, { - child: localChild.clone().setId(1), + child: localChild.setId(1), }), ); return { fory, child, root }; From 28b4b4f3a0e031d8e0d69dd77d6b8c690f98af07 Mon Sep 17 00:00:00 2001 From: chaokunyang Date: Sun, 30 Aug 2026 03:17:41 +0800 Subject: [PATCH 101/168] fix(javascript): clear failed roots immediately --- javascript/packages/core/lib/fory.ts | 42 +++++++++++++++++++--------- javascript/test/rootCleanup.test.ts | 8 +++++- 2 files changed, 36 insertions(+), 14 deletions(-) diff --git a/javascript/packages/core/lib/fory.ts b/javascript/packages/core/lib/fory.ts index cf0f33ff2a..86b85867d8 100644 --- a/javascript/packages/core/lib/fory.ts +++ b/javascript/packages/core/lib/fory.ts @@ -40,6 +40,7 @@ const DEFAULT_MAX_SCHEMA_VERSIONS_PER_TYPE = 10 as const; const DEFAULT_MAX_AVERAGE_SCHEMA_VERSIONS_PER_TYPE = 3 as const; const DEFAULT_MAX_GRAPH_MEMORY_BYTES = 128 * 1024 * 1024; const DEFAULT_MAX_UNBACKED_CONTAINER_ITEMS = 8192 as const; +const EMPTY_BYTES = new Uint8Array(0); export default class Fory { readonly typeResolver: TypeResolver; readonly anySerializer: Serializer; @@ -179,12 +180,17 @@ export default class Fory { deserialize(bytes: Uint8Array, serializer: Serializer = this.anySerializer): T | null { this.registrationFrozen = true; this.readContext.reset(bytes); - const reader = this.readContext.reader; - const bitmap = reader.readUint8(); - if (bitmap !== ConfigFlags.isCrossLanguageFlag) { - this.throwInvalidRootHeader(bitmap); + try { + const reader = this.readContext.reader; + const bitmap = reader.readUint8(); + if (bitmap !== ConfigFlags.isCrossLanguageFlag) { + this.throwInvalidRootHeader(bitmap); + } + return serializer.readRef(); + } catch (error) { + this.readContext.reset(EMPTY_BYTES); + throw error; } - return serializer.readRef(); } private throwInvalidRootHeader(bitmap: number): never { @@ -210,10 +216,15 @@ export default class Fory { this.registrationFrozen = true; // The entry reset releases state from the previous root before this context is reused. writeContext.reset(); - writer.writeUint8(rootHeader); - writer.reserve(serializer.fixedSize); - serializer.writeRef(data); - return writer.dump(); + try { + writer.writeUint8(rootHeader); + writer.reserve(serializer.fixedSize); + serializer.writeRef(data); + return writer.dump(); + } catch (error) { + writeContext.reset(); + throw error; + } }; this.rootSerializers.set(serializer, rootSerializer); return rootSerializer; @@ -233,11 +244,16 @@ export default class Fory { rootDeserializer = (bytes: Uint8Array) => { this.registrationFrozen = true; readContext.reset(bytes); - const bitmap = reader.readUint8(); - if (bitmap !== rootHeader) { - this.throwInvalidRootHeader(bitmap); + try { + const bitmap = reader.readUint8(); + if (bitmap !== rootHeader) { + this.throwInvalidRootHeader(bitmap); + } + return rootSerializer.readRef(); + } catch (error) { + readContext.reset(EMPTY_BYTES); + throw error; } - return rootSerializer.readRef(); }; this.rootDeserializers.set(serializer, rootDeserializer); return rootDeserializer; diff --git a/javascript/test/rootCleanup.test.ts b/javascript/test/rootCleanup.test.ts index fb96e0458f..243d17fed5 100644 --- a/javascript/test/rootCleanup.test.ts +++ b/javascript/test/rootCleanup.test.ts @@ -63,6 +63,7 @@ describe.each([ if (outcome === "failure") { expect(read).toThrow(); + expectRootStateCleared(readerFory.readContext); expect(invoke(readerFory, reader, bytes)).toEqual({ value: 7 }); } else { const first = read(); @@ -94,6 +95,7 @@ describe.each([ const read = () => invoke(fory, registered, new Uint8Array([1])); if (outcome === "failure") { expect(read).toThrow(); + expectRootStateCleared(readContext); registered.serializer.readRef = () => { expectRootStateCleared(readContext); return 7; @@ -127,6 +129,9 @@ test.each(["success", "failure"] as const)("restores root write state for %s", ( if (outcome === "failure") { expect(() => registered.serialize(value)).toThrow(); + expect(writeContext.refWriter.writeObjects.size).toBe(0); + expect(name.dynamicWriteStringId).toBe(-1); + expect(typeMeta.dynamicTypeId).toBe(-1); expect(fory.serialize(7)).toBeDefined(); } else { expect(registered.serialize(value)).toBeDefined(); @@ -237,8 +242,8 @@ test("releases failed write buffer", () => { }; expect(() => registered.serialize({})).toThrow(); - expect(fory.serialize(7)).toBeDefined(); expect(writer.getPlatformBuffer().byteLength).toBeLessThan(4 * 1024 * 1024); + expect(fory.serialize(7)).toBeDefined(); }); test("clears failed read refs", () => { @@ -251,6 +256,7 @@ test("clears failed read refs", () => { throw new Error("root read failed"); }; expect(() => registered.deserialize(new Uint8Array([1]))).toThrow(); + expect(refReader.readObjects).toHaveLength(0); registered.serializer.readRef = () => { expect(refReader.readObjects).toHaveLength(0); From 02a14de7c35aadd934835672ab854dbc36ed919c Mon Sep 17 00:00:00 2001 From: chaokunyang Date: Sun, 30 Aug 2026 03:18:10 +0800 Subject: [PATCH 102/168] fix(python): publish automatic type IDs on success --- python/pyfory/registry.py | 23 ++++++++++++++++++----- python/pyfory/serializer.py | 10 +++++----- python/pyfory/tests/test_serializer.py | 3 +++ 3 files changed, 26 insertions(+), 10 deletions(-) diff --git a/python/pyfory/registry.py b/python/pyfory/registry.py index ecb60d51fd..f3e4c8d4fc 100644 --- a/python/pyfory/registry.py +++ b/python/pyfory/registry.py @@ -618,15 +618,19 @@ def register_union( self._check_registry_mutable() if typename is not None and type_id is not None: raise TypeError(f"type name {typename} and id {type_id} should not be set at the same time") + auto_type_id = typename is None and type_id is None if typename is None and type_id is None: - type_id = self._next_type_id() + type_id = self._type_id_counter + 1 + while type_id in self._used_user_type_ids: + type_id += 1 + assigned_type_id = type_id if type_id not in {0, None}: user_type_id = type_id type_id = TypeId.TYPED_UNION else: user_type_id = NO_USER_TYPE_ID type_id = TypeId.NAMED_UNION - return self.__register_type( + typeinfo = self.__register_type( cls, type_id=type_id, user_type_id=user_type_id, @@ -635,6 +639,9 @@ def register_union( serializer=serializer, internal=False, ) + if auto_type_id: + self._type_id_counter = assigned_type_id + return typeinfo def _register_type( self, @@ -680,13 +687,16 @@ def _register_type( if typeinfo is not None: return typeinfo n_params = len({typename, type_id, None}) - 1 - if n_params == 0 and typename is None: - type_id = self._next_type_id() + auto_type_id = n_params == 0 and typename is None + if auto_type_id: + type_id = self._type_id_counter + 1 + while type_id in self._used_user_type_ids: + type_id += 1 if n_params == 2: raise TypeError(f"type name {typename} and id {type_id} should not be set at the same time") if cls in self._types_info: raise TypeError(f"{cls} registered already") - return self._register_xtype( + typeinfo = self._register_xtype( cls, type_id=type_id, user_type_id=user_type_id, @@ -695,6 +705,9 @@ def _register_type( serializer=serializer, internal=internal, ) + if auto_type_id: + self._type_id_counter = type_id + return typeinfo def _register_xtype( self, diff --git a/python/pyfory/serializer.py b/python/pyfory/serializer.py index a2146b91c2..9360761492 100644 --- a/python/pyfory/serializer.py +++ b/python/pyfory/serializer.py @@ -382,7 +382,7 @@ def _resolve_validated_bound_method(policy, obj, method_name, is_local): class NoneSerializer(Serializer): def __init__(self, type_resolver): - super().__init__(type_resolver, type(None)) + super().__init__(type_resolver, None) self.need_to_write_ref = False def write(self, buffer, value): @@ -525,6 +525,8 @@ def read(self, read_context): class PandasRangeIndexSerializer(Serializer): + __slots__ = "_cached" + def __init__(self, type_resolver): import pandas as pd @@ -561,9 +563,7 @@ def write(self, write_context, value): else: write_context.write_int8(NOT_NULL_VALUE_FLAG) write_context.write_no_ref(step) - # Concrete dtype classes differ across NumPy versions. Keep this wire slot owned by - # RangeIndex and encode NumPy's stable descriptor instead of a version-specific object ref. - write_context.write_string(value.dtype.str) + write_context.write_ref(value.dtype) write_context.write_ref(value.name) def read(self, read_context): @@ -579,7 +579,7 @@ def read(self, read_context): step = None else: step = read_context.read_no_ref() - dtype = np.dtype(read_context.read_string()) + dtype = read_context.read_ref() name = read_context.read_ref() return self.type_(start, stop, step, dtype=dtype, name=name) diff --git a/python/pyfory/tests/test_serializer.py b/python/pyfory/tests/test_serializer.py index 8dc98ede2b..a2f244a570 100644 --- a/python/pyfory/tests/test_serializer.py +++ b/python/pyfory/tests/test_serializer.py @@ -959,6 +959,7 @@ def test_registry_freezes_at_root(root): def test_factory_root_freeze(): fory = Fory(xlang=True, compatible=False) + next_type_id = getattr(fory.type_resolver, "_type_id_counter", None) def factory(type_resolver, cls): fory.serialize(None) @@ -967,6 +968,8 @@ def factory(type_resolver, cls): with pytest.raises(RuntimeError): fory.register_type(FrozenRegistration, serializer=factory) assert fory.type_resolver.get_type_info(FrozenRegistration, create=False) is None + if next_type_id is not None: + assert fory.type_resolver._type_id_counter == next_type_id def test_inferred_registration_freeze(): From dde640c7e5865a927dac9e214c535b5b7a0ce540 Mon Sep 17 00:00:00 2001 From: chaokunyang Date: Sun, 30 Aug 2026 03:20:40 +0800 Subject: [PATCH 103/168] test(swift): remove unrelated superclass restriction --- .../ExternalTypeSerializationTests.swift | 21 ------------------- 1 file changed, 21 deletions(-) diff --git a/swift/Tests/ForyTests/ExternalTypeSerializationTests.swift b/swift/Tests/ForyTests/ExternalTypeSerializationTests.swift index 3605173049..bed6ebf62b 100644 --- a/swift/Tests/ForyTests/ExternalTypeSerializationTests.swift +++ b/swift/Tests/ForyTests/ExternalTypeSerializationTests.swift @@ -164,19 +164,6 @@ private final class LocalNode { required init() {} } -private class SuperclassBase { - required init() {} -} - -@ForyStruct -private final class SuperclassChild: SuperclassBase { - var local: Int32 = 0 - - required init() { - super.init() - } -} - @ForyStruct private struct LocalNamedValue: NamedValue, Equatable { var name: String @@ -1185,14 +1172,6 @@ func hiddenCarrierAliasIsRejected() throws { } } -@Test -func superclassIsRejected() throws { - let fory = Fory() - #expect(throws: ForyError.self) { - try fory.register(SuperclassChild.self, id: 133) - } -} - @Test func numericIDConflictIsAtomic() throws { let fory = Fory() From fb9aed04a0b9a4e31d5c53dbb246b2575c66112a Mon Sep 17 00:00:00 2001 From: chaokunyang Date: Sun, 30 Aug 2026 03:22:08 +0800 Subject: [PATCH 104/168] fix(csharp): preserve checked metadata across failed roots --- csharp/src/Fory/Fory.cs | 6 +++--- csharp/src/Fory/ReadContext.cs | 10 ---------- csharp/tests/Fory.Tests/RuntimeEdgeCaseTests.cs | 16 ++++++++-------- 3 files changed, 11 insertions(+), 21 deletions(-) diff --git a/csharp/src/Fory/Fory.cs b/csharp/src/Fory/Fory.cs index 9afc47c6cd..e3dfa6b55d 100644 --- a/csharp/src/Fory/Fory.cs +++ b/csharp/src/Fory/Fory.cs @@ -215,7 +215,7 @@ public T Deserialize(ReadOnlySpan payload) T value = DeserializeFromReader(reader); if (reader.Remaining != 0) { - _readContext.ResetAfterFailure(); + _readContext.Reset(); ThrowUnexpectedTrailingBytes(); } @@ -236,7 +236,7 @@ public T Deserialize(byte[] payload) T value = DeserializeFromReader(reader); if (reader.Remaining != 0) { - _readContext.ResetAfterFailure(); + _readContext.Reset(); ThrowUnexpectedTrailingBytes(); } @@ -308,7 +308,7 @@ internal T DeserializeFromReader(ByteReader reader) { // Failed roots can leave partially published refs, metadata refs, or graph-budget state. // Keep the success path minimal, but fully reset failed reads before this context is reused. - readContext.ResetAfterFailure(); + readContext.Reset(); throw; } } diff --git a/csharp/src/Fory/ReadContext.cs b/csharp/src/Fory/ReadContext.cs index dd83200539..3bf7e1d75f 100644 --- a/csharp/src/Fory/ReadContext.cs +++ b/csharp/src/Fory/ReadContext.cs @@ -565,14 +565,4 @@ internal void Reset() _remainingUnbackedContainerItems = 0; } - internal void ResetAfterFailure() - { - Reset(); - // Remote metadata may be accepted before the owning value body fails. Failed roots must - // not accumulate that decoded state across operations, while successful roots keep using - // this map as the sole checked-cache owner. - _typeMetasByHash.Clear(); - _remoteSchemaVersionsByType.Clear(); - _totalAcceptedSchemaVersions = 0; - } } diff --git a/csharp/tests/Fory.Tests/RuntimeEdgeCaseTests.cs b/csharp/tests/Fory.Tests/RuntimeEdgeCaseTests.cs index 4ae1a26d04..f6073d7585 100644 --- a/csharp/tests/Fory.Tests/RuntimeEdgeCaseTests.cs +++ b/csharp/tests/Fory.Tests/RuntimeEdgeCaseTests.cs @@ -1008,7 +1008,7 @@ public void TrailingBytesResetReadState(bool useSpan) } [Fact] - public void RootHeaderFailureClearsMetaCache() + public void RootHeaderFailureKeepsMetaCache() { ForyRuntime fory = ForyRuntime.Builder() .Compatible(false) @@ -1020,15 +1020,15 @@ public void RootHeaderFailureClearsMetaCache() Assert.ThrowsAny(() => fory.Deserialize([0])); - Assert.False(context.TryGetTypeMetaByHash(firstHash, out _)); - TypeMeta second = ReadAndStoreTypeMeta(context, RemoteStructTypeMeta(901, "second")); - Assert.True(context.TryGetTypeMetaByHash(EncodedTypeMetaHash(second), out _)); + Assert.True(context.TryGetTypeMetaByHash(firstHash, out _)); + Assert.Throws( + () => ReadAndStoreTypeMeta(context, RemoteStructTypeMeta(901, "second"))); } [Theory] [InlineData(false)] [InlineData(true)] - public void TrailingFailureClearsTypeMetaCache(bool useSpan) + public void TrailingFailureKeepsTypeMetaCache(bool useSpan) { ForyRuntime fory = ForyRuntime.Builder() .Compatible(false) @@ -1048,9 +1048,9 @@ public void TrailingFailureClearsTypeMetaCache(bool useSpan) Assert.ThrowsAny(() => fory.Deserialize(invalidPayload)); } - Assert.False(context.TryGetTypeMetaByHash(firstHash, out _)); - TypeMeta second = ReadAndStoreTypeMeta(context, RemoteStructTypeMeta(901, "second")); - Assert.True(context.TryGetTypeMetaByHash(EncodedTypeMetaHash(second), out _)); + Assert.True(context.TryGetTypeMetaByHash(firstHash, out _)); + Assert.Throws( + () => ReadAndStoreTypeMeta(context, RemoteStructTypeMeta(901, "second"))); } private static ForyRuntime NewCompatibleTimeFory() From 63bf2a711d8853e283eeec3942c7526feb7f54c3 Mon Sep 17 00:00:00 2001 From: chaokunyang Date: Sun, 30 Aug 2026 03:22:33 +0800 Subject: [PATCH 105/168] test(python): prove failed registration keeps type IDs --- python/pyfory/tests/test_serializer.py | 21 ++++++++++++++++++--- 1 file changed, 18 insertions(+), 3 deletions(-) diff --git a/python/pyfory/tests/test_serializer.py b/python/pyfory/tests/test_serializer.py index a2f244a570..ec64076d41 100644 --- a/python/pyfory/tests/test_serializer.py +++ b/python/pyfory/tests/test_serializer.py @@ -800,6 +800,10 @@ class RejectedRegistration: pass +class RegistrationAfterFailure: + pass + + @dataclass class FrozenChild: value: int @@ -959,7 +963,6 @@ def test_registry_freezes_at_root(root): def test_factory_root_freeze(): fory = Fory(xlang=True, compatible=False) - next_type_id = getattr(fory.type_resolver, "_type_id_counter", None) def factory(type_resolver, cls): fory.serialize(None) @@ -968,8 +971,20 @@ def factory(type_resolver, cls): with pytest.raises(RuntimeError): fory.register_type(FrozenRegistration, serializer=factory) assert fory.type_resolver.get_type_info(FrozenRegistration, create=False) is None - if next_type_id is not None: - assert fory.type_resolver._type_id_counter == next_type_id + + +def test_failed_factory_keeps_type_id(): + fory = Fory(xlang=True, compatible=False) + first = fory.register_type(FrozenRegistration) + + def factory(_type_resolver, _cls): + raise RuntimeError("serializer construction failed") + + with pytest.raises(RuntimeError): + fory.register_type(RejectedRegistration, serializer=factory) + + following = fory.register_type(RegistrationAfterFailure) + assert following.user_type_id == first.user_type_id + 1 def test_inferred_registration_freeze(): From 149d235960f18b745531b5661ab0ae481d22141f Mon Sep 17 00:00:00 2001 From: chaokunyang Date: Sun, 30 Aug 2026 03:24:32 +0800 Subject: [PATCH 106/168] fix(python): keep pooled serializers child-owned --- python/pyfory/_fory.py | 40 +++++++++++++++++++------ python/pyfory/tests/test_thread_safe.py | 31 +++++++++++++++++++ 2 files changed, 62 insertions(+), 9 deletions(-) diff --git a/python/pyfory/_fory.py b/python/pyfory/_fory.py index d14dd536b7..797f071644 100644 --- a/python/pyfory/_fory.py +++ b/python/pyfory/_fory.py @@ -702,13 +702,13 @@ def _get_fory(self): self._registry_frozen = True if self._pool: return self._pool.pop() - if self._fory_factory is not None: - fory = self._fory_factory() - else: - fory = self._fory_class(**self._config) - for callback in self._callbacks: - callback(fory) - return fory + if self._fory_factory is not None: + fory = self._fory_factory() + else: + fory = self._fory_class(**self._config) + for callback in self._callbacks: + callback(fory) + return fory def _return_fory(self, fory): with self._lock: @@ -720,6 +720,15 @@ def _register_callback(self, callback): raise RuntimeError("Cannot register types after the first root serialization or deserialization operation has started.") self._callbacks.append(callback) + @staticmethod + def _check_serializer_factory(serializer): + if serializer is None: + return + from pyfory.registry import CythonSerializer, Serializer + + if isinstance(serializer, (Serializer, CythonSerializer)) or not callable(serializer): + raise TypeError("ThreadSafeFory requires a serializer class or factory") + def register( self, cls, @@ -728,6 +737,7 @@ def register( name: str = None, serializer=None, ): + self._check_serializer_factory(serializer) self._register_callback(lambda f: f.register(cls, type_id=type_id, name=name, serializer=serializer)) def register_type( @@ -738,6 +748,7 @@ def register_type( name: str = None, serializer=None, ): + self._check_serializer_factory(serializer) self._register_callback(lambda f: f.register_type(cls, type_id=type_id, name=name, serializer=serializer)) def register_union( @@ -748,10 +759,21 @@ def register_union( name: str = None, serializer=None, ): + self._check_serializer_factory(serializer) self._register_callback(lambda f: f.register_union(cls, type_id=type_id, name=name, serializer=serializer)) - def register_serializer(self, cls: type, serializer): - self._register_callback(lambda f: f.register_serializer(cls, serializer)) + def register_serializer(self, cls: type, serializer_factory): + self._check_serializer_factory(serializer_factory) + if serializer_factory is None: + raise TypeError("ThreadSafeFory requires a serializer class or factory") + + def register(fory): + from pyfory.registry import _construct_serializer + + serializer = _construct_serializer(serializer_factory, fory.type_resolver, cls) + fory.register_serializer(cls, serializer) + + self._register_callback(register) def serialize( self, diff --git a/python/pyfory/tests/test_thread_safe.py b/python/pyfory/tests/test_thread_safe.py index d7111283cd..d952d81177 100644 --- a/python/pyfory/tests/test_thread_safe.py +++ b/python/pyfory/tests/test_thread_safe.py @@ -20,6 +20,7 @@ import pytest +import pyfory from pyfory import ThreadSafeFory @@ -35,6 +36,10 @@ class Address: country: str +class PersonSerializer(pyfory.Serializer): + pass + + def test_thread_safe_fory_basic_serialization(): fory = ThreadSafeFory( xlang=False, @@ -191,3 +196,29 @@ def test_thread_safe_fory_register_after_use(): with pytest.raises(RuntimeError): fory.register(Address) + + +def test_thread_safe_serializer_factory_owns_children(): + fory = ThreadSafeFory(xlang=False, compatible=False) + fory.register(Person, serializer=PersonSerializer) + + first = fory._get_fory() + second = fory._get_fory() + try: + first_serializer = first.type_resolver.get_serializer(Person) + second_serializer = second.type_resolver.get_serializer(Person) + assert first_serializer is not second_serializer + assert first_serializer.type_resolver is first.type_resolver + assert second_serializer.type_resolver is second.type_resolver + finally: + fory._return_fory(first) + fory._return_fory(second) + + +def test_thread_safe_rejects_serializer_instance(): + runtime = pyfory.Fory(xlang=False, compatible=False) + serializer = PersonSerializer(runtime.type_resolver, Person) + fory = ThreadSafeFory(xlang=False, compatible=False) + + with pytest.raises(TypeError, match="serializer class or factory"): + fory.register(Person, serializer=serializer) From 62089dec267a72cb5d61ea29e727461a0d510a47 Mon Sep 17 00:00:00 2001 From: chaokunyang Date: Sun, 30 Aug 2026 03:24:59 +0800 Subject: [PATCH 107/168] test(python): rely on native iterator discovery --- python/pyfory/tests/test_reduce_serializer.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/python/pyfory/tests/test_reduce_serializer.py b/python/pyfory/tests/test_reduce_serializer.py index 3fa13339f5..29ed458c91 100644 --- a/python/pyfory/tests/test_reduce_serializer.py +++ b/python/pyfory/tests/test_reduce_serializer.py @@ -516,7 +516,6 @@ def test_reduce_with_list_items(): fory = Fory(xlang=False, ref=True, strict=False, compatible=False) obj = ReduceWithListItems([1, 2, 3, 4]) - fory.register_type(type(iter([]))) # Verify ReduceSerializer is used serializer = fory.type_resolver.get_serializer(ReduceWithListItems) @@ -536,7 +535,6 @@ def test_reduce_with_dict_items(): fory = Fory(xlang=False, ref=True, strict=False, compatible=False) obj = ReduceWithDictItems({"a": 1, "b": 2}) - fory.register_type(type(iter({}.items()))) # Verify ReduceSerializer is used serializer = fory.type_resolver.get_serializer(ReduceWithDictItems) From afed4796bf547c08cdcf86e8b8386b2fbf5f0a71 Mon Sep 17 00:00:00 2001 From: chaokunyang Date: Sun, 30 Aug 2026 03:25:13 +0800 Subject: [PATCH 108/168] refactor(csharp): keep read context unchanged --- csharp/src/Fory/ReadContext.cs | 1 - 1 file changed, 1 deletion(-) diff --git a/csharp/src/Fory/ReadContext.cs b/csharp/src/Fory/ReadContext.cs index 3bf7e1d75f..34ea7ff2f4 100644 --- a/csharp/src/Fory/ReadContext.cs +++ b/csharp/src/Fory/ReadContext.cs @@ -564,5 +564,4 @@ internal void Reset() _readMetaStrings.Clear(); _remainingUnbackedContainerItems = 0; } - } From 5cdb1e430317ba54149477d81432363e4dec3c5e Mon Sep 17 00:00:00 2001 From: chaokunyang Date: Sun, 30 Aug 2026 03:27:20 +0800 Subject: [PATCH 109/168] refactor(csharp): replay registrations only at child creation --- csharp/src/Fory/ThreadSafeFory.cs | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/csharp/src/Fory/ThreadSafeFory.cs b/csharp/src/Fory/ThreadSafeFory.cs index 89fddf80cf..eca4bfecf7 100644 --- a/csharp/src/Fory/ThreadSafeFory.cs +++ b/csharp/src/Fory/ThreadSafeFory.cs @@ -35,7 +35,7 @@ public sealed class ThreadSafeFory : IDisposable internal ThreadSafeFory(Config config) { _config = config; - _threadLocalFory = new ThreadLocal(CreatePerThreadFory, trackAllValues: true); + _threadLocalFory = new ThreadLocal(CreatePerThreadFory); } /// @@ -232,10 +232,6 @@ private void ApplyRegistration(Action registration) } _registrations.Add(registration); - foreach (Fory fory in _threadLocalFory.Values) - { - registration(fory); - } } } From 4616639ab0e4fe26b011eca9fb13c0b8f82dc7cd Mon Sep 17 00:00:00 2001 From: chaokunyang Date: Sun, 30 Aug 2026 03:28:39 +0800 Subject: [PATCH 110/168] refactor(jvm): simplify registry freeze ownership --- .../fory/idl_tests/KotlinIdlRoundTripPeer.kt | 13 +- .../idl_tests/ScalaIdlRoundTripTest.scala | 8 +- .../apache/fory/AbstractThreadSafeFory.java | 5 + .../main/java/org/apache/fory/BaseFory.java | 6 + .../apache/fory/FacadeRegistrationGate.java | 171 ---- .../src/main/java/org/apache/fory/Fory.java | 91 +-- .../java/org/apache/fory/ThreadLocalFory.java | 96 ++- .../java/org/apache/fory/ThreadSafeFory.java | 11 +- .../org/apache/fory/config/ForyBuilder.java | 2 - .../apache/fory/io/ForyReadableChannel.java | 2 +- .../org/apache/fory/logging/LogOnceState.java | 2 - .../org/apache/fory/pool/ThreadPoolFory.java | 83 +- .../fory/resolver/AllowListChecker.java | 5 +- .../apache/fory/resolver/ClassResolver.java | 313 ++++---- .../org/apache/fory/resolver/TypeInfo.java | 2 - .../apache/fory/resolver/TypeResolver.java | 310 +------- .../apache/fory/resolver/XtypeResolver.java | 218 ++---- .../serializer/AbstractObjectSerializer.java | 3 +- .../apache/fory/serializer/FieldGroups.java | 20 +- .../fory/serializer/ObjectSerializer.java | 5 +- .../serializer/ReplaceResolveSerializer.java | 2 +- .../apache/fory/serializer/Serializers.java | 4 +- .../StaticGeneratedStructSerializer.java | 9 +- .../java/org/apache/fory/ForyCopyTest.java | 9 - .../test/java/org/apache/fory/ForyTest.java | 4 - .../org/apache/fory/ThreadSafeForyTest.java | 710 +++++------------ .../StaticCompatibleCodecBuilderTest.java | 43 - .../fory/resolver/ClassResolverTest.java | 16 +- .../apache/fory/serializer/RegisterTest.java | 732 +----------------- .../fory/kotlin/xlang/KotlinXlangPeer.kt | 71 +- .../serializer/kotlin/KotlinSerializers.java | 310 ++++---- .../org/apache/fory/kotlin/ForyExtensions.kt | 32 +- .../kotlin/BuiltinClassSerializerTests.kt | 146 ---- .../serializer/scala/ScalaEnumSerializer.java | 6 - .../serializer/scala/ScalaSerializers.java | 308 ++++---- .../apache/fory/scala/ForyExtensions.scala | 40 +- .../apache/fory/scala/ForySerializer.scala | 129 +-- .../scala/ForySerializerDerivationTest.scala | 134 +--- .../fory/serializer/scala/ScalaEnumTest.scala | 50 -- 39 files changed, 1094 insertions(+), 3027 deletions(-) delete mode 100644 java/fory-core/src/main/java/org/apache/fory/FacadeRegistrationGate.java diff --git a/integration_tests/idl_tests/kotlin/src/main/kotlin/org/apache/fory/idl_tests/KotlinIdlRoundTripPeer.kt b/integration_tests/idl_tests/kotlin/src/main/kotlin/org/apache/fory/idl_tests/KotlinIdlRoundTripPeer.kt index 4b29634f8c..f6d4bddcee 100644 --- a/integration_tests/idl_tests/kotlin/src/main/kotlin/org/apache/fory/idl_tests/KotlinIdlRoundTripPeer.kt +++ b/integration_tests/idl_tests/kotlin/src/main/kotlin/org/apache/fory/idl_tests/KotlinIdlRoundTripPeer.kt @@ -23,7 +23,6 @@ package org.apache.fory.idl_tests import addressbook.AddressBook import addressbook.AddressbookForyModule -import basic.BasicForyModule import basic.Money import example.ExampleForyModule import example.ExampleMessage @@ -110,10 +109,10 @@ private fun runGeneratedSurfaceChecks() { assertRoundTrip(fory, ExampleMessageUnion.Uint32Array(uintArrayOf(1u, UInt.MAX_VALUE))) assertRoundTrip(fory, ExampleMessageUnion.Float16Array(Float16Array.of(1.0f, -2.0f))) assertRoundTrip(fory, ExampleMessageUnion.Bfloat16Array(BFloat16Array.of(3.0f, -4.0f))) - assertRegistrationOwners() + assertBaseForyExtensionReceivers() } -private fun assertRegistrationOwners() { +private fun assertBaseForyExtensionReceivers() { val expected = Money(BigDecimal("12.34"), "USD") val direct = ForyKotlin.builder().withXlang(true).build() @@ -121,13 +120,13 @@ private fun assertRegistrationOwners() { val directBytes = direct.serialize(expected) require(direct.deserialize(directBytes, Money::class.java) == expected) - val threadLocal = - ForyKotlin.builder().withXlang(true).withModule(BasicForyModule).buildThreadLocalFory() + val threadLocal = ForyKotlin.builder().withXlang(true).buildThreadLocalFory() + threadLocal.register(130L) val threadLocalBytes = threadLocal.serialize(expected) require(threadLocal.deserialize(threadLocalBytes, Money::class.java) == expected) - val pooled = - ForyKotlin.builder().withXlang(true).withModule(BasicForyModule).buildThreadSafeForyPool(1) + val pooled = ForyKotlin.builder().withXlang(true).buildThreadSafeForyPool(1) + pooled.register(130L) val pooledBytes = pooled.serialize(expected) require(pooled.deserialize(pooledBytes, Money::class.java) == expected) } diff --git a/integration_tests/idl_tests/scala/src/test/scala/org/apache/fory/idl_tests/ScalaIdlRoundTripTest.scala b/integration_tests/idl_tests/scala/src/test/scala/org/apache/fory/idl_tests/ScalaIdlRoundTripTest.scala index 0908fd3065..1a74795397 100644 --- a/integration_tests/idl_tests/scala/src/test/scala/org/apache/fory/idl_tests/ScalaIdlRoundTripTest.scala +++ b/integration_tests/idl_tests/scala/src/test/scala/org/apache/fory/idl_tests/ScalaIdlRoundTripTest.scala @@ -62,17 +62,19 @@ final class ScalaIdlRoundTripTest extends AnyWordSpec with Matchers { fory.deserialize(fory.serialize(envelope)) shouldEqual envelope } - "register generated serializers through their lifecycle owners" in { + "register generated serializers through BaseFory extension receivers" in { val expected = Money(new BigDecimal("12.34"), "USD") val direct = ForyScala.builder().withXlang(true).build() direct.register[Money](130L) direct.deserialize(direct.serialize(expected)).asInstanceOf[Money] shouldEqual expected - val threadLocal = ForyScala.builder().withXlang(true).withModule(BasicForyModule).buildThreadLocalFory() + val threadLocal = ForyScala.builder().withXlang(true).buildThreadLocalFory() + threadLocal.register[Money](130L) threadLocal.deserialize(threadLocal.serialize(expected)).asInstanceOf[Money] shouldEqual expected - val pooled = ForyScala.builder().withXlang(true).withModule(BasicForyModule).buildThreadSafeForyPool(1) + val pooled = ForyScala.builder().withXlang(true).buildThreadSafeForyPool(1) + pooled.register[Money](130L) pooled.deserialize(pooled.serialize(expected)).asInstanceOf[Money] shouldEqual expected } diff --git a/java/fory-core/src/main/java/org/apache/fory/AbstractThreadSafeFory.java b/java/fory-core/src/main/java/org/apache/fory/AbstractThreadSafeFory.java index 1fa4aa2daa..0098854119 100644 --- a/java/fory-core/src/main/java/org/apache/fory/AbstractThreadSafeFory.java +++ b/java/fory-core/src/main/java/org/apache/fory/AbstractThreadSafeFory.java @@ -66,6 +66,11 @@ public void register(String className, String namespace, String typeName) { registerCallback(fory -> fory.register(className, namespace, typeName)); } + @Override + public void register(ForyModule module) { + registerCallback(fory -> fory.register(module)); + } + public void registerUnion( Class cls, int id, org.apache.fory.serializer.Serializer serializer) { registerCallback(fory -> fory.registerUnion(cls, id, serializer)); diff --git a/java/fory-core/src/main/java/org/apache/fory/BaseFory.java b/java/fory-core/src/main/java/org/apache/fory/BaseFory.java index e6c7ce3ab9..cc3bfd024b 100644 --- a/java/fory-core/src/main/java/org/apache/fory/BaseFory.java +++ b/java/fory-core/src/main/java/org/apache/fory/BaseFory.java @@ -89,6 +89,12 @@ public interface BaseFory { */ void register(String className, String namespace, String typeName); + /** + * Register a runtime module. Direct {@link Fory} instances install the module immediately; + * thread-safe runtimes install it into every current and future underlying runtime instance. + */ + void register(ForyModule module); + void registerUnion(Class cls, int id, Serializer serializer); /** diff --git a/java/fory-core/src/main/java/org/apache/fory/FacadeRegistrationGate.java b/java/fory-core/src/main/java/org/apache/fory/FacadeRegistrationGate.java deleted file mode 100644 index 49f168b170..0000000000 --- a/java/fory-core/src/main/java/org/apache/fory/FacadeRegistrationGate.java +++ /dev/null @@ -1,171 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -package org.apache.fory; - -import java.util.function.Consumer; -import java.util.function.Supplier; -import org.apache.fory.annotation.Internal; -import org.apache.fory.exception.ForyException; -import org.apache.fory.util.ExceptionUtils; - -/** Owns registration/root linearization and the permanent freeze at the first facade root. */ -@Internal -public final class FacadeRegistrationGate { - private enum RegistrationState { - OPEN, - REGISTERING, - FINALIZING, - FROZEN, - FAILED - } - - private final Object lock = new Object(); - private final Runnable finishChildren; - private boolean childInitializing; - private volatile RegistrationState state = RegistrationState.OPEN; - - public FacadeRegistrationGate(Runnable finishChildren) { - this.finishChildren = finishChildren; - } - - public void applyRegistration(Runnable action) { - synchronized (lock) { - beginRegistration(); - try { - action.run(); - finishRegistration(); - } catch (Throwable e) { - state = RegistrationState.FAILED; - throw ExceptionUtils.throwException(e); - } - } - } - - public void applyRegistration(Runnable prepare, Runnable publish) { - synchronized (lock) { - beginRegistration(); - try { - prepare.run(); - requireRegistrationActive(); - publish.run(); - finishRegistration(); - } catch (Throwable e) { - state = RegistrationState.FAILED; - throw ExceptionUtils.throwException(e); - } - } - } - - /** Initializes and publishes a child while registration callbacks cannot change. */ - public Fory initializeChild(Supplier initializer, Consumer publisher) { - synchronized (lock) { - if (state == RegistrationState.FAILED) { - throw registrationClosed(); - } - // The monitor is reentrant, so an active initialization here is necessarily the same - // thread reentering the facade before the current child initialization completes. - if (childInitializing) { - throw new IllegalStateException( - "ThreadSafeFory cannot start a root while a child is being initialized."); - } - childInitializing = true; - try { - Fory child = initializer.get(); - if (state == RegistrationState.FROZEN) { - child.getTypeResolver().finishRegistration(); - } else if (state != RegistrationState.OPEN) { - throw registrationClosed(); - } - publisher.accept(child); - return child; - } catch (Throwable e) { - state = RegistrationState.FAILED; - throw ExceptionUtils.throwException(e); - } finally { - childInitializing = false; - } - } - } - - public void freeze() { - RegistrationState current = state; - if (current == RegistrationState.FROZEN) { - return; - } - synchronized (lock) { - current = state; - if (current == RegistrationState.FROZEN) { - return; - } - if (current == RegistrationState.FAILED) { - throw new ForyException("ThreadSafeFory registration finalization previously failed."); - } - if (current == RegistrationState.REGISTERING) { - state = RegistrationState.FAILED; - throw new ForyException( - "Cannot start a root operation while ThreadSafeFory registration is in progress."); - } - if (current == RegistrationState.FINALIZING) { - throw new ForyException("ThreadSafeFory registration finalization is already in progress."); - } - state = RegistrationState.FINALIZING; - try { - finishChildren.run(); - state = RegistrationState.FROZEN; - } catch (Throwable e) { - // Registration remains permanently closed after a failed first finalization. - state = RegistrationState.FAILED; - throw ExceptionUtils.throwException(e); - } - } - } - - private void beginRegistration() { - RegistrationState current = state; - if (current == RegistrationState.OPEN) { - state = RegistrationState.REGISTERING; - return; - } - // The lock is reentrant, so REGISTERING here can only be same-thread facade reentry. Applying - // its callback now would give existing children and future replay different registration order. - if (current == RegistrationState.REGISTERING) { - state = RegistrationState.FAILED; - } - throw registrationClosed(); - } - - private void finishRegistration() { - requireRegistrationActive(); - state = RegistrationState.OPEN; - } - - private void requireRegistrationActive() { - if (state != RegistrationState.REGISTERING) { - throw registrationClosed(); - } - } - - private ForyException registrationClosed() { - return new ForyException( - "Cannot register class/serializer after registration has been frozen or failed. Please " - + "register all classes before invoking top-level `serialize/deserialize/copy` " - + "methods of ThreadSafeFory."); - } -} diff --git a/java/fory-core/src/main/java/org/apache/fory/Fory.java b/java/fory-core/src/main/java/org/apache/fory/Fory.java index d65f4d0874..16a374a1da 100644 --- a/java/fory-core/src/main/java/org/apache/fory/Fory.java +++ b/java/fory-core/src/main/java/org/apache/fory/Fory.java @@ -22,9 +22,7 @@ import java.io.IOException; import java.io.OutputStream; import java.nio.ByteBuffer; -import java.util.Collections; import java.util.IdentityHashMap; -import java.util.Set; import java.util.function.Consumer; import java.util.function.Function; import javax.annotation.concurrent.NotThreadSafe; @@ -101,8 +99,7 @@ public final class Fory implements BaseFory { private final WriteContext writeContext; private final ReadContext readContext; private final CopyContext copyContext; - private final Set moduleRegistrations = - Collections.newSetFromMap(new IdentityHashMap<>()); + private final IdentityHashMap installedModules = new IdentityHashMap<>(); private final byte headerBitmap; private MemoryBuffer buffer; @@ -172,13 +169,13 @@ public Fory(ForyBuilder builder, ClassLoader classLoader, SharedRegistry sharedR @Override public void register(Class cls) { - checkRegisterAllowed(); + typeResolver.checkRegistrationOpen(); getTypeResolver().register(cls); } @Override public void register(Class cls, int id) { - checkRegisterAllowed(); + typeResolver.checkRegistrationOpen(); getTypeResolver().register(cls, Integer.toUnsignedLong(id)); } @@ -188,71 +185,67 @@ public void register(Class cls, int id) { */ @Override public void register(Class cls, String name) { - checkRegisterAllowed(); + typeResolver.checkRegistrationOpen(); String[] parts = splitRegistrationName(name); register(cls, parts[0], parts[1]); } public void register(Class cls, String namespace, String typeName) { - checkRegisterAllowed(); + typeResolver.checkRegistrationOpen(); getTypeResolver().register(cls, namespace, typeName); } @Override public void register(String className) { - checkRegisterAllowed(); + typeResolver.checkRegistrationOpen(); getTypeResolver().register(className); } @Override public void register(String className, int classId) { - checkRegisterAllowed(); + typeResolver.checkRegistrationOpen(); getTypeResolver().register(className, Integer.toUnsignedLong(classId)); } @Override public void register(String className, String name) { - checkRegisterAllowed(); + typeResolver.checkRegistrationOpen(); String[] parts = splitRegistrationName(name); getTypeResolver().register(className, parts[0], parts[1]); } @Override public void register(String className, String namespace, String typeName) { - checkRegisterAllowed(); + typeResolver.checkRegistrationOpen(); getTypeResolver().register(className, namespace, typeName); } - /** - * Installs a module into this runtime before its first root operation. Configure modules for a - * thread-safe facade through {@link ForyBuilder#withModule(ForyModule)} before building it. - */ + /** Installs a module into this runtime before its first root operation. */ + @Override public void register(ForyModule module) { Preconditions.checkNotNull(module); - checkRegisterAllowed(); - if (!moduleRegistrations.add(module)) { + typeResolver.checkRegistrationOpen(); + if (installedModules.containsKey(module)) { return; } + installedModules.put(module, Boolean.TRUE); try { - // Publishing the identity before the callback breaks self and mutual installation cycles. - // A failure removes it, while a successful installation retains the identity for idempotence. module.install(this); - checkRegisterAllowed(); - } catch (Throwable e) { - moduleRegistrations.remove(module); - throw ExceptionUtils.throwException(e); + } catch (RuntimeException | Error e) { + installedModules.remove(module); + throw e; } } @Override public void registerUnion(Class cls, int id, Serializer serializer) { - checkRegisterAllowed(); + typeResolver.checkRegistrationOpen(); getTypeResolver().registerUnion(cls, Integer.toUnsignedLong(id), serializer); } @Override public void registerUnion(Class cls, String name, Serializer serializer) { - checkRegisterAllowed(); + typeResolver.checkRegistrationOpen(); String[] parts = splitRegistrationName(name); getTypeResolver().registerUnion(cls, parts[0], parts[1], serializer); } @@ -260,52 +253,52 @@ public void registerUnion(Class cls, String name, Serializer serializer) { @Override public void registerUnion( Class cls, String namespace, String typeName, Serializer serializer) { - checkRegisterAllowed(); + typeResolver.checkRegistrationOpen(); getTypeResolver().registerUnion(cls, namespace, typeName, serializer); } @Override public void registerSerializer(Class type, Class serializerClass) { - checkRegisterAllowed(); + typeResolver.checkRegistrationOpen(); getTypeResolver().registerSerializer(type, serializerClass); } @Override public void registerSerializer(Class type, Serializer serializer) { - checkRegisterAllowed(); + typeResolver.checkRegistrationOpen(); getTypeResolver().registerSerializer(type, serializer); } @Override public void registerSerializer( Class type, Function> serializerCreator) { - checkRegisterAllowed(); - getTypeResolver().registerSerializer(type, serializerCreator); + typeResolver.checkRegistrationOpen(); + getTypeResolver().registerSerializer(type, serializerCreator.apply(typeResolver)); } @Override public void registerSerializerAndType( Class type, Class serializerClass) { - checkRegisterAllowed(); + typeResolver.checkRegistrationOpen(); getTypeResolver().registerSerializerAndType(type, serializerClass); } @Override public void registerSerializerAndType(Class type, Serializer serializer) { - checkRegisterAllowed(); + typeResolver.checkRegistrationOpen(); getTypeResolver().registerSerializerAndType(type, serializer); } @Override public void registerSerializerAndType( Class type, Function> serializerCreator) { - checkRegisterAllowed(); - getTypeResolver().registerSerializerAndType(type, serializerCreator); + typeResolver.checkRegistrationOpen(); + getTypeResolver().registerSerializerAndType(type, serializerCreator.apply(typeResolver)); } @Override public void registerSerializerFactory(SerializerFactory serializerFactory) { - checkRegisterAllowed(); + typeResolver.checkRegistrationOpen(); typeResolver.registerSerializerFactory(serializerFactory); } @@ -314,12 +307,6 @@ public Serializer getSerializer(Class cls) { return typeResolver.getSerializer(cls); } - private void ensureRegistrationFinished() { - if (!typeResolver.isRegistrationFinished()) { - typeResolver.finishRegistration(); - } - } - @Override public byte[] serialize(Object obj) { MemoryBuffer buf = getBuffer(); @@ -347,7 +334,7 @@ public MemoryBuffer serialize(MemoryBuffer buffer, Object obj) { @Override public MemoryBuffer serialize(MemoryBuffer buffer, Object obj, BufferCallback callback) { - ensureRegistrationFinished(); + typeResolver.freezeRegistration(); writeContext.prepare(buffer, callback); try { byte bitmap = headerBitmap; @@ -433,7 +420,7 @@ public T deserialize(byte[] bytes, Class type) { @Override public T deserialize(MemoryBuffer buffer, Class type) { - ensureRegistrationFinished(); + typeResolver.freezeRegistration(); byte bitmap = buffer.readByte(); if (bitmap != headerBitmap) { checkHeaderBitmapWithoutOutOfBand(bitmap); @@ -458,6 +445,7 @@ public T deserialize(MemoryBuffer buffer, Class type) { @Override public T deserialize(ForyInputStream inputStream, Class type) { + typeResolver.freezeRegistration(); try { return deserialize(inputStream.getBuffer(), type); } finally { @@ -467,6 +455,7 @@ public T deserialize(ForyInputStream inputStream, Class type) { @Override public T deserialize(ForyReadableChannel channel, Class type) { + typeResolver.freezeRegistration(); try { return deserialize(channel.getBuffer(), type); } finally { @@ -499,7 +488,7 @@ public Object deserialize(MemoryBuffer buffer) { */ @Override public Object deserialize(MemoryBuffer buffer, Iterable outOfBandBuffers) { - ensureRegistrationFinished(); + typeResolver.freezeRegistration(); byte bitmap = buffer.readByte(); boolean peerOutOfBandEnabled = false; if (bitmap != headerBitmap) { @@ -542,6 +531,7 @@ public Object deserialize(ForyInputStream inputStream) { @Override public Object deserialize(ForyInputStream inputStream, Iterable outOfBandBuffers) { + typeResolver.freezeRegistration(); try { MemoryBuffer buf = inputStream.getBuffer(); return deserialize(buf, outOfBandBuffers); @@ -557,6 +547,7 @@ public Object deserialize(ForyReadableChannel channel) { @Override public Object deserialize(ForyReadableChannel channel, Iterable outOfBandBuffers) { + typeResolver.freezeRegistration(); try { MemoryBuffer buf = channel.getBuffer(); return deserialize(buf, outOfBandBuffers); @@ -608,7 +599,6 @@ private boolean checkHeaderBitmap(byte bitmap) { @Override public T copy(T obj) { - ensureRegistrationFinished(); try { return copyContext.copyObject(obj); } catch (Throwable e) { @@ -717,15 +707,6 @@ SharedRegistry getSharedRegistry() { return sharedRegistry; } - private void checkRegisterAllowed() { - if (typeResolver.isRegistrationFrozen()) { - throw new ForyException( - "Cannot register class/serializer after registration has been frozen. Please register " - + "all classes before invoking top-level `serialize/deserialize/copy` methods of " - + "Fory."); - } - } - public Config getConfig() { return config; } diff --git a/java/fory-core/src/main/java/org/apache/fory/ThreadLocalFory.java b/java/fory-core/src/main/java/org/apache/fory/ThreadLocalFory.java index 51935ece79..ec85bccfcb 100644 --- a/java/fory-core/src/main/java/org/apache/fory/ThreadLocalFory.java +++ b/java/fory-core/src/main/java/org/apache/fory/ThreadLocalFory.java @@ -30,6 +30,7 @@ import javax.annotation.concurrent.ThreadSafe; import org.apache.fory.annotation.Internal; import org.apache.fory.config.ForyBuilder; +import org.apache.fory.exception.ForyException; import org.apache.fory.io.ForyInputStream; import org.apache.fory.io.ForyReadableChannel; import org.apache.fory.memory.MemoryBuffer; @@ -47,14 +48,14 @@ public class ThreadLocalFory extends AbstractThreadSafeFory { private final ThreadLocal foryThreadLocal; private Consumer factoryCallback; private final Map allFory; - private final FacadeRegistrationGate registrationGate; + private final Object callbackLock = new Object(); + private volatile boolean registrationFrozen; public ThreadLocalFory(Function factory) { SharedRegistry sharedRegistry = new SharedRegistry(); foryFactory = () -> factory.apply(Fory.builder().withSharedRegistry(sharedRegistry)); factoryCallback = f -> {}; allFory = Collections.synchronizedMap(new WeakHashMap<>()); - registrationGate = new FacadeRegistrationGate(this::finishChildRegistration); foryThreadLocal = ThreadLocal.withInitial(this::newFory); // 1. init and warm for current thread. // Fory creation took about 1~2 ms, but first creation @@ -64,53 +65,71 @@ public ThreadLocalFory(Function factory) { } private Fory newFory() { - return registrationGate.initializeChild( - () -> { - Fory child = foryFactory.get(); - factoryCallback.accept(child); - if (child.getTypeResolver().isRegistrationFrozen()) { - throw new IllegalStateException( - "A ThreadSafeFory child started a root operation during registration replay."); - } - return child; - }, - child -> allFory.put(child, null)); - } - - private void finishChildRegistration() { - synchronized (allFory) { - for (Fory fory : allFory.keySet()) { - fory.getTypeResolver().finishRegistration(); + synchronized (callbackLock) { + Fory fory = foryFactory.get(); + factoryCallback.accept(fory); + if (registrationFrozen) { + fory.getTypeResolver().freezeRegistration(); } + allFory.put(fory, null); + return fory; } } private Fory currentFory() { - registrationGate.freeze(); + freezeRegistration(); return foryThreadLocal.get(); } + private void freezeRegistration() { + if (!registrationFrozen) { + synchronized (callbackLock) { + if (!registrationFrozen) { + registrationFrozen = true; + } + } + } + } + @Internal @Override public void registerCallback(Consumer callback) { - registrationGate.applyRegistration( - () -> { - synchronized (allFory) { - for (Fory fory : allFory.keySet()) { - callback.accept(fory); - if (fory.getTypeResolver().isRegistrationFrozen()) { - throw new IllegalStateException( - "A ThreadSafeFory child started a root operation during registration."); - } - } - } - }, - () -> factoryCallback = factoryCallback.andThen(callback)); + synchronized (callbackLock) { + checkRegistrationOpen(); + synchronized (allFory) { + allFory.keySet().forEach(callback); + } + factoryCallback = factoryCallback.andThen(callback); + } } @Override public R execute(Function action) { - return action.apply(currentFory()); + Fory fory = foryThreadLocal.get(); + if (!registrationFrozen) { + synchronized (callbackLock) { + if (!registrationFrozen) { + try { + return action.apply(fory); + } finally { + if (fory.getTypeResolver().isRegistrationFrozen()) { + registrationFrozen = true; + } + } + } + } + } + fory.getTypeResolver().freezeRegistration(); + return action.apply(fory); + } + + private void checkRegistrationOpen() { + if (registrationFrozen) { + throw new ForyException( + "Cannot register class/serializer after registration has been frozen. Please register " + + "all classes before invoking top-level `serialize/deserialize` methods of " + + "ThreadSafeFory."); + } } @Override @@ -210,6 +229,13 @@ public Object deserialize(ForyReadableChannel channel, Iterable ou @Override public T copy(T obj) { - return currentFory().copy(obj); + if (!registrationFrozen) { + synchronized (callbackLock) { + if (!registrationFrozen) { + return foryThreadLocal.get().copy(obj); + } + } + } + return foryThreadLocal.get().copy(obj); } } diff --git a/java/fory-core/src/main/java/org/apache/fory/ThreadSafeFory.java b/java/fory-core/src/main/java/org/apache/fory/ThreadSafeFory.java index 6f436ef40f..1f2c1f3f20 100644 --- a/java/fory-core/src/main/java/org/apache/fory/ThreadSafeFory.java +++ b/java/fory-core/src/main/java/org/apache/fory/ThreadSafeFory.java @@ -22,7 +22,6 @@ import java.util.function.Consumer; import java.util.function.Function; import org.apache.fory.annotation.Internal; -import org.apache.fory.config.ForyBuilder; import org.apache.fory.resolver.TypeChecker; /** @@ -31,18 +30,12 @@ * *

The runtime class loader is fixed when the thread-safe serializer is built. If you need a * different class loader, build a different {@link ThreadSafeFory} instance. - * - *

Configure runtime modules through {@link ForyBuilder#withModule(ForyModule)} before building - * the facade. */ public interface ThreadSafeFory extends BaseFory { /** - * Executes {@code action} with an underlying {@link Fory} instance and returns its result. - * - *

Calling this method permanently freezes registration before {@code action} runs. Complete - * all facade registration first; the supplied instance is already frozen and remains frozen if - * the callback returns or retains it. + * Provide a context to execution operations on {@link Fory} directly and return the executed + * result. */ R execute(Function action); diff --git a/java/fory-core/src/main/java/org/apache/fory/config/ForyBuilder.java b/java/fory-core/src/main/java/org/apache/fory/config/ForyBuilder.java index c9c796b241..e66b0b7df8 100644 --- a/java/fory-core/src/main/java/org/apache/fory/config/ForyBuilder.java +++ b/java/fory-core/src/main/java/org/apache/fory/config/ForyBuilder.java @@ -426,8 +426,6 @@ public ForyBuilder withSerializerFactory(SerializerFactory serializerFactory) { *

Each created Fory instance ignores repeated registration of the same module object. Dedupe * uses identity, not {@link Object#equals(Object)}, so distinct module instances are installed * independently. - * - *

Thread-safe facades accept modules only through this builder configuration. */ public ForyBuilder withModule(ForyModule module) { ForyModule checkedModule = Objects.requireNonNull(module); diff --git a/java/fory-core/src/main/java/org/apache/fory/io/ForyReadableChannel.java b/java/fory-core/src/main/java/org/apache/fory/io/ForyReadableChannel.java index 348129f1a1..78a5f99d86 100644 --- a/java/fory-core/src/main/java/org/apache/fory/io/ForyReadableChannel.java +++ b/java/fory-core/src/main/java/org/apache/fory/io/ForyReadableChannel.java @@ -310,7 +310,7 @@ public void compactBuffer() { } ByteBuffer byteBuf = byteBuffer; // A read method may compute its post-fill absolute cursor before invoking fillBuffer, so moving - // bytes during a fill invalidates that pending cursor. Root finalization is the safe owner for + // bytes during a fill invalidates that pending cursor. The root cleanup boundary owns // compaction and still preserves bytes prefetched from the following root. int dataEnd = byteBuf.position(); int dataStart = dataEnd - memoryBuf.size(); diff --git a/java/fory-core/src/main/java/org/apache/fory/logging/LogOnceState.java b/java/fory-core/src/main/java/org/apache/fory/logging/LogOnceState.java index f999c0c31a..4878dceb76 100644 --- a/java/fory-core/src/main/java/org/apache/fory/logging/LogOnceState.java +++ b/java/fory-core/src/main/java/org/apache/fory/logging/LogOnceState.java @@ -28,8 +28,6 @@ final class LogOnceState { static final Object[] NO_ARGS = new Object[0]; - // Keys live as long as the logger. Read paths selected by untrusted names must use a fixed - // message without name-derived arguments so input cannot grow this set without bound. private final Set logged = Collections.newSetFromMap(new ConcurrentHashMap()); diff --git a/java/fory-core/src/main/java/org/apache/fory/pool/ThreadPoolFory.java b/java/fory-core/src/main/java/org/apache/fory/pool/ThreadPoolFory.java index 544d05d9d3..6ed88d8d71 100644 --- a/java/fory-core/src/main/java/org/apache/fory/pool/ThreadPoolFory.java +++ b/java/fory-core/src/main/java/org/apache/fory/pool/ThreadPoolFory.java @@ -29,10 +29,10 @@ import java.util.function.Supplier; import javax.annotation.concurrent.ThreadSafe; import org.apache.fory.AbstractThreadSafeFory; -import org.apache.fory.FacadeRegistrationGate; import org.apache.fory.Fory; import org.apache.fory.annotation.Internal; import org.apache.fory.config.ForyBuilder; +import org.apache.fory.exception.ForyException; import org.apache.fory.io.ForyInputStream; import org.apache.fory.io.ForyReadableChannel; import org.apache.fory.memory.MemoryBuffer; @@ -53,7 +53,8 @@ public class ThreadPoolFory extends AbstractThreadSafeFory { private final Fory[] pooledFory; private final Semaphore waiterSignal = new Semaphore(0); private final AtomicInteger waitingBorrowers = new AtomicInteger(); - private final FacadeRegistrationGate registrationGate; + private final Object callbackLock = new Object(); + private volatile boolean registrationFrozen; public ThreadPoolFory(Function foryFactory, int poolSize) { if (poolSize <= 0) { @@ -71,17 +72,14 @@ public ThreadPoolFory(Function foryFactory, int poolSize) { pooledFory[i] = fory; slots.set(i, new PooledEntry(fory, i)); } - registrationGate = new FacadeRegistrationGate(this::finishChildRegistration); } - private void finishChildRegistration() { - for (Fory fory : pooledFory) { - fory.getTypeResolver().finishRegistration(); - } + private PooledEntry acquire() { + freezeRegistration(); + return acquireEntry(); } - private PooledEntry acquire() { - registrationGate.freeze(); + private PooledEntry acquireEntry() { int slotIndex = slotIndexForCurrentThread(); PooledEntry entry = tryBorrowPreferredSlots(slotIndex); if (entry != null) { @@ -156,28 +154,59 @@ private static int spread(int hash) { @Internal @Override public void registerCallback(Consumer callback) { - registrationGate.applyRegistration( - () -> { - for (Fory fory : pooledFory) { - callback.accept(fory); - if (fory.getTypeResolver().isRegistrationFrozen()) { - throw new IllegalStateException( - "A ThreadSafeFory child started a root operation during registration."); - } - } - }); + synchronized (callbackLock) { + checkRegistrationOpen(); + for (Fory fory : pooledFory) { + callback.accept(fory); + } + } } @Override public R execute(Function action) { - PooledEntry entry = acquire(); + if (!registrationFrozen) { + synchronized (callbackLock) { + if (!registrationFrozen) { + PooledEntry entry = acquireEntry(); + try { + return action.apply(entry.fory); + } finally { + if (entry.fory.getTypeResolver().isRegistrationFrozen()) { + registrationFrozen = true; + } + release(entry); + } + } + } + } + PooledEntry entry = acquireEntry(); try { + entry.fory.getTypeResolver().freezeRegistration(); return action.apply(entry.fory); } finally { release(entry); } } + private void freezeRegistration() { + if (!registrationFrozen) { + synchronized (callbackLock) { + if (!registrationFrozen) { + registrationFrozen = true; + } + } + } + } + + private void checkRegistrationOpen() { + if (registrationFrozen) { + throw new ForyException( + "Cannot register class/serializer after registration has been frozen. Please register " + + "all classes before invoking top-level `serialize/deserialize` methods of " + + "ThreadSafeFory."); + } + } + @Override public byte[] serialize(Object obj) { PooledEntry entry = acquire(); @@ -370,7 +399,19 @@ public Object deserialize(ForyReadableChannel channel, Iterable ou @Override public T copy(T obj) { - PooledEntry entry = acquire(); + if (!registrationFrozen) { + synchronized (callbackLock) { + if (!registrationFrozen) { + PooledEntry entry = acquireEntry(); + try { + return entry.fory.copy(obj); + } finally { + release(entry); + } + } + } + } + PooledEntry entry = acquireEntry(); try { return entry.fory.copy(obj); } finally { diff --git a/java/fory-core/src/main/java/org/apache/fory/resolver/AllowListChecker.java b/java/fory-core/src/main/java/org/apache/fory/resolver/AllowListChecker.java index 85b02453e8..d21546de27 100644 --- a/java/fory-core/src/main/java/org/apache/fory/resolver/AllowListChecker.java +++ b/java/fory-core/src/main/java/org/apache/fory/resolver/AllowListChecker.java @@ -130,8 +130,9 @@ private boolean check(String className) { } if (!allowed) { LOG.warnOnce( - "A class is not in the allow list. Check whether its objects are allowed for " - + "serialization or deserialization."); + "Class {} not in allow list, please check whether objects of this class " + + "are allowed for serialization or deserialization.", + className); } return true; case STRICT: diff --git a/java/fory-core/src/main/java/org/apache/fory/resolver/ClassResolver.java b/java/fory-core/src/main/java/org/apache/fory/resolver/ClassResolver.java index 644d1c5bbe..72842d6c60 100644 --- a/java/fory-core/src/main/java/org/apache/fory/resolver/ClassResolver.java +++ b/java/fory-core/src/main/java/org/apache/fory/resolver/ClassResolver.java @@ -266,9 +266,8 @@ private void clearTypeInfoCache() { } @Override - @Internal public void initialize() { - checkRegisterAllowed(); + checkRegistrationOpen(); extRegistry.objectGenericType = buildGenericType(OBJECT_TYPE); registerInternal(LambdaSerializer.ReplaceStub.class, LAMBDA_STUB_ID); registerInternal(JdkProxySerializer.ReplaceStub.class, JDK_PROXY_STUB_ID); @@ -481,15 +480,40 @@ private void registerDefaultClasses() { */ @Override public void register(Class cls) { - checkRegisterAllowed(); + checkRegistrationOpen(); if (!extRegistry.registeredClassIdMap.containsKey(cls)) { while (containsUserTypeId(extRegistry.userIdGenerator)) { extRegistry.userIdGenerator++; } - registerUserImpl(cls, extRegistry.userIdGenerator); + register(cls, extRegistry.userIdGenerator); } } + /** + * Registers a class by its fully qualified name with an auto-assigned user ID. + * + * @param className the fully qualified class name + * @see #register(Class) + */ + @Override + public void register(String className) { + checkRegistrationOpen(); + register(loadClassFromLoader(className)); + } + + /** + * Registers a class by its fully qualified name with a specified user ID. + * + * @param className the fully qualified class name + * @param classId the user ID to assign (0-based, in user ID space) + * @see #register(Class, long) + */ + @Override + public void register(String className, long classId) { + checkRegistrationOpen(); + register(loadClassFromLoader(className), classId); + } + /** * Registers a class with a user-specified ID. * @@ -502,7 +526,7 @@ public void register(Class cls) { */ @Override public void register(Class cls, long id) { - checkRegisterAllowed(); + checkRegistrationOpen(); registerUserImpl(cls, toUserTypeId(id)); } @@ -513,7 +537,7 @@ public void register(Class cls, long id) { */ @Override public void register(Class cls, String namespace, String name) { - checkRegisterAllowed(); + checkRegistrationOpen(); Preconditions.checkArgument(!Functions.isLambda(cls)); Preconditions.checkArgument(!ReflectionUtils.isJdkProxy(cls)); Preconditions.checkArgument(!cls.isArray()); @@ -546,7 +570,7 @@ public void register(Class cls, String namespace, String name) { @Override public void registerUnion(Class cls, long userId, Serializer serializer) { - checkRegisterAllowed(); + checkRegistrationOpen(); int checkedUserId = toUserTypeId(userId); Preconditions.checkNotNull(serializer); checkRegistration(cls, checkedUserId, cls.getName(), false); @@ -566,7 +590,7 @@ public void registerUnion(Class cls, long userId, Serializer serializer) { @Override public void registerUnion(Class cls, String namespace, String name, Serializer serializer) { - checkRegisterAllowed(); + checkRegistrationOpen(); Preconditions.checkNotNull(serializer); Preconditions.checkArgument(!Functions.isLambda(cls)); Preconditions.checkArgument(!ReflectionUtils.isJdkProxy(cls)); @@ -595,7 +619,7 @@ public void registerUnion(Class cls, String namespace, String name, Serialize @Override public void registerEnum(Class cls, long userId, Serializer serializer) { - checkRegisterAllowed(); + checkRegistrationOpen(); int checkedUserId = toUserTypeId(userId); Preconditions.checkNotNull(serializer); checkRegistration(cls, checkedUserId, cls.getName(), false); @@ -614,7 +638,7 @@ public void registerEnum(Class cls, long userId, Serializer serializer) { @Override public void registerEnum(Class cls, String namespace, String name, Serializer serializer) { - checkRegisterAllowed(); + checkRegistrationOpen(); Preconditions.checkNotNull(serializer); Preconditions.checkArgument(!Functions.isLambda(cls)); Preconditions.checkArgument(!ReflectionUtils.isJdkProxy(cls)); @@ -648,11 +672,10 @@ public void registerEnum(Class cls, String namespace, String name, Serializer * * @param classes the classes to register */ - @Internal public void registerInternal(Class... classes) { - checkRegisterAllowed(); + checkRegistrationOpen(); for (Class cls : classes) { - registerInternalType(cls); + registerInternal(cls); } } @@ -664,10 +687,23 @@ public void registerInternal(Class... classes) { * * @param cls the class to register */ - @Internal public void registerInternal(Class cls) { - checkRegisterAllowed(); - registerInternalType(cls); + checkRegistrationOpen(); + if (!extRegistry.registeredClassIdMap.containsKey(cls)) { + Preconditions.checkArgument( + extRegistry.classIdGenerator < INTERNAL_NATIVE_ID_LIMIT, + "Internal type id overflow: %s", + extRegistry.classIdGenerator); + while (extRegistry.classIdGenerator < typeIdToTypeInfo.length + && typeIdToTypeInfo[extRegistry.classIdGenerator] != null) { + extRegistry.classIdGenerator++; + } + Preconditions.checkArgument( + extRegistry.classIdGenerator < INTERNAL_NATIVE_ID_LIMIT, + "Internal type id overflow: %s", + extRegistry.classIdGenerator); + registerInternal(cls, extRegistry.classIdGenerator); + } } /** @@ -682,32 +718,14 @@ public void registerInternal(Class cls) { * @param classId the internal ID, must be in range [0, 255] * @throws IllegalArgumentException if the ID is out of range or already in use */ - @Internal public void registerInternal(Class cls, int classId) { - checkRegisterAllowed(); + checkRegistrationOpen(); Preconditions.checkArgument(classId >= 0 && classId < INTERNAL_NATIVE_ID_LIMIT); registerInternalImpl(cls, classId); } - private void registerInternalType(Class cls) { - if (!extRegistry.registeredClassIdMap.containsKey(cls)) { - Preconditions.checkArgument( - extRegistry.classIdGenerator < INTERNAL_NATIVE_ID_LIMIT, - "Internal type id overflow: %s", - extRegistry.classIdGenerator); - while (extRegistry.classIdGenerator < typeIdToTypeInfo.length - && typeIdToTypeInfo[extRegistry.classIdGenerator] != null) { - extRegistry.classIdGenerator++; - } - Preconditions.checkArgument( - extRegistry.classIdGenerator < INTERNAL_NATIVE_ID_LIMIT, - "Internal type id overflow: %s", - extRegistry.classIdGenerator); - registerInternalImpl(cls, extRegistry.classIdGenerator); - } - } - private void registerInternalImpl(Class cls, int typeId) { + checkRegistrationOpen(); Preconditions.checkArgument(typeId >= 0 && typeId < INTERNAL_NATIVE_ID_LIMIT); checkRegistration(cls, typeId, cls.getName(), true); extRegistry.registeredClassIdMap.put(cls, typeId); @@ -723,6 +741,7 @@ private void registerInternalImpl(Class cls, int typeId) { } private void registerUserImpl(Class cls, int userId) { + checkRegistrationOpen(); Preconditions.checkArgument(userId != -1, "User type id 0xffffffff is reserved"); checkRegistration(cls, userId, cls.getName(), false); extRegistry.registeredClassIdMap.put(cls, userId); @@ -932,7 +951,6 @@ public String getTypeAlias(Class cls) { * Compute the typeId used in TypeDef without forcing serializer creation. This avoids recursive * serializer construction while building class metadata. */ - @Internal public int getTypeIdForTypeDef(Class cls) { TypeInfo typeInfo = classInfoMap.get(cls); if (typeInfo != null) { @@ -959,7 +977,6 @@ && checkType(cls.getName()) return typeId; } - @Internal public int getTypeDefRootTypeId(Class cls, boolean hasFieldMetadata) { if (hasFieldMetadata) { // Preserve the normal TypeInfo/name cache so locally generated or dynamically registered @@ -1207,14 +1224,14 @@ public static boolean requireJavaSerialization(Class clz) { * @param type of class */ public void registerSerializer(Class type, Class serializerClass) { - checkRegisterAllowed(); + checkRegistrationOpen(); checkSerializerRegistration(type, serializerClass); - registerSerializer(type, resolver -> resolver.newSerializer(type, serializerClass)); + registerSerializerImpl(type, Serializers.newSerializer(this, type, serializerClass)); } @Override public void registerSerializer(Class type, Serializer serializer) { - checkRegisterAllowed(); + checkRegistrationOpen(); checkSerializerRegistration(type, serializer.getClass()); registerSerializerImpl(type, serializer); } @@ -1226,9 +1243,8 @@ public void registerSerializer(Class type, Serializer serializer) { * @param serializer serializer for object of {@code type} */ @Override - @Internal public void registerInternalSerializer(Class type, Serializer serializer) { - checkRegisterAllowed(); + checkRegistrationOpen(); Integer classId = extRegistry.registeredClassIdMap.get(type); if (classId != null && !isInternalRegisteredClassId(type, classId)) { throw new IllegalArgumentException( @@ -1243,7 +1259,7 @@ public void registerInternalSerializer(Class type, Serializer serializer) classId); } if (classId == null) { - registerInternalType(type); + registerInternal(type); } // Internal serializers are owned by the resolver path, not by their runtime package name. // Android/R8 may obfuscate Fory packages, so package text is not a stable internal marker. @@ -1251,12 +1267,39 @@ public void registerInternalSerializer(Class type, Serializer serializer) } private void registerSerializerImpl(Class type, Serializer serializer) { - TypeInfo typeInfo = newSerializerTypeInfo(type, serializer, false); - publishSerializerTypeInfo(typeInfo, false, true); + checkRegistrationOpen(); + // Serializer registration trusts the Java name, but must not replace an existing custom name. + if (extRegistry.registeredClasses.inverse().get(type) == null) { + extRegistry.registeredClasses.put(type.getName(), type); + } + TypeInfo existingTypeInfo = classInfoMap.get(type); + boolean localOverride = existingTypeInfo != null && existingTypeInfo.serializer != null; + boolean shareable = serializer instanceof Shareable; + if (shareable && !localOverride) { + serializer = sharedRegistry.cacheRegisteredSerializer(type, serializer); + } + addSerializer(type, serializer); + TypeInfo typeInfo = classInfoMap.get(type); + if (shareable && !localOverride) { + TypeInfo sharedTypeInfo = sharedRegistry.cacheRegisteredTypeInfo(type, typeInfo); + if (sharedTypeInfo != typeInfo) { + typeInfo = sharedTypeInfo; + updateTypeInfo(type, typeInfo); + clearTypeInfoCache(); + } + } + if (typeInfo.namespace != null && typeInfo.typeName != null) { + compositeNameBytes2TypeInfo.put( + new TypeNameBytes(typeInfo.namespace, typeInfo.typeName), typeInfo); + } + // in order to support custom serializer for abstract or interface. + if (!type.isPrimitive() && (ReflectionUtils.isAbstract(type) || type.isInterface())) { + extRegistry.abstractTypeInfo.put(type, typeInfo); + extRegistry.registeredTypeInfos.add(typeInfo); + } } - @Override - protected void checkSerializerRegistration(Class type, Class serializerClass) { + private void checkSerializerRegistration(Class type, Class serializerClass) { boolean replaceResolveSerializer = ReplaceResolveSerializer.class.isAssignableFrom(serializerClass) && useReplaceResolveSerializer(type); @@ -1283,105 +1326,6 @@ protected void checkSerializerRegistration(Class type, Class serializerCla } } - @Override - protected TypeInfo newSerializerTypeInfo( - Class type, Serializer serializer, boolean registerType) { - TypeInfo existingInfo = classInfoMap.get(type); - if (registerType && !isRegistered(type)) { - int userId = extRegistry.userIdGenerator; - while (containsUserTypeId(userId)) { - userId++; - } - checkRegistration(type, userId, type.getName(), false); - return new TypeInfo(this, type, serializer, buildUserTypeId(type, serializer), userId); - } - int typeId; - int userTypeId = INVALID_USER_TYPE_ID; - Integer registeredId = extRegistry.registeredClassIdMap.get(type); - if (registeredId != null) { - boolean internal = isInternalRegisteredClassId(type, registeredId); - typeId = internal ? registeredId : buildUserTypeId(type, serializer); - userTypeId = internal ? INVALID_USER_TYPE_ID : registeredId; - } else { - typeId = buildUnregisteredTypeId(type, serializer); - } - TypeInfo typeInfo; - if (existingInfo == null) { - typeInfo = new TypeInfo(this, type, serializer, typeId, userTypeId); - } else { - typeInfo = - new TypeInfo( - type, existingInfo.namespace, existingInfo.typeName, serializer, typeId, userTypeId); - typeInfo.typeDef = existingInfo.typeDef; - typeInfo.setSerializer(this, serializer); - } - return typeInfo; - } - - @Override - @Internal - protected TypeInfo publishSerializerTypeInfo( - TypeInfo typeInfo, boolean registerType, boolean explicitRegistration) { - Class type = typeInfo.type; - TypeInfo currentInfo = classInfoMap.get(type); - TypeInfo publishedInfo = typeInfo; - boolean retainedLocalOwner = false; - boolean localOverride = currentInfo != null && currentInfo.serializer != null; - boolean shareable = explicitRegistration && typeInfo.serializer instanceof Shareable; - if (shareable && !localOverride) { - Serializer serializer = - sharedRegistry.cacheRegisteredSerializer(type, typeInfo.serializer); - typeInfo.setSerializer(this, serializer); - } - if (currentInfo != null - && currentInfo.typeId == typeInfo.typeId - && currentInfo.userTypeId == typeInfo.userTypeId) { - currentInfo.setSerializer(this, typeInfo.serializer); - typeInfo = currentInfo; - publishedInfo = typeInfo; - retainedLocalOwner = true; - } - if (shareable && !localOverride) { - TypeInfo sharedInfo = sharedRegistry.cacheRegisteredTypeInfo(type, typeInfo); - if (!retainedLocalOwner - && sharedInfo.typeId == typeInfo.typeId - && sharedInfo.userTypeId == typeInfo.userTypeId) { - publishedInfo = sharedInfo; - } - } - if (registerType && typeInfo.userTypeId != INVALID_USER_TYPE_ID) { - extRegistry.registeredClassIdMap.put(type, typeInfo.userTypeId); - } - if (publishedInfo.typeId == REPLACE_STUB_ID) { - classInfoMap.put(type, publishedInfo); - } else { - updateTypeInfo(type, publishedInfo); - } - if (explicitRegistration && extRegistry.registeredClasses.inverse().get(type) == null) { - extRegistry.registeredClasses.put(type.getName(), type); - } - boolean publishName = - explicitRegistration - || (!config.requireClassRegistration() - && (extRegistry.typeChecker == DEFAULT_TYPE_CHECKER - || sharedRegistry.isTypeAccepted(type.getName()))); - if (publishName && publishedInfo.namespace != null && publishedInfo.typeName != null) { - compositeNameBytes2TypeInfo.put( - new TypeNameBytes(publishedInfo.namespace, publishedInfo.typeName), publishedInfo); - } - if (explicitRegistration - && !type.isPrimitive() - && (ReflectionUtils.isAbstract(type) || type.isInterface())) { - extRegistry.abstractTypeInfo.put(type, publishedInfo); - extRegistry.registeredTypeInfos.add(publishedInfo); - } - if (registerType) { - registerGraalvmClass(type); - } - clearTypeInfoCache(); - return publishedInfo; - } - /** * Set the serializer for cls, overwrite serializer if exists. Note if class info is * already related with a class, this method should try to reuse that class info, otherwise jit @@ -1389,12 +1333,7 @@ protected TypeInfo publishSerializerTypeInfo( * classinfo. */ @Override - @Internal public void setSerializer(Class cls, Serializer serializer) { - if (isConstructingSerializer()) { - bindConstructedSerializer(cls, serializer); - return; - } addSerializer(cls, serializer); } @@ -1405,24 +1344,63 @@ public void setSerializer(Class cls, Serializer serializer) { * creating a data serializer for serialization of parts fields of a class. */ @Override - @Internal public void setSerializerIfAbsent(Class cls, Serializer serializer) { - if (isConstructingSerializer()) { - if (!hasConstructedSerializer(cls)) { - bindConstructedSerializer(cls, serializer); - } - return; - } Serializer s = getSerializer(cls, false); if (s == null) { setSerializer(cls, serializer); } } - private void addSerializer(Class type, Serializer serializer) { + /** Clear serializer associated with cls if not null. */ + public void clearSerializer(Class cls) { + TypeInfo typeInfo = classInfoMap.get(cls); + if (typeInfo != null) { + typeInfo.setSerializer(this, null); + } + } + + /** Add serializer for specified class. */ + public void addSerializer(Class type, Serializer serializer) { Preconditions.checkNotNull(serializer); - TypeInfo typeInfo = newAutomaticTypeInfo(type, serializer); - publishSerializerTypeInfo(typeInfo, false, false); + TypeInfo typeInfo; + Integer classId = extRegistry.registeredClassIdMap.get(type); + boolean registered = classId != null; + if (registered) { + int id = classId; + boolean internal = isInternalRegisteredClassId(type, id); + int typeId = internal ? id : buildUserTypeId(type, serializer); + typeInfo = classInfoMap.get(type); + if (typeInfo == null) { + typeInfo = new TypeInfo(this, type, null, typeId, internal ? INVALID_USER_TYPE_ID : id); + } else { + typeInfo = typeInfo.copy(typeId); + } + updateTypeInfo(type, typeInfo); + } else { + int typeId = buildUnregisteredTypeId(type, serializer); + typeInfo = classInfoMap.get(type); + if (typeInfo == null) { + typeInfo = new TypeInfo(this, type, null, typeId, INVALID_USER_TYPE_ID); + } else { + typeInfo = typeInfo.copy(typeId); + } + if (typeId == REPLACE_STUB_ID) { + classInfoMap.put(type, typeInfo); + } else { + updateTypeInfo(type, typeInfo); + } + // Automatic serializer creation may publish only a name accepted earlier by isSecure. + // Explicit registerSerializer publishes below in registerSerializerImpl as a trust event. + if (!config.requireClassRegistration() + && (extRegistry.typeChecker == DEFAULT_TYPE_CHECKER + || sharedRegistry.isTypeAccepted(type.getName())) + && typeInfo.namespace != null + && typeInfo.typeName != null) { + compositeNameBytes2TypeInfo.put( + new TypeNameBytes(typeInfo.namespace, typeInfo.typeName), typeInfo); + } + } + typeInfo.setSerializer(this, serializer); } @SuppressWarnings("unchecked") @@ -1693,7 +1671,6 @@ public TypeInfo getTypeInfo(Class cls) { return typeInfo; } - @Internal public TypeInfo getTypeInfo(short classId) { TypeInfo typeInfo = typeIdToTypeInfo[classId]; assert typeInfo != null : classId; @@ -1750,9 +1727,7 @@ private TypeInfo getOrUpdateTypeInfo(Class cls, int depth) { if (typeInfo == null || typeInfo.serializer == null) { typeInfo = createTypeInfo(cls); } - if (!isConstructingSerializer()) { - typeInfoCache[depth] = typeInfo; - } + typeInfoCache[depth] = typeInfo; } return typeInfo; } @@ -1764,11 +1739,7 @@ private TypeInfo createTypeInfo(Class cls) { // the declaring enum, so its registered ID or class name must be used instead of `$1`. return getTypeInfo(enumClass); } - Serializer serializer = createSerializer(cls); - if (isConstructingSerializer()) { - return bindConstructedSerializer(cls, serializer); - } - addSerializer(cls, serializer); + addSerializer(cls, createSerializer(cls)); return Objects.requireNonNull(classInfoMap.get(cls)); } diff --git a/java/fory-core/src/main/java/org/apache/fory/resolver/TypeInfo.java b/java/fory-core/src/main/java/org/apache/fory/resolver/TypeInfo.java index 874ff091b2..862c24c3a8 100644 --- a/java/fory-core/src/main/java/org/apache/fory/resolver/TypeInfo.java +++ b/java/fory-core/src/main/java/org/apache/fory/resolver/TypeInfo.java @@ -22,7 +22,6 @@ import static org.apache.fory.meta.Encoders.PACKAGE_DECODER; import static org.apache.fory.meta.Encoders.TYPE_NAME_DECODER; -import org.apache.fory.annotation.Internal; import org.apache.fory.collection.Tuple2; import org.apache.fory.meta.EncodedMetaString; import org.apache.fory.meta.Encoders; @@ -164,7 +163,6 @@ public Serializer getSerializer() { return (Serializer) serializer; } - @Internal public void setSerializer(Serializer serializer) { this.serializer = serializer; } diff --git a/java/fory-core/src/main/java/org/apache/fory/resolver/TypeResolver.java b/java/fory-core/src/main/java/org/apache/fory/resolver/TypeResolver.java index bfd022c21b..efb6812072 100644 --- a/java/fory-core/src/main/java/org/apache/fory/resolver/TypeResolver.java +++ b/java/fory-core/src/main/java/org/apache/fory/resolver/TypeResolver.java @@ -140,23 +140,6 @@ private static final class TransformedTypeInfo { } } - private static final class SerializerConstruction { - // Recursive fields must capture their final TypeInfo owner during construction, while the - // candidate serializers remain unpublished until the whole construction succeeds. Mixing the - // two makes fields retain temporary metadata or lets a failed constructor mutate a canonical - // owner. - final Class registrationType; - final boolean registerType; - final IdentityHashMap, TypeInfo> typeInfos = new IdentityHashMap<>(); - final IdentityHashMap, Serializer> serializers = new IdentityHashMap<>(); - boolean rejected; - - SerializerConstruction(Class registrationType, boolean registerType) { - this.registrationType = registrationType; - this.registerType = registerType; - } - } - final Config config; final boolean metaContextShareEnabled; final SharedRegistry sharedRegistry; @@ -173,8 +156,6 @@ private static final class SerializerConstruction { // dynamically created classes that can't be found by Class.forName private final TypeInfo[] typeInfoCache; private boolean registrationFrozen; - private boolean registrationFinished; - private SerializerConstruction serializerConstruction; protected TypeResolver( Config config, @@ -212,20 +193,11 @@ public final JITContext getJITContext() { return jitContext; } - public final boolean isRegistrationFinished() { - return registrationFinished; - } - @Internal public final boolean isRegistrationFrozen() { return registrationFrozen; } - protected final void setRegistrationFinished() { - registrationFrozen = true; - registrationFinished = true; - } - public final boolean isCrossLanguage() { return config.isXlang(); } @@ -254,12 +226,12 @@ public final Class getDefaultJDKStreamSerializerType() { return config.getDefaultJDKStreamSerializerType(); } - protected final void checkRegisterAllowed() { - checkRegistrationOpen(); - if (serializerConstruction != null) { - serializerConstruction.rejected = true; + @Internal + public final void checkRegistrationOpen() { + if (registrationFrozen) { throw new ForyException( - "Cannot start an independent registration while a serializer is being constructed."); + "Cannot register class/serializer after registration has been frozen. Please register " + + "all classes before invoking top-level `serialize/deserialize` methods of Fory."); } } @@ -290,13 +262,13 @@ protected final void checkRegisterAllowed() { /** Registers a class by name with an auto-assigned user ID. */ public void register(String className) { - checkRegisterAllowed(); + checkRegistrationOpen(); register(loadClassFromLoader(className)); } /** Registers a class by name with a user-specified ID. */ public void register(String className, long classId) { - checkRegisterAllowed(); + checkRegistrationOpen(); register(loadClassFromLoader(className), classId); } @@ -304,7 +276,7 @@ public void register(String className, long classId) { * Registers a class by name with a namespace and type name. The type name must not contain `.`. */ public void register(String className, String namespace, String typeName) { - checkRegisterAllowed(); + checkRegistrationOpen(); register(loadClassFromLoader(className), namespace, typeName); } @@ -318,7 +290,7 @@ public void register(String className, String namespace, String typeName) { */ @Internal public final void registerRuntimeTypeAlias(Class runtimeType, Class canonicalType) { - checkRegisterAllowed(); + checkRegistrationOpen(); Preconditions.checkNotNull(runtimeType, "runtimeType"); Preconditions.checkNotNull(canonicalType, "canonicalType"); if (runtimeType == canonicalType) { @@ -406,14 +378,6 @@ public final ObjectInstantiator getObjectInstantiator(Class type) { public abstract void registerSerializer( Class type, Class serializerClass); - /** Registers a serializer produced by {@code serializerCreator}. */ - @Internal - public final void registerSerializer( - Class type, Function> serializerCreator) { - checkRegisterAllowed(); - constructSerializer(type, serializerCreator, false); - } - /** * Registers a serializer for internal types (those with fixed IDs in the type system). This * method is used for built-in types like ArrayList, HashMap, etc. @@ -433,22 +397,15 @@ public final void registerSerializer( * later callers adopt those same maps. This method is idempotent so top-level runtime entry * points can call it defensively. */ - public final void finishRegistration() { - if (registrationFinished) { + public final void freezeRegistration() { + if (registrationFrozen) { return; } registrationFrozen = true; - boolean constructionActive = serializerConstruction != null; - if (constructionActive) { - serializerConstruction.rejected = true; - throw new ForyException( - "Cannot start a root operation while a serializer is being constructed."); - } sharedRegistry.setRegistrationIfAbsent( extRegistry.registeredClassIdMap, extRegistry.registeredClasses); - extRegistry.finishRegistration( + extRegistry.freezeRegistration( sharedRegistry.getRegisteredClassIdMap(), sharedRegistry.getRegisteredClasses()); - setRegistrationFinished(); } /** @@ -460,22 +417,11 @@ public final void finishRegistration() { */ public void registerSerializerAndType( Class type, Class serializerClass) { - checkRegisterAllowed(); - checkSerializerRegistration(type, serializerClass); - if (StaticGeneratedStructSerializer.class.isAssignableFrom(serializerClass)) { - throw new ForyException( - "Static generated serializers require registering the type first, then installing a " - + "constructed serializer instance with registerSerializer."); + checkRegistrationOpen(); + if (!isRegistered(type)) { + register(type); } - constructSerializer(type, resolver -> resolver.newSerializer(type, serializerClass), true); - } - - /** Registers a type and a serializer produced by {@code serializerCreator}. */ - @Internal - public final void registerSerializerAndType( - Class type, Function> serializerCreator) { - checkRegisterAllowed(); - constructSerializer(type, serializerCreator, true); + registerSerializer(type, serializerClass); } /** @@ -485,111 +431,13 @@ public final void registerSerializerAndType( * @param serializer the serializer instance to use */ public void registerSerializerAndType(Class type, Serializer serializer) { - checkRegisterAllowed(); - checkSerializerRegistration(type, serializer.getClass()); - TypeInfo typeInfo = newSerializerTypeInfo(type, serializer, true); checkRegistrationOpen(); - publishSerializerTypeInfo(typeInfo, true, true); - } - - private void constructSerializer( - Class type, - Function> serializerCreator, - boolean registerType) { - jitContext.lock(); - try { - SerializerConstruction construction = new SerializerConstruction(type, registerType); - serializerConstruction = construction; - try { - Serializer serializer = Preconditions.checkNotNull(serializerCreator.apply(this)); - checkSerializerRegistration(type, serializer.getClass()); - bindConstructedSerializer(type, serializer); - checkRegistrationOpen(); - if (construction.rejected) { - throw new ForyException( - "Serializer construction attempted an independent registration for " - + type.getName()); - } - publishConstruction(construction); - } finally { - serializerConstruction = null; - } - } finally { - jitContext.unlock(); - } - } - - private void publishConstruction(SerializerConstruction construction) { - TypeInfo registrationInfo = construction.typeInfos.get(construction.registrationType); - Preconditions.checkNotNull(registrationInfo); - Serializer constructedSerializer = - Preconditions.checkNotNull(construction.serializers.get(construction.registrationType)); - TypeInfo preparedInfo = - prepareConstructionTypeInfo( - construction.registrationType, - registrationInfo, - constructedSerializer, - construction.registerType); - // The target may still fail shareable-serializer conflict validation. Publish it first so a - // rejected target cannot leave otherwise complete recursive dependencies in canonical maps. - TypeInfo publishedInfo = - publishSerializerTypeInfo(preparedInfo, construction.registerType, true); - // Reusing a shared serializer discards the candidate constructor and every dependency it - // discovered. Only dependencies retained by the published serializer belong in this resolver. - if (publishedInfo.serializer != constructedSerializer) { - return; - } - construction.typeInfos.forEach( - (type, typeInfo) -> { - if (type != construction.registrationType) { - publishSerializerTypeInfo( - prepareConstructionTypeInfo( - type, typeInfo, construction.serializers.get(type), false), - false, - false); - } - }); - } - - private TypeInfo prepareConstructionTypeInfo( - Class type, TypeInfo typeInfo, Serializer serializer, boolean registerType) { - if (classInfoMap.get(type) == typeInfo) { - return registerType - ? newSerializerTypeInfo(type, serializer, true) - : newAutomaticTypeInfo(type, serializer); + if (!isRegistered(type)) { + register(type); } - typeInfo.setSerializer(this, serializer); - return typeInfo; - } - - private void checkRegistrationOpen() { - if (registrationFrozen) { - throw new ForyException( - "Cannot register class/serializer after registration has been frozen. Please register " - + "all classes before invoking top-level `serialize/deserialize/copy` methods of " - + "Fory."); - } - } - - protected abstract void checkSerializerRegistration(Class type, Class serializerClass); - - protected abstract TypeInfo newSerializerTypeInfo( - Class type, Serializer serializer, boolean registerType); - - protected TypeInfo newAutomaticTypeInfo(Class type, Serializer serializer) { - return newSerializerTypeInfo(type, serializer, false); + registerSerializer(type, serializer); } - /** - * Publishes prepared serializer metadata through the resolver's canonical commit path. - * - *

When the wire and user IDs are unchanged, the resolver must update the existing {@link - * TypeInfo} owner because generated serializers and field metadata may already retain it. - */ - @Internal - protected abstract TypeInfo publishSerializerTypeInfo( - TypeInfo typeInfo, boolean registerType, boolean explicitRegistration); - /** * Whether to track reference for this type. If false, reference tracing of subclasses may be * ignored too. @@ -1653,7 +1501,7 @@ final Class loadClass( } catch (IllegalStateException e) { if (deserializeUnknownClass) { if (!config.suppressClassRegistrationWarnings()) { - LOG.warnOnce("A class could not be loaded and will be read as an unknown class."); + LOG.warnOnce(e.getMessage()); } return UnknownClass.getUnknowClass(className, isEnum, arrayDims, metaContextShareEnabled); } @@ -1751,108 +1599,14 @@ private Serializer getNativeTypedValueSerializer(int typeId, Class rawType public abstract Serializer getRawSerializer(Class cls); - @Internal public abstract void setSerializer(Class cls, Serializer serializer); - @Internal public abstract void setSerializerIfAbsent(Class cls, Serializer serializer); - /** Returns the final metadata owner for a declared field during serializer construction. */ - @Internal - public final TypeInfo getFieldTypeInfo(Class type) { - if (serializerConstruction != null) { - TypeInfo typeInfo = serializerConstruction.typeInfos.get(type); - if (typeInfo != null) { - return typeInfo; - } - } - return getTypeInfo(type); - } - - /** Returns the serializer-construction owner without creating type metadata. */ - @Internal - public final TypeInfo getConstructionTypeInfo(Class type) { - TypeInfo typeInfo = getConstructedTypeInfo(type); - return typeInfo == null ? getTypeInfo(type, false) : typeInfo; - } - - /** Returns the serializer visible to the active construction without creating type metadata. */ - @Internal - public final Serializer getConstructionSerializer(Class type) { - if (serializerConstruction != null && serializerConstruction.serializers.containsKey(type)) { - return serializerConstruction.serializers.get(type); - } - TypeInfo typeInfo = getTypeInfo(type, false); - return typeInfo == null ? null : typeInfo.serializer; - } - - protected final TypeInfo getConstructedTypeInfo(Class type) { - return serializerConstruction == null ? null : serializerConstruction.typeInfos.get(type); - } - - protected final boolean isConstructingSerializer() { - return serializerConstruction != null; - } - - protected final boolean hasConstructedSerializer(Class type) { - return serializerConstruction != null && serializerConstruction.serializers.containsKey(type); - } - - protected final TypeInfo bindConstructedSerializer(Class type, Serializer serializer) { - SerializerConstruction construction = Preconditions.checkNotNull(serializerConstruction); - TypeInfo typeInfo = construction.typeInfos.get(type); - if (typeInfo == null) { - TypeInfo preparedInfo; - if (type == construction.registrationType) { - preparedInfo = newSerializerTypeInfo(type, serializer, construction.registerType); - } else { - preparedInfo = newAutomaticTypeInfo(type, serializer); - } - TypeInfo currentInfo = classInfoMap.get(type); - if (currentInfo != null - && (serializer instanceof StaticGeneratedStructSerializer - || (currentInfo.typeId == preparedInfo.typeId - && currentInfo.userTypeId == preparedInfo.userTypeId))) { - // Static-generated construction starts after canonical type registration. Its base - // constructor binds early for recursion, but that candidate must retain the registered - // Struct identity instead of being reclassified as an explicit EXT serializer. - typeInfo = currentInfo; - } else { - typeInfo = preparedInfo; - } - construction.typeInfos.put(type, typeInfo); - } - construction.serializers.put(type, serializer); - if (classInfoMap.get(type) != typeInfo) { - typeInfo.setSerializer(this, serializer); - } - return typeInfo; - } - - protected final TypeInfo stageConstructedTypeInfo(Class type, TypeInfo typeInfo) { - Preconditions.checkArgument(typeInfo.type == type); - SerializerConstruction construction = Preconditions.checkNotNull(serializerConstruction); - construction.typeInfos.put(type, typeInfo); - construction.serializers.put(type, typeInfo.serializer); - return typeInfo; - } - /** * Reset serializer if {@code serializer} is not null, otherwise clear serializer for {@code cls}. */ - @Internal public void resetSerializer(Class cls, Serializer serializer) { - TypeInfo constructedTypeInfo = getConstructedTypeInfo(cls); - if (constructedTypeInfo != null) { - serializerConstruction.serializers.put(cls, serializer); - if (classInfoMap.get(cls) != constructedTypeInfo) { - constructedTypeInfo.setSerializer(this, serializer); - } - return; - } - if (serializerConstruction != null) { - return; - } if (serializer == null) { TypeInfo typeInfo = getTypeInfo(cls, false); if (typeInfo != null) { @@ -1921,7 +1675,6 @@ public GenericType getGenericTypeInStruct(Class cls, String genericTypeStr) { return map.getOrDefault(genericTypeStr, OBJECT_GENERIC_TYPE); } - @Internal public abstract void initialize(); public abstract void ensureSerializersCompiled(); @@ -2140,9 +1893,10 @@ public final DescriptorGrouper groupDescriptors( } private List buildFieldDescriptors(Class clz, boolean searchParent) { - List ownedStaticDescriptors = getOwnedStaticGeneratedDescriptors(clz); - if (ownedStaticDescriptors != null) { - return normalizeFieldDescriptors(clz, searchParent, ownedStaticDescriptors); + List registeredStaticDescriptors = + getRegisteredStaticGeneratedStructDescriptors(clz); + if (registeredStaticDescriptors != null) { + return normalizeFieldDescriptors(clz, searchParent, registeredStaticDescriptors); } if (shouldPreferStaticGeneratedSerializer(clz)) { List staticDescriptors = getStaticGeneratedStructDescriptors(clz); @@ -2278,14 +2032,14 @@ private List getStaticGeneratedStructDescriptors(Class cls) { cls, isCrossLanguage()); } - private List getOwnedStaticGeneratedDescriptors(Class cls) { - Serializer serializer = getConstructionSerializer(cls); - if (!(serializer instanceof StaticGeneratedStructSerializer)) { + private List getRegisteredStaticGeneratedStructDescriptors(Class cls) { + TypeInfo typeInfo = getTypeInfo(cls, false); + if (typeInfo == null + || !(typeInfo.getSerializer() instanceof StaticGeneratedStructSerializer)) { return null; } - // Generated descriptors are immutable constructor input. Let TypeDef construction see them - // without publishing the serializer candidate that owns the active construction. - return ((StaticGeneratedStructSerializer) serializer).getGeneratedDescriptors(); + return ((StaticGeneratedStructSerializer) typeInfo.getSerializer()) + .getGeneratedDescriptors(); } private StaticGeneratedStructSerializer copyRegisteredStaticGeneratedStructSerializer( @@ -2589,7 +2343,7 @@ final void clearCheckerCache() { } public void registerSerializerFactory(SerializerFactory serializerFactory) { - checkRegisterAllowed(); + checkRegistrationOpen(); extRegistry.serializerFactories.add(Preconditions.checkNotNull(serializerFactory)); } @@ -2785,7 +2539,7 @@ class ExtRegistry { codeGeneratorMap = sharedRegistry.codeGeneratorMap; } - void finishRegistration( + void freezeRegistration( IdentityHashMap, Integer> sharedRegisteredClassIdMap, BiMap> sharedRegisteredClasses) { registeredClassIdMap = sharedRegisteredClassIdMap; diff --git a/java/fory-core/src/main/java/org/apache/fory/resolver/XtypeResolver.java b/java/fory-core/src/main/java/org/apache/fory/resolver/XtypeResolver.java index 5c2a8a51fd..66efe50a9f 100644 --- a/java/fory-core/src/main/java/org/apache/fory/resolver/XtypeResolver.java +++ b/java/fory-core/src/main/java/org/apache/fory/resolver/XtypeResolver.java @@ -168,9 +168,8 @@ public XtypeResolver( } @Override - @Internal public void initialize() { - checkRegisterAllowed(); + checkRegistrationOpen(); registerDefaultTypes(); Serializers.registerDefaultSerializers(this); if (shareMeta) { @@ -206,7 +205,7 @@ protected void updateTypeInfo(Class cls, TypeInfo typeInfo) { @Override public void register(Class type) { - checkRegisterAllowed(); + checkRegistrationOpen(); while (containsUserTypeId(xtypeIdGenerator)) { xtypeIdGenerator++; } @@ -215,7 +214,7 @@ public void register(Class type) { @Override public void register(Class type, long userTypeId) { - checkRegisterAllowed(); + checkRegistrationOpen(); int checkedUserTypeId = toUserTypeId(userTypeId); Preconditions.checkArgument( !containsUserTypeId(checkedUserTypeId), "Type id %s has been registered", userTypeId); @@ -268,7 +267,7 @@ public void register(Class type, long userTypeId) { @Override public void register(Class type, String namespace, String typeName) { - checkRegisterAllowed(); + checkRegistrationOpen(); Preconditions.checkArgument( !typeName.isEmpty() && !typeName.contains("."), "Type name %s must be non-empty and must not contain `.` when namespace is provided", @@ -363,7 +362,7 @@ private void register( @Override public void registerUnion(Class type, long userTypeId, Serializer serializer) { - checkRegisterAllowed(); + checkRegistrationOpen(); Preconditions.checkNotNull(serializer); int checkedUserTypeId = toUserTypeId(userTypeId); Preconditions.checkArgument( @@ -386,7 +385,7 @@ public void registerUnion(Class type, long userTypeId, Serializer serializ @Override public void registerUnion( Class type, String namespace, String typeName, Serializer serializer) { - checkRegisterAllowed(); + checkRegistrationOpen(); Preconditions.checkNotNull(serializer); Preconditions.checkArgument( !typeName.isEmpty() && !typeName.contains("."), @@ -409,7 +408,7 @@ public void registerUnion( @Override public void registerEnum(Class type, long userTypeId, Serializer serializer) { - checkRegisterAllowed(); + checkRegistrationOpen(); Preconditions.checkNotNull(serializer); int checkedUserTypeId = toUserTypeId(userTypeId); Preconditions.checkArgument( @@ -431,7 +430,7 @@ public void registerEnum(Class type, long userTypeId, Serializer serialize @Override public void registerEnum( Class type, String namespace, String typeName, Serializer serializer) { - checkRegisterAllowed(); + checkRegistrationOpen(); Preconditions.checkNotNull(serializer); if (namespace == null) { namespace = ""; @@ -467,7 +466,7 @@ public void registerEnum( */ @Internal public void registerForyType(Class type, Serializer serializer, int typeId) { - checkRegisterAllowed(); + checkRegistrationOpen(); Preconditions.checkArgument(typeId < MAX_TYPE_ID, "Too big type id %s", typeId); register( type, @@ -518,21 +517,40 @@ private TypeInfo newTypeInfo( } public void registerSerializer(Class type, Class serializerClass) { - checkRegisterAllowed(); - checkSerializerRegistration(type, serializerClass); - registerSerializer(type, resolver -> resolver.newSerializer(type, serializerClass)); + checkRegistrationOpen(); + registerSerializer(type, newSerializer(type, serializerClass)); } public void registerSerializer(Class type, Serializer serializer) { - checkRegisterAllowed(); - checkClassRegistration(type); + checkRegistrationOpen(); + TypeInfo typeInfo = checkClassRegistration(type); checkSerializerRegistration(type, serializer.getClass()); - TypeInfo typeInfo = newSerializerTypeInfo(type, serializer, false); - publishSerializerTypeInfo(typeInfo, false, true); + boolean localOverride = typeInfo.serializer != null; + boolean shouldShare = serializer instanceof Shareable && !localOverride; + if (shouldShare) { + serializer = sharedRegistry.cacheRegisteredSerializer(type, serializer); + } + int oldTypeId = typeInfo.typeId; + int foryId = oldTypeId; + + if (foryId == Types.STRUCT || foryId == Types.COMPATIBLE_STRUCT) { + foryId = Types.EXT; + } else if (foryId == Types.NAMED_STRUCT || foryId == Types.NAMED_COMPATIBLE_STRUCT) { + foryId = Types.NAMED_EXT; + } + typeInfo = typeInfo.copy(foryId); + typeInfo.setSerializer(this, serializer); + if (shouldShare) { + typeInfo = sharedRegistry.cacheRegisteredTypeInfo(type, typeInfo); + } + updateTypeInfo(type, typeInfo); + if (typeInfo.typeName != null) { + TypeNameBytes typeNameBytes = new TypeNameBytes(typeInfo.namespace, typeInfo.typeName); + compositeClassNameBytes2TypeInfo.put(typeNameBytes, typeInfo); + } } - @Override - protected void checkSerializerRegistration(Class type, Class serializerClass) { + private void checkSerializerRegistration(Class type, Class serializerClass) { if (isCollection(type) || Collection.class.isAssignableFrom(type)) { if (!CollectionLikeSerializer.class.isAssignableFrom(serializerClass)) { throw new IllegalArgumentException( @@ -552,118 +570,8 @@ protected void checkSerializerRegistration(Class type, Class serializerCla } @Override - protected TypeInfo newSerializerTypeInfo( - Class type, Serializer serializer, boolean registerType) { - TypeInfo existingInfo = classInfoMap.get(type); - if (!registerType && existingInfo == null) { - checkClassRegistration(type); - } - if (registerType && existingInfo == null) { - if (type.isArray()) { - return newTypeInfo(type, serializer, determineTypeIdForClass(type)); - } - int userTypeId = xtypeIdGenerator; - while (containsUserTypeId(userTypeId)) { - userTypeId++; - } - int typeId = type.isEnum() ? Types.ENUM : Types.EXT; - return newTypeInfo(type, serializer, typeId, userTypeId); - } - int typeId = existingInfo == null ? determineTypeIdForClass(type) : existingInfo.typeId; - if (typeId == Types.STRUCT || typeId == Types.COMPATIBLE_STRUCT) { - typeId = Types.EXT; - } else if (typeId == Types.NAMED_STRUCT || typeId == Types.NAMED_COMPATIBLE_STRUCT) { - typeId = Types.NAMED_EXT; - } - TypeInfo typeInfo; - if (existingInfo == null) { - typeInfo = newTypeInfo(type, serializer, typeId); - } else { - typeInfo = - new TypeInfo( - type, - existingInfo.namespace, - existingInfo.typeName, - serializer, - typeId, - existingInfo.userTypeId); - typeInfo.typeDef = existingInfo.typeDef; - typeInfo.setSerializer(this, serializer); - } - return typeInfo; - } - - @Override - protected TypeInfo newAutomaticTypeInfo(Class type, Serializer serializer) { - TypeInfo existingInfo = classInfoMap.get(type); - if (existingInfo == null) { - return newTypeInfo(type, serializer, determineTypeIdForClass(type)); - } - TypeInfo typeInfo = - new TypeInfo( - type, - existingInfo.namespace, - existingInfo.typeName, - serializer, - existingInfo.typeId, - existingInfo.userTypeId); - typeInfo.typeDef = existingInfo.typeDef; - typeInfo.setSerializer(this, serializer); - return typeInfo; - } - - @Override - @Internal - protected TypeInfo publishSerializerTypeInfo( - TypeInfo typeInfo, boolean registerType, boolean explicitRegistration) { - Class type = typeInfo.type; - TypeInfo currentInfo = classInfoMap.get(type); - TypeInfo publishedInfo = typeInfo; - boolean retainedLocalOwner = false; - boolean localOverride = currentInfo != null && currentInfo.serializer != null; - boolean shareable = explicitRegistration && typeInfo.serializer instanceof Shareable; - if (shareable && !localOverride) { - Serializer serializer = - sharedRegistry.cacheRegisteredSerializer(type, typeInfo.serializer); - typeInfo.setSerializer(this, serializer); - } - if (currentInfo != null - && currentInfo.typeId == typeInfo.typeId - && currentInfo.userTypeId == typeInfo.userTypeId) { - currentInfo.setSerializer(this, typeInfo.serializer); - typeInfo = currentInfo; - publishedInfo = typeInfo; - retainedLocalOwner = true; - } - if (shareable && !localOverride) { - TypeInfo sharedInfo = sharedRegistry.cacheRegisteredTypeInfo(type, typeInfo); - if (!retainedLocalOwner - && sharedInfo.typeId == typeInfo.typeId - && sharedInfo.userTypeId == typeInfo.userTypeId) { - publishedInfo = sharedInfo; - } - } - updateTypeInfo(type, publishedInfo); - if (explicitRegistration && typeInfo.typeName != null) { - compositeClassNameBytes2TypeInfo.put( - new TypeNameBytes(publishedInfo.namespace, publishedInfo.typeName), publishedInfo); - } - if (registerType && !(type.isArray() && typeInfo.userTypeId == INVALID_USER_TYPE_ID)) { - if (typeInfo.userTypeId != INVALID_USER_TYPE_ID && typeInfo.userTypeId >= xtypeIdGenerator) { - xtypeIdGenerator = typeInfo.userTypeId + 1; - } - String namespace = publishedInfo.decodeNamespace(); - String typeName = publishedInfo.decodeTypeName(); - extRegistry.registeredClasses.put(qualifiedName(namespace, typeName), type); - registerGraalvmClass(type); - } - return publishedInfo; - } - - @Override - @Internal public void registerInternalSerializer(Class type, Serializer serializer) { - checkRegisterAllowed(); + checkRegistrationOpen(); Class unwrapped = TypeUtils.unwrap(type); if (unwrapped == char.class || unwrapped == void.class @@ -843,16 +751,12 @@ public boolean isMonomorphic(Class clz) { if (clz == UnknownStruct.class) { return false; } - TypeInfo typeInfo = getConstructedTypeInfo(clz); - if (typeInfo == null) { - typeInfo = getTypeInfo(clz, false); - } + TypeInfo typeInfo = getTypeInfo(clz, false); if (typeInfo != null) { if (Types.isEnumType(typeInfo.typeId) || Types.isUnionType(typeInfo.typeId)) { return true; } - Serializer s = - isConstructingSerializer() ? getConstructionSerializer(clz) : typeInfo.serializer; + Serializer s = typeInfo.serializer; if (s instanceof TimeSerializers.TimeSerializer || s instanceof MapLikeSerializer || s instanceof CollectionLikeSerializer @@ -936,10 +840,6 @@ public TypeInfo getUserTypeInfo(int userTypeId) { // buildGenericType methods are inherited from TypeResolver private TypeInfo buildTypeInfo(Class cls) { - TypeInfo constructedTypeInfo = getConstructedTypeInfo(cls); - if (constructedTypeInfo != null && hasConstructedSerializer(cls)) { - return constructedTypeInfo; - } TypeInfo typeInfo = classInfoMap.get(cls); if (typeInfo != null && typeInfo.serializer != null) { return typeInfo; @@ -947,11 +847,7 @@ private TypeInfo buildTypeInfo(Class cls) { if (typeInfo != null) { Class serializerClass = getSerializerClassFromGraalvmRegistry(cls); if (serializerClass != null) { - Serializer serializer = Serializers.newSerializer(this, cls, serializerClass); - if (isConstructingSerializer()) { - return bindConstructedSerializer(cls, serializer); - } - typeInfo.setSerializer(this, serializer); + typeInfo.setSerializer(this, Serializers.newSerializer(this, cls, serializerClass)); return typeInfo; } } @@ -986,7 +882,11 @@ private TypeInfo buildTypeInfo(Class cls) { typeId = Types.MAP; } else if (UnknownClass.class.isAssignableFrom(cls)) { serializer = UnknownClassSerializers.getSerializer(this, "Unknown", cls); - typeId = cls.isEnum() ? Types.ENUM : shareMeta ? Types.COMPATIBLE_STRUCT : Types.STRUCT; + if (cls.isEnum()) { + typeId = Types.ENUM; + } else { + typeId = shareMeta ? Types.COMPATIBLE_STRUCT : Types.STRUCT; + } } else if (cls == Object.class) { // Object.class is handled as unknown type in xlang return getTypeInfo(cls); @@ -994,9 +894,6 @@ private TypeInfo buildTypeInfo(Class cls) { Class enclosingClass = (Class) cls.getEnclosingClass(); if (enclosingClass != null && enclosingClass.isEnum()) { TypeInfo enumInfo = getTypeInfo(enclosingClass); - if (isConstructingSerializer()) { - return enumInfo; - } classInfoMap.put(cls, enumInfo); return enumInfo; } else { @@ -1004,15 +901,8 @@ private TypeInfo buildTypeInfo(Class cls) { } } TypeInfo info = newTypeInfo(cls, serializer, typeId); - if (isConstructingSerializer()) { - TypeInfo constructedInfo = getConstructedTypeInfo(cls); - if (constructedInfo != null) { - return bindConstructedSerializer(cls, serializer); - } - return stageConstructedTypeInfo(cls, info); - } - publishSerializerTypeInfo(info, false, false); - return classInfoMap.get(cls); + classInfoMap.put(cls, info); + return info; } private Serializer getCollectionSerializer(Class cls) { @@ -1352,24 +1242,12 @@ public Serializer getRawSerializer(Class cls) { } @Override - @Internal public void setSerializer(Class cls, Serializer serializer) { - if (isConstructingSerializer()) { - bindConstructedSerializer(cls, serializer); - return; - } getTypeInfo(cls).setSerializer(this, serializer); } @Override - @Internal public void setSerializerIfAbsent(Class cls, Serializer serializer) { - if (isConstructingSerializer()) { - if (!hasConstructedSerializer(cls)) { - bindConstructedSerializer(cls, serializer); - } - return; - } TypeInfo typeInfo = classInfoMap.get(cls); Preconditions.checkNotNull(typeInfo); if (typeInfo.serializer == null) { diff --git a/java/fory-core/src/main/java/org/apache/fory/serializer/AbstractObjectSerializer.java b/java/fory-core/src/main/java/org/apache/fory/serializer/AbstractObjectSerializer.java index 7a3f4e4124..422ccdaeeb 100644 --- a/java/fory-core/src/main/java/org/apache/fory/serializer/AbstractObjectSerializer.java +++ b/java/fory-core/src/main/java/org/apache/fory/serializer/AbstractObjectSerializer.java @@ -931,12 +931,11 @@ private T copyRecord(CopyContext copyContext, T originObj) { fieldValues = RecordUtils.remapping(copyRecordInfo, fieldValues); try { T t = objectInstantiator.newInstanceWithArguments(fieldValues); + Arrays.fill(copyRecordInfo.getRecordComponents(), null); copyContext.reference(originObj, t); return t; } catch (Throwable e) { ExceptionUtils.throwException(e); - } finally { - Arrays.fill(copyRecordInfo.getRecordComponents(), null); } return originObj; } diff --git a/java/fory-core/src/main/java/org/apache/fory/serializer/FieldGroups.java b/java/fory-core/src/main/java/org/apache/fory/serializer/FieldGroups.java index a4291ded61..92e0ec4f6b 100644 --- a/java/fory-core/src/main/java/org/apache/fory/serializer/FieldGroups.java +++ b/java/fory-core/src/main/java/org/apache/fory/serializer/FieldGroups.java @@ -208,24 +208,26 @@ public SerializationFieldInfo(TypeResolver resolver, Descriptor d) { boolean primitiveListCollection = TypeUtils.isPrimitiveListClass(typeRef.getRawType()) && resolver.isCollectionDescriptor(d); - Serializer fieldSerializer; + // invoke `copy` to avoid ObjectSerializer construct clear serializer by `clearSerializer`. if (resolver.isMonomorphic(descriptor)) { - typeInfo = resolver.getFieldTypeInfo(typeRef.getRawType()); - fieldSerializer = resolver.getConstructionSerializer(typeInfo.getType()); + typeInfo = resolver.getTypeInfo(typeRef.getRawType()); if (!resolver.isShareMeta() && !resolver.isCompatible() - && fieldSerializer instanceof ReplaceResolveSerializer) { + && typeInfo.getSerializer() instanceof ReplaceResolveSerializer) { // overwrite replace resolve serializer for final field - fieldSerializer = new FinalFieldReplaceResolveSerializer(resolver, typeInfo.getType()); - resolver.setSerializer(typeInfo.getType(), fieldSerializer); + typeInfo.setSerializer( + new FinalFieldReplaceResolveSerializer(resolver, typeInfo.getType())); } } else { typeInfo = null; - fieldSerializer = null; } useDeclaredTypeInfo = typeInfo != null && resolver.isMonomorphic(descriptor) && !primitiveListCollection; - serializer = fieldSerializer; + if (typeInfo != null) { + serializer = typeInfo.getSerializer(); + } else { + serializer = null; + } this.qualifiedFieldName = d.getDeclaringClass() + "." + d.getName(); if (d.getField() != null) { @@ -299,7 +301,7 @@ public SerializationFieldInfo(TypeResolver resolver, Descriptor d) { } else { if (!primitiveListCollection && (resolver.isMap(cls) || resolver.isCollection(cls) || resolver.isSet(cls))) { - containerTypeInfo = resolver.getFieldTypeInfo(cls); + containerTypeInfo = resolver.getTypeInfo(cls); } else { containerTypeInfo = null; } diff --git a/java/fory-core/src/main/java/org/apache/fory/serializer/ObjectSerializer.java b/java/fory-core/src/main/java/org/apache/fory/serializer/ObjectSerializer.java index 4d21a4d270..333366172d 100644 --- a/java/fory-core/src/main/java/org/apache/fory/serializer/ObjectSerializer.java +++ b/java/fory-core/src/main/java/org/apache/fory/serializer/ObjectSerializer.java @@ -85,8 +85,9 @@ public ObjectSerializer( trackingRef = config.trackingRef(); checkClassVersion = typeResolver.checkClassVersion(); if (resolveParent) { - // Recursive field construction must see this serializer before its fields are resolved. The - // resolver keeps this binding construction-local during combined registration. + // avoid recursive building serializers. + // Use `setSerializerIfAbsent` to avoid overwriting existing serializer for class when used + // as data serializer. typeResolver.setSerializerIfAbsent(cls, this); } Collection descriptors; diff --git a/java/fory-core/src/main/java/org/apache/fory/serializer/ReplaceResolveSerializer.java b/java/fory-core/src/main/java/org/apache/fory/serializer/ReplaceResolveSerializer.java index 6ce2dd2eca..c07ae31326 100644 --- a/java/fory-core/src/main/java/org/apache/fory/serializer/ReplaceResolveSerializer.java +++ b/java/fory-core/src/main/java/org/apache/fory/serializer/ReplaceResolveSerializer.java @@ -293,7 +293,7 @@ private static Class dataSerializerClass( private static Serializer createDataSerializer( TypeResolver typeResolver, Class cls, Class sc) { ClassResolver classResolver = (ClassResolver) typeResolver; - Serializer prev = classResolver.getConstructionSerializer(cls); + Serializer prev = classResolver.getSerializer(cls, false); Serializer serializer = Serializers.newSerializer(typeResolver, cls, sc); classResolver.resetSerializer(cls, prev); return serializer; diff --git a/java/fory-core/src/main/java/org/apache/fory/serializer/Serializers.java b/java/fory-core/src/main/java/org/apache/fory/serializer/Serializers.java index 9b57e51bf5..9612d09754 100644 --- a/java/fory-core/src/main/java/org/apache/fory/serializer/Serializers.java +++ b/java/fory-core/src/main/java/org/apache/fory/serializer/Serializers.java @@ -55,6 +55,7 @@ import org.apache.fory.platform.internal._JDKAccess; import org.apache.fory.reflect.ReflectionUtils; import org.apache.fory.resolver.ClassResolver; +import org.apache.fory.resolver.TypeInfo; import org.apache.fory.resolver.TypeResolver; import org.apache.fory.serializer.CodegenSerializer.LazyInitBeanSerializer; import org.apache.fory.serializer.collection.ChildContainerSerializers; @@ -107,7 +108,8 @@ public static Serializer newSerializer( */ public static Serializer newSerializer( TypeResolver typeResolver, Class type, Class serializerClass) { - Serializer serializer = typeResolver.getConstructionSerializer(type); + TypeInfo typeInfo = typeResolver.getTypeInfo(type, false); + Serializer serializer = typeInfo == null ? null : typeInfo.getSerializer(); try { return buildSerializer(typeResolver, type, serializerClass); } catch (Throwable t) { diff --git a/java/fory-core/src/main/java/org/apache/fory/serializer/StaticGeneratedStructSerializer.java b/java/fory-core/src/main/java/org/apache/fory/serializer/StaticGeneratedStructSerializer.java index e1ed8fc2f1..1f599f3951 100644 --- a/java/fory-core/src/main/java/org/apache/fory/serializer/StaticGeneratedStructSerializer.java +++ b/java/fory-core/src/main/java/org/apache/fory/serializer/StaticGeneratedStructSerializer.java @@ -93,14 +93,13 @@ public StaticGeneratedStructSerializer( } private void setSerializerIfAbsent(TypeResolver typeResolver, Class type) { - TypeInfo typeInfo = typeResolver.getConstructionTypeInfo(type); - Serializer serializer = typeResolver.getConstructionSerializer(type); + TypeInfo typeInfo = typeResolver.getTypeInfo(type, false); if (!typeResolver.isCrossLanguage() || typeInfo != null) { // Field-group construction resolves monomorphic field serializers. A generated serializer can // therefore encounter its own type before the subclass constructor has finished, just like - // ObjectSerializer. The resolver routes combined registration to its construction owner so - // recursive fields never observe an incomplete serializer in the runtime registry. - if (typeInfo != null && serializer instanceof DeferedLazySerializer) { + // ObjectSerializer. Install this instance early so recursive fields reuse it instead of + // constructing another serializer for the same type. + if (typeInfo != null && typeInfo.getSerializer() instanceof DeferedLazySerializer) { typeResolver.setSerializer(type, this); } else { typeResolver.setSerializerIfAbsent(type, this); diff --git a/java/fory-core/src/test/java/org/apache/fory/ForyCopyTest.java b/java/fory-core/src/test/java/org/apache/fory/ForyCopyTest.java index bd4cd7d418..525d63826b 100644 --- a/java/fory-core/src/test/java/org/apache/fory/ForyCopyTest.java +++ b/java/fory-core/src/test/java/org/apache/fory/ForyCopyTest.java @@ -191,15 +191,6 @@ public void threadpoolCopyTest() throws InterruptedException { Assert.assertFalse(flag.get()); } - @Test - public void testCopyFinalizesRegistrationPhase() { - Fory fory = - builder().withCodegen(false).withRefCopy(true).requireClassRegistration(true).build(); - fory.register(BeanA.class); - assertEquals(fory.copy(BeanA.createBeanA(2)), BeanA.createBeanA(2)); - Assert.assertThrows(ForyException.class, () -> fory.register(BeanB.class)); - } - @Test public void testCopyOnlySerializerStillRejectsSerialize() { Fory fory = diff --git a/java/fory-core/src/test/java/org/apache/fory/ForyTest.java b/java/fory-core/src/test/java/org/apache/fory/ForyTest.java index 462c37d32f..4b4db16174 100644 --- a/java/fory-core/src/test/java/org/apache/fory/ForyTest.java +++ b/java/fory-core/src/test/java/org/apache/fory/ForyTest.java @@ -128,10 +128,6 @@ public void testRegistrationFreezesOnUse() { Fory reader = newNativeFory(); reader.deserialize(bytes); assertRegistrationFrozen(reader); - - Fory copier = newNativeFory(); - copier.copy(1); - assertRegistrationFrozen(copier); } private static Fory newNativeFory() { diff --git a/java/fory-core/src/test/java/org/apache/fory/ThreadSafeForyTest.java b/java/fory-core/src/test/java/org/apache/fory/ThreadSafeForyTest.java index 7a2291fb64..39584eef84 100644 --- a/java/fory-core/src/test/java/org/apache/fory/ThreadSafeForyTest.java +++ b/java/fory-core/src/test/java/org/apache/fory/ThreadSafeForyTest.java @@ -26,19 +26,15 @@ import static org.testng.Assert.assertTrue; import java.nio.ByteBuffer; -import java.util.Map; import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.CountDownLatch; -import java.util.concurrent.ExecutionException; import java.util.concurrent.ExecutorService; import java.util.concurrent.Executors; -import java.util.concurrent.Future; import java.util.concurrent.TimeUnit; -import java.util.concurrent.TimeoutException; import java.util.concurrent.atomic.AtomicInteger; import java.util.concurrent.atomic.AtomicReference; -import java.util.concurrent.atomic.AtomicReferenceArray; import lombok.Data; +import org.apache.fory.context.CopyContext; import org.apache.fory.context.MetaReadContext; import org.apache.fory.context.MetaWriteContext; import org.apache.fory.context.ReadContext; @@ -46,13 +42,11 @@ import org.apache.fory.exception.ForyException; import org.apache.fory.memory.MemoryBuffer; import org.apache.fory.pool.ThreadPoolFory; -import org.apache.fory.resolver.ClassResolver; import org.apache.fory.resolver.SharedRegistry; import org.apache.fory.resolver.TypeResolver; import org.apache.fory.serializer.Serializer; import org.apache.fory.test.bean.BeanA; import org.apache.fory.test.bean.BeanB; -import org.apache.fory.util.ExceptionUtils; import org.testng.Assert; import org.testng.annotations.Test; @@ -155,6 +149,7 @@ public void testThreadSafeRuntimesShareRegistry() throws Exception { AtomicReference threadPoolRegistry1 = new AtomicReference<>(); AtomicReference threadPoolRegistry2 = new AtomicReference<>(); AtomicReference error = new AtomicReference<>(); + threadPool.serialize("warm"); Thread poolThread1 = new Thread( () -> { @@ -527,6 +522,37 @@ public Foo read(ReadContext readContext) { } } + private static final class BlockingCopyValue {} + + private static final class BlockingCopySerializer extends Serializer { + private final CountDownLatch copyStarted; + private final CountDownLatch finishCopy; + + private BlockingCopySerializer( + TypeResolver resolver, CountDownLatch copyStarted, CountDownLatch finishCopy) { + super(resolver.getConfig(), BlockingCopyValue.class); + this.copyStarted = copyStarted; + this.finishCopy = finishCopy; + } + + @Override + public void write(WriteContext writeContext, BlockingCopyValue value) { + throw new UnsupportedOperationException("unused"); + } + + @Override + public BlockingCopyValue read(ReadContext readContext) { + throw new UnsupportedOperationException("unused"); + } + + @Override + public BlockingCopyValue copy(CopyContext copyContext, BlockingCopyValue value) { + copyStarted.countDown(); + awaitUnchecked(finishCopy); + return new BlockingCopyValue(); + } + } + public static class CustomClassLoader extends ClassLoader { public CustomClassLoader(ClassLoader parent) { super(parent); @@ -626,555 +652,203 @@ public void testPoolRegisterAfterSerializeThrows() { } @Test - public void testExecuteFreezesThreadLocal() throws Exception { - ThreadSafeFory fory = - Fory.builder() - .withXlang(false) - .requireClassRegistration(true) - .withCompatible(false) - .buildThreadLocalFory(); - fory.register(BeanA.class); - - Fory escaped = fory.execute(value -> value); - Assert.assertThrows(ForyException.class, () -> escaped.register(BeanB.class)); - assertNull(((ClassResolver) escaped.getTypeResolver()).getRegisteredClassId(BeanB.class)); - - ExecutorService executor = Executors.newSingleThreadExecutor(); - try { - Fory otherThreadFory = - executor.submit(() -> fory.execute(value -> value)).get(10, TimeUnit.SECONDS); - ClassResolver otherResolver = (ClassResolver) otherThreadFory.getTypeResolver(); - assertNotNull(otherResolver.getRegisteredClassId(BeanA.class)); - assertTrue(otherResolver.isRegistrationFinished()); - Assert.assertThrows(ForyException.class, () -> otherThreadFory.register(BeanB.class)); - assertNull(otherResolver.getRegisteredClassId(BeanB.class)); - } finally { - executor.shutdownNow(); + public void testFailedRootFreezesRegistration() { + for (ThreadSafeFory fory : newThreadSafeRuntimes()) { + Assert.assertThrows(RuntimeException.class, () -> fory.deserialize(new byte[0])); + Assert.assertThrows(ForyException.class, () -> fory.register(BeanB.class)); } } @Test - public void testRegistrationGateLinearization() throws Exception { - FacadeRegistrationGate gate = new FacadeRegistrationGate(() -> {}); - CountDownLatch registrationEntered = new CountDownLatch(1); - CountDownLatch freezeEntered = new CountDownLatch(1); - CountDownLatch releaseRegistration = new CountDownLatch(1); - ExecutorService executor = Executors.newFixedThreadPool(2); - try { - Future registration = - executor.submit( - () -> - gate.applyRegistration( - () -> { - registrationEntered.countDown(); - awaitUnchecked(releaseRegistration); - })); - assertTrue(registrationEntered.await(10, TimeUnit.SECONDS)); - Future freeze = - executor.submit( - () -> { - freezeEntered.countDown(); - gate.freeze(); - }); - assertTrue(freezeEntered.await(10, TimeUnit.SECONDS)); - Assert.assertThrows(TimeoutException.class, () -> freeze.get(100, TimeUnit.MILLISECONDS)); + public void testExecuteRootFreezesFacade() { + for (ThreadSafeFory fory : newThreadSafeRuntimes()) { + fory.execute(child -> child.serialize("value")); + AtomicInteger callbacks = new AtomicInteger(); - releaseRegistration.countDown(); - registration.get(10, TimeUnit.SECONDS); - freeze.get(10, TimeUnit.SECONDS); Assert.assertThrows( - ForyException.class, () -> gate.applyRegistration(() -> Assert.fail("must not run"))); - } finally { - releaseRegistration.countDown(); - executor.shutdownNow(); + ForyException.class, () -> fory.registerCallback(child -> callbacks.incrementAndGet())); + assertEquals(callbacks.get(), 0); } } @Test - public void testFreezeWaitsForChildPublish() throws Exception { - AtomicReference published = new AtomicReference<>(); - AtomicInteger finishSawChild = new AtomicInteger(); - FacadeRegistrationGate gate = - new FacadeRegistrationGate( - () -> { - Fory child = published.get(); - assertNotNull(child); - finishSawChild.incrementAndGet(); - child.getTypeResolver().finishRegistration(); - }); - Fory child = - Fory.builder() - .withXlang(false) - .requireClassRegistration(true) - .withCompatible(false) - .build(); - CountDownLatch publishEntered = new CountDownLatch(1); - CountDownLatch releasePublish = new CountDownLatch(1); - CountDownLatch freezeStarted = new CountDownLatch(1); - ExecutorService executor = Executors.newFixedThreadPool(2); - try { - Future initialization = - executor.submit( - () -> - gate.initializeChild( - () -> child, - value -> { - publishEntered.countDown(); - awaitUnchecked(releasePublish); - published.set(value); - })); - assertTrue(publishEntered.await(10, TimeUnit.SECONDS)); - Future freeze = - executor.submit( + public void testExecuteRootRegistrationRace() throws InterruptedException { + for (ThreadSafeFory fory : newThreadSafeRuntimes()) { + CountDownLatch rootStarted = new CountDownLatch(1); + CountDownLatch registrationStarted = new CountDownLatch(1); + CountDownLatch finishRoot = new CountDownLatch(1); + AtomicInteger callbacks = new AtomicInteger(); + AtomicReference rootError = new AtomicReference<>(); + AtomicReference registrationError = new AtomicReference<>(); + Thread rootThread = + new Thread( () -> { - freezeStarted.countDown(); - gate.freeze(); + try { + fory.execute( + child -> { + child.serialize("value"); + rootStarted.countDown(); + awaitUnchecked(finishRoot); + return null; + }); + } catch (Throwable t) { + rootError.set(t); + } }); - assertTrue(freezeStarted.await(10, TimeUnit.SECONDS)); - Assert.assertThrows(TimeoutException.class, () -> freeze.get(100, TimeUnit.MILLISECONDS)); - - releasePublish.countDown(); - assertSame(initialization.get(10, TimeUnit.SECONDS), child); - freeze.get(10, TimeUnit.SECONDS); - assertSame(published.get(), child); - assertEquals(finishSawChild.get(), 1); - assertTrue(child.getTypeResolver().isRegistrationFinished()); - } finally { - releasePublish.countDown(); - executor.shutdownNow(); - executor.awaitTermination(10, TimeUnit.SECONDS); - } - } - - @Test - public void testFreezeWaitsForChildren() throws Exception { - CountDownLatch finishEntered = new CountDownLatch(1); - CountDownLatch releaseFinish = new CountDownLatch(1); - FacadeRegistrationGate gate = - new FacadeRegistrationGate( - () -> { - finishEntered.countDown(); - awaitUnchecked(releaseFinish); - }); - ExecutorService executor = Executors.newFixedThreadPool(2); - try { - Future first = executor.submit(gate::freeze); - assertTrue(finishEntered.await(10, TimeUnit.SECONDS)); - CountDownLatch secondStarted = new CountDownLatch(1); - Future second = - executor.submit( + Thread registrationThread = + new Thread( () -> { - secondStarted.countDown(); - gate.freeze(); + try { + registrationStarted.countDown(); + fory.registerCallback(child -> callbacks.incrementAndGet()); + } catch (Throwable t) { + registrationError.set(t); + } }); - assertTrue(secondStarted.await(10, TimeUnit.SECONDS)); - Assert.assertThrows(TimeoutException.class, () -> second.get(100, TimeUnit.MILLISECONDS)); - - releaseFinish.countDown(); - first.get(10, TimeUnit.SECONDS); - second.get(10, TimeUnit.SECONDS); - } finally { - releaseFinish.countDown(); - executor.shutdownNow(); - } - } - - @Test - public void testFailedFreezeStaysClosed() { - AtomicInteger finishCalls = new AtomicInteger(); - FacadeRegistrationGate gate = - new FacadeRegistrationGate( - () -> { - finishCalls.incrementAndGet(); - throw new IllegalStateException("failed"); - }); - - Assert.assertThrows(IllegalStateException.class, gate::freeze); - Assert.assertThrows(ForyException.class, gate::freeze); - Assert.assertThrows(ForyException.class, () -> gate.applyRegistration(() -> {})); - assertEquals(finishCalls.get(), 1); - } - - @Test - public void testCheckedFailureStaysClosed() { - FacadeRegistrationGate gate = new FacadeRegistrationGate(() -> {}); - - Assert.assertThrows( - Exception.class, - () -> - gate.applyRegistration( - () -> { - throw ExceptionUtils.throwException(new Exception("failed")); - })); - Assert.assertThrows(ForyException.class, gate::freeze); - Assert.assertThrows(ForyException.class, () -> gate.applyRegistration(() -> {})); - } - - @Test - public void testRejectedCallbackNotReplayed() throws Exception { - ThreadLocalFory facade = - Fory.builder() - .withXlang(false) - .requireClassRegistration(true) - .withCompatible(false) - .buildThreadLocalFory(); - AtomicInteger callbackCalls = new AtomicInteger(); - Assert.assertThrows( - ForyException.class, - () -> - facade.registerCallback( - child -> { - callbackCalls.incrementAndGet(); - facade.serialize("freeze"); - })); - assertEquals(callbackCalls.get(), 1); - Assert.assertThrows(ForyException.class, () -> facade.serialize("closed")); - assertEquals(callbackCalls.get(), 1); - } - - @Test - public void testNestedRegistrationCloses() throws Exception { - ThreadLocalFory facade = - Fory.builder() - .withXlang(false) - .requireClassRegistration(true) - .withCompatible(false) - .buildThreadLocalFory(); - threadLocalChildren(facade); - AtomicInteger callbackCalls = new AtomicInteger(); - - Assert.assertThrows( - ForyException.class, - () -> - facade.registerCallback( - child -> { - callbackCalls.incrementAndGet(); - try { - facade.register(BeanB.class); - } catch (ForyException ignored) { - // The outer callback must still observe the failed gate before publication. - } - child.register(BeanA.class); - })); - - assertTrue(callbackCalls.get() > 0); - Assert.assertThrows(ForyException.class, () -> facade.serialize("closed")); - Assert.assertThrows(ForyException.class, () -> facade.register(BeanA.class)); - } - - @Test - public void testReentrantRegistrationFreeze() throws Exception { - ThreadLocalFory threadLocal = - Fory.builder() - .withXlang(false) - .requireClassRegistration(true) - .withCompatible(false) - .buildThreadLocalFory(); - assertReentrantRegistrationRejected(threadLocal, threadLocalChildren(threadLocal)); - - ThreadPoolFory threadPool = - (ThreadPoolFory) - Fory.builder() - .withXlang(false) - .requireClassRegistration(true) - .withCompatible(false) - .buildThreadSafeForyPool(2); - Fory[] pooledFory = TestUtils.getFieldValue(threadPool, "pooledFory"); - assertReentrantRegistrationRejected(threadPool, pooledFory); - } - - private static void assertReentrantRegistrationRejected(ThreadSafeFory facade, Fory[] children) { - AtomicInteger creatorCalls = new AtomicInteger(); - Assert.assertThrows( - ForyException.class, - () -> - facade.registerSerializerAndType( - Foo.class, - resolver -> { - creatorCalls.incrementAndGet(); - facade.serialize("freeze"); - return new FooSerializer(resolver, Foo.class); - })); - Assert.assertEquals(creatorCalls.get(), 1); - for (Fory child : children) { - TypeResolver resolver = child.getTypeResolver(); - assertNull(((ClassResolver) resolver).getRegisteredClassId(Foo.class)); - } - Assert.assertThrows(ForyException.class, () -> facade.serialize("closed")); - } - - private static Fory[] threadLocalChildren(ThreadLocalFory facade) throws Exception { - ThreadLocal local = TestUtils.getFieldValue(facade, "foryThreadLocal"); - Fory first = local.get(); - ExecutorService executor = Executors.newSingleThreadExecutor(); - try { - Fory second = executor.submit(local::get).get(10, TimeUnit.SECONDS); - return new Fory[] {first, second}; - } finally { - executor.shutdownNow(); - } - } - @Test - public void testBuilderModuleForLateChild() throws Exception { - AtomicInteger installs = new AtomicInteger(); - ForyModule module = - child -> { - installs.incrementAndGet(); - child.registerSerializerAndType(Foo.class, FooSerializer.class); - }; - ThreadLocalFory facade = - Fory.builder() - .withXlang(false) - .requireClassRegistration(true) - .withModule(module) - .withCompatible(false) - .buildThreadLocalFory(); - assertEquals(installs.get(), 1); - facade.serialize("freeze"); - - ExecutorService executor = Executors.newSingleThreadExecutor(); - try { - Foo value = new Foo(); - value.f1 = 42; - Foo result = - executor - .submit(() -> facade.deserialize(facade.serialize(value), Foo.class)) - .get(10, TimeUnit.SECONDS); - assertEquals(result, value); - assertEquals(installs.get(), 2); - } finally { - executor.shutdownNow(); - } - } - - @Test - public void testLateChildReplaysRegistration() throws Exception { - ThreadLocalFory facade = - Fory.builder() - .withXlang(false) - .requireClassRegistration(true) - .withCompatible(false) - .buildThreadLocalFory(); - facade.registerSerializerAndType(Foo.class, FooSerializer.class); - facade.serialize("freeze"); - - ExecutorService executor = Executors.newSingleThreadExecutor(); - try { - Foo value = new Foo(); - value.f1 = 42; - Foo result = - executor - .submit(() -> facade.deserialize(facade.serialize(value), Foo.class)) - .get(10, TimeUnit.SECONDS); - assertEquals(result, value); - Class serializerType = - executor - .submit( - () -> - facade.execute( - child -> child.getTypeResolver().getSerializer(Foo.class).getClass())) - .get(10, TimeUnit.SECONDS); - assertSame(serializerType, FooSerializer.class); - } finally { - executor.shutdownNow(); + rootThread.start(); + assertTrue(rootStarted.await(30, TimeUnit.SECONDS)); + registrationThread.start(); + assertTrue(registrationStarted.await(30, TimeUnit.SECONDS)); + finishRoot.countDown(); + rootThread.join(); + registrationThread.join(); + + assertNull(rootError.get()); + assertTrue(registrationError.get() instanceof ForyException); + assertEquals(callbacks.get(), 0); } } @Test - public void testReentrantReplayCleanup() throws Exception { - ThreadLocalFory facade = - Fory.builder() - .withXlang(false) - .requireClassRegistration(true) - .withCompatible(false) - .buildThreadLocalFory(); - Map children = TestUtils.getFieldValue(facade, "allFory"); - AtomicInteger callbackCalls = new AtomicInteger(); - AtomicInteger replayPublished = new AtomicInteger(-1); - facade.registerCallback( - child -> { - if (callbackCalls.incrementAndGet() > 1) { - replayPublished.set(children.containsKey(child) ? 1 : 0); - facade.serialize("nested"); - } - child.register(BeanA.class); - }); - facade.serialize("freeze"); + public void testExecuteChildRegisterRace() throws InterruptedException { + for (ThreadSafeFory fory : newThreadSafeRuntimes()) { + CountDownLatch rootStarted = new CountDownLatch(1); + CountDownLatch finishRoot = new CountDownLatch(1); + AtomicReference rootError = new AtomicReference<>(); + AtomicReference registrationError = new AtomicReference<>(); + Thread rootThread = + new Thread( + () -> { + try { + fory.execute( + child -> { + child.serialize("value"); + rootStarted.countDown(); + awaitUnchecked(finishRoot); + return null; + }); + } catch (Throwable t) { + rootError.set(t); + } + }); + Thread registrationThread = + new Thread( + () -> { + try { + fory.execute( + child -> { + child.register(BeanB.class); + return null; + }); + } catch (Throwable t) { + registrationError.set(t); + } + }); - ExecutorService executor = Executors.newSingleThreadExecutor(); - try { - Assert.expectThrows( - ExecutionException.class, - () -> executor.submit(() -> facade.execute(child -> child)).get(10, TimeUnit.SECONDS)); - assertEquals(children.size(), 1); - assertEquals(callbackCalls.get(), 2); - assertEquals(replayPublished.get(), 0); + rootThread.start(); + assertTrue(rootStarted.await(30, TimeUnit.SECONDS)); + registrationThread.start(); + finishRoot.countDown(); + rootThread.join(); + registrationThread.join(); - Assert.assertThrows(ForyException.class, () -> facade.serialize("closed")); - assertEquals(callbackCalls.get(), 2); - } finally { - executor.shutdownNow(); + assertNull(rootError.get()); + assertTrue(registrationError.get() instanceof ForyException); } } @Test - public void testLateChildFailureRace() throws Exception { - ThreadLocalFory facade = - Fory.builder() - .withXlang(false) - .requireClassRegistration(true) - .withCompatible(false) - .buildThreadLocalFory(); - Map children = TestUtils.getFieldValue(facade, "allFory"); - AtomicInteger callbackCalls = new AtomicInteger(); - CountDownLatch replayEntered = new CountDownLatch(1); - CountDownLatch releaseReplay = new CountDownLatch(1); - facade.registerCallback( - child -> { - int call = callbackCalls.incrementAndGet(); - if (call == 2) { - replayEntered.countDown(); - awaitUnchecked(releaseReplay); - throw new IllegalStateException("failed replay"); - } - child.register(BeanA.class); - }); - facade.serialize("freeze"); - - ExecutorService executor = Executors.newFixedThreadPool(2); - AtomicReference waitingThread = new AtomicReference<>(); - CountDownLatch waitingStarted = new CountDownLatch(1); - try { - Future failing = executor.submit(() -> facade.execute(child -> child)); - assertTrue(replayEntered.await(10, TimeUnit.SECONDS)); - Future waiting = - executor.submit( + public void testCopyRegistrationRace() throws InterruptedException { + for (ThreadSafeFory fory : newThreadSafeRuntimes()) { + CountDownLatch copyStarted = new CountDownLatch(1); + CountDownLatch finishCopy = new CountDownLatch(1); + CountDownLatch registrationStarted = new CountDownLatch(1); + CountDownLatch registrationDone = new CountDownLatch(1); + AtomicInteger callbacks = new AtomicInteger(); + AtomicReference copyError = new AtomicReference<>(); + AtomicReference registrationError = new AtomicReference<>(); + fory.registerSerializer( + BlockingCopyValue.class, + resolver -> new BlockingCopySerializer(resolver, copyStarted, finishCopy)); + Thread copyThread = + new Thread( () -> { - waitingThread.set(Thread.currentThread()); - waitingStarted.countDown(); - return facade.execute(child -> child); + try { + fory.copy(new BlockingCopyValue()); + } catch (Throwable t) { + copyError.set(t); + } }); - assertTrue(waitingStarted.await(10, TimeUnit.SECONDS)); - awaitBlocked(waitingThread.get()); - - releaseReplay.countDown(); - ExecutionException replayFailure = - Assert.expectThrows(ExecutionException.class, () -> failing.get(10, TimeUnit.SECONDS)); - assertTrue(replayFailure.getCause() instanceof IllegalStateException); - ExecutionException waitingFailure = - Assert.expectThrows(ExecutionException.class, () -> waiting.get(10, TimeUnit.SECONDS)); - assertTrue(waitingFailure.getCause() instanceof ForyException); - assertEquals(callbackCalls.get(), 2); - assertEquals(children.size(), 1); - } finally { - releaseReplay.countDown(); - executor.shutdownNow(); - executor.awaitTermination(10, TimeUnit.SECONDS); - } - } - - @Test - public void testPoolGatePrecedesBorrow() throws Exception { - ThreadPoolFory facade = - (ThreadPoolFory) - Fory.builder() - .withXlang(false) - .requireClassRegistration(true) - .withCompatible(false) - .buildThreadSafeForyPool(1); - AtomicReferenceArray slots = TestUtils.getFieldValue(facade, "slots"); - CountDownLatch callbackEntered = new CountDownLatch(1); - CountDownLatch allowReentrantRoot = new CountDownLatch(1); - CountDownLatch rootStarted = new CountDownLatch(1); - AtomicReference rootThread = new AtomicReference<>(); - ExecutorService executor = Executors.newFixedThreadPool(2); - try { - Future registration = - executor.submit( - () -> - facade.registerCallback( - child -> { - callbackEntered.countDown(); - awaitUnchecked(allowReentrantRoot); - facade.execute(value -> null); - })); - assertTrue(callbackEntered.await(10, TimeUnit.SECONDS)); - Future root = - executor.submit( + Thread registrationThread = + new Thread( () -> { - rootThread.set(Thread.currentThread()); - rootStarted.countDown(); - return facade.execute(value -> null); + registrationStarted.countDown(); + try { + fory.registerCallback(child -> callbacks.incrementAndGet()); + } catch (Throwable t) { + registrationError.set(t); + } finally { + registrationDone.countDown(); + } }); - assertTrue(rootStarted.await(10, TimeUnit.SECONDS)); - awaitBlocked(rootThread.get()); - assertNotNull(slots.get(0)); - allowReentrantRoot.countDown(); - ExecutionException registrationFailure = - Assert.expectThrows( - ExecutionException.class, () -> registration.get(10, TimeUnit.SECONDS)); - assertTrue(registrationFailure.getCause() instanceof ForyException); - ExecutionException rootFailure = - Assert.expectThrows(ExecutionException.class, () -> root.get(10, TimeUnit.SECONDS)); - assertTrue(rootFailure.getCause() instanceof ForyException); - } finally { - allowReentrantRoot.countDown(); - executor.shutdownNow(); - executor.awaitTermination(10, TimeUnit.SECONDS); + copyThread.start(); + assertTrue(copyStarted.await(30, TimeUnit.SECONDS)); + registrationThread.start(); + assertTrue(registrationStarted.await(30, TimeUnit.SECONDS)); + Assert.assertFalse(registrationDone.await(100, TimeUnit.MILLISECONDS)); + finishCopy.countDown(); + copyThread.join(); + registrationThread.join(); + + assertNull(copyError.get()); + assertNull(registrationError.get()); + assertTrue(callbacks.get() > 0); } } - private static void awaitBlocked(Thread thread) { - long deadline = System.nanoTime() + TimeUnit.SECONDS.toNanos(10); - while (thread.getState() != Thread.State.BLOCKED && System.nanoTime() < deadline) { - Thread.yield(); - } - Assert.assertEquals(thread.getState(), Thread.State.BLOCKED); - } - @Test - public void testExecuteFreezesPool() { - ThreadPoolFory fory = - (ThreadPoolFory) - Fory.builder() - .withXlang(false) - .requireClassRegistration(true) - .withCompatible(false) - .buildThreadSafeForyPool(2); - fory.register(BeanA.class); - - Fory escaped = fory.execute(value -> value); - Assert.assertThrows(ForyException.class, () -> escaped.register(BeanB.class)); + public void testNonRootKeepsRegistrationOpen() { + Fory direct = + Fory.builder() + .withXlang(false) + .requireClassRegistration(true) + .withCompatible(false) + .build(); + direct.copy("value"); + direct.register(BeanA.class); - Fory[] pooledFory = TestUtils.getFieldValue(fory, "pooledFory"); - for (Fory child : pooledFory) { - ClassResolver resolver = (ClassResolver) child.getTypeResolver(); - assertNotNull(resolver.getRegisteredClassId(BeanA.class)); - assertNull(resolver.getRegisteredClassId(BeanB.class)); + for (ThreadSafeFory fory : newThreadSafeRuntimes()) { + fory.copy("value"); + fory.execute(child -> child.getConfig()); + fory.register(BeanA.class); } } - @Test - public void testFailedRootFreezesFacade() { - ThreadSafeFory[] runtimes = - new ThreadSafeFory[] { - Fory.builder() - .withXlang(false) - .requireClassRegistration(true) - .withCompatible(false) - .buildThreadLocalFory(), - Fory.builder() - .withXlang(false) - .requireClassRegistration(true) - .withCompatible(false) - .buildThreadSafeForyPool(2) - }; - for (ThreadSafeFory fory : runtimes) { - Assert.assertThrows(RuntimeException.class, () -> fory.deserialize(new byte[0])); - Assert.assertThrows(ForyException.class, () -> fory.register(BeanB.class)); - } + private static ThreadSafeFory[] newThreadSafeRuntimes() { + return new ThreadSafeFory[] { + Fory.builder() + .withXlang(false) + .requireClassRegistration(true) + .withCompatible(false) + .buildThreadLocalFory(), + Fory.builder() + .withXlang(false) + .requireClassRegistration(true) + .withCompatible(false) + .buildThreadSafeForyPool(2) + }; } private void assertConcurrentRoundTrip(ThreadSafeFory fory, BeanA beanA) diff --git a/java/fory-core/src/test/java/org/apache/fory/builder/StaticCompatibleCodecBuilderTest.java b/java/fory-core/src/test/java/org/apache/fory/builder/StaticCompatibleCodecBuilderTest.java index 97bfa3c7b0..b22d0cc2b9 100644 --- a/java/fory-core/src/test/java/org/apache/fory/builder/StaticCompatibleCodecBuilderTest.java +++ b/java/fory-core/src/test/java/org/apache/fory/builder/StaticCompatibleCodecBuilderTest.java @@ -47,7 +47,6 @@ import org.apache.fory.platform.GraalvmSupport; import org.apache.fory.reflect.TypeRef; import org.apache.fory.resolver.TypeResolver; -import org.apache.fory.serializer.AbstractObjectSerializer; import org.apache.fory.serializer.CompatibleSerializer; import org.apache.fory.serializer.FieldGroups.FieldCodecCategory; import org.apache.fory.serializer.ObjectSerializer; @@ -433,48 +432,6 @@ public void testRecordFailureClearsArgs() throws Exception { } } - @Test - public void testRecordCopyClearsArgs() throws Exception { - assumeRecordSupport(); - CompilationResult result = - compile( - "test.CopyFailingRecord", - "package test;\n" - + "public record CopyFailingRecord(String value) {\n" - + " public static boolean fail;\n" - + " public CopyFailingRecord {\n" - + " if (fail) throw new IllegalStateException(\"expected\");\n" - + " }\n" - + "}\n"); - Assert.assertTrue(result.success, result.diagnostics()); - try (URLClassLoader loader = result.classLoader()) { - Class type = loader.loadClass("test.CopyFailingRecord"); - Fory fory = - Fory.builder() - .withClassLoader(loader) - .withXlang(false) - .withRefTracking(true) - .withRefCopy(true) - .withCodegen(false) - .requireClassRegistration(false) - .build(); - Object value = type.getConstructor(String.class).newInstance("retained-value"); - Serializer serializer = fory.getTypeResolver().getSerializer(type); - Assert.assertTrue(serializer instanceof ObjectSerializer); - - setField(type, null, "fail", true); - Assert.assertThrows(RuntimeException.class, () -> fory.copy(value)); - - Field copyRecordInfoField = AbstractObjectSerializer.class.getDeclaredField("copyRecordInfo"); - copyRecordInfoField.setAccessible(true); - RecordInfo recordInfo = (RecordInfo) copyRecordInfoField.get(serializer); - Assert.assertEquals(recordInfo.getRecordComponents(), new Object[] {null}); - - setField(type, null, "fail", false); - Assert.assertEquals(invoke(type, fory.copy(value), "value"), "retained-value"); - } - } - @Test public void testCompatibleRecordSerializerConvertsRemoteField() throws Exception { assumeRecordSupport(); diff --git a/java/fory-core/src/test/java/org/apache/fory/resolver/ClassResolverTest.java b/java/fory-core/src/test/java/org/apache/fory/resolver/ClassResolverTest.java index c626dbd356..748b79d7b3 100644 --- a/java/fory-core/src/test/java/org/apache/fory/resolver/ClassResolverTest.java +++ b/java/fory-core/src/test/java/org/apache/fory/resolver/ClassResolverTest.java @@ -851,8 +851,8 @@ public void testSharedRegistryCachesFieldDescriptorsAndDescriptorGrouper() { ClassResolver resolver1 = (ClassResolver) fory1.getTypeResolver(); ClassResolver resolver2 = (ClassResolver) fory2.getTypeResolver(); - resolver1.finishRegistration(); - resolver2.finishRegistration(); + resolver1.freezeRegistration(); + resolver2.freezeRegistration(); List descriptors1 = resolver1.getFieldDescriptors(BeanB.class, true); List descriptors2 = resolver2.getFieldDescriptors(BeanB.class, true); @@ -887,8 +887,8 @@ public void testSharedRegistryCachesTypeDefDescriptorsAndDescriptorGrouperBySema ClassResolver resolver1 = (ClassResolver) fory1.getTypeResolver(); ClassResolver resolver2 = (ClassResolver) fory2.getTypeResolver(); - resolver1.finishRegistration(); - resolver2.finishRegistration(); + resolver1.freezeRegistration(); + resolver2.freezeRegistration(); TypeDef canonicalTypeDef = resolver1.getTypeDef(BeanB.class, true); MemoryBuffer buffer1 = MemoryBuffer.newHeapBuffer(256); @@ -978,11 +978,11 @@ public void testFinishRegisterPublishesAndAdoptsSharedRegistration() { assertNull(resolver2.getRegisteredClassId(BeanB.class)); assertNull(resolver2.getRegisteredClass("ns.C1")); - resolver1.finishRegistration(); + resolver1.freezeRegistration(); assertEquals(sharedRegistry.registeredClassIdMap.get(BeanB.class), Integer.valueOf(1)); assertEquals(sharedRegistry.registeredClasses.get("ns.C1"), C1.class); - resolver2.finishRegistration(); + resolver2.freezeRegistration(); assertEquals(resolver2.getRegisteredClassId(BeanB.class), Integer.valueOf(1)); assertEquals(resolver2.getRegisteredClass("ns.C1"), C1.class); } @@ -1489,8 +1489,8 @@ public void testShareableSerializerSharedAcrossRuntimes() { resolver2.register(Foo.class, 101); resolver1.registerSerializer(Foo.class, ShareableFooSerializer.class); resolver2.registerSerializer(Foo.class, ShareableFooSerializer.class); - resolver1.finishRegistration(); - resolver2.finishRegistration(); + resolver1.freezeRegistration(); + resolver2.freezeRegistration(); Serializer serializer1 = resolver1.getSerializer(Foo.class); TypeInfo sharedTypeInfo = sharedRegistry.registeredTypeInfoCache.get(Foo.class); diff --git a/java/fory-core/src/test/java/org/apache/fory/serializer/RegisterTest.java b/java/fory-core/src/test/java/org/apache/fory/serializer/RegisterTest.java index 4e7594e01a..b0d8adbc86 100644 --- a/java/fory-core/src/test/java/org/apache/fory/serializer/RegisterTest.java +++ b/java/fory-core/src/test/java/org/apache/fory/serializer/RegisterTest.java @@ -19,35 +19,18 @@ package org.apache.fory.serializer; -import java.io.Externalizable; -import java.io.ObjectInput; -import java.io.ObjectOutput; -import java.util.ArrayList; -import java.util.Arrays; -import java.util.Collections; -import java.util.List; -import java.util.Set; +import java.io.ByteArrayInputStream; import java.util.concurrent.atomic.AtomicBoolean; -import java.util.concurrent.atomic.AtomicInteger; -import java.util.concurrent.atomic.AtomicReference; import org.apache.fory.Fory; import org.apache.fory.ForyModule; import org.apache.fory.ForyTestBase; -import org.apache.fory.TestUtils; -import org.apache.fory.builder.Generated; import org.apache.fory.config.ForyBuilder; import org.apache.fory.context.ReadContext; import org.apache.fory.context.WriteContext; import org.apache.fory.exception.ForyException; -import org.apache.fory.meta.TypeDef; -import org.apache.fory.resolver.ClassResolver; -import org.apache.fory.resolver.SharedRegistry; -import org.apache.fory.resolver.TypeInfo; +import org.apache.fory.io.ForyInputStream; +import org.apache.fory.memory.MemoryBuffer; import org.apache.fory.resolver.TypeResolver; -import org.apache.fory.resolver.XtypeResolver; -import org.apache.fory.type.Descriptor; -import org.apache.fory.type.Types; -import org.apache.fory.util.ExceptionUtils; import org.testng.Assert; import org.testng.annotations.Test; @@ -187,331 +170,6 @@ public static class MyExt { public String id; } - public static class ParentValue { - public int parent; - } - - public static class ChildValue extends ParentValue { - public int child; - } - - public static final class RecursiveValue { - public int value; - public RecursiveValue next; - } - - public static class LeftValue { - public int value; - public RightValue right; - } - - public static class RightValue { - public int value; - public LeftValue left; - } - - public static class CustomList extends ArrayList {} - - @Test - public void testCombinedObjectFields() { - Fory fory = - Fory.builder() - .withXlang(false) - .withCodegen(false) - .requireClassRegistration(true) - .withCompatible(false) - .build(); - fory.registerSerializerAndType(ChildValue.class, ObjectSerializer.class); - ChildValue value = new ChildValue(); - value.parent = 1; - value.child = 2; - - ChildValue result = fory.deserialize(fory.serialize(value), ChildValue.class); - - Assert.assertEquals(result.parent, 1); - Assert.assertEquals(result.child, 2); - } - - @Test - public void testCombinedRecursiveObject() { - Fory fory = newStrictNativeFory(); - fory.registerSerializerAndType(RecursiveValue.class, ObjectSerializer.class); - RecursiveValue value = new RecursiveValue(); - value.value = 1; - value.next = value; - - RecursiveValue result = fory.deserialize(fory.serialize(value), RecursiveValue.class); - - Assert.assertEquals(result.value, 1); - Assert.assertSame(result.next, result); - } - - @Test - public void testCombinedMutualObject() { - Fory fory = newStrictNativeFory(); - fory.register(RightValue.class); - fory.registerSerializerAndType( - LeftValue.class, resolver -> new ObjectSerializer<>(resolver, LeftValue.class)); - LeftValue value = new LeftValue(); - value.value = 1; - value.right = new RightValue(); - value.right.value = 2; - value.right.left = value; - - LeftValue result = fory.deserialize(fory.serialize(value), LeftValue.class); - - Assert.assertEquals(result.value, 1); - Assert.assertEquals(result.right.value, 2); - Assert.assertSame(result.right.left, result); - } - - @Test(dataProvider = "xlang") - public void testCombinedInstanceValidation(boolean xlang) { - Fory fory = - Fory.builder() - .withXlang(xlang) - .withCodegen(false) - .requireClassRegistration(true) - .withCompatible(false) - .build(); - Serializer serializer = new MyExtSerializer(fory.getTypeResolver()); - - Assert.assertThrows( - IllegalArgumentException.class, - () -> fory.registerSerializerAndType(CustomList.class, serializer)); - Assert.assertFalse(fory.getTypeResolver().isRegistered(CustomList.class)); - } - - @Test - public void testSerializerKeepsTypeOwner() { - Fory fory = - Fory.builder() - .withXlang(false) - .withCodegen(false) - .requireClassRegistration(true) - .withCompatible(false) - .build(); - fory.register(RecursiveValue.class); - TypeInfo typeInfo = fory.getTypeResolver().getTypeInfo(RecursiveValue.class, false); - - fory.registerSerializer(RecursiveValue.class, ObjectSerializer.class); - - Assert.assertSame(fory.getTypeResolver().getTypeInfo(RecursiveValue.class, false), typeInfo); - ObjectSerializer serializer = - (ObjectSerializer) fory.getTypeResolver().getRawSerializer(RecursiveValue.class); - FieldGroups.SerializationFieldInfo[] fields = TestUtils.getFieldValue(serializer, "allFields"); - FieldGroups.SerializationFieldInfo nextField = - Arrays.stream(fields) - .filter(field -> field.descriptor.getName().equals("next")) - .findFirst() - .orElseThrow(AssertionError::new); - Assert.assertSame(nextField.typeInfo, typeInfo); - Assert.assertSame(nextField.serializer, serializer); - } - - @Test - public void testSharedSerializerKeepsLocalOwner() { - ForyBuilder builder = - Fory.builder() - .withSharedRegistry(new SharedRegistry()) - .withXlang(false) - .withCodegen(false) - .requireClassRegistration(true) - .withCompatible(false); - Fory first = builder.build(); - Fory second = builder.build(); - first.register(ExternalValue.class, 201); - second.register(ExternalValue.class, 201); - TypeInfo firstInfo = first.getTypeResolver().getTypeInfo(ExternalValue.class, false); - TypeInfo secondInfo = second.getTypeResolver().getTypeInfo(ExternalValue.class, false); - - first.registerSerializer(ExternalValue.class, ShareableExternalSerializer.class); - second.registerSerializer(ExternalValue.class, ShareableExternalSerializer.class); - - Assert.assertSame(first.getTypeResolver().getTypeInfo(ExternalValue.class, false), firstInfo); - Assert.assertSame(second.getTypeResolver().getTypeInfo(ExternalValue.class, false), secondInfo); - Assert.assertNotSame(firstInfo, secondInfo); - Assert.assertSame(firstInfo.getSerializer(), secondInfo.getSerializer()); - } - - @Test - public void testFailedConstructorKeepsOwner() { - Fory fory = - Fory.builder() - .withXlang(false) - .withCodegen(false) - .requireClassRegistration(true) - .withCompatible(false) - .build(); - fory.register(ExternalValue.class, 203); - TypeInfo typeInfo = fory.getTypeResolver().getTypeInfo(ExternalValue.class, false); - Assert.assertNull(typeInfo.getSerializer()); - - Assert.assertThrows( - IllegalStateException.class, - () -> fory.registerSerializer(ExternalValue.class, FailingExternalSerializer.class)); - - Assert.assertSame(fory.getTypeResolver().getTypeInfo(ExternalValue.class, false), typeInfo); - Assert.assertNull(typeInfo.getSerializer()); - } - - @Test(dataProvider = "xlang") - public void testNestedCombinedRegistration(boolean xlang) { - Fory fory = - Fory.builder() - .withXlang(xlang) - .withCodegen(false) - .requireClassRegistration(true) - .withCompatible(false) - .build(); - - Assert.assertThrows( - ForyException.class, - () -> - fory.registerSerializerAndType( - MyExt.class, - resolver -> { - resolver.register(ObjectField.class); - return new MyExtSerializer(resolver); - })); - Assert.assertFalse(fory.getTypeResolver().isRegistered(MyExt.class)); - Assert.assertFalse(fory.getTypeResolver().isRegistered(ObjectField.class)); - } - - @Test(dataProvider = "xlang") - public void testCombinedConstructorFailure(boolean xlang) { - Fory fory = - Fory.builder() - .withXlang(xlang) - .withCodegen(false) - .requireClassRegistration(true) - .withCompatible(false) - .build(); - - Assert.assertThrows( - IllegalStateException.class, - () -> fory.registerSerializerAndType(MyExt.class, FailingSerializer.class)); - Assert.assertFalse(fory.getTypeResolver().isRegistered(MyExt.class)); - Assert.assertNull(fory.getTypeResolver().getTypeInfo(MyExt.class, false)); - } - - @Test - public void testFailedDependencyCache() { - Fory fory = newStrictNativeFory(); - AtomicReference> stagedSerializer = new AtomicReference<>(); - - Assert.assertThrows( - IllegalStateException.class, - () -> - fory.registerSerializerAndType( - MyExt.class, - resolver -> { - stagedSerializer.set(resolver.getSerializer(ObjectField.class)); - throw new IllegalStateException("failed"); - })); - Assert.assertNull(fory.getTypeResolver().getTypeInfo(ObjectField.class, false)); - Assert.assertNotSame( - fory.getTypeResolver().getSerializer(ObjectField.class), stagedSerializer.get()); - } - - @Test - public void testShareConflictIsAtomic() { - SharedRegistry sharedRegistry = new SharedRegistry(); - ForyBuilder builder = - Fory.builder() - .withSharedRegistry(sharedRegistry) - .withXlang(false) - .withCodegen(false) - .requireClassRegistration(false) - .withCompatible(false); - Fory first = builder.build(); - first.registerSerializer(MyExt.class, new FirstShareableSerializer(first.getTypeResolver())); - Serializer sharedSerializer = - first.getTypeResolver().getTypeInfo(MyExt.class, false).getSerializer(); - Fory registered = builder.build(); - registered.registerSerializerAndType(MyExt.class, FirstShareableSerializer.class); - Assert.assertTrue(registered.getTypeResolver().isRegistered(MyExt.class)); - TypeInfo registeredInfo = registered.getTypeResolver().getTypeInfo(MyExt.class, false); - Assert.assertTrue(registeredInfo.getUserTypeId() >= 0); - Assert.assertSame(registeredInfo.getSerializer(), sharedSerializer); - Assert.assertNull(registered.getTypeResolver().getTypeInfo(ObjectField.class, false)); - Fory second = builder.build(); - - Assert.assertThrows( - IllegalArgumentException.class, - () -> second.registerSerializerAndType(MyExt.class, SecondShareableSerializer.class)); - Assert.assertFalse(second.getTypeResolver().isRegistered(MyExt.class)); - Assert.assertNull(second.getTypeResolver().getTypeInfo(MyExt.class, false)); - Assert.assertNull(second.getTypeResolver().getTypeInfo(ObjectField.class, false)); - } - - @Test(dataProvider = "xlang") - public void testReentrantSerializerCreator(boolean xlang) { - Fory fory = - Fory.builder() - .withXlang(xlang) - .withCodegen(false) - .requireClassRegistration(true) - .withCompatible(false) - .build(); - fory.register(MyExt.class); - TypeInfo typeInfo = fory.getTypeResolver().getTypeInfo(MyExt.class, false); - Serializer serializer = typeInfo.getSerializer(); - - Assert.assertThrows( - ForyException.class, - () -> - fory.registerSerializer( - MyExt.class, - resolver -> { - Assert.assertThrows(ForyException.class, () -> fory.serialize("freeze")); - Assert.assertThrows(ForyException.class, () -> fory.serialize("still frozen")); - Assert.assertThrows(ForyException.class, () -> fory.register(runtime -> {})); - return new MyExtSerializer(resolver); - })); - Assert.assertTrue(fory.getTypeResolver().isRegistrationFrozen()); - Assert.assertFalse(fory.getTypeResolver().isRegistrationFinished()); - Assert.assertSame(fory.getTypeResolver().getTypeInfo(MyExt.class, false), typeInfo); - Assert.assertSame(typeInfo.getSerializer(), serializer); - } - - @Test(dataProvider = "xlang") - public void testReentrantSerializerClass(boolean xlang) { - Fory fory = - Fory.builder() - .withXlang(xlang) - .withCodegen(false) - .requireClassRegistration(true) - .withCompatible(false) - .build(); - fory.register(MyExt.class); - TypeInfo typeInfo = fory.getTypeResolver().getTypeInfo(MyExt.class, false); - Serializer serializer = typeInfo.getSerializer(); - ReentrantSerializer.CONSTRUCTION.set(() -> fory.serialize("freeze")); - try { - Assert.assertThrows( - ForyException.class, - () -> fory.registerSerializer(MyExt.class, ReentrantSerializer.class)); - } finally { - ReentrantSerializer.CONSTRUCTION.set(null); - } - - Assert.assertTrue(fory.getTypeResolver().isRegistrationFrozen()); - Assert.assertFalse(fory.getTypeResolver().isRegistrationFinished()); - Assert.assertSame(fory.getTypeResolver().getTypeInfo(MyExt.class, false), typeInfo); - Assert.assertSame(typeInfo.getSerializer(), serializer); - } - - private static Fory newStrictNativeFory() { - return Fory.builder() - .withXlang(false) - .withCodegen(false) - .withRefTracking(true) - .requireClassRegistration(true) - .withCompatible(false) - .build(); - } - @Test public void testFrozenFacadeRegistration() { Fory fory = @@ -542,378 +200,26 @@ public void testFrozenFacadeRegistration() { Assert.assertFalse(creatorCalled.get()); } - @Test(dataProvider = "xlang") - public void testFrozenResolverRegistration(boolean xlang) { - Fory fory = - Fory.builder() - .withXlang(xlang) - .withCodegen(false) - .requireClassRegistration(true) - .withCompatible(false) - .build(); - TypeResolver resolver = fory.getTypeResolver(); - TypeInfo stringTypeInfo = resolver.getTypeInfo(String.class, false); - Serializer stringSerializer = stringTypeInfo.getSerializer(); - fory.serialize("freeze"); - int factoryCount = serializerFactoryCount(resolver); - - Assert.assertThrows( - ForyException.class, () -> resolver.registerSerializerFactory((r, type) -> null)); - Assert.assertEquals(serializerFactoryCount(resolver), factoryCount); - Assert.assertThrows( - ForyException.class, () -> resolver.registerRuntimeTypeAlias(String.class, String.class)); - Assert.assertThrows(ForyException.class, () -> resolver.register("missing.FrozenType")); - Assert.assertThrows(ForyException.class, () -> resolver.register(ObjectField.class, -1L)); - Assert.assertThrows(ForyException.class, resolver::initialize); - Assert.assertSame(resolver.getTypeInfo(String.class, false), stringTypeInfo); - Assert.assertSame(stringTypeInfo.getSerializer(), stringSerializer); - - if (xlang) { - Assert.assertThrows( - ForyException.class, - () -> - resolver.registerInternalSerializer(char.class, new ObjectFieldSerializer(resolver))); - Assert.assertNull(resolver.getTypeInfo(ObjectField.class, false)); - Assert.assertThrows( - ForyException.class, - () -> - ((XtypeResolver) resolver) - .registerForyType( - ObjectField.class, new ObjectFieldSerializer(resolver), Types.EXT)); - Assert.assertNull(resolver.getTypeInfo(ObjectField.class, false)); - } else { - ClassResolver classResolver = (ClassResolver) resolver; - Object extRegistry = TestUtils.getFieldValue(resolver, "extRegistry"); - int classIdGenerator = TestUtils.getFieldValue(extRegistry, "classIdGenerator"); - Assert.assertThrows( - ForyException.class, () -> classResolver.registerInternal(ObjectField.class)); - Assert.assertThrows(ForyException.class, () -> classResolver.registerInternal(String.class)); - Assert.assertThrows( - ForyException.class, () -> classResolver.registerInternal(new Class[0])); - Assert.assertThrows( - ForyException.class, - () -> - classResolver.registerInternalSerializer( - String.class, new ObjectFieldSerializer(resolver))); - int currentClassIdGenerator = TestUtils.getFieldValue(extRegistry, "classIdGenerator"); - Assert.assertEquals(currentClassIdGenerator, classIdGenerator); - Assert.assertNull(resolver.getTypeInfo(ObjectField.class, false)); - } - } - - @Test(dataProvider = "xlang") - public void testFactoryRegistrationReentry(boolean xlang) { - Fory fory = - Fory.builder() - .withXlang(xlang) - .withCodegen(false) - .requireClassRegistration(true) - .withCompatible(false) - .build(); - TypeResolver resolver = fory.getTypeResolver(); - int factoryCount = serializerFactoryCount(resolver); - FactoryRegisteringSerializer.ATTEMPTED.set(false); - FactoryRegisteringSerializer.FACTORY.set((r, type) -> null); - try { - Assert.assertThrows( - ForyException.class, - () -> fory.registerSerializerAndType(MyExt.class, FactoryRegisteringSerializer.class)); - } finally { - FactoryRegisteringSerializer.FACTORY.set(null); - } - - Assert.assertTrue(FactoryRegisteringSerializer.ATTEMPTED.get()); - Assert.assertEquals(serializerFactoryCount(resolver), factoryCount); - Assert.assertFalse(resolver.isRegistered(MyExt.class)); - Assert.assertNull(resolver.getTypeInfo(MyExt.class, false)); - } - - private static int serializerFactoryCount(TypeResolver resolver) { - Object extRegistry = TestUtils.getFieldValue(resolver, "extRegistry"); - List factories = TestUtils.getFieldValue(extRegistry, "serializerFactories"); - return factories.size(); - } - @Test - public void testReentrantModuleFreeze() { - Fory fory = - Fory.builder() - .withXlang(false) - .withCodegen(false) - .requireClassRegistration(false) - .withCompatible(false) - .build(); - AtomicBoolean installReturned = new AtomicBoolean(); - ForyModule module = - runtime -> { - runtime.serialize("freeze"); - installReturned.set(true); - }; - - Assert.assertThrows(ForyException.class, () -> fory.register(module)); - Assert.assertTrue(installReturned.get()); - Set modules = TestUtils.getFieldValue(fory, "moduleRegistrations"); - Assert.assertFalse(modules.contains(module)); - } - - @Test - public void testCheckedModuleFailure() { - Fory fory = Fory.builder().withXlang(false).requireClassRegistration(false).build(); - AtomicBoolean fail = new AtomicBoolean(true); - ForyModule module = - runtime -> { - if (fail.getAndSet(false)) { - throw ExceptionUtils.throwException(new Exception("failed")); + public void testStreamRootFreezesBeforeBuffer() { + Fory reader = Fory.builder().requireClassRegistration(false).build(); + byte[] bytes = Fory.builder().requireClassRegistration(false).build().serialize("value"); + AtomicBoolean registrationRejected = new AtomicBoolean(); + ForyInputStream inputStream = + new ForyInputStream(new ByteArrayInputStream(bytes)) { + @Override + public MemoryBuffer getBuffer() { + try { + reader.register(MyExt.class); + } catch (ForyException e) { + registrationRejected.set(true); + } + return super.getBuffer(); } }; - Assert.assertThrows(Exception.class, () -> fory.register(module)); - Set modules = TestUtils.getFieldValue(fory, "moduleRegistrations"); - Assert.assertFalse(modules.contains(module)); - - fory.register(module); - Assert.assertTrue(modules.contains(module)); - } - - @Test - public void testFrozenModuleDuplicateRejected() { - Fory fory = - Fory.builder() - .withXlang(false) - .withCodegen(false) - .requireClassRegistration(false) - .withCompatible(false) - .build(); - AtomicInteger installs = new AtomicInteger(); - ForyModule module = runtime -> installs.incrementAndGet(); - fory.register(module); - fory.serialize("freeze"); - - Assert.assertThrows(ForyException.class, () -> fory.register(module)); - Assert.assertEquals(installs.get(), 1); - } - - @Test - public void testModuleCycle() { - Fory fory = Fory.builder().withXlang(false).requireClassRegistration(false).build(); - AtomicInteger firstInstalls = new AtomicInteger(); - AtomicInteger secondInstalls = new AtomicInteger(); - ForyModule[] modules = new ForyModule[2]; - modules[0] = - runtime -> { - firstInstalls.incrementAndGet(); - runtime.register(modules[1]); - }; - modules[1] = - runtime -> { - secondInstalls.incrementAndGet(); - runtime.register(modules[0]); - }; - - fory.register(modules[0]); - fory.register(modules[1]); - - Assert.assertEquals(firstInstalls.get(), 1); - Assert.assertEquals(secondInstalls.get(), 1); - } - - @Test(dataProvider = "xlang") - public void testReentrantCombinedRegistration(boolean xlang) { - Fory fory = - Fory.builder() - .withXlang(xlang) - .withCodegen(false) - .requireClassRegistration(true) - .withCompatible(false) - .build(); - ReentrantSerializer.CONSTRUCTION.set(() -> fory.serialize("freeze")); - try { - Assert.assertThrows( - ForyException.class, - () -> fory.registerSerializerAndType(MyExt.class, ReentrantSerializer.class)); - } finally { - ReentrantSerializer.CONSTRUCTION.set(null); - } - - Assert.assertTrue(fory.getTypeResolver().isRegistrationFrozen()); - Assert.assertFalse(fory.getTypeResolver().isRegistrationFinished()); - Assert.assertFalse(fory.getTypeResolver().isRegistered(MyExt.class)); - Assert.assertNull(fory.getTypeResolver().getTypeInfo(MyExt.class, false)); - } - - @Test - public void testReentrantObjectRegistration() { - Fory fory = - Fory.builder() - .withXlang(false) - .withCodegen(false) - .requireClassRegistration(false) - .withCompatible(false) - .build(); - AtomicInteger factoryCalls = new AtomicInteger(); - fory.registerSerializerFactory( - (resolver, type) -> { - if (type != ObjectField.class) { - return null; - } - factoryCalls.incrementAndGet(); - fory.serialize("freeze"); - return new ObjectFieldSerializer(resolver); - }); - - Assert.assertThrows( - ForyException.class, - () -> fory.registerSerializerAndType(ObjectHolder.class, ObjectSerializer.class)); - Assert.assertEquals(factoryCalls.get(), 1); - Assert.assertTrue(fory.getTypeResolver().isRegistrationFrozen()); - Assert.assertFalse(fory.getTypeResolver().isRegistrationFinished()); - Assert.assertFalse(fory.getTypeResolver().isRegistered(ObjectHolder.class)); - Assert.assertNull(fory.getTypeResolver().getTypeInfo(ObjectHolder.class, false)); - Assert.assertNull(fory.getTypeResolver().getTypeInfo(ObjectField.class, false)); - } - - @Test(dataProvider = "xlang") - public void testStaticGeneratedClassRejected(boolean xlang) { - Fory fory = - Fory.builder() - .withXlang(xlang) - .withCodegen(false) - .requireClassRegistration(false) - .withCompatible(false) - .build(); - RejectedStaticSerializer.CONSTRUCTIONS.set(0); - - Assert.assertThrows( - ForyException.class, - () -> fory.registerSerializerAndType(ObjectHolder.class, RejectedStaticSerializer.class)); - Assert.assertEquals(RejectedStaticSerializer.CONSTRUCTIONS.get(), 0); - Assert.assertFalse(fory.getTypeResolver().isRegistered(ObjectHolder.class)); - Assert.assertNull(fory.getTypeResolver().getTypeInfo(ObjectHolder.class, false)); - } - - public static class ReentrantSerializer extends MyExtSerializer { - private static final AtomicReference CONSTRUCTION = new AtomicReference<>(); - - public ReentrantSerializer(TypeResolver typeResolver) { - super(typeResolver); - CONSTRUCTION.get().run(); - } - } - - public static class FactoryRegisteringSerializer extends MyExtSerializer { - private static final AtomicBoolean ATTEMPTED = new AtomicBoolean(); - private static final AtomicReference FACTORY = new AtomicReference<>(); - - public FactoryRegisteringSerializer(TypeResolver typeResolver) { - super(typeResolver); - ATTEMPTED.set(true); - try { - typeResolver.registerSerializerFactory(FACTORY.get()); - } catch (ForyException ignored) { - // The enclosing registration must remain rejected after this callback returns. - } - } - } - - public static class FailingSerializer extends MyExtSerializer { - public FailingSerializer(TypeResolver typeResolver) { - super(typeResolver); - typeResolver.setSerializer(MyExt.class, this); - throw new IllegalStateException("failed"); - } - } - - public static final class FirstShareableSerializer extends MyExtSerializer implements Shareable { - public FirstShareableSerializer(TypeResolver typeResolver) { - super(typeResolver); - typeResolver.getTypeInfo(ObjectField.class); - } - } - - public static final class SecondShareableSerializer extends MyExtSerializer implements Shareable { - public SecondShareableSerializer(TypeResolver typeResolver) { - super(typeResolver); - typeResolver.getTypeInfo(ObjectField.class); - } - } - - public static final class ExternalValue implements Externalizable { - @Override - public void writeExternal(ObjectOutput out) {} - - @Override - public void readExternal(ObjectInput in) {} - } - - public static final class ShareableExternalSerializer extends Serializer - implements Shareable { - public ShareableExternalSerializer(TypeResolver typeResolver) { - super(typeResolver.getConfig(), ExternalValue.class); - } - - @Override - public void write(WriteContext writeContext, ExternalValue value) {} - - @Override - public ExternalValue read(ReadContext readContext) { - return new ExternalValue(); - } - } - - public static final class FailingExternalSerializer extends Serializer { - public FailingExternalSerializer(TypeResolver typeResolver) { - super(typeResolver.getConfig(), ExternalValue.class); - typeResolver.setSerializer(ExternalValue.class, this); - throw new IllegalStateException("failed"); - } - - @Override - public void write(WriteContext writeContext, ExternalValue value) {} - - @Override - public ExternalValue read(ReadContext readContext) { - return new ExternalValue(); - } - } - - public static class ObjectHolder { - public ObjectField field; - } - - public static final class ObjectField {} - - public static class ObjectFieldSerializer extends Serializer { - public ObjectFieldSerializer(TypeResolver typeResolver) { - super(typeResolver.getConfig(), ObjectField.class); - } - - @Override - public void write(WriteContext writeContext, ObjectField value) {} - - @Override - public ObjectField read(ReadContext readContext) { - return new ObjectField(); - } - } - - public static final class RejectedStaticSerializer - extends Generated.GeneratedStaticCompatibleSerializer { - private static final AtomicInteger CONSTRUCTIONS = new AtomicInteger(); - - public RejectedStaticSerializer(TypeResolver resolver, Class type, TypeDef typeDef) { - super(resolver, type, typeDef, Collections.emptyList()); - CONSTRUCTIONS.incrementAndGet(); - } - - @Override - public List getGeneratedDescriptors() { - return Collections.emptyList(); - } - - @Override - public Object readCompatible(ReadContext readContext) { - throw new UnsupportedOperationException(); - } + Assert.assertEquals(reader.deserialize(inputStream, String.class), "value"); + Assert.assertTrue(registrationRejected.get()); } public static class MyExtSerializer extends Serializer { diff --git a/kotlin/fory-kotlin-tests/src/main/kotlin/org/apache/fory/kotlin/xlang/KotlinXlangPeer.kt b/kotlin/fory-kotlin-tests/src/main/kotlin/org/apache/fory/kotlin/xlang/KotlinXlangPeer.kt index a46ef3f764..92e1904118 100644 --- a/kotlin/fory-kotlin-tests/src/main/kotlin/org/apache/fory/kotlin/xlang/KotlinXlangPeer.kt +++ b/kotlin/fory-kotlin-tests/src/main/kotlin/org/apache/fory/kotlin/xlang/KotlinXlangPeer.kt @@ -32,7 +32,6 @@ import kotlin.time.Duration import kotlin.time.Duration.Companion.milliseconds import org.apache.fory.BaseFory import org.apache.fory.Fory -import org.apache.fory.ForyModule import org.apache.fory.annotation.ArrayType import org.apache.fory.annotation.ForyCase import org.apache.fory.annotation.ForyField @@ -306,7 +305,6 @@ private fun staticSerializerRoundTrip(dataFile: String) { compatibleScalarContainerRefs() compatibleDenseUIntList() trackedDenseArrayRefs() - serializerRegistrationFreezes() val fory = newFory() fory.register("kotlin.KotlinUser") @@ -755,62 +753,6 @@ private fun trackedDenseArrayRefs() { check(noRefDecoded.added == "reader-default") } -private fun serializerRegistrationFreezes() { - val registeredFory = newFory() - registeredFory.register("kotlin.KotlinUser") - registeredFory.serialize(KotlinUser(1u, "freeze", 2L)) - val serializer = registeredFory.getSerializer(KotlinUser::class.java) - val typeId = registeredFory.typeResolver.getTypeInfo(KotlinUser::class.java).typeId - check( - runCatching { KotlinSerializers.registerSerializer(registeredFory, KotlinUser::class.java) } - .isFailure - ) - check(registeredFory.getSerializer(KotlinUser::class.java) === serializer) - check(registeredFory.typeResolver.getTypeInfo(KotlinUser::class.java).typeId == typeId) - - val unregisteredFory = newFory() - unregisteredFory.serialize("freeze") - check(!unregisteredFory.typeResolver.isRegistered(KotlinUser::class.java)) - check( - runCatching { KotlinSerializers.registerSerializer(unregisteredFory, KotlinUser::class.java) } - .isFailure - ) - check(!unregisteredFory.typeResolver.isRegistered(KotlinUser::class.java)) - - val failedRootFory = newFory() - check(runCatching { failedRootFory.deserialize(byteArrayOf()) }.isFailure) - for (frozenFory in listOf(unregisteredFory, failedRootFory)) { - val resolver = frozenFory.typeResolver - check( - runCatching { - KotlinSerializers.registerUnion( - frozenFory, - KotlinPet::class.java, - "kotlin.LateKotlinPet", - ) - } - .isFailure - ) - check(!resolver.isRegistered(KotlinPet::class.java)) - } - - val compatibleFory = newCompatibleFory() - KotlinSerializers.registerType( - compatibleFory, - KotlinUser::class.java, - "kotlin.KotlinUserCompatible", - ) - check( - compatibleFory.typeResolver.getTypeInfo(KotlinUser::class.java).typeId == - Types.NAMED_COMPATIBLE_STRUCT - ) - KotlinSerializers.registerSerializer(compatibleFory, KotlinUser::class.java) - check( - compatibleFory.typeResolver.getTypeInfo(KotlinUser::class.java).typeId == - Types.NAMED_COMPATIBLE_STRUCT - ) -} - private fun checkUnionListBudget(values: List) { val writer = newFory() writer.register("kotlin.KotlinUser") @@ -942,18 +884,15 @@ private fun compatibleDefaultRoundTrip() { private fun checkNoArgRegisterReceivers() { checkNoArgRegister(newFory()) - val module = ForyModule { it.register() } - checkNoArgRegistered( + checkNoArgRegister( ForyKotlin.builder() - .withModule(module) .withXlang(true) .requireClassRegistration(true) .withRefTracking(false) .buildThreadLocalFory() ) - checkNoArgRegistered( + checkNoArgRegister( ForyKotlin.builder() - .withModule(module) .withXlang(true) .requireClassRegistration(true) .withRefTracking(false) @@ -961,12 +900,8 @@ private fun checkNoArgRegisterReceivers() { ) } -private fun checkNoArgRegister(fory: Fory) { +private fun checkNoArgRegister(fory: BaseFory) { fory.register() - checkNoArgRegistered(fory) -} - -private fun checkNoArgRegistered(fory: BaseFory) { val value = KotlinInternalUser(id = 7u, name = "receiver") check(fory.deserialize(fory.serialize(value), KotlinInternalUser::class.java) == value) } diff --git a/kotlin/fory-kotlin/src/main/java/org/apache/fory/serializer/kotlin/KotlinSerializers.java b/kotlin/fory-kotlin/src/main/java/org/apache/fory/serializer/kotlin/KotlinSerializers.java index 35e69f0e52..c6d93fa31f 100644 --- a/kotlin/fory-kotlin/src/main/java/org/apache/fory/serializer/kotlin/KotlinSerializers.java +++ b/kotlin/fory-kotlin/src/main/java/org/apache/fory/serializer/kotlin/KotlinSerializers.java @@ -36,6 +36,7 @@ import kotlin.time.TimedValue; import kotlin.uuid.Uuid; import org.apache.fory.Fory; +import org.apache.fory.ThreadSafeFory; import org.apache.fory.annotation.Internal; import org.apache.fory.codegen.GeneratedClassNames; import org.apache.fory.config.Config; @@ -53,115 +54,118 @@ public class KotlinSerializers { private static final String XLANG_GENERATED_SERIALIZER_SUFFIX = "_ForySerializer"; + public static void registerSerializers(ThreadSafeFory fory) { + fory.register(ForyKotlin.INSTANCE); + } + public static void registerSerializers(Fory fory) { fory.register(ForyKotlin.INSTANCE); } @Internal public static void installSerializers(Fory fory) { + DefaultValueUtils.setKotlinDefaultValueSupport(KotlinDefaultValueSupport.INSTANCE); TypeResolver resolver = fory.getTypeResolver(); - checkRegistrationOpen(resolver); - if (!resolver.isCrossLanguage()) { - Config config = resolver.getConfig(); - - // UByte - Class ubyteClass = KotlinToJavaClass.INSTANCE.getUByteClass(); - registerIfAbsent(resolver, ubyteClass); - resolver.registerSerializer(ubyteClass, new UByteSerializer(config)); - - // UShort - Class ushortClass = KotlinToJavaClass.INSTANCE.getUShortClass(); - registerIfAbsent(resolver, ushortClass); - resolver.registerSerializer(ushortClass, new UShortSerializer(config)); - - // UInt - Class uintClass = KotlinToJavaClass.INSTANCE.getUIntClass(); - registerIfAbsent(resolver, uintClass); - resolver.registerSerializer(uintClass, new UIntSerializer(config)); - - // ULong - Class ulongClass = KotlinToJavaClass.INSTANCE.getULongClass(); - registerIfAbsent(resolver, ulongClass); - resolver.registerSerializer(ulongClass, new ULongSerializer(config)); - - // EmptyList - Class emptyListClass = KotlinToJavaClass.INSTANCE.getEmptyListClass(); - registerIfAbsent(resolver, emptyListClass); - resolver.registerSerializer( - emptyListClass, new CollectionSerializers.EmptyListSerializer(resolver, emptyListClass)); - - // EmptySet - Class emptySetClass = KotlinToJavaClass.INSTANCE.getEmptySetClass(); - registerIfAbsent(resolver, emptySetClass); - resolver.registerSerializer( - emptySetClass, new CollectionSerializers.EmptySetSerializer(resolver, emptySetClass)); - - // EmptyMap - Class emptyMapClass = KotlinToJavaClass.INSTANCE.getEmptyMapClass(); - registerIfAbsent(resolver, emptyMapClass); - resolver.registerSerializer( - emptyMapClass, new MapSerializers.EmptyMapSerializer(resolver, emptyMapClass)); - - // Non-Java collection implementation in kotlin stdlib. - Class arrayDequeClass = KotlinToJavaClass.INSTANCE.getArrayDequeClass(); - registerIfAbsent(resolver, arrayDequeClass); - resolver.registerSerializer( - arrayDequeClass, new KotlinArrayDequeSerializer(resolver, arrayDequeClass)); - - // Unsigned array classes: UByteArray, UShortArray, UIntArray, ULongArray. - registerIfAbsent(resolver, UByteArray.class); - resolver.registerSerializer(UByteArray.class, new UByteArraySerializer(resolver)); - registerIfAbsent(resolver, UShortArray.class); - resolver.registerSerializer(UShortArray.class, new UShortArraySerializer(resolver)); - registerIfAbsent(resolver, UIntArray.class); - resolver.registerSerializer(UIntArray.class, new UIntArraySerializer(resolver)); - registerIfAbsent(resolver, ULongArray.class); - resolver.registerSerializer(ULongArray.class, new ULongArraySerializer(resolver)); - - // Ranges and Progressions. - registerIfAbsent(resolver, kotlin.ranges.CharRange.class); - registerIfAbsent(resolver, kotlin.ranges.CharProgression.class); - registerIfAbsent(resolver, kotlin.ranges.IntRange.class); - registerIfAbsent(resolver, kotlin.ranges.IntProgression.class); - registerIfAbsent(resolver, kotlin.ranges.LongRange.class); - registerIfAbsent(resolver, kotlin.ranges.LongProgression.class); - registerIfAbsent(resolver, kotlin.ranges.UIntRange.class); - registerIfAbsent(resolver, kotlin.ranges.UIntProgression.class); - registerIfAbsent(resolver, kotlin.ranges.ULongRange.class); - registerIfAbsent(resolver, kotlin.ranges.ULongProgression.class); - - // Built-in classes. - registerIfAbsent(resolver, kotlin.Pair.class); - registerIfAbsent(resolver, kotlin.Triple.class); - registerIfAbsent(resolver, kotlin.Result.class); - registerIfAbsent(resolver, Result.Failure.class); - - // kotlin.random - registerIfAbsent(resolver, KotlinToJavaClass.INSTANCE.getRandomDefaultClass()); - registerIfAbsent(resolver, KotlinToJavaClass.INSTANCE.getRandomInternalClass()); - registerIfAbsent(resolver, KotlinToJavaClass.INSTANCE.getRandomSerializedClass()); - - // kotlin.text - registerIfAbsent(resolver, Regex.class); - registerIfAbsent(resolver, KotlinToJavaClass.INSTANCE.getRegexSerializedClass()); - registerIfAbsent(resolver, RegexOption.class); - registerIfAbsent(resolver, CharCategory.class); - registerIfAbsent(resolver, CharDirectionality.class); - registerIfAbsent(resolver, HexFormat.class); - registerIfAbsent(resolver, MatchGroup.class); - - // kotlin.time - registerIfAbsent(resolver, DurationUnit.class); - registerIfAbsent(resolver, Duration.class); - resolver.registerSerializer(Duration.class, new DurationSerializer(config)); - registerIfAbsent(resolver, TimedValue.class); - - // kotlin.uuid - registerIfAbsent(resolver, Uuid.class); - resolver.registerSerializer(Uuid.class, new UuidSerializer(config)); + if (resolver.isCrossLanguage()) { + return; } - checkRegistrationOpen(resolver); - DefaultValueUtils.setKotlinDefaultValueSupport(KotlinDefaultValueSupport.INSTANCE); + Config config = resolver.getConfig(); + + // UByte + Class ubyteClass = KotlinToJavaClass.INSTANCE.getUByteClass(); + registerIfAbsent(resolver, ubyteClass); + resolver.registerSerializer(ubyteClass, new UByteSerializer(config)); + + // UShort + Class ushortClass = KotlinToJavaClass.INSTANCE.getUShortClass(); + registerIfAbsent(resolver, ushortClass); + resolver.registerSerializer(ushortClass, new UShortSerializer(config)); + + // UInt + Class uintClass = KotlinToJavaClass.INSTANCE.getUIntClass(); + registerIfAbsent(resolver, uintClass); + resolver.registerSerializer(uintClass, new UIntSerializer(config)); + + // ULong + Class ulongClass = KotlinToJavaClass.INSTANCE.getULongClass(); + registerIfAbsent(resolver, ulongClass); + resolver.registerSerializer(ulongClass, new ULongSerializer(config)); + + // EmptyList + Class emptyListClass = KotlinToJavaClass.INSTANCE.getEmptyListClass(); + registerIfAbsent(resolver, emptyListClass); + resolver.registerSerializer( + emptyListClass, new CollectionSerializers.EmptyListSerializer(resolver, emptyListClass)); + + // EmptySet + Class emptySetClass = KotlinToJavaClass.INSTANCE.getEmptySetClass(); + registerIfAbsent(resolver, emptySetClass); + resolver.registerSerializer( + emptySetClass, new CollectionSerializers.EmptySetSerializer(resolver, emptySetClass)); + + // EmptyMap + Class emptyMapClass = KotlinToJavaClass.INSTANCE.getEmptyMapClass(); + registerIfAbsent(resolver, emptyMapClass); + resolver.registerSerializer( + emptyMapClass, new MapSerializers.EmptyMapSerializer(resolver, emptyMapClass)); + + // Non-Java collection implementation in kotlin stdlib. + Class arrayDequeClass = KotlinToJavaClass.INSTANCE.getArrayDequeClass(); + registerIfAbsent(resolver, arrayDequeClass); + resolver.registerSerializer( + arrayDequeClass, new KotlinArrayDequeSerializer(resolver, arrayDequeClass)); + + // Unsigned array classes: UByteArray, UShortArray, UIntArray, ULongArray. + registerIfAbsent(resolver, UByteArray.class); + resolver.registerSerializer(UByteArray.class, new UByteArraySerializer(resolver)); + registerIfAbsent(resolver, UShortArray.class); + resolver.registerSerializer(UShortArray.class, new UShortArraySerializer(resolver)); + registerIfAbsent(resolver, UIntArray.class); + resolver.registerSerializer(UIntArray.class, new UIntArraySerializer(resolver)); + registerIfAbsent(resolver, ULongArray.class); + resolver.registerSerializer(ULongArray.class, new ULongArraySerializer(resolver)); + + // Ranges and Progressions. + registerIfAbsent(resolver, kotlin.ranges.CharRange.class); + registerIfAbsent(resolver, kotlin.ranges.CharProgression.class); + registerIfAbsent(resolver, kotlin.ranges.IntRange.class); + registerIfAbsent(resolver, kotlin.ranges.IntProgression.class); + registerIfAbsent(resolver, kotlin.ranges.LongRange.class); + registerIfAbsent(resolver, kotlin.ranges.LongProgression.class); + registerIfAbsent(resolver, kotlin.ranges.UIntRange.class); + registerIfAbsent(resolver, kotlin.ranges.UIntProgression.class); + registerIfAbsent(resolver, kotlin.ranges.ULongRange.class); + registerIfAbsent(resolver, kotlin.ranges.ULongProgression.class); + + // Built-in classes. + registerIfAbsent(resolver, kotlin.Pair.class); + registerIfAbsent(resolver, kotlin.Triple.class); + registerIfAbsent(resolver, kotlin.Result.class); + registerIfAbsent(resolver, Result.Failure.class); + + // kotlin.random + registerIfAbsent(resolver, KotlinToJavaClass.INSTANCE.getRandomDefaultClass()); + registerIfAbsent(resolver, KotlinToJavaClass.INSTANCE.getRandomInternalClass()); + registerIfAbsent(resolver, KotlinToJavaClass.INSTANCE.getRandomSerializedClass()); + + // kotlin.text + registerIfAbsent(resolver, Regex.class); + registerIfAbsent(resolver, KotlinToJavaClass.INSTANCE.getRegexSerializedClass()); + registerIfAbsent(resolver, RegexOption.class); + registerIfAbsent(resolver, CharCategory.class); + registerIfAbsent(resolver, CharDirectionality.class); + registerIfAbsent(resolver, HexFormat.class); + registerIfAbsent(resolver, MatchGroup.class); + + // kotlin.time + registerIfAbsent(resolver, DurationUnit.class); + registerIfAbsent(resolver, Duration.class); + resolver.registerSerializer(Duration.class, new DurationSerializer(config)); + registerIfAbsent(resolver, TimedValue.class); + + // kotlin.uuid + registerIfAbsent(resolver, Uuid.class); + resolver.registerSerializer(Uuid.class, new UuidSerializer(config)); } private static void registerIfAbsent(TypeResolver resolver, Class cls) { @@ -206,87 +210,106 @@ public static void registerType(Fory fory, Class cls, String namespace, Strin } public static void register(Fory fory, Class cls) { - fory.register(cls); - registerSerializer(fory, cls); + TypeResolver resolver = fory.getTypeResolver(); + resolver.checkRegistrationOpen(); + resolver.register(cls); + Serializer serializer = newGeneratedSerializer(resolver, cls); + resolver.checkRegistrationOpen(); + resolver.setSerializer(cls, serializer); } public static void register(Fory fory, Class cls, long typeId) { - registerType(fory, cls, typeId); - registerSerializer(fory, cls); + TypeResolver resolver = fory.getTypeResolver(); + resolver.checkRegistrationOpen(); + resolver.register(cls, typeId); + Serializer serializer = newGeneratedSerializer(resolver, cls); + resolver.checkRegistrationOpen(); + resolver.setSerializer(cls, serializer); } public static void register(Fory fory, Class cls, String name) { - registerType(fory, cls, name); - registerSerializer(fory, cls); + TypeResolver resolver = fory.getTypeResolver(); + resolver.checkRegistrationOpen(); + fory.register(cls, name); + Serializer serializer = newGeneratedSerializer(resolver, cls); + resolver.checkRegistrationOpen(); + resolver.setSerializer(cls, serializer); } public static void register(Fory fory, Class cls, String namespace, String typeName) { - registerType(fory, cls, namespace, typeName); - registerSerializer(fory, cls); + checkTypeName(typeName); + TypeResolver resolver = fory.getTypeResolver(); + resolver.checkRegistrationOpen(); + resolver.register(cls, namespace, typeName); + Serializer serializer = newGeneratedSerializer(resolver, cls); + resolver.checkRegistrationOpen(); + resolver.setSerializer(cls, serializer); } public static void registerSerializer(Fory fory, Class cls) { TypeResolver resolver = fory.getTypeResolver(); - checkRegistrationOpen(resolver); - if (!resolver.isRegistered(cls) || resolver.getTypeInfo(cls, false) == null) { - throw new IllegalArgumentException( - "Generated Kotlin serializer requires registering the type first: " + cls.getName()); - } - resolver.registerSerializer(cls, owner -> newGeneratedSerializer(owner, cls)); - } - - private static void checkRegistrationOpen(TypeResolver resolver) { - if (resolver.isRegistrationFrozen()) { - throw new ForyException( - "Cannot register class/serializer after registration has been frozen. Please register " - + "all classes before invoking top-level `serialize/deserialize/copy` methods of " - + "Fory."); - } + resolver.checkRegistrationOpen(); + Serializer serializer = newGeneratedSerializer(resolver, cls); + resolver.checkRegistrationOpen(); + resolver.setSerializer(cls, serializer); } public static void registerEnum(Fory fory, Class cls, long typeId) { TypeResolver resolver = fory.getTypeResolver(); - checkRegistrationOpen(resolver); - resolver.registerEnum(cls, typeId, new EnumSerializer(resolver.getConfig(), enumClass(cls))); + resolver.checkRegistrationOpen(); + Serializer serializer = new EnumSerializer(resolver.getConfig(), enumClass(cls)); + resolver.checkRegistrationOpen(); + resolver.registerEnum(cls, typeId, serializer); } public static void registerEnum(Fory fory, Class cls, String namespace, String typeName) { checkTypeName(typeName); TypeResolver resolver = fory.getTypeResolver(); - checkRegistrationOpen(resolver); - resolver.registerEnum( - cls, namespace, typeName, new EnumSerializer(resolver.getConfig(), enumClass(cls))); + resolver.checkRegistrationOpen(); + Serializer serializer = new EnumSerializer(resolver.getConfig(), enumClass(cls)); + resolver.checkRegistrationOpen(); + resolver.registerEnum(cls, namespace, typeName, serializer); } public static void registerEnum(Fory fory, Class cls, String name) { TypeResolver resolver = fory.getTypeResolver(); + resolver.checkRegistrationOpen(); String[] parts = splitName(name); - checkRegistrationOpen(resolver); - resolver.registerEnum( - cls, parts[0], parts[1], new EnumSerializer(resolver.getConfig(), enumClass(cls))); + Serializer serializer = new EnumSerializer(resolver.getConfig(), enumClass(cls)); + resolver.checkRegistrationOpen(); + resolver.registerEnum(cls, parts[0], parts[1], serializer); } public static void registerUnion(Fory fory, Class cls, long typeId) { TypeResolver resolver = fory.getTypeResolver(); - checkRegistrationOpen(resolver); - resolver.registerUnion(cls, typeId, newGeneratedSerializer(resolver, cls)); - registerCaseAliases(fory, cls); + resolver.checkRegistrationOpen(); + Serializer serializer = newGeneratedSerializer(resolver, cls); + Class[] caseClasses = cls.getDeclaredClasses(); + resolver.checkRegistrationOpen(); + resolver.registerUnion(cls, typeId, serializer); + registerCaseAliases(fory, cls, caseClasses); } public static void registerUnion(Fory fory, Class cls, String namespace, String typeName) { checkTypeName(typeName); TypeResolver resolver = fory.getTypeResolver(); - checkRegistrationOpen(resolver); - resolver.registerUnion(cls, namespace, typeName, newGeneratedSerializer(resolver, cls)); - registerCaseAliases(fory, cls); + resolver.checkRegistrationOpen(); + Serializer serializer = newGeneratedSerializer(resolver, cls); + Class[] caseClasses = cls.getDeclaredClasses(); + resolver.checkRegistrationOpen(); + resolver.registerUnion(cls, namespace, typeName, serializer); + registerCaseAliases(fory, cls, caseClasses); } public static void registerUnion(Fory fory, Class cls, String name) { TypeResolver resolver = fory.getTypeResolver(); + resolver.checkRegistrationOpen(); String[] parts = splitName(name); - checkRegistrationOpen(resolver); - resolver.registerUnion(cls, parts[0], parts[1], newGeneratedSerializer(resolver, cls)); - registerCaseAliases(fory, cls); + Serializer serializer = newGeneratedSerializer(resolver, cls); + Class[] caseClasses = cls.getDeclaredClasses(); + resolver.checkRegistrationOpen(); + resolver.registerUnion(cls, parts[0], parts[1], serializer); + registerCaseAliases(fory, cls, caseClasses); } private static Serializer newGeneratedSerializer(TypeResolver resolver, Class cls) { @@ -333,8 +356,9 @@ private static Class enumClass(Class cls) { return (Class) cls; } - private static void registerCaseAliases(Fory fory, Class canonicalClass) { - for (Class nestedClass : canonicalClass.getDeclaredClasses()) { + private static void registerCaseAliases( + Fory fory, Class canonicalClass, Class[] caseClasses) { + for (Class nestedClass : caseClasses) { if (canonicalClass.isAssignableFrom(nestedClass)) { fory.getTypeResolver().registerRuntimeTypeAlias(nestedClass, canonicalClass); } diff --git a/kotlin/fory-kotlin/src/main/kotlin/org/apache/fory/kotlin/ForyExtensions.kt b/kotlin/fory-kotlin/src/main/kotlin/org/apache/fory/kotlin/ForyExtensions.kt index 463a2fa5f7..84cb13ead1 100644 --- a/kotlin/fory-kotlin/src/main/kotlin/org/apache/fory/kotlin/ForyExtensions.kt +++ b/kotlin/fory-kotlin/src/main/kotlin/org/apache/fory/kotlin/ForyExtensions.kt @@ -19,40 +19,46 @@ package org.apache.fory.kotlin +import org.apache.fory.BaseFory import org.apache.fory.Fory +import org.apache.fory.ForyModule import org.apache.fory.serializer.kotlin.KotlinSerializers -public inline fun Fory.register() { +public inline fun BaseFory.register() { registerKotlin(this, T::class.java, null, null, null, null) } -public inline fun Fory.register(typeId: Long) { +public inline fun BaseFory.register(typeId: Long) { registerKotlin(this, T::class.java, typeId, null, null, null) } -public inline fun Fory.register(name: String) { +public inline fun BaseFory.register(name: String) { registerKotlin(this, T::class.java, null, name, null, null) } -public inline fun Fory.register(namespace: String, typeName: String) { +public inline fun BaseFory.register(namespace: String, typeName: String) { registerKotlin(this, T::class.java, null, null, namespace, typeName) } @PublishedApi internal fun registerKotlin( - fory: Fory, + fory: BaseFory, cls: Class<*>, typeId: Long?, name: String?, namespace: String?, typeName: String?, ) { - fory.register(ForyKotlin) - when { - typeId != null -> KotlinSerializers.register(fory, cls, typeId) - name != null -> KotlinSerializers.register(fory, cls, name) - namespace != null && typeName != null -> - KotlinSerializers.register(fory, cls, namespace, typeName) - else -> KotlinSerializers.register(fory, cls) - } + fory.register( + ForyModule { runtime: Fory -> + runtime.register(ForyKotlin) + when { + typeId != null -> KotlinSerializers.register(runtime, cls, typeId) + name != null -> KotlinSerializers.register(runtime, cls, name) + namespace != null && typeName != null -> + KotlinSerializers.register(runtime, cls, namespace, typeName) + else -> KotlinSerializers.register(runtime, cls) + } + }, + ) } diff --git a/kotlin/fory-kotlin/src/test/kotlin/org/apache/fory/serializer/kotlin/BuiltinClassSerializerTests.kt b/kotlin/fory-kotlin/src/test/kotlin/org/apache/fory/serializer/kotlin/BuiltinClassSerializerTests.kt index 684a4b285c..4ffa61fa15 100644 --- a/kotlin/fory-kotlin/src/test/kotlin/org/apache/fory/serializer/kotlin/BuiltinClassSerializerTests.kt +++ b/kotlin/fory-kotlin/src/test/kotlin/org/apache/fory/serializer/kotlin/BuiltinClassSerializerTests.kt @@ -34,156 +34,10 @@ import kotlin.time.Duration.Companion.seconds import kotlin.uuid.ExperimentalUuidApi import kotlin.uuid.Uuid import org.apache.fory.Fory -import org.apache.fory.context.CopyContext -import org.apache.fory.context.ReadContext -import org.apache.fory.context.WriteContext -import org.apache.fory.exception.ForyException import org.apache.fory.kotlin.ForyKotlin -import org.apache.fory.resolver.TypeInfo -import org.apache.fory.resolver.TypeResolver -import org.apache.fory.serializer.Serializer -import org.apache.fory.serializer.StaticGeneratedStructSerializer -import org.apache.fory.type.Descriptor -import org.apache.fory.util.DefaultValueUtils import org.testng.Assert -import org.testng.Assert.assertThrows - -private object ReentrantRegistration { - var fory: Fory? = null - var canonical: TypeInfo? = null - var canonicalSerializer: Serializer<*>? = null - var candidate: Serializer<*>? = null - var constructions: Int = 0 - - fun reset() { - fory = null - canonical = null - canonicalSerializer = null - candidate = null - constructions = 0 - } -} - -class ReentrantStruct - -@Suppress("UNCHECKED_CAST") -class ReentrantStruct_ForySerializer(resolver: TypeResolver, cls: Class<*>) : - StaticGeneratedStructSerializer(resolver, cls as Class) { - init { - ReentrantRegistration.constructions++ - ReentrantRegistration.canonical = resolver.getTypeInfo(cls, false) - ReentrantRegistration.canonicalSerializer = - ReentrantRegistration.canonical?.getSerializer() - ReentrantRegistration.candidate = this - checkNotNull(ReentrantRegistration.fory).serialize("freeze") - } - - override fun write(writeContext: WriteContext, value: ReentrantStruct) = Unit - - override fun read(readContext: ReadContext): ReentrantStruct = ReentrantStruct() - - override fun readCompatible(readContext: ReadContext): ReentrantStruct = ReentrantStruct() - - override fun copy(copyContext: CopyContext, value: ReentrantStruct): ReentrantStruct = - ReentrantStruct() - - override fun getGeneratedDescriptors(): List = emptyList() -} class BuiltinClassSerializerTests { - @Test - fun testLateBootstrapIsReadOnly() { - val fory = ForyKotlin.builder().withXlang(false).requireClassRegistration(true).build() - fory.serialize(1) - val defaultValueSupport = DefaultValueUtils.getKotlinDefaultValueSupport() - - assertThrows(ForyException::class.java) { KotlinSerializers.registerSerializers(fory) } - Assert.assertSame(DefaultValueUtils.getKotlinDefaultValueSupport(), defaultValueSupport) - } - - @Test - fun testCombinedFreezeRecheck() { - val fory = - ForyKotlin.builder() - .withXlang(true) - .requireClassRegistration(true) - .withRefTracking(false) - .build() - ReentrantRegistration.reset() - ReentrantRegistration.fory = fory - - try { - assertThrows(ForyException::class.java) { - KotlinSerializers.register(fory, ReentrantStruct::class.java, "kotlin.ReentrantStruct") - } - val resolver = fory.typeResolver - val canonical = checkNotNull(ReentrantRegistration.canonical) - val candidate = checkNotNull(ReentrantRegistration.candidate) - Assert.assertEquals(ReentrantRegistration.constructions, 1) - Assert.assertTrue(candidate is StaticGeneratedStructSerializer<*>) - Assert.assertTrue(resolver.isRegistered(ReentrantStruct::class.java)) - Assert.assertSame(resolver.getTypeInfo(ReentrantStruct::class.java, false), canonical) - Assert.assertSame( - canonical.getSerializer(), - ReentrantRegistration.canonicalSerializer, - ) - Assert.assertNotSame(canonical.getSerializer(), candidate) - } finally { - ReentrantRegistration.reset() - } - } - - @Test - fun testMissingGeneratedType() { - val fory = - ForyKotlin.builder() - .withXlang(true) - .requireClassRegistration(true) - .withRefTracking(false) - .build() - ReentrantRegistration.reset() - - assertThrows(IllegalArgumentException::class.java) { - KotlinSerializers.registerSerializer(fory, ReentrantStruct::class.java) - } - Assert.assertEquals(ReentrantRegistration.constructions, 0) - Assert.assertFalse(fory.typeResolver.isRegistered(ReentrantStruct::class.java)) - Assert.assertNull(fory.typeResolver.getTypeInfo(ReentrantStruct::class.java, false)) - } - - @Test - fun testFrozenUnionSkipsConstruction() { - val fory = - ForyKotlin.builder() - .withXlang(true) - .requireClassRegistration(true) - .withRefTracking(false) - .build() - fory.serialize("freeze") - ReentrantRegistration.reset() - - assertThrows(ForyException::class.java) { - KotlinSerializers.registerUnion( - fory, - ReentrantStruct::class.java, - "kotlin.ReentrantStruct", - ) - } - Assert.assertEquals(ReentrantRegistration.constructions, 0) - Assert.assertFalse(fory.typeResolver.isRegistered(ReentrantStruct::class.java)) - } - - @Test - fun testSharedDefaultValueSupport() { - ForyKotlin.builder().withXlang(false).requireClassRegistration(true).build() - val support = checkNotNull(DefaultValueUtils.getKotlinDefaultValueSupport()) - Assert.assertEquals(support.getDefaultValue(ClassWithDefaults::class.java, "x"), 1) - - ForyKotlin.builder().withXlang(false).requireClassRegistration(true).build() - Assert.assertSame(DefaultValueUtils.getKotlinDefaultValueSupport(), support) - Assert.assertEquals(support.getDefaultValue(ClassWithDefaults::class.java, "x"), 1) - } - @Test fun testSerializePair() { val fory: Fory = ForyKotlin.builder().withXlang(false).requireClassRegistration(true).build() diff --git a/scala/fory-scala/src/main/java/org/apache/fory/serializer/scala/ScalaEnumSerializer.java b/scala/fory-scala/src/main/java/org/apache/fory/serializer/scala/ScalaEnumSerializer.java index c8f3909d5c..f6f6e3a7a7 100644 --- a/scala/fory-scala/src/main/java/org/apache/fory/serializer/scala/ScalaEnumSerializer.java +++ b/scala/fory-scala/src/main/java/org/apache/fory/serializer/scala/ScalaEnumSerializer.java @@ -138,12 +138,6 @@ static Object[] loadValues(Class cls) { } } - // Registration reuses the values already discovered here so enum companion code cannot run - // after the canonical type has been published. - Object[] getEnumConstants() { - return enumConstants; - } - static boolean canSerialize(Class cls) { Class enumClass = ScalaTypes.resolveScalaEnumClass(cls); if (enumClass == null) { diff --git a/scala/fory-scala/src/main/java/org/apache/fory/serializer/scala/ScalaSerializers.java b/scala/fory-scala/src/main/java/org/apache/fory/serializer/scala/ScalaSerializers.java index e9a9ff3459..b5541f496c 100644 --- a/scala/fory-scala/src/main/java/org/apache/fory/serializer/scala/ScalaSerializers.java +++ b/scala/fory-scala/src/main/java/org/apache/fory/serializer/scala/ScalaSerializers.java @@ -24,15 +24,20 @@ import java.util.Objects; import org.apache.fory.Fory; +import org.apache.fory.ThreadSafeFory; import org.apache.fory.annotation.Internal; import org.apache.fory.config.Config; -import org.apache.fory.exception.ForyException; import org.apache.fory.resolver.TypeResolver; import org.apache.fory.scala.ForyScala$; +import org.apache.fory.serializer.Serializer; import scala.collection.immutable.NumericRange; import scala.collection.immutable.Range; public class ScalaSerializers { + public static void registerSerializers(ThreadSafeFory fory) { + fory.register(ForyScala$.MODULE$); + } + public static void registerSerializers(Fory fory) { fory.register(ForyScala$.MODULE$); } @@ -40,150 +45,147 @@ public static void registerSerializers(Fory fory) { @Internal public static void installSerializers(Fory fory) { TypeResolver resolver = fory.getTypeResolver(); - checkRegistrationOpen(resolver); - if (!resolver.isCrossLanguage()) { - Config config = resolver.getConfig(); - - resolver.registerSerializer( - IterableToFactoryClass, new ToFactorySerializers.IterableToFactorySerializer(config)); - resolver.registerSerializer( - MapToFactoryClass, new ToFactorySerializers.MapToFactorySerializer(config)); - - // Seq - resolver.register(scala.collection.immutable.Seq.class); - resolver.register(scala.collection.immutable.Nil$.class); - resolver.register(scala.collection.immutable.List$.class); - resolver.register(scala.collection.immutable.$colon$colon.class); - // StrictOptimizedSeqFactory -> ... extends -> IterableFactory - resolver.register(scala.collection.immutable.Vector$.class); - resolver.register("scala.collection.immutable.VectorImpl"); - resolver.register("scala.collection.immutable.Vector0"); - resolver.register("scala.collection.immutable.Vector1"); - resolver.register("scala.collection.immutable.Vector2"); - resolver.register("scala.collection.immutable.Vector3"); - resolver.register("scala.collection.immutable.Vector4"); - resolver.register("scala.collection.immutable.Vector5"); - resolver.register("scala.collection.immutable.Vector6"); - resolver.register(scala.collection.immutable.Queue.class); - resolver.register(scala.collection.immutable.Queue$.class); - resolver.register(scala.collection.immutable.LazyList.class); - resolver.register(scala.collection.immutable.LazyList$.class); - resolver.register(scala.collection.immutable.ArraySeq.class); - resolver.register(scala.collection.immutable.ArraySeq$.class); - - // Set - resolver.register(scala.collection.immutable.Set.class); - // IterableFactory - resolver.register(scala.collection.immutable.Set$.class); - resolver.register(scala.collection.immutable.Set.Set1.class); - resolver.register(scala.collection.immutable.Set.Set2.class); - resolver.register(scala.collection.immutable.Set.Set3.class); - resolver.register(scala.collection.immutable.Set.Set4.class); - resolver.register(scala.collection.immutable.HashSet.class); - resolver.register(scala.collection.immutable.TreeSet.class); - // SortedIterableFactory - resolver.register(scala.collection.immutable.TreeSet$.class); - // IterableFactory - resolver.register(scala.collection.immutable.HashSet$.class); - resolver.register(scala.collection.immutable.ListSet.class); - resolver.register(scala.collection.immutable.ListSet$.class); - resolver.register("scala.collection.immutable.Set$EmptySet$"); - resolver.register("scala.collection.immutable.SetBuilderImpl"); - resolver.register("scala.collection.immutable.SortedMapOps$ImmutableKeySortedSet"); - - // Map - resolver.register(scala.collection.immutable.Map.class); - resolver.register(scala.collection.immutable.Map$.class); - resolver.register(scala.collection.immutable.Map.Map1.class); - resolver.register(scala.collection.immutable.Map.Map2.class); - resolver.register(scala.collection.immutable.Map.Map3.class); - resolver.register(scala.collection.immutable.Map.Map4.class); - resolver.register(scala.collection.immutable.Map.WithDefault.class); - resolver.register("scala.collection.immutable.MapBuilderImpl"); - resolver.register("scala.collection.immutable.Map$EmptyMap$"); - resolver.register("scala.collection.immutable.SeqMap$EmptySeqMap$"); - resolver.register(scala.collection.immutable.HashMap.class); - resolver.register(scala.collection.immutable.HashMap$.class); - resolver.register(scala.collection.immutable.TreeMap.class); - resolver.register(scala.collection.immutable.TreeMap$.class); - resolver.register(scala.collection.immutable.SortedMap$.class); - resolver.register(scala.collection.immutable.TreeSeqMap.class); - resolver.register(scala.collection.immutable.TreeSeqMap$.class); - resolver.register(scala.collection.immutable.ListMap.class); - resolver.register(scala.collection.immutable.ListMap$.class); - resolver.register(scala.collection.immutable.IntMap.class); - resolver.register(scala.collection.immutable.IntMap$.class); - resolver.register(scala.collection.immutable.LongMap.class); - resolver.register(scala.collection.immutable.LongMap$.class); - - // Range - resolver.register("scala.math.Numeric$IntIsIntegral$"); - resolver.register("scala.math.Numeric$LongIsIntegral$"); - resolver.registerSerializerAndType( - Range.Inclusive.class, new RangeSerializer(resolver, Range.Inclusive.class)); - resolver.registerSerializerAndType( - Range.Exclusive.class, new RangeSerializer(resolver, Range.Exclusive.class)); - resolver.registerSerializerAndType( - NumericRange.class, new NumericRangeSerializer<>(resolver, NumericRange.class)); - resolver.registerSerializerAndType( - NumericRange.Exclusive.class, - new NumericRangeSerializer<>(resolver, NumericRange.Exclusive.class)); - resolver.registerSerializerAndType( - NumericRange.Inclusive.class, - new NumericRangeSerializer<>(resolver, NumericRange.Inclusive.class)); - - resolver.register(scala.collection.generic.SerializeEnd$.class); - resolver.register(scala.collection.generic.DefaultSerializationProxy.class); - resolver.register(scala.runtime.ModuleSerializationProxy.class); - - // mutable collection types - resolver.register(scala.collection.mutable.StringBuilder.class); - resolver.register(scala.collection.mutable.ArrayBuffer.class); - resolver.register(scala.collection.mutable.ArrayBuffer$.class); - resolver.register(scala.collection.mutable.ArraySeq.class); - resolver.register(scala.collection.mutable.ArraySeq$.class); - resolver.register(scala.collection.mutable.ListBuffer.class); - resolver.register(scala.collection.mutable.ListBuffer$.class); - resolver.register(scala.collection.mutable.Buffer$.class); - resolver.register(scala.collection.mutable.ArrayDeque.class); - resolver.register(scala.collection.mutable.ArrayDeque$.class); - - resolver.register(scala.collection.mutable.HashSet.class); - resolver.register(scala.collection.mutable.HashSet$.class); - resolver.register(scala.collection.mutable.TreeSet.class); - resolver.register(scala.collection.mutable.TreeSet$.class); - - resolver.register(scala.collection.mutable.HashMap.class); - resolver.register(scala.collection.mutable.HashMap$.class); - resolver.register(scala.collection.mutable.TreeMap.class); - resolver.register(scala.collection.mutable.TreeMap$.class); - resolver.register(scala.collection.mutable.LinkedHashMap.class); - resolver.register(scala.collection.mutable.LinkedHashMap$.class); - resolver.register(scala.collection.mutable.LinkedHashSet.class); - resolver.register(scala.collection.mutable.LinkedHashSet$.class); - resolver.register(scala.collection.mutable.LongMap.class); - resolver.register(scala.collection.mutable.LongMap$.class); - - resolver.register(scala.collection.mutable.Queue.class); - resolver.register(scala.collection.mutable.Queue$.class); - resolver.register(scala.collection.mutable.Stack.class); - resolver.register(scala.collection.mutable.Stack$.class); - resolver.register(scala.collection.mutable.BitSet.class); - resolver.register(scala.collection.mutable.BitSet$.class); - } - checkRegistrationOpen(resolver); - // Install the factory only after the repeatable registrations above have completed, so a - // failed module installation cannot append the same factory again on retry. fory.registerSerializerFactory(new ScalaSerializerFactory()); + if (resolver.isCrossLanguage()) { + return; + } + Config config = resolver.getConfig(); + + resolver.registerSerializer( + IterableToFactoryClass, new ToFactorySerializers.IterableToFactorySerializer(config)); + resolver.registerSerializer( + MapToFactoryClass, new ToFactorySerializers.MapToFactorySerializer(config)); + + // Seq + resolver.register(scala.collection.immutable.Seq.class); + resolver.register(scala.collection.immutable.Nil$.class); + resolver.register(scala.collection.immutable.List$.class); + resolver.register(scala.collection.immutable.$colon$colon.class); + // StrictOptimizedSeqFactory -> ... extends -> IterableFactory + resolver.register(scala.collection.immutable.Vector$.class); + resolver.register("scala.collection.immutable.VectorImpl"); + resolver.register("scala.collection.immutable.Vector0"); + resolver.register("scala.collection.immutable.Vector1"); + resolver.register("scala.collection.immutable.Vector2"); + resolver.register("scala.collection.immutable.Vector3"); + resolver.register("scala.collection.immutable.Vector4"); + resolver.register("scala.collection.immutable.Vector5"); + resolver.register("scala.collection.immutable.Vector6"); + resolver.register(scala.collection.immutable.Queue.class); + resolver.register(scala.collection.immutable.Queue$.class); + resolver.register(scala.collection.immutable.LazyList.class); + resolver.register(scala.collection.immutable.LazyList$.class); + resolver.register(scala.collection.immutable.ArraySeq.class); + resolver.register(scala.collection.immutable.ArraySeq$.class); + + // Set + resolver.register(scala.collection.immutable.Set.class); + // IterableFactory + resolver.register(scala.collection.immutable.Set$.class); + resolver.register(scala.collection.immutable.Set.Set1.class); + resolver.register(scala.collection.immutable.Set.Set2.class); + resolver.register(scala.collection.immutable.Set.Set3.class); + resolver.register(scala.collection.immutable.Set.Set4.class); + resolver.register(scala.collection.immutable.HashSet.class); + resolver.register(scala.collection.immutable.TreeSet.class); + // SortedIterableFactory + resolver.register(scala.collection.immutable.TreeSet$.class); + // IterableFactory + resolver.register(scala.collection.immutable.HashSet$.class); + resolver.register(scala.collection.immutable.ListSet.class); + resolver.register(scala.collection.immutable.ListSet$.class); + resolver.register("scala.collection.immutable.Set$EmptySet$"); + resolver.register("scala.collection.immutable.SetBuilderImpl"); + resolver.register("scala.collection.immutable.SortedMapOps$ImmutableKeySortedSet"); + + // Map + resolver.register(scala.collection.immutable.Map.class); + resolver.register(scala.collection.immutable.Map$.class); + resolver.register(scala.collection.immutable.Map.Map1.class); + resolver.register(scala.collection.immutable.Map.Map2.class); + resolver.register(scala.collection.immutable.Map.Map3.class); + resolver.register(scala.collection.immutable.Map.Map4.class); + resolver.register(scala.collection.immutable.Map.WithDefault.class); + resolver.register("scala.collection.immutable.MapBuilderImpl"); + resolver.register("scala.collection.immutable.Map$EmptyMap$"); + resolver.register("scala.collection.immutable.SeqMap$EmptySeqMap$"); + resolver.register(scala.collection.immutable.HashMap.class); + resolver.register(scala.collection.immutable.HashMap$.class); + resolver.register(scala.collection.immutable.TreeMap.class); + resolver.register(scala.collection.immutable.TreeMap$.class); + resolver.register(scala.collection.immutable.SortedMap$.class); + resolver.register(scala.collection.immutable.TreeSeqMap.class); + resolver.register(scala.collection.immutable.TreeSeqMap$.class); + resolver.register(scala.collection.immutable.ListMap.class); + resolver.register(scala.collection.immutable.ListMap$.class); + resolver.register(scala.collection.immutable.IntMap.class); + resolver.register(scala.collection.immutable.IntMap$.class); + resolver.register(scala.collection.immutable.LongMap.class); + resolver.register(scala.collection.immutable.LongMap$.class); + + // Range + resolver.register("scala.math.Numeric$IntIsIntegral$"); + resolver.register("scala.math.Numeric$LongIsIntegral$"); + resolver.registerSerializerAndType( + Range.Inclusive.class, new RangeSerializer(resolver, Range.Inclusive.class)); + resolver.registerSerializerAndType( + Range.Exclusive.class, new RangeSerializer(resolver, Range.Exclusive.class)); + resolver.registerSerializerAndType( + NumericRange.class, new NumericRangeSerializer<>(resolver, NumericRange.class)); + resolver.registerSerializerAndType( + NumericRange.Exclusive.class, + new NumericRangeSerializer<>(resolver, NumericRange.Exclusive.class)); + resolver.registerSerializerAndType( + NumericRange.Inclusive.class, + new NumericRangeSerializer<>(resolver, NumericRange.Inclusive.class)); + + resolver.register(scala.collection.generic.SerializeEnd$.class); + resolver.register(scala.collection.generic.DefaultSerializationProxy.class); + resolver.register(scala.runtime.ModuleSerializationProxy.class); + + // mutable collection types + resolver.register(scala.collection.mutable.StringBuilder.class); + resolver.register(scala.collection.mutable.ArrayBuffer.class); + resolver.register(scala.collection.mutable.ArrayBuffer$.class); + resolver.register(scala.collection.mutable.ArraySeq.class); + resolver.register(scala.collection.mutable.ArraySeq$.class); + resolver.register(scala.collection.mutable.ListBuffer.class); + resolver.register(scala.collection.mutable.ListBuffer$.class); + resolver.register(scala.collection.mutable.Buffer$.class); + resolver.register(scala.collection.mutable.ArrayDeque.class); + resolver.register(scala.collection.mutable.ArrayDeque$.class); + + resolver.register(scala.collection.mutable.HashSet.class); + resolver.register(scala.collection.mutable.HashSet$.class); + resolver.register(scala.collection.mutable.TreeSet.class); + resolver.register(scala.collection.mutable.TreeSet$.class); + + resolver.register(scala.collection.mutable.HashMap.class); + resolver.register(scala.collection.mutable.HashMap$.class); + resolver.register(scala.collection.mutable.TreeMap.class); + resolver.register(scala.collection.mutable.TreeMap$.class); + resolver.register(scala.collection.mutable.LinkedHashMap.class); + resolver.register(scala.collection.mutable.LinkedHashMap$.class); + resolver.register(scala.collection.mutable.LinkedHashSet.class); + resolver.register(scala.collection.mutable.LinkedHashSet$.class); + resolver.register(scala.collection.mutable.LongMap.class); + resolver.register(scala.collection.mutable.LongMap$.class); + + resolver.register(scala.collection.mutable.Queue.class); + resolver.register(scala.collection.mutable.Queue$.class); + resolver.register(scala.collection.mutable.Stack.class); + resolver.register(scala.collection.mutable.Stack$.class); + resolver.register(scala.collection.mutable.BitSet.class); + resolver.register(scala.collection.mutable.BitSet$.class); } public static void registerEnum(Fory fory, Class cls, long typeId) { TypeResolver resolver = fory.getTypeResolver(); - checkRegistrationOpen(resolver); - ScalaEnumSerializer serializer = new ScalaEnumSerializer(resolver, cls); - Object[] values = serializer.getEnumConstants(); + resolver.checkRegistrationOpen(); + Serializer serializer = new ScalaEnumSerializer(resolver, cls); + resolver.checkRegistrationOpen(); resolver.registerEnum(cls, typeId, serializer); - registerEnumRuntimeAliases(fory, cls, values); + registerEnumRuntimeAliases(fory, cls); } private static String[] splitName(String name) { @@ -210,22 +212,22 @@ private static void checkTypeName(String typeName) { public static void registerEnum(Fory fory, Class cls, String name) { TypeResolver resolver = fory.getTypeResolver(); + resolver.checkRegistrationOpen(); String[] parts = splitName(name); - checkRegistrationOpen(resolver); - ScalaEnumSerializer serializer = new ScalaEnumSerializer(resolver, cls); - Object[] values = serializer.getEnumConstants(); + Serializer serializer = new ScalaEnumSerializer(resolver, cls); + resolver.checkRegistrationOpen(); resolver.registerEnum(cls, parts[0], parts[1], serializer); - registerEnumRuntimeAliases(fory, cls, values); + registerEnumRuntimeAliases(fory, cls); } public static void registerEnum(Fory fory, Class cls, String namespace, String typeName) { checkTypeName(typeName); TypeResolver resolver = fory.getTypeResolver(); - checkRegistrationOpen(resolver); - ScalaEnumSerializer serializer = new ScalaEnumSerializer(resolver, cls); - Object[] values = serializer.getEnumConstants(); + resolver.checkRegistrationOpen(); + Serializer serializer = new ScalaEnumSerializer(resolver, cls); + resolver.checkRegistrationOpen(); resolver.registerEnum(cls, namespace, typeName, serializer); - registerEnumRuntimeAliases(fory, cls, values); + registerEnumRuntimeAliases(fory, cls); } @Internal @@ -234,8 +236,8 @@ public static void registerRuntimeTypeAlias( fory.getTypeResolver().registerRuntimeTypeAlias(runtimeClass, canonicalClass); } - private static void registerEnumRuntimeAliases(Fory fory, Class cls, Object[] values) { - for (Object value : values) { + private static void registerEnumRuntimeAliases(Fory fory, Class cls) { + for (Object value : ScalaEnumSerializer.loadValues(cls)) { Class runtimeClass = value.getClass(); if (runtimeClass != cls) { registerRuntimeTypeAlias(fory, runtimeClass, cls); @@ -243,12 +245,4 @@ private static void registerEnumRuntimeAliases(Fory fory, Class cls, Object[] } } - private static void checkRegistrationOpen(TypeResolver resolver) { - if (resolver.isRegistrationFrozen()) { - throw new ForyException( - "Cannot register class/serializer after registration has been frozen. Please register " - + "all classes before invoking top-level `serialize/deserialize/copy` methods of " - + "Fory."); - } - } } diff --git a/scala/fory-scala/src/main/scala-3/org/apache/fory/scala/ForyExtensions.scala b/scala/fory-scala/src/main/scala-3/org/apache/fory/scala/ForyExtensions.scala index 8b587301e6..8a3d23076e 100644 --- a/scala/fory-scala/src/main/scala-3/org/apache/fory/scala/ForyExtensions.scala +++ b/scala/fory-scala/src/main/scala-3/org/apache/fory/scala/ForyExtensions.scala @@ -19,33 +19,37 @@ package org.apache.fory.scala -import org.apache.fory.Fory +import org.apache.fory.BaseFory import scala.reflect.ClassTag -extension (fory: Fory) - def register[T](using serializer: ForySerializer[T], tag: ClassTag[T]): Unit = { - fory.register(ForyScala) - ForySerializer.register(fory, tag.runtimeClass.asInstanceOf[Class[T]]) - } +extension (fory: BaseFory) + def register[T](using serializer: ForySerializer[T], tag: ClassTag[T]): Unit = + ForySerializer.registerModule(fory, tag.runtimeClass.asInstanceOf[Class[T]], null, null, null) - def register[T](typeId: Long)(using serializer: ForySerializer[T], tag: ClassTag[T]): Unit = { - fory.register(ForyScala) - ForySerializer.register(fory, tag.runtimeClass.asInstanceOf[Class[T]], typeId) - } + def register[T](typeId: Long)(using serializer: ForySerializer[T], tag: ClassTag[T]): Unit = + ForySerializer.registerModule( + fory, + tag.runtimeClass.asInstanceOf[Class[T]], + java.lang.Long.valueOf(typeId), + null, + null) - def register[T](name: String)(using serializer: ForySerializer[T], tag: ClassTag[T]): Unit = { - fory.register(ForyScala) - ForySerializer.register(fory, tag.runtimeClass.asInstanceOf[Class[T]], name) - } + def register[T](name: String)(using serializer: ForySerializer[T], tag: ClassTag[T]): Unit = + val (namespace, typeName) = ForySerializer.splitName(name) + ForySerializer.registerModule( + fory, + tag.runtimeClass.asInstanceOf[Class[T]], + null, + namespace, + typeName) def register[T](namespace: String, typeName: String)(using serializer: ForySerializer[T], - tag: ClassTag[T]): Unit = { - fory.register(ForyScala) - ForySerializer.register( + tag: ClassTag[T]): Unit = + ForySerializer.registerModule( fory, tag.runtimeClass.asInstanceOf[Class[T]], + null, namespace, typeName) - } diff --git a/scala/fory-scala/src/main/scala-3/org/apache/fory/scala/ForySerializer.scala b/scala/fory-scala/src/main/scala-3/org/apache/fory/scala/ForySerializer.scala index 88d271d76a..fbc98d9e16 100644 --- a/scala/fory-scala/src/main/scala-3/org/apache/fory/scala/ForySerializer.scala +++ b/scala/fory-scala/src/main/scala-3/org/apache/fory/scala/ForySerializer.scala @@ -19,9 +19,8 @@ package org.apache.fory.scala -import org.apache.fory.Fory +import org.apache.fory.{BaseFory, Fory, ForyModule, ThreadSafeFory} import org.apache.fory.annotation.Internal -import org.apache.fory.exception.ForyException import org.apache.fory.meta.TypeDef import org.apache.fory.resolver.TypeResolver import org.apache.fory.serializer.Serializer @@ -62,14 +61,6 @@ object ForySerializer { } } - private def checkRegistrationOpen(resolver: TypeResolver): Unit = { - if resolver.isRegistrationFrozen then { - throw new ForyException( - "Cannot register class/serializer after registration has been frozen. Please register " + - "all classes before invoking top-level `serialize/deserialize/copy` methods of Fory.") - } - } - def register[T](fory: Fory, cls: Class[T])(using serializer: ForySerializer[T]): Unit = { register(fory, cls, null, null) } @@ -116,19 +107,13 @@ object ForySerializer { @Internal def registerSerializer[T](fory: Fory, cls: Class[T])(using serializer: ForySerializer[T]): Unit = { val resolver = fory.getTypeResolver - checkRegistrationOpen(resolver) - val union = serializer.isUnion - checkRegistrationOpen(resolver) - if union then { + if serializer.isUnion then { throw new IllegalArgumentException("Use ForySerializer.register for Scala union serializers") } - if !resolver.isRegistered(cls) || resolver.getTypeInfo(cls, false) == null then { - throw new IllegalArgumentException( - "Generated Scala serializer requires registering the type first: " + cls.getName) - } - resolver.registerSerializer( - cls, - (owner: TypeResolver) => serializer.createSerializer(owner)) + resolver.checkRegistrationOpen() + val runtimeSerializer = serializer.createSerializer(resolver) + resolver.checkRegistrationOpen() + resolver.setSerializer(cls, runtimeSerializer) } private def register[T]( @@ -141,36 +126,80 @@ object ForySerializer { checkTypeName(typeName) } val resolver = fory.getTypeResolver - checkRegistrationOpen(resolver) - val union = serializer.isUnion - checkRegistrationOpen(resolver) - if union then { - // Union construction does not require canonical registration, so finish the remaining user - // methods before publishing any type state. - val generatedSerializer = serializer.createSerializer(resolver) - checkRegistrationOpen(resolver) - val runtimeClasses = serializer.handledRuntimeClasses(cls) - if typeId != null then { - resolver.registerUnion(cls, typeId.longValue(), generatedSerializer) - } else { - val unionNamespace = - if namespace != null then namespace else Option(cls.getPackage).map(_.getName).orNull - val unionTypeName = if typeName != null then typeName else cls.getSimpleName - fory.registerUnion( - cls, - if unionNamespace == null then "" else unionNamespace, - unionTypeName, - generatedSerializer) - } - runtimeClasses.foreach { runtimeClass => - ScalaSerializers.registerRuntimeTypeAlias(fory, runtimeClass, cls) - } - } else { - registerType(fory, cls, typeId, namespace, typeName) - resolver.registerSerializer( - cls, - (owner: TypeResolver) => serializer.createSerializer(owner)) + resolver.checkRegistrationOpen() + serializer match { + case _ if serializer.isUnion => + val runtimeSerializer = serializer.createSerializer(resolver) + val runtimeClasses = serializer.handledRuntimeClasses(cls) + resolver.checkRegistrationOpen() + if typeId != null then { + resolver.registerUnion(cls, typeId.longValue(), runtimeSerializer) + } else { + val unionNamespace = + if namespace != null then namespace else Option(cls.getPackage).map(_.getName).orNull + val unionTypeName = if typeName != null then typeName else cls.getSimpleName + fory.registerUnion( + cls, + if unionNamespace == null then "" else unionNamespace, + unionTypeName, + runtimeSerializer) + } + runtimeClasses.foreach { runtimeClass => + ScalaSerializers.registerRuntimeTypeAlias(fory, runtimeClass, cls) + } + case _ => + registerType(fory, cls, typeId, namespace, typeName) + val runtimeSerializer = serializer.createSerializer(resolver) + resolver.checkRegistrationOpen() + resolver.setSerializer(cls, runtimeSerializer) + } + } + + def register[T]( + fory: ThreadSafeFory, + cls: Class[T])(using serializer: ForySerializer[T]): Unit = { + registerModule(fory, cls, null, null, null) + } + + def register[T]( + fory: ThreadSafeFory, + cls: Class[T], + typeId: Long)(using serializer: ForySerializer[T]): Unit = { + registerModule(fory, cls, java.lang.Long.valueOf(typeId), null, null) + } + + def register[T]( + fory: ThreadSafeFory, + cls: Class[T], + name: String)(using serializer: ForySerializer[T]): Unit = { + val (namespace, typeName) = splitName(name) + registerModule(fory, cls, null, namespace, typeName) + } + + def register[T]( + fory: ThreadSafeFory, + cls: Class[T], + namespace: String, + typeName: String)(using serializer: ForySerializer[T]): Unit = { + checkTypeName(typeName) + registerModule(fory, cls, null, namespace, typeName) + } + + private[scala] def registerModule[T]( + fory: BaseFory, + cls: Class[T], + typeId: java.lang.Long, + namespace: String, + typeName: String)(using serializer: ForySerializer[T]): Unit = { + if typeName != null then { + checkTypeName(typeName) } + fory.register(new ForyModule { + override def install(runtime: Fory): Unit = { + runtime.register(ForyScala) + register(runtime, cls, typeId, namespace, typeName)(using serializer) + } + }) } private def registerType[T]( diff --git a/scala/fory-scala/src/test/scala-3/org/apache/fory/serializer/scala/ForySerializerDerivationTest.scala b/scala/fory-scala/src/test/scala-3/org/apache/fory/serializer/scala/ForySerializerDerivationTest.scala index 1eb34f982d..49d731882a 100644 --- a/scala/fory-scala/src/test/scala-3/org/apache/fory/serializer/scala/ForySerializerDerivationTest.scala +++ b/scala/fory-scala/src/test/scala-3/org/apache/fory/serializer/scala/ForySerializerDerivationTest.scala @@ -31,15 +31,14 @@ import org.apache.fory.annotation.{ UInt8Type } import org.apache.fory.config.Int64Encoding -import org.apache.fory.exception.{ForyException, InsecureException} +import org.apache.fory.exception.InsecureException import org.apache.fory.memory.MemoryBuffer import org.apache.fory.meta.TypeDef import org.apache.fory.reflect.{FieldAccessor, ObjectInstantiators} -import org.apache.fory.resolver.TypeInfo import org.apache.fory.scala.ForySerializer import org.apache.fory.scala.ForyScala import org.apache.fory.scala.register -import org.apache.fory.serializer.{GraphMemoryEstimates, Serializer, StaticGeneratedStructSerializer} +import org.apache.fory.serializer.{GraphMemoryEstimates, StaticGeneratedStructSerializer} import org.apache.fory.`type`.{Types, TypeUtils} import org.apache.fory.`type`.union.UnknownCase import org.scalatest.matchers.should.Matchers @@ -324,135 +323,6 @@ class ForySerializerDerivationTest extends AnyWordSpec with Matchers { fory.deserialize(fory.serialize(fixed)) shouldEqual fixed } - "freeze public registration helpers" in { - var personUnionCheckCalled = false - given ForySerializer[Person] with { - override def isUnion: Boolean = { - personUnionCheckCalled = true - false - } - - override def createSerializer( - typeResolver: org.apache.fory.resolver.TypeResolver, - typeDef: TypeDef): org.apache.fory.serializer.Serializer[Person] = - throw new IllegalStateException("Late person serializer creation") - } - - var searchTargetUnionCheckCalled = false - var searchTargetSerializerCreated = false - given ForySerializer[SearchTarget] with { - override def isUnion: Boolean = { - searchTargetUnionCheckCalled = true - true - } - - override def createSerializer( - typeResolver: org.apache.fory.resolver.TypeResolver, - typeDef: TypeDef): org.apache.fory.serializer.Serializer[SearchTarget] = { - searchTargetSerializerCreated = true - throw new IllegalStateException("Late union serializer creation") - } - } - - Seq( - () => { - val runtime = xlangFory() - runtime.serialize(Person("Ada", 36, None)) - runtime - }, - () => { - val runtime = xlangFory() - intercept[RuntimeException] { - runtime.deserialize(Array.emptyByteArray) - } - runtime - }).foreach { runtime => - val frozenRuntime = runtime() - val serializer = frozenRuntime.getSerializer(classOf[Person]) - intercept[RuntimeException] { - ForySerializer.registerSerializer(frozenRuntime, classOf[Person]) - } - personUnionCheckCalled shouldBe false - frozenRuntime.getSerializer(classOf[Person]) shouldBe theSameInstanceAs(serializer) - frozenRuntime.getTypeResolver.isRegistered(classOf[StoredState]) shouldBe false - intercept[RuntimeException] { - ForySerializer.register( - frozenRuntime, - classOf[StoredState], - "scala_test.LateStoredState") - } - frozenRuntime.getTypeResolver.isRegistered(classOf[StoredState]) shouldBe false - intercept[RuntimeException] { - ForySerializer.register( - frozenRuntime, - classOf[SearchTarget], - "scala_test.LateSearchTarget") - } - searchTargetUnionCheckCalled shouldBe false - searchTargetSerializerCreated shouldBe false - } - } - - "reject serializer replacement frozen during creation" in { - val runtime = xlangFory() - val resolver = runtime.getTypeResolver - val originalSerializer = runtime.getSerializer(classOf[Person]) - val factory = summon[ForySerializer[Person]] - var canonical: TypeInfo = null - var canonicalSerializer: Serializer[Person] = null - var candidate: Serializer[Person] = null - val reentrantSerializer = new ForySerializer[Person] { - override def createSerializer( - typeResolver: org.apache.fory.resolver.TypeResolver, - typeDef: TypeDef): org.apache.fory.serializer.Serializer[Person] = { - canonical = typeResolver.getTypeInfo(classOf[Person], false) - canonicalSerializer = canonical.getSerializer - candidate = factory.createSerializer(typeResolver, typeDef) - runtime.serialize(Person("Ada", 36, None)) - candidate - } - } - - intercept[ForyException] { - ForySerializer.registerSerializer(runtime, classOf[Person])(using reentrantSerializer) - } - candidate.isInstanceOf[StaticGeneratedStructSerializer[?]] shouldBe true - resolver.getTypeInfo(classOf[Person], false) shouldBe theSameInstanceAs(canonical) - canonical.getSerializer shouldBe theSameInstanceAs(canonicalSerializer) - canonical.getSerializer should not be theSameInstanceAs(candidate) - runtime.getSerializer(classOf[Person]) shouldBe theSameInstanceAs(originalSerializer) - } - - "reject combined registration frozen during creation" in { - val runtime = xlangFory() - val resolver = runtime.getTypeResolver - val factory = summon[ForySerializer[StoredState]] - var canonical: TypeInfo = null - var canonicalSerializer: Serializer[StoredState] = null - var candidate: Serializer[StoredState] = null - val reentrantFactory = new ForySerializer[StoredState] { - override def createSerializer( - typeResolver: org.apache.fory.resolver.TypeResolver, - typeDef: TypeDef): org.apache.fory.serializer.Serializer[StoredState] = { - canonical = typeResolver.getTypeInfo(classOf[StoredState], false) - canonicalSerializer = canonical.getSerializer - candidate = factory.createSerializer(typeResolver, typeDef) - runtime.serialize("freeze") - candidate - } - } - - intercept[ForyException] { - ForySerializer.register(runtime, classOf[StoredState], "scala_test.ReentrantStoredState")( - using reentrantFactory) - } - candidate.isInstanceOf[StaticGeneratedStructSerializer[?]] shouldBe true - resolver.isRegistered(classOf[StoredState]) shouldBe true - resolver.getTypeInfo(classOf[StoredState], false) shouldBe theSameInstanceAs(canonical) - canonical.getSerializer shouldBe theSameInstanceAs(canonicalSerializer) - canonical.getSerializer should not be theSameInstanceAs(candidate) - } - "serialize derived case classes with Scala collection fields" in { val fory = xlangFory() val box = CollectionBox(List("a", "b"), Set("x", "y"), Map("a" -> 1, "b" -> 2)) diff --git a/scala/fory-scala/src/test/scala-3/org/apache/fory/serializer/scala/ScalaEnumTest.scala b/scala/fory-scala/src/test/scala-3/org/apache/fory/serializer/scala/ScalaEnumTest.scala index 35190ef863..91c9b33df6 100644 --- a/scala/fory-scala/src/test/scala-3/org/apache/fory/serializer/scala/ScalaEnumTest.scala +++ b/scala/fory-scala/src/test/scala-3/org/apache/fory/serializer/scala/ScalaEnumTest.scala @@ -22,8 +22,6 @@ package org.apache.fory.serializer.scala import org.apache.fory.Fory import org.apache.fory.scala.ForyScala import org.apache.fory.annotation.ForyEnumId -import org.apache.fory.exception.ForyException -import org.apache.fory.resolver.TypeResolver import org.scalatest.matchers.should.Matchers import org.scalatest.wordspec.AnyWordSpec @@ -44,16 +42,6 @@ object ScalaEnumTest { case Red } - object EnumDiscoveryProbe { - var initialized: Int = 0 - } - - enum CountingEnum { case Value } - - object CountingEnum { - EnumDiscoveryProbe.initialized += 1 - } - case class Colors(set: Set[ColorEnum]) } @@ -94,43 +82,5 @@ class ScalaEnumTest extends AnyWordSpec with Matchers { reader.deserialize(writer.serialize(StableColorV1.Green)) shouldBe StableColorV2.Green } - "reject late bootstrap before enum discovery" in { - val frozen = ForyScala.builder() - .withXlang(false) - .requireClassRegistration(false) - .build() - frozen.serialize(1) - - intercept[ForyException] { - ScalaSerializers.registerSerializers(frozen) - } - EnumDiscoveryProbe.initialized = 0 - intercept[ForyException] { - ScalaSerializers.registerEnum(frozen, classOf[CountingEnum], 712L) - } - EnumDiscoveryProbe.initialized shouldBe 0 - } - "reject discovery after failed finalization" in { - val failed = ForyScala.builder() - .withXlang(false) - .requireClassRegistration(false) - .build() - intercept[ForyException] { - failed.registerSerializerAndType( - classOf[Colors], - (_: TypeResolver) => { - failed.serialize("freeze") - null - }) - } - failed.getTypeResolver.isRegistrationFrozen shouldBe true - failed.getTypeResolver.isRegistrationFinished shouldBe false - - EnumDiscoveryProbe.initialized = 0 - intercept[ForyException] { - ScalaSerializers.registerEnum(failed, classOf[CountingEnum], 713L) - } - EnumDiscoveryProbe.initialized shouldBe 0 - } } } From 7bce016d0fe95fe1433a6360169ec18c5bc4009e Mon Sep 17 00:00:00 2001 From: chaokunyang Date: Sun, 30 Aug 2026 03:35:50 +0800 Subject: [PATCH 111/168] fix(python): keep explicit serializer ownership local --- python/pyfory/registry.py | 43 ++++++++++++++++--------- python/pyfory/tests/test_thread_safe.py | 25 ++++++++++++++ 2 files changed, 53 insertions(+), 15 deletions(-) diff --git a/python/pyfory/registry.py b/python/pyfory/registry.py index f3e4c8d4fc..b7da5a2179 100644 --- a/python/pyfory/registry.py +++ b/python/pyfory/registry.py @@ -223,8 +223,27 @@ def _construct_serializer(serializer_factory, type_resolver, cls): (0, ()), ): if _accepts_n_positional_args(serializer_factory, nargs): - return serializer_factory(*args) - raise TypeError(f"Unsupported serializer constructor for {serializer_factory!r}; expected `(type_resolver, cls)`, `(type_resolver)`, or `()`.") + serializer = serializer_factory(*args) + break + else: + raise TypeError( + f"Unsupported serializer constructor for {serializer_factory!r}; " + "expected `(type_resolver, cls)`, `(type_resolver)`, or `()`." + ) + if not isinstance(serializer, (Serializer, CythonSerializer)): + raise TypeError("Serializer factory must return a Fory serializer") + if serializer.type_resolver is not type_resolver: + raise TypeError("Serializer factory returned a serializer for another resolver") + return serializer + + +def _check_serializer_owner(serializer, type_resolver, cls): + if not isinstance(serializer, (Serializer, CythonSerializer)): + raise TypeError("Expected a Fory serializer") + if serializer.type_resolver is not type_resolver: + raise TypeError("Serializer belongs to another resolver") + if normalize_fory_type(serializer.type_) != normalize_fory_type(cls): + raise TypeError("Serializer belongs to another type") def _split_registration_name(name: str): @@ -609,13 +628,14 @@ def register_union( namespace, typename = _split_registration_name(name) if serializer is None: raise TypeError("register_union requires a serializer") - if serializer is not None and not isinstance(serializer, Serializer): + if serializer is not None and not isinstance(serializer, (Serializer, CythonSerializer)): serializer = _construct_serializer( serializer, self._actual_type_resolver, cls, ) self._check_registry_mutable() + _check_serializer_owner(serializer, self._actual_type_resolver, cls) if typename is not None and type_id is not None: raise TypeError(f"type name {typename} and id {type_id} should not be set at the same time") auto_type_id = typename is None and type_id is None @@ -663,7 +683,7 @@ def _register_type( else: if user_type_id not in {None, NO_USER_TYPE_ID} and (user_type_id < 0 or user_type_id > 0xFFFFFFFE): raise ValueError(f"user_type_id must be in range [0, 0xfffffffe], got {user_type_id}") - if serializer is not None and not isinstance(serializer, Serializer): + if serializer is not None and not isinstance(serializer, (Serializer, CythonSerializer)): serializer = _construct_serializer( serializer, self._actual_type_resolver, @@ -671,6 +691,8 @@ def _register_type( ) if not internal: self._check_registry_mutable() + if serializer is not None and not internal: + _check_serializer_owner(serializer, self._actual_type_resolver, cls) if ( cls in self._types_info and type_id is None @@ -680,12 +702,6 @@ def _register_type( and user_type_id in {None, NO_USER_TYPE_ID} ): return self._types_info[cls] - if not internal and not self.xlang and not self.strict and type_id is None and typename is None and namespace is None and serializer is None: - # Native carriers keep their reserved discovery identity when users - # configure them explicitly; application classes retain struct registration. - typeinfo = self._register_inferred_type(cls, native_only=True) - if typeinfo is not None: - return typeinfo n_params = len({typename, type_id, None}) - 1 auto_type_id = n_params == 0 and typename is None if auto_type_id: @@ -851,6 +867,7 @@ def register_serializer(self, cls, serializer): self._check_registry_mutable() cls = normalize_fory_type(cls) assert isinstance(cls, type) or type(cls) is int, cls + _check_serializer_owner(serializer, self._actual_type_resolver, cls) if cls not in self._types_info: raise TypeUnregisteredError(f"{cls} not registered") typeinfo = self._types_info[cls] @@ -899,15 +916,11 @@ def get_type_info(self, cls, create=True): logger.info("Type %s not registered", cls) return self._register_inferred_type(cls) - def _register_inferred_type(self, cls, native_only=False): + def _register_inferred_type(self, cls): serializer = self._create_serializer(cls) - if native_only: - self._check_registry_mutable() native_registration = self._internal_py_serializer_map.get(type(serializer)) if native_registration is not None: type_id = native_registration[1] - elif native_only: - return None elif not self.xlang and isinstance(serializer, EnumSerializer): type_id = TypeId.NAMED_ENUM elif not self.xlang and isinstance(serializer, (ObjectSerializer, StatefulSerializer)): diff --git a/python/pyfory/tests/test_thread_safe.py b/python/pyfory/tests/test_thread_safe.py index d952d81177..3f2641d78b 100644 --- a/python/pyfory/tests/test_thread_safe.py +++ b/python/pyfory/tests/test_thread_safe.py @@ -222,3 +222,28 @@ def test_thread_safe_rejects_serializer_instance(): with pytest.raises(TypeError, match="serializer class or factory"): fory.register(Person, serializer=serializer) + + +def test_thread_safe_rejects_foreign_factory_serializer(): + runtime = pyfory.Fory(xlang=False, compatible=False) + serializer = PersonSerializer(runtime.type_resolver, Person) + fory = ThreadSafeFory(xlang=False, compatible=False) + fory.register(Person, serializer=lambda: serializer) + + with pytest.raises(TypeError, match="another resolver"): + fory._get_fory() + + +@pytest.mark.parametrize( + "serializer_factory, error", + [ + (lambda: object(), "must return a Fory serializer"), + (lambda resolver: PersonSerializer(resolver, Address), "another type"), + ], +) +def test_thread_safe_validates_serializer_factory(serializer_factory, error): + fory = ThreadSafeFory(xlang=False, compatible=False) + fory.register(Person, serializer=serializer_factory) + + with pytest.raises(TypeError, match=error): + fory._get_fory() From 5ca463c2d40b22cad8d8cd09a404755af0a9aae7 Mon Sep 17 00:00:00 2001 From: chaokunyang Date: Sun, 30 Aug 2026 03:36:40 +0800 Subject: [PATCH 112/168] test(python): exercise the registered serializer owner --- python/pyfory/tests/test_serializer.py | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/python/pyfory/tests/test_serializer.py b/python/pyfory/tests/test_serializer.py index ec64076d41..8514d2e020 100644 --- a/python/pyfory/tests/test_serializer.py +++ b/python/pyfory/tests/test_serializer.py @@ -822,11 +822,9 @@ def write(self, write_context, value): write_context.write_int32(value.f1) def read(self, read_context): - a = A() - a.f1 = read_context.read_int32() - return a + return RegisterClass(read_context.read_int32()) - fory.register_type(A, serializer=Serializer(fory.type_resolver, RegisterClass)) + fory.register_type(RegisterClass, serializer=Serializer(fory.type_resolver, RegisterClass)) assert fory.deserialize(fory.serialize(RegisterClass(100))).f1 == 100 From de197b30fc7a35c81ae622f025266e76a1230411 Mon Sep 17 00:00:00 2001 From: chaokunyang Date: Sun, 30 Aug 2026 04:03:33 +0800 Subject: [PATCH 113/168] refactor(cpp): complete type metadata in context owners --- .agents/languages/cpp.md | 28 +--- cpp/fory/serialization/context.cc | 39 ++++- cpp/fory/serialization/fory.h | 32 ++-- cpp/fory/serialization/serialization_test.cc | 150 +++++++++++------- cpp/fory/serialization/skip.cc | 27 +++- cpp/fory/serialization/struct_serializer.h | 69 +++++--- cpp/fory/serialization/type_info.h | 10 +- cpp/fory/serialization/type_resolver.cc | 122 +++++--------- cpp/fory/serialization/type_resolver.h | 42 ++--- .../cpp/type-registration.md | 5 +- 10 files changed, 272 insertions(+), 252 deletions(-) diff --git a/.agents/languages/cpp.md b/.agents/languages/cpp.md index 74d9152ed1..8a26d10b4a 100644 --- a/.agents/languages/cpp.md +++ b/.agents/languages/cpp.md @@ -18,26 +18,14 @@ Load this file when changing `cpp/`, Cython build plumbing, or C++ xlang behavio resource amplification, publish reference or cache state that survives root cleanup, or return success past the required safepoint. Do not add per-field checks, cursor rollback, or tests that pin the first detection point solely to make an error earlier or more precise. -- Every public `Fory` and `ThreadSafeFory` root overload must enter a root owner whose first action - freezes facade registration, before constructing stream wrappers, accessing stream buffers, - validating arguments, or acquiring pooled instances. Overloads for the same byte input shape - share that one owner instead of duplicating the hot-path gate. `BaseFory` and the source - `TypeResolver` keep separate owner-local - freeze gates so direct resolver registration cannot bypass the facade gate. Both reject before - mutation; do not collapse these gates or describe permanent registry freeze as finalization. -- Keep `TypeResolver::check_registration()` as the single out-of-line owner of the frozen and - registration-thread checks. Do not add a layout-preserving helper, padding, or call-shape - workaround. Resolver finalization prepares metadata completely and publishes it only after the - completed resolver clone succeeds; failed finalization must not expose partial metadata. -- `TypeResolver` owns its registration mutex indirectly because runtime serializers retain and - query the same resolver object after registration closes. Keep this cold synchronization object - off the resolver's hot lookup footprint; do not replace it with padding, alignment fields, - platform locks, or benchmark-shape logic. -- `TypeResolver::register_type_internal` owns the bidirectional C++ type-to-wire identity preflight - for struct, enum, extension, and union registration. Reject an existing compile-time type owner - or wire ID/name before publication; numeric user type IDs are unique across all four families, - and exact repeated registration remains rejected. Do not add rollback, rebuild, or parallel - identity state. +- Every public `Fory` and `ThreadSafeFory` root serialization or deserialization overload freezes + explicit registration before codec work. Route facade and resolver registration through one + authoritative frozen flag for that registry. Do not add another lifecycle flag or a multi-state + machine around the existing read/write-context resolver construction. +- Freezing clones the registered resolver tables but must not eagerly complete every registered + `TypeInfo`. Complete metadata only when a read/write context clone first uses that type for + metadata, struct-version, or skip behavior; keep ordinary type lookup free of completion work. +- Keep the registration check out of normal runtime lookup hot paths. - Put private methods last in class definitions, immediately before private fields. - Do not redesign alias-based or low-level public type shapes to add convenience methods unless the user explicitly asks for that API change. - For cross-language feature ports, match protocol behavior but use idiomatic C++ ownership and layering instead of mirroring Java structure literally. diff --git a/cpp/fory/serialization/context.cc b/cpp/fory/serialization/context.cc index 35c2a8c153..803057cac1 100644 --- a/cpp/fory/serialization/context.cc +++ b/cpp/fory/serialization/context.cc @@ -80,11 +80,19 @@ WriteContext::write_type_meta(const std::type_index &type_id) { // either type_index or TypeInfo* path FORY_TRY(type_info, type_resolver_->get_type_info(type_id)); write_type_meta(type_info); + if (FORY_PREDICT_FALSE(has_error())) { + return Unexpected(error()); + } return Result(); } void WriteContext::write_type_meta(const TypeInfo *type_info) { if (first_type_info_ == nullptr) { + auto result = type_resolver_->ensure_type_meta(type_info); + if (FORY_PREDICT_FALSE(!result.ok())) { + set_error(std::move(result).error()); + return; + } first_type_info_ = type_info; buffer_.write_uint8(0); // (index << 1), index=0 buffer_.write_bytes(type_info->type_def.data(), type_info->type_def.size()); @@ -109,6 +117,11 @@ void WriteContext::write_type_meta(const TypeInfo *type_info) { } // New type: index << 1, LSB=0, followed by TypeDef bytes inline + auto result = type_resolver_->ensure_type_meta(type_info); + if (FORY_PREDICT_FALSE(!result.ok())) { + set_error(std::move(result).error()); + return; + } uint32_t index = static_cast(write_type_info_index_map_.size() + 1); uint32_t marker = static_cast(index << 1); if (marker < 0x80) { @@ -190,6 +203,9 @@ WriteContext::write_enum_type_info(const TypeInfo *type_info) { if (config_->compatible) { // write type meta inline using streaming protocol write_type_meta(type_info); + if (FORY_PREDICT_FALSE(has_error())) { + return Unexpected(error()); + } } else { // write pre-encoded namespace and type_name if (type_info->encoded_namespace && type_info->encoded_type_name) { @@ -282,6 +298,9 @@ WriteContext::write_any_type_info(const TypeInfo *type_info) { case TypeId::NAMED_COMPATIBLE_STRUCT: // write type meta inline using streaming protocol write_type_meta(type_info); + if (FORY_PREDICT_FALSE(has_error())) { + return Unexpected(error()); + } break; case TypeId::NAMED_ENUM: case TypeId::NAMED_EXT: @@ -290,6 +309,9 @@ WriteContext::write_any_type_info(const TypeInfo *type_info) { if (config_->compatible) { // write type meta inline using streaming protocol write_type_meta(type_info); + if (FORY_PREDICT_FALSE(has_error())) { + return Unexpected(error()); + } } else { // write pre-encoded namespace and type_name if (type_info->encoded_namespace && type_info->encoded_type_name) { @@ -369,11 +391,17 @@ WriteContext::write_struct_type_info(const TypeInfo *type_info) { case TypeId::NAMED_COMPATIBLE_STRUCT: // write type meta inline using streaming protocol write_type_meta(type_info); + if (FORY_PREDICT_FALSE(has_error())) { + return Unexpected(error()); + } break; case TypeId::NAMED_STRUCT: if (config_->compatible) { // write type meta inline using streaming protocol write_type_meta(type_info); + if (FORY_PREDICT_FALSE(has_error())) { + return Unexpected(error()); + } } else { // write pre-encoded namespace and type_name if (type_info->encoded_namespace && type_info->encoded_type_name) { @@ -698,6 +726,16 @@ ReadContext::read_type_meta_owner(const TypeInfo *expected_type_info) { ReadTypeInfo{local_type_info, local_type_info}); return local_type_info; } + if (FORY_PREDICT_FALSE( + !local_type_info->type_meta && + is_struct_type(static_cast(local_type_info->type_id)))) { + FORY_RETURN_NOT_OK(type_resolver_->ensure_type_meta(local_type_info)); + if (has_local_meta_hash(local_type_info, meta_hash)) { + reading_type_infos_.push_back( + ReadTypeInfo{local_type_info, local_type_info}); + return local_type_info; + } + } } FORY_TRY(remote_schema_key, check_remote_type_meta_limit(*parsed_meta)); @@ -708,7 +746,6 @@ ReadContext::read_type_meta_owner(const TypeInfo *expected_type_info) { cached->concrete_owner = local_type_info; if (local_type_info) { // Have local type - assign dispatch IDs by comparing schemas. - // Note: Extension types don't have type_meta (only structs do) if (local_type_info->type_meta) { FORY_RETURN_NOT_OK(TypeMeta::assign_local_dispatch_ids( local_type_info->type_meta.get(), parsed_meta->field_infos)); diff --git a/cpp/fory/serialization/fory.h b/cpp/fory/serialization/fory.h index 12d84e6248..cf247e4570 100644 --- a/cpp/fory/serialization/fory.h +++ b/cpp/fory/serialization/fory.h @@ -705,29 +705,25 @@ class Fory : public BaseFory { : BaseFory(config, std::move(resolver)), precomputed_header_(compute_header(config.xlang)) {} - /// Constructor for ThreadSafeFory pool - resolver metadata is ready. - struct PreparedResolver {}; + /// Constructor for ThreadSafeFory pool - registration is already frozen. + struct FrozenResolver {}; explicit Fory(const Config &config, std::shared_ptr resolver, - PreparedResolver) + FrozenResolver) : BaseFory(config, std::move(resolver)), precomputed_header_(compute_header(config.xlang)) { write_ctx_.emplace(config_, type_resolver_->clone()); read_ctx_.emplace(config_, type_resolver_->clone()); } - /// Initialize operation contexts from the registered type metadata. + /// Freeze registration and initialize operation contexts. void ensure_contexts_initialized() { if (!write_ctx_.has_value()) { FORY_CHECK(!read_ctx_.has_value()); - auto context_result = type_resolver_->build_context_type_resolver(); - FORY_CHECK(context_result.ok()) - << "Failed to build context TypeResolver: " - << context_result.error().to_string(); - auto prepared_resolver = std::move(context_result).value(); + auto context_resolver = type_resolver_->build_context_type_resolver(); // Create contexts with cloned resolvers - write_ctx_.emplace(config_, prepared_resolver->clone()); - read_ctx_.emplace(config_, prepared_resolver->clone()); - type_resolver_ = std::move(prepared_resolver); + write_ctx_.emplace(config_, context_resolver->clone()); + read_ctx_.emplace(config_, context_resolver->clone()); + type_resolver_ = std::move(context_resolver); } } @@ -1017,19 +1013,15 @@ class ThreadSafeFory : public BaseFory { std::shared_ptr resolver) : BaseFory(config, std::move(resolver)), shared_resolver_(), resolver_once_flag_(), fory_pool_([this]() { - // Every public root prepares the resolver before pool acquisition. + // Every public root freezes the resolver before pool acquisition. // The pooled Fory constructor owns its context clones. return std::unique_ptr( - new Fory(config_, shared_resolver_, Fory::PreparedResolver{})); + new Fory(config_, shared_resolver_, Fory::FrozenResolver{})); }) {} void ensure_resolver_initialized() const { std::call_once(resolver_once_flag_, [this]() { - auto context_result = type_resolver_->build_context_type_resolver(); - FORY_CHECK(context_result.ok()) - << "Failed to build context TypeResolver: " - << context_result.error().to_string(); - shared_resolver_ = std::move(context_result).value(); + shared_resolver_ = type_resolver_->build_context_type_resolver(); }); } @@ -1059,7 +1051,7 @@ inline ThreadSafeFory ForyBuilder::build_thread_safe() { type_resolver_ = std::make_shared(); } type_resolver_->apply_config(normalized_config()); - // ThreadSafeFory prepares shared resolver metadata on its first root. + // ThreadSafeFory freezes and clones its shared resolver on the first root. return ThreadSafeFory(config_, type_resolver_); } diff --git a/cpp/fory/serialization/serialization_test.cc b/cpp/fory/serialization/serialization_test.cc index d34b7368e6..7428fe66a5 100644 --- a/cpp/fory/serialization/serialization_test.cc +++ b/cpp/fory/serialization/serialization_test.cc @@ -77,6 +77,16 @@ struct NestedStruct { FORY_STRUCT(NestedStruct, point, label); }; +struct UnregisteredField { + int32_t value = 0; + FORY_STRUCT(UnregisteredField, value); +}; + +struct MissingFieldOwner { + UnregisteredField field; + FORY_STRUCT(MissingFieldOwner, field); +}; + enum class Color { RED, GREEN, BLUE }; enum class SignedScopedStatus : int32_t { NEG = -3, ZERO = 0, LARGE = 42 }; FORY_ENUM(SignedScopedStatus, NEG, ZERO, LARGE); @@ -1417,6 +1427,42 @@ TEST(SerializationTest, RegistrationByNameFailureDoesNotLeakTypeInfo) { EXPECT_EQ(dotted_type_name.error().code(), ErrorCode::Invalid); } +TEST(SerializationTest, UnusedTypeMetaStaysLazy) { + auto fory = Fory::builder().xlang(true).compatible(true).build(); + ASSERT_TRUE(fory.register_struct("demo", "UsedStruct").ok()); + ASSERT_TRUE(fory.register_struct("demo", "UnusedStruct").ok()); + + auto serialized = fory.serialize(SimpleStruct{1, 2}); + ASSERT_TRUE(serialized.ok()) << serialized.error().to_string(); + + auto used = + fory.write_context().type_resolver().get_type_info(); + auto unused = + fory.write_context().type_resolver().get_type_info(); + ASSERT_TRUE(used.ok()); + ASSERT_TRUE(unused.ok()); + EXPECT_NE(used.value()->type_meta, nullptr); + EXPECT_EQ(unused.value()->type_meta, nullptr); + EXPECT_TRUE(unused.value()->type_def.empty()); +} + +TEST(SerializationTest, TypeMetaFailureIsAtomic) { + auto fory = Fory::builder().xlang(true).compatible(true).build(); + ASSERT_TRUE( + fory.register_struct("demo", "MissingOwner").ok()); + + auto serialized = fory.serialize(MissingFieldOwner{}); + ASSERT_FALSE(serialized.ok()); + + auto owner = + fory.write_context().type_resolver().get_type_info(); + ASSERT_TRUE(owner.ok()); + EXPECT_EQ(owner.value()->type_meta, nullptr); + EXPECT_TRUE(owner.value()->type_def.empty()); + EXPECT_FALSE( + fory.register_struct("demo", "MissingField").ok()); +} + static std::vector make_remote_type_meta(const std::string &type_name, const std::string &field) { std::vector fields; @@ -1744,7 +1790,7 @@ TEST(SerializationTest, ExpectedLocalTypeMetaStaysRootLocal) { auto serialized = fory.serialize(SimpleStruct{}); ASSERT_TRUE(serialized.ok()) << serialized.error().to_string(); - ReadContext ctx(fory.config(), fory.type_resolver().clone()); + ReadContext ctx(fory.config(), fory.write_context().type_resolver().clone()); auto struct_info = ctx.type_resolver().get_type_info(); auto enum_info = ctx.type_resolver().get_type_info(); auto ext_info = ctx.type_resolver().get_type_info(); @@ -1856,6 +1902,7 @@ TEST(SerializationTest, StaticTypeMetaChecksOwner) { ASSERT_TRUE(struct_b.ok()); ASSERT_TRUE(enum_b.ok()); ASSERT_TRUE(union_b.ok()); + ASSERT_TRUE(ctx.type_resolver().ensure_type_meta(struct_b.value()).ok()); expect_static_meta_ref_mismatch(ctx, struct_b.value()); expect_static_meta_ref_mismatch(ctx, enum_b.value()); @@ -2142,7 +2189,7 @@ TEST(SerializationTest, IdExtDoesNotUseTypeMetaLimits) { EXPECT_EQ(decoded.value(), IdLimitExt{42}); } -TEST(SerializationTest, LocalTypeMetaCompletionIgnoresReceiveBodyLimit) { +TEST(SerializationTest, LocalMetaIgnoresReceiveLimit) { auto fory = Fory::builder() .xlang(true) .compatible(true) @@ -2523,7 +2570,7 @@ TEST(SerializationTest, ConfigurationBuilder) { } // ============================================================================ -// Thread Safety Tests +// Registration Lifecycle Tests // ============================================================================ TEST(SerializationTest, DirectFailedRootFreezes) { @@ -2556,47 +2603,6 @@ TEST(SerializationTest, DirectFailedRootFreezes) { .ok()); } -TEST(SerializationTest, ThreadSafeForyMultiThread) { - auto fory = Fory::builder() - .xlang(true) - .compatible(false) - .track_ref(false) - .build_thread_safe(); - fory.register_struct<::ComplexStruct>(1); - - constexpr int k_num_threads = 8; - constexpr int k_iterations_per_thread = 100; - std::vector threads; - std::atomic success_count{0}; - - for (int t = 0; t < k_num_threads; ++t) { - threads.emplace_back([&, t]() { - for (int i = 0; i < k_iterations_per_thread; ++i) { - ::ComplexStruct original{"thread" + std::to_string(t) + "_iter" + - std::to_string(i), - t * 1000 + i, - {"hobby1", "hobby2"}}; - - auto bytes_result = fory.serialize(original); - if (!bytes_result.ok()) - continue; - - auto deser_result = fory.deserialize<::ComplexStruct>( - bytes_result.value().data(), bytes_result.value().size()); - if (deser_result.ok() && deser_result.value() == original) { - success_count.fetch_add(1); - } - } - }); - } - - for (auto &t : threads) { - t.join(); - } - - EXPECT_EQ(success_count.load(), k_num_threads * k_iterations_per_thread); -} - TEST(SerializationTest, ThreadSafeRegistrationFreezes) { auto fory = Fory::builder() .xlang(true) @@ -2638,19 +2644,53 @@ TEST(SerializationTest, ThreadSafeFailedRootFreezes) { auto facade_registration = fory.register_struct<::SimpleStruct>(1); ASSERT_FALSE(facade_registration.ok()); - auto type_info = source_resolver->get_type_info_by_id( - static_cast(TypeId::STRING)); - ASSERT_TRUE(type_info.ok()); - ASSERT_EQ(type_info.value()->harness.any_write_fn, nullptr); - ASSERT_EQ(type_info.value()->harness.any_read_fn, nullptr); - auto late_registration = register_any_type(*source_resolver); ASSERT_FALSE(late_registration.ok()); - EXPECT_EQ(type_info.value()->harness.any_write_fn, nullptr); - EXPECT_EQ(type_info.value()->harness.any_read_fn, nullptr); - EXPECT_FALSE( - source_resolver->get_type_info(std::type_index(typeid(std::string))) - .ok()); +} + +// ============================================================================ +// Thread Safety Tests +// ============================================================================ + +TEST(SerializationTest, ThreadSafeForyMultiThread) { + auto fory = Fory::builder() + .xlang(true) + .compatible(false) + .track_ref(false) + .build_thread_safe(); + fory.register_struct<::ComplexStruct>(1); + + constexpr int k_num_threads = 8; + constexpr int k_iterations_per_thread = 100; + std::vector threads; + std::atomic success_count{0}; + + for (int t = 0; t < k_num_threads; ++t) { + threads.emplace_back([&, t]() { + for (int i = 0; i < k_iterations_per_thread; ++i) { + ::ComplexStruct original{"thread" + std::to_string(t) + "_iter" + + std::to_string(i), + t * 1000 + i, + {"hobby1", "hobby2"}}; + + auto bytes_result = fory.serialize(original); + if (!bytes_result.ok()) + continue; + + auto deser_result = fory.deserialize<::ComplexStruct>( + bytes_result.value().data(), bytes_result.value().size()); + if (deser_result.ok() && deser_result.value() == original) { + success_count.fetch_add(1); + } + } + }); + } + + for (auto &t : threads) { + t.join(); + } + + EXPECT_EQ(success_count.load(), k_num_threads * k_iterations_per_thread); } TEST(SerializationTest, TemporalCarriersAreHashable) { diff --git a/cpp/fory/serialization/skip.cc b/cpp/fory/serialization/skip.cc index 6d79c34a0c..e3c904f6fc 100644 --- a/cpp/fory/serialization/skip.cc +++ b/cpp/fory/serialization/skip.cc @@ -98,8 +98,11 @@ void skip_fields(ReadContext &ctx, const std::vector &field_infos) { void skip_struct_data(ReadContext &ctx, const TypeInfo &type_info) { if (!type_info.type_meta) { - ctx.set_error(Error::type_error("TypeMeta not found for struct skip")); - return; + auto result = ctx.type_resolver().ensure_type_meta(&type_info); + if (FORY_PREDICT_FALSE(!result.ok())) { + ctx.set_error(std::move(result).error()); + return; + } } if (ctx.check_struct_version()) { (void)ctx.read_int32(ctx.error()); @@ -537,11 +540,17 @@ void skip_struct(ReadContext &ctx, const FieldType &) { } } - if (!type_info || !type_info->type_meta) { - ctx.set_error( - Error::type_error("TypeInfo or TypeMeta not found for struct skip")); + if (!type_info) { + ctx.set_error(Error::type_error("TypeInfo not found for struct skip")); return; } + if (!type_info->type_meta) { + auto result = ctx.type_resolver().ensure_type_meta(type_info); + if (FORY_PREDICT_FALSE(!result.ok())) { + ctx.set_error(std::move(result).error()); + return; + } + } skip_fields(ctx, type_info->type_meta->get_field_infos()); } @@ -673,9 +682,11 @@ void skip_unknown(ReadContext &ctx) { case TypeId::NAMED_COMPATIBLE_STRUCT: { // For struct types, we already have the type_info with field_infos if (!type_info->type_meta) { - ctx.set_error( - Error::type_error("TypeMeta not found for UNKNOWN struct skip")); - return; + auto result = ctx.type_resolver().ensure_type_meta(type_info); + if (FORY_PREDICT_FALSE(!result.ok())) { + ctx.set_error(std::move(result).error()); + return; + } } skip_fields(ctx, type_info->type_meta->get_field_infos()); return; diff --git a/cpp/fory/serialization/struct_serializer.h b/cpp/fory/serialization/struct_serializer.h index d04707935a..643bc6b4e6 100644 --- a/cpp/fory/serialization/struct_serializer.h +++ b/cpp/fory/serialization/struct_serializer.h @@ -4682,8 +4682,9 @@ struct Serializer>> { ctx.write_struct_type_id_direct(tid, type_info->user_type_id); } else { // Complex type (NAMED_STRUCT, COMPATIBLE_STRUCT, etc.) - use TypeInfo* - ctx.write_struct_type_info(type_info); - if (FORY_PREDICT_FALSE(ctx.has_error())) { + auto result = ctx.write_struct_type_info(type_info); + if (FORY_PREDICT_FALSE(!result.ok())) { + ctx.set_error(std::move(result).error()); return; } } @@ -4701,10 +4702,12 @@ struct Serializer>> { return; } const TypeInfo *type_info = type_info_res.value(); - if (!type_info->type_meta) { - ctx.set_error( - Error::type_error("Type metadata not initialized for struct")); - return; + if (FORY_PREDICT_FALSE(!type_info->type_meta)) { + auto meta_result = ctx.type_resolver().ensure_type_meta(type_info); + if (FORY_PREDICT_FALSE(!meta_result.ok())) { + ctx.set_error(std::move(meta_result).error()); + return; + } } int32_t local_version = TypeMeta::compute_struct_version(*type_info->type_meta); @@ -4733,10 +4736,12 @@ struct Serializer>> { return; } const TypeInfo *type_info = type_info_res.value(); - if (!type_info->type_meta) { - ctx.set_error( - Error::type_error("Type metadata not initialized for struct")); - return; + if (FORY_PREDICT_FALSE(!type_info->type_meta)) { + auto meta_result = ctx.type_resolver().ensure_type_meta(type_info); + if (FORY_PREDICT_FALSE(!meta_result.ok())) { + ctx.set_error(std::move(meta_result).error()); + return; + } } int32_t local_version = TypeMeta::compute_struct_version(*type_info->type_meta); @@ -4952,10 +4957,13 @@ struct Serializer>> { return T{}; } local_type_info = local_type_info_res.value(); - if (!local_type_info->type_meta) { - ctx.set_error(Error::type_error( - "Type metadata not initialized for requested struct")); - return T{}; + if (FORY_PREDICT_FALSE(!local_type_info->type_meta)) { + auto meta_result = + ctx.type_resolver().ensure_type_meta(local_type_info); + if (FORY_PREDICT_FALSE(!meta_result.ok())) { + ctx.set_error(std::move(meta_result).error()); + return T{}; + } } int32_t local_version = TypeMeta::compute_struct_version(*local_type_info->type_meta); @@ -4973,10 +4981,13 @@ struct Serializer>> { return T{}; } local_type_info = local_type_info_res.value(); - if (!local_type_info->type_meta) { - ctx.set_error(Error::type_error( - "Type metadata not initialized for requested struct")); - return T{}; + if (FORY_PREDICT_FALSE(!local_type_info->type_meta)) { + auto meta_result = + ctx.type_resolver().ensure_type_meta(local_type_info); + if (FORY_PREDICT_FALSE(!meta_result.ok())) { + ctx.set_error(std::move(meta_result).error()); + return T{}; + } } } @@ -5043,10 +5054,13 @@ struct Serializer>> { return T{}; } const TypeInfo *local_type_info = local_type_info_res.value(); - if (!local_type_info->type_meta) { - ctx.set_error(Error::type_error( - "Type metadata not initialized for requested struct")); - return T{}; + if (FORY_PREDICT_FALSE(!local_type_info->type_meta)) { + auto meta_result = + ctx.type_resolver().ensure_type_meta(local_type_info); + if (FORY_PREDICT_FALSE(!meta_result.ok())) { + ctx.set_error(std::move(meta_result).error()); + return T{}; + } } int32_t local_version = TypeMeta::compute_struct_version(*local_type_info->type_meta); @@ -5086,8 +5100,15 @@ struct Serializer>> { return T{}; } - // In compatible mode with type info provided, use schema evolution path - if (ctx.is_compatible() && type_info.type_meta) { + // In compatible mode with type info provided, use schema evolution path. + if (ctx.is_compatible()) { + if (FORY_PREDICT_FALSE(!type_info.type_meta)) { + auto meta_result = ctx.type_resolver().ensure_type_meta(&type_info); + if (FORY_PREDICT_FALSE(!meta_result.ok())) { + ctx.set_error(std::move(meta_result).error()); + return T{}; + } + } return read_compatible(ctx, &type_info); } diff --git a/cpp/fory/serialization/type_info.h b/cpp/fory/serialization/type_info.h index 48564019f5..37441f737e 100644 --- a/cpp/fory/serialization/type_info.h +++ b/cpp/fory/serialization/type_info.h @@ -136,14 +136,14 @@ struct TypeInfo { std::string type_name; bool register_by_name = false; bool is_external = false; - std::unique_ptr type_meta; + mutable std::unique_ptr type_meta; std::vector sorted_indices; fory::flat_hash_map name_to_index; - std::vector type_def; + mutable std::vector type_def; Harness harness; - // TypeInfo and its harness are immutable after registration. Cache the last - // read target so repeated root operations avoid walking the declared base - // graph; ThreadSafeFory uses distinct cloned TypeInfo owners per pooled Fory. + // Registration data and the harness are immutable after registration. + // Operation-context clones complete TypeMeta lazily and cache the last read + // target; ThreadSafeFory never shares these mutable caches between workers. mutable const std::type_info *cached_read_target = nullptr; mutable Harness::ReadAsFn cached_read_as_fn = nullptr; // Pre-encoded meta strings for efficient writing (avoids re-encoding on each diff --git a/cpp/fory/serialization/type_resolver.cc b/cpp/fory/serialization/type_resolver.cc index 7a4b729eb7..8d927b20e0 100644 --- a/cpp/fory/serialization/type_resolver.cc +++ b/cpp/fory/serialization/type_resolver.cc @@ -1773,94 +1773,49 @@ Result TypeResolver::check_registration() { return Result(); } -Result, Error> -TypeResolver::build_context_type_resolver() { +std::unique_ptr TypeResolver::build_context_type_resolver() { std::lock_guard lock(registration_mutex_); registry_frozen_ = true; - auto context_resolver = std::make_unique(); - - // copy configuration - context_resolver->compatible_ = compatible_; - context_resolver->xlang_ = xlang_; - context_resolver->check_struct_version_ = check_struct_version_; - context_resolver->track_ref_ = track_ref_; - context_resolver->registry_frozen_ = true; - - // Build mapping from old pointers to new pointers for rebuilding lookup maps - fory::flat_hash_map ptr_map; - - // Deep clone all existing TypeInfo objects - for (const auto &info : type_infos_) { - auto cloned = info->deep_clone(); - TypeInfo *new_ptr = cloned.get(); - ptr_map[info.get()] = new_ptr; - context_resolver->type_infos_.push_back(std::move(cloned)); - } - auto remap_type_info = [&ptr_map](const TypeInfo *old_ptr) { - auto *entry = ptr_map.find(old_ptr); - FORY_CHECK(entry != nullptr); - return entry->second; - }; + return clone(); +} - // Rebuild lookup maps with new pointers - for (const auto &[key, old_ptr] : type_info_by_ctid_) { - context_resolver->type_info_by_ctid_.put(key, remap_type_info(old_ptr)); - } - for (const auto &[key, old_ptr] : type_info_by_id_) { - context_resolver->type_info_by_id_.put(key, remap_type_info(old_ptr)); +Result TypeResolver::ensure_type_meta(const TypeInfo *type_info) { + if (FORY_PREDICT_TRUE(type_info != nullptr && type_info->type_meta)) { + return Result(); } - for (const auto &[key, old_ptr] : user_type_info_by_id_) { - context_resolver->user_type_info_by_id_.put(key, remap_type_info(old_ptr)); - } - for (const auto &[key, old_ptr] : type_info_by_name_) { - context_resolver->type_info_by_name_[key] = remap_type_info(old_ptr); - } - for (const auto &[key, old_ptr] : type_info_by_runtime_type_) { - context_resolver->type_info_by_runtime_type_[key] = - remap_type_info(old_ptr); + if (FORY_PREDICT_FALSE(type_info == nullptr)) { + return Unexpected(Error::invalid("TypeInfo is null")); } - - for (const auto &[key, old_ptr] : partial_type_infos_) { - context_resolver->partial_type_infos_.put(key, remap_type_info(old_ptr)); - } - - // Process all partial type infos to build complete type metadata - for (const auto &[rust_type_id, partial_ptr] : - context_resolver->partial_type_infos_) { - // Call the harness's sorted_field_infos function to get complete field info - FORY_TRY(sorted_fields, - partial_ptr->harness.sorted_field_infos_fn(*context_resolver)); - - // Build complete TypeMeta - TypeMeta meta = TypeMeta::from_fields( - partial_ptr->type_id, partial_ptr->namespace_name, - partial_ptr->type_name, partial_ptr->register_by_name, - partial_ptr->user_type_id, std::move(sorted_fields)); - - // Serialize TypeMeta to bytes - FORY_TRY(type_def, meta.to_bytes()); - - // Update the TypeInfo in place - partial_ptr->type_def = std::move(type_def); - - // Parse the serialized TypeMeta back to create unique_ptr - Buffer buffer(partial_ptr->type_def.data(), - static_cast(partial_ptr->type_def.size()), false); - buffer.writer_index(static_cast(partial_ptr->type_def.size())); - // This metadata was just generated from local registration state. Remote - // receive limits are enforced only on remote metadata parse/cache-miss - // paths, so large trusted local schemas do not fail metadata completion. - FORY_TRY(parsed_meta, - TypeMeta::from_bytes(buffer, nullptr, - std::numeric_limits::max(), - std::numeric_limits::max())); - partial_ptr->type_meta = std::move(parsed_meta); + if (FORY_PREDICT_FALSE(!registry_frozen_)) { + return Unexpected(Error::invalid( + "Type metadata is available only after registration is frozen")); } - - // The context resolver retains only completed metadata. - context_resolver->partial_type_infos_.clear(); - - return context_resolver; + if (FORY_PREDICT_FALSE(type_info->harness.sorted_field_infos_fn == nullptr)) { + return Unexpected( + Error::type_error("Type metadata builder is not available")); + } + + FORY_TRY(sorted_fields, type_info->harness.sorted_field_infos_fn(*this)); + TypeMeta meta = + TypeMeta::from_fields(type_info->type_id, type_info->namespace_name, + type_info->type_name, type_info->register_by_name, + type_info->user_type_id, std::move(sorted_fields)); + FORY_TRY(type_def, meta.to_bytes()); + + Buffer buffer(type_def.data(), static_cast(type_def.size()), false); + buffer.writer_index(static_cast(type_def.size())); + // Local registration metadata is trusted. Remote receive limits apply only + // to remote cache misses, not to this context-local completion path. + FORY_TRY(parsed_meta, + TypeMeta::from_bytes(buffer, nullptr, + std::numeric_limits::max(), + std::numeric_limits::max())); + + // Publish only after every fallible step succeeds. The TypeMeta pointer is + // the completion condition, so a failed attempt leaves no partial state. + type_info->type_def = std::move(type_def); + type_info->type_meta = std::move(parsed_meta); + return Result(); } std::unique_ptr TypeResolver::clone() const { @@ -1905,9 +1860,6 @@ std::unique_ptr TypeResolver::clone() const { for (const auto &[key, old_ptr] : type_info_by_runtime_type_) { cloned->type_info_by_runtime_type_[key] = remap_type_info(old_ptr); } - // Note: Don't copy partial_type_infos_ - clone is used only after metadata - // completion. - return cloned; } diff --git a/cpp/fory/serialization/type_resolver.h b/cpp/fory/serialization/type_resolver.h index 02b05dcac3..0e78a857fb 100644 --- a/cpp/fory/serialization/type_resolver.h +++ b/cpp/fory/serialization/type_resolver.h @@ -1360,23 +1360,12 @@ class TypeResolver { template Result register_any_type(); - /// Builds the TypeResolver used by operation contexts by completing all - /// partial type infos created during registration. - /// - /// This method processes all types that were registered. During registration, - /// types are stored in `partial_type_infos` without their complete - /// type metadata to avoid circular dependencies. This method: - /// - /// 1. Iterates through all partial type infos - /// 2. Calls their `sorted_field_infos` function to get complete field - /// information - /// 3. Builds complete TypeMeta and serializes it to bytes - /// 4. Returns a new TypeResolver with complete metadata - /// - /// Registration is permanently frozen before metadata construction starts. - /// - /// @return A TypeResolver ready to create operation contexts. - Result, Error> build_context_type_resolver(); + /// Permanently freezes registration and clones the resolver for operation + /// contexts. Type metadata is completed by the context clone when used. + std::unique_ptr build_context_type_resolver(); + + /// Complete one TypeInfo's metadata after registration is frozen. + Result ensure_type_meta(const TypeInfo *type_info); /// Deep clones the TypeResolver for use in a new context. /// @@ -1542,7 +1531,6 @@ class TypeResolver { util::U32PtrMap type_info_by_id_{256}; util::U64PtrMap user_type_info_by_id_{256}; fory::flat_hash_map type_info_by_name_; - util::U64PtrMap partial_type_infos_{256}; // For runtime polymorphic lookups (smart pointers) - uses std::type_index fory::flat_hash_map type_info_by_runtime_type_; @@ -1708,8 +1696,8 @@ template const TypeMeta &TypeResolver::struct_meta() { constexpr uint64_t ctid = type_index(); TypeInfo *info = type_info_by_ctid_.get_or_default(ctid, nullptr); FORY_CHECK(info != nullptr) << "Type not registered"; - FORY_CHECK(info->type_meta) - << "Type metadata not initialized for requested struct"; + auto result = ensure_type_meta(info); + FORY_CHECK(result.ok()) << result.error().to_string(); return *info->type_meta; } @@ -1717,8 +1705,8 @@ template TypeMeta TypeResolver::clone_struct_meta() { constexpr uint64_t ctid = type_index(); TypeInfo *info = type_info_by_ctid_.get_or_default(ctid, nullptr); FORY_CHECK(info != nullptr) << "Type not registered"; - FORY_CHECK(info->type_meta) - << "Type metadata not initialized for requested struct"; + auto result = ensure_type_meta(info); + FORY_CHECK(result.ok()) << result.error().to_string(); return *info->type_meta; } @@ -1812,8 +1800,7 @@ Result TypeResolver::register_by_id(uint32_t type_id) { // Register and get back the stored pointer FORY_TRY(stored_ptr, register_type_internal(ctid, std::move(info))); - // Also register for runtime polymorphic lookups and partial type infos - partial_type_infos_.put(ctid, stored_ptr); + // Also register for runtime polymorphic lookups. register_type_internal_runtime(std::type_index(typeid(T)), stored_ptr); return Result(); } else if constexpr (std::is_enum_v) { @@ -1828,7 +1815,6 @@ Result TypeResolver::register_by_id(uint32_t type_id) { } FORY_TRY(stored_ptr, register_type_internal(ctid, std::move(info))); - partial_type_infos_.put(ctid, stored_ptr); register_type_internal_runtime(std::type_index(typeid(T)), stored_ptr); return Result(); } else { @@ -1870,7 +1856,6 @@ TypeResolver::register_by_name(const std::string &ns, } FORY_TRY(stored_ptr, register_type_internal(ctid, std::move(info))); - partial_type_infos_.put(ctid, stored_ptr); register_type_internal_runtime(std::type_index(typeid(T)), stored_ptr); return Result(); } else if constexpr (std::is_enum_v) { @@ -1883,7 +1868,6 @@ TypeResolver::register_by_name(const std::string &ns, } FORY_TRY(stored_ptr, register_type_internal(ctid, std::move(info))); - partial_type_infos_.put(ctid, stored_ptr); register_type_internal_runtime(std::type_index(typeid(T)), stored_ptr); return Result(); } else { @@ -1912,7 +1896,6 @@ Result TypeResolver::register_ext_type_by_id(uint32_t type_id) { build_ext_type_info(actual_type_id, user_type_id, "", "", false)); FORY_TRY(stored_ptr, register_type_internal(ctid, std::move(info))); - partial_type_infos_.put(ctid, stored_ptr); register_type_internal_runtime(std::type_index(typeid(T)), stored_ptr); return Result(); } @@ -1939,7 +1922,6 @@ TypeResolver::register_ext_type_by_name(const std::string &ns, type_name, true)); FORY_TRY(stored_ptr, register_type_internal(ctid, std::move(info))); - partial_type_infos_.put(ctid, stored_ptr); register_type_internal_runtime(std::type_index(typeid(T)), stored_ptr); return Result(); } @@ -1961,7 +1943,6 @@ Result TypeResolver::register_union_by_id(uint32_t type_id) { false)); FORY_TRY(stored_ptr, register_type_internal(ctid, std::move(info))); - partial_type_infos_.put(ctid, stored_ptr); register_type_internal_runtime(std::type_index(typeid(T)), stored_ptr); return Result(); } @@ -1988,7 +1969,6 @@ TypeResolver::register_union_by_name(const std::string &ns, ns, type_name, true)); FORY_TRY(stored_ptr, register_type_internal(ctid, std::move(info))); - partial_type_infos_.put(ctid, stored_ptr); register_type_internal_runtime(std::type_index(typeid(T)), stored_ptr); return Result(); } diff --git a/docs/object-serialization/cpp/type-registration.md b/docs/object-serialization/cpp/type-registration.md index 1d6f2b34ef..13169fc4bf 100644 --- a/docs/object-serialization/cpp/type-registration.md +++ b/docs/object-serialization/cpp/type-registration.md @@ -66,9 +66,8 @@ int main() { Type IDs must be: -1. **Unique**: Each numeric ID identifies one C++ type across structs, enums, extension types, and unions within a Fory instance. Each registered name also identifies one C++ type -2. **Single binding**: Register each C++ type once. It cannot be rebound to another ID or name, and repeating the same registration also returns an error -3. **Consistent**: Use the same ID across all languages and versions +1. **Unique**: Each type must have a unique ID within a Fory instance +2. **Consistent**: Same ID must be used across all languages and versions User-registered type IDs are in a separate namespace from built-in type IDs, so you can start from 0: From f1fdfb68631de642cab17cbb91f2b77b4ab9fec4 Mon Sep 17 00:00:00 2001 From: chaokunyang Date: Sun, 30 Aug 2026 04:03:44 +0800 Subject: [PATCH 114/168] fix(csharp): restore failed root state immediately --- .agents/languages/csharp.md | 19 +++++-------------- csharp/src/Fory/Fory.cs | 12 +++++++++++- .../tests/Fory.Tests/RuntimeEdgeCaseTests.cs | 5 +++-- .../csharp/thread-safety.md | 3 +-- .../csharp/type-registration.md | 4 ---- 5 files changed, 20 insertions(+), 23 deletions(-) diff --git a/.agents/languages/csharp.md b/.agents/languages/csharp.md index 449a70451f..0c4445b7d3 100644 --- a/.agents/languages/csharp.md +++ b/.agents/languages/csharp.md @@ -9,20 +9,11 @@ Load this file when changing `csharp/` or C# xlang behavior. - C# code must build without compiler or analyzer warnings. Treat warnings as blockers in project, test, and generated code. - Fory C# requires .NET SDK `8.0+` and C# `12+`. - Use `dotnet format` to keep C# code style consistent. -- A direct C# `Fory` owns permanent registration freeze at first root entry, including a failed - root. `ThreadSafeFory` linearizes root entry and registration with its registration lock and - frozen state, validates registration on its staging `Fory`, and appends only successful actions - to the replay log. Serializer construction and generated factories may reenter a root, so direct - registration must recheck the facade after resolving the serializer and before resolver mutation. - `ThreadSafeFory` must recheck disposal and freeze after staging registration and before replay-log - publication. Resolver registration must prepare all serializer and MetaString state before its - single map commit, so failed validation needs no staging rebuild or identity workaround. - Before serializer resolution and again before commit, the resolver must enforce a one-to-one - mapping between CLR types and wire IDs or names. The same mapping is idempotent, including the - same concrete custom serializer when one is explicit; neither identity side nor an explicit - serializer may be rebound to a different owner. - New per-thread runtimes replay that log; do not mutate existing runtimes or introduce another - freeze owner. +- A direct C# `Fory` registry and the `ThreadSafeFory` public registration boundary each own one + authoritative frozen flag. The first root sets the owning flag before codec work and leaves it + set after failure. Explicit registration checks that flag before mutation. `ThreadSafeFory` keeps + its existing successful-registration list only to configure newly created child runtimes; do not + turn that list into another registry lifecycle state. - Generated C# gRPC service companions are compiler-owned files that depend on application-provided gRPC packages, not `csharp/src/Fory`. Keep gRPC package references out of the Fory runtime package. - C# generated schema modules are source-file owners. Service companions must use that module's `ThreadSafeFory` and must not introduce namespace-owned aliases or duplicate serializer registration paths. - C# external-type serialization is target-keyed. A local diff --git a/csharp/src/Fory/Fory.cs b/csharp/src/Fory/Fory.cs index e3dfa6b55d..d57509dd25 100644 --- a/csharp/src/Fory/Fory.cs +++ b/csharp/src/Fory/Fory.cs @@ -183,7 +183,15 @@ public byte[] Serialize(in T value) Serializer serializer = _typeResolver.GetSerializer(); WriteHead(writer); RefMode refMode = Config.TrackRef ? RefMode.Tracking : RefMode.NullOnly; - serializer.Write(_writeContext, value, refMode, true, false); + try + { + serializer.Write(_writeContext, value, refMode, true, false); + } + catch + { + _writeContext.Reset(); + throw; + } _writeContext.RefWriter.Reset(); return writer.ToArray(); @@ -210,6 +218,7 @@ public void Serialize(IBufferWriter output, in T value) /// Thrown when trailing bytes remain after decoding. public T Deserialize(ReadOnlySpan payload) { + _registryFrozen = true; ByteReader reader = _readContext.Reader; reader.Reset(payload); T value = DeserializeFromReader(reader); @@ -231,6 +240,7 @@ public T Deserialize(ReadOnlySpan payload) /// Thrown when trailing bytes remain after decoding. public T Deserialize(byte[] payload) { + _registryFrozen = true; ByteReader reader = _readContext.Reader; reader.Reset(payload); T value = DeserializeFromReader(reader); diff --git a/csharp/tests/Fory.Tests/RuntimeEdgeCaseTests.cs b/csharp/tests/Fory.Tests/RuntimeEdgeCaseTests.cs index f6073d7585..0faeb5bf01 100644 --- a/csharp/tests/Fory.Tests/RuntimeEdgeCaseTests.cs +++ b/csharp/tests/Fory.Tests/RuntimeEdgeCaseTests.cs @@ -873,18 +873,19 @@ public void FailedRootFreezesRegistry() { ForyRuntime fory = ForyRuntime.Builder().Build(); - Assert.ThrowsAny(() => fory.Deserialize(Array.Empty())); + Assert.ThrowsAny(() => fory.Deserialize((byte[])null!)); Assert.Throws(() => fory.Register(713)); } [Fact] - public void FailedWriteRestoresNextRoot() + public void FailedWriteRestoresState() { ForyRuntime fory = ForyRuntime.Builder().TrackRef(true).Build(); fory.Register(718); FailingWritePayload value = new() { Value = 1 }; Assert.Throws(() => fory.Serialize(value)); + Assert.Equal(0u, WriteContextFor(fory).RefWriter.ReserveRefId()); Assert.Throws(() => fory.Register(719)); Assert.Throws(() => fory.Serialize(value)); diff --git a/docs/object-serialization/csharp/thread-safety.md b/docs/object-serialization/csharp/thread-safety.md index 8a7f105bbd..fa83b6b6c3 100644 --- a/docs/object-serialization/csharp/thread-safety.md +++ b/docs/object-serialization/csharp/thread-safety.md @@ -55,8 +55,7 @@ Parallel.For(0, 64, i => - Register every type before the first serialization or deserialization attempt. - Starting the first root permanently freezes registration, including when that root fails. -- `ThreadSafeFory` serializes registration against the first root. If the root wins a concurrent - race, registration throws `InvalidOperationException` before changing any runtime. +- Later registration throws `InvalidOperationException` before changing the registry. ## Disposal diff --git a/docs/object-serialization/csharp/type-registration.md b/docs/object-serialization/csharp/type-registration.md index 7a42403bb3..7f275c6e47 100644 --- a/docs/object-serialization/csharp/type-registration.md +++ b/docs/object-serialization/csharp/type-registration.md @@ -92,10 +92,6 @@ fory.Register(101); - Register user-defined types on both writer and reader sides. - Keep ID/name mappings consistent across services and languages. -- Within one `Fory` instance, each numeric ID or full type name belongs to one CLR type, and each - CLR type has one wire identity. Repeating the same mapping is idempotent. Reusing either side for - a different mapping fails and leaves the existing registration unchanged. Repeating an explicit - custom serializer registration must use the same concrete serializer type. - For external-type serialization, register the third-party target, such as `fory.Register(100)`, not the local serializer declaration. - Register a concrete derived class by its concrete type. Annotated abstract From 3853d220f407fea3c003fff775104224a6b97034 Mon Sep 17 00:00:00 2001 From: chaokunyang Date: Sun, 30 Aug 2026 04:04:50 +0800 Subject: [PATCH 115/168] refactor(python): keep registry freeze to one flag --- .agents/languages/python.md | 59 ++++---------- python/pyfory/registry.py | 102 ++++++++++-------------- python/pyfory/tests/test_serializer.py | 37 +-------- python/pyfory/tests/test_thread_safe.py | 12 +-- 4 files changed, 66 insertions(+), 144 deletions(-) diff --git a/.agents/languages/python.md b/.agents/languages/python.md index ee060c6e0d..e280ee9cdc 100644 --- a/.agents/languages/python.md +++ b/.agents/languages/python.md @@ -11,53 +11,26 @@ Load this file when changing `python/`, Cython serialization, or Python xlang be - Python mode is the pure-Python xlang implementation and is mainly for debugging and testing. - Cython mode is the default high-performance implementation. - Cython mode owns the hot runtime path. Do not duplicate core runtime types between Python and Cython, tunnel Python facade methods into hidden Cython internals, or keep dead shims unless the user explicitly needs a compatibility module path. -- Python `TypeResolver` separately owns permanent registry freeze, active finalization, and - successful finalization. A root may bypass that owner only after successful completion; roots - entered during finalization or after failed finalization must fail before codec work. Its Cython - companion may cache completion only after the Python owner succeeds and every native resolver - table is synchronized; the `Fory` facade must not mirror that state. Cython roots call the - resolver owner directly until then. The Python owner permanently rejects its own incomplete - finalization; if native synchronization fails, the companion records only permanent failure, - never the exception, and rejects later roots without retrying partial synchronization. - Serializer construction may reenter registration or a root, so the resolver rechecks both - registration conflicts and its frozen state after construction and before publishing type, - serializer, name, or ID state. Allocate automatic type IDs only after those checks at the common - publication point; do not reserve IDs before callbacks or maintain rollback state. - `ThreadSafeFory` validates registrations before retaining their semantic replay descriptors, and - it must not execute application factories or registrations while holding its pool lock. Its - registration linearization is reentrant so nested facade registrations share the same - publication order. A root started during registration must not reuse the staging instance, and - root reentry from a running user `fory_factory` or retained registration replay fails without - recursively building another instance. The build thread must be rejected before pool acquisition - even when another instance becomes available during that build. The non-reentrant pool lock owns - pool publication, root-started state, registration depth, and the staging instance; the separate - instance-build boundary covers the factory and complete registration replay. During child replay, a nested - facade registration is a no-op only when it exactly matches an accepted descriptor in the prefix - already applied to that child; reject every unknown or different request before that request - mutates the child. - Retained descriptors may contain a serializer class or factory, but never a resolver-bound - serializer instance. A serializer factory must return a supported serializer carrier bound to - the provided child resolver and normalized declared type; singleton serializers cannot be shared - across children. Instance-specific serializer configuration belongs in `fory_factory`, which - creates and configures each child. +- A direct Python `TypeResolver` owns one authoritative `_registry_frozen` flag. Pure Python and + Cython roots set that owner before codec work and never clear it, including after failure. + `ThreadSafeFory` owns its own `_registry_frozen` flag for the public registration boundary over + pooled children. Its existing callback list configures newly created children; it is not another + lifecycle state. +- Explicit type, serializer, name, and ID registration checks the frozen flag before mutation. + Automatic IDs remain registration-owned and must not turn native runtime discovery into explicit + registration. +- `ThreadSafeFory` accepts serializer classes or factories and constructs a serializer for each + child resolver. It must reject resolver-bound serializer instances instead of replaying one + instance across pooled children. - Registry freeze prohibits explicit type and serializer registration after the first root; it - does not prohibit policy-authorized native runtime type resolution. Non-strict native roots may - resolve module-global classes or callables and materialize resolver-owned type information or - serializer cache entries without creating or changing an explicit type, serializer, ID, name, or - policy registration. Do not describe these operations as late registration. -- In non-strict native mode, public unqualified `register_type` for a built-in native carrier uses - the same reserved type identity as pre-root discovery. Ordinary application classes and - dataclasses retain their struct registration identity. Configure both through public registration; - do not prewarm private resolver state or enumerate version-specific transitive object shapes. + does not prohibit native runtime type resolution. Non-strict native writes may discover runtime + classes or callables, and reads may resolve those authorized by the deserialization policy. Both + paths may materialize resolver-owned type information or serializer cache entries without + creating or changing an explicit type, serializer, ID, name, or policy registration. Do not + describe these operations as late registration. - Function serialization writes captured globals as a data-only exact `dict`. Keep the reader's exact-type check before sizing or merging the namespace; a dict subclass or other mapping must not introduce runtime behavior into function reconstruction. -- Python reduction list-item and dict-item iterators remain native carrier values. Register their - concrete iterator types before the first root; do not materialize them into lists in the serializer, - which changes the established carrier path and allocates storage proportional to their contents. -- Pandas `RangeIndex` owns its dtype wire slot. Encode `dtype.str` and reconstruct it with - `numpy.dtype`; do not serialize the dtype object as a reference because concrete NumPy dtype - classes vary across versions and would make the wire depend on version-specific registration. - Use explicit Cython fields and methods for fixed hot-path shapes. Avoid `__getattr__`, generic `object` fields, public bridge internals, or `Fory` backreferences where ownership can stay explicit. - Keep Python and Cython context/ref-tracking branch conditions and stack mutations semantically aligned unless a documented intentional difference exists. - Root deserialization graph memory budget state belongs to pure-Python and Cython `ReadContext`. diff --git a/python/pyfory/registry.py b/python/pyfory/registry.py index b7da5a2179..f975a62ae8 100644 --- a/python/pyfory/registry.py +++ b/python/pyfory/registry.py @@ -220,16 +220,12 @@ def _construct_serializer(serializer_factory, type_resolver, cls): for nargs, args in ( (2, (type_resolver, cls)), (1, (type_resolver,)), - (0, ()), ): if _accepts_n_positional_args(serializer_factory, nargs): serializer = serializer_factory(*args) break else: - raise TypeError( - f"Unsupported serializer constructor for {serializer_factory!r}; " - "expected `(type_resolver, cls)`, `(type_resolver)`, or `()`." - ) + raise TypeError(f"Unsupported serializer constructor for {serializer_factory!r}; expected `(type_resolver, cls)` or `(type_resolver)`.") if not isinstance(serializer, (Serializer, CythonSerializer)): raise TypeError("Serializer factory must return a Fory serializer") if serializer.type_resolver is not type_resolver: @@ -267,13 +263,13 @@ def _split_registration_name(name: str): class TypeInfo: __slots__ = ( "cls", - "type_id", - "user_type_id", - "serializer", - "namespace_bytes", - "typename_bytes", "dynamic_type", + "namespace_bytes", + "serializer", "type_def", + "type_id", + "typename_bytes", + "user_type_id", ) def __init__( @@ -311,7 +307,7 @@ def decode_typename(self) -> str: class SharedRegistry: - __slots__ = ("_metastr_to_bytes", "_encoded_metastrings") + __slots__ = ("_encoded_metastrings", "_metastr_to_bytes") def __init__(self): self._metastr_to_bytes = {} @@ -360,38 +356,38 @@ def get_or_create_encoded_meta_string(self, data: bytes, hashcode: int) -> Encod class TypeResolver: __slots__ = ( - "xlang", - "track_ref", - "strict", - "compatible", - "field_nullable", - "policy", - "config", - "shared_registry", - "_type_id_counter", - "_types_info", - "_python_name_to_type", - "_metastr_to_type", + "_actual_type_resolver", "_hash_to_type_info", - "_ns_type_to_type_info", - "_named_type_to_type_info", - "namespace_encoder", - "namespace_decoder", - "typename_encoder", - "typename_decoder", - "meta_compressor", - "require_registration", - "_type_id_to_type_info", - "_user_type_id_to_type_info", - "_used_user_type_ids", + "_internal_py_serializer_map", "_local_type_info_by_hash", "_meta_shared_type_info", + "_metastr_to_type", + "_named_type_to_type_info", + "_ns_type_to_type_info", + "_python_name_to_type", + "_registry_frozen", "_remote_schema_versions_by_type", "_total_accepted_schema_versions", + "_type_id_counter", + "_type_id_to_type_info", + "_types_info", + "_used_user_type_ids", + "_user_type_id_to_type_info", + "compatible", + "config", + "field_nullable", + "meta_compressor", "meta_share", - "_internal_py_serializer_map", - "_actual_type_resolver", - "_registry_frozen", + "namespace_decoder", + "namespace_encoder", + "policy", + "require_registration", + "shared_registry", + "strict", + "track_ref", + "typename_decoder", + "typename_encoder", + "xlang", ) def __init__(self, config, *, shared_registry): @@ -638,19 +634,15 @@ def register_union( _check_serializer_owner(serializer, self._actual_type_resolver, cls) if typename is not None and type_id is not None: raise TypeError(f"type name {typename} and id {type_id} should not be set at the same time") - auto_type_id = typename is None and type_id is None if typename is None and type_id is None: - type_id = self._type_id_counter + 1 - while type_id in self._used_user_type_ids: - type_id += 1 - assigned_type_id = type_id + type_id = self._next_type_id() if type_id not in {0, None}: user_type_id = type_id type_id = TypeId.TYPED_UNION else: user_type_id = NO_USER_TYPE_ID type_id = TypeId.NAMED_UNION - typeinfo = self.__register_type( + return self.__register_type( cls, type_id=type_id, user_type_id=user_type_id, @@ -659,9 +651,6 @@ def register_union( serializer=serializer, internal=False, ) - if auto_type_id: - self._type_id_counter = assigned_type_id - return typeinfo def _register_type( self, @@ -703,16 +692,13 @@ def _register_type( ): return self._types_info[cls] n_params = len({typename, type_id, None}) - 1 - auto_type_id = n_params == 0 and typename is None - if auto_type_id: - type_id = self._type_id_counter + 1 - while type_id in self._used_user_type_ids: - type_id += 1 + if n_params == 0 and typename is None: + type_id = self._next_type_id() if n_params == 2: raise TypeError(f"type name {typename} and id {type_id} should not be set at the same time") if cls in self._types_info: raise TypeError(f"{cls} registered already") - typeinfo = self._register_xtype( + return self._register_xtype( cls, type_id=type_id, user_type_id=user_type_id, @@ -721,9 +707,6 @@ def _register_type( serializer=serializer, internal=internal, ) - if auto_type_id: - self._type_id_counter = type_id - return typeinfo def _register_xtype( self, @@ -804,10 +787,6 @@ def __register_type( serializer = self._create_serializer(cls) if not internal: self._check_registry_mutable() - if needs_user_type_id(type_id) and user_type_id not in {None, NO_USER_TYPE_ID}: - existing = self._user_type_id_to_type_info.get(user_type_id) - if existing is not None and existing.cls is not cls: - raise TypeError(f"user_type_id {user_type_id} already registered for {existing.cls}") if serializer is not None and type_id in _NO_REF_NUMERIC_TYPE_IDS: serializer.need_to_write_ref = False @@ -827,11 +806,14 @@ def __register_type( type_metastr = self.typename_encoder.encode(typename) type_meta_bytes = self.shared_registry.get_encoded_meta_string(type_metastr) typeinfo = TypeInfo(cls, type_id, user_type_id, serializer, ns_meta_bytes, type_meta_bytes, dynamic_type) - if typename is not None: self._named_type_to_type_info[(namespace, typename)] = typeinfo self._ns_type_to_type_info[(ns_meta_bytes, type_meta_bytes)] = typeinfo self._types_info[cls] = typeinfo if type_id is not None and type_id != 0: + if needs_user_type_id(type_id) and user_type_id not in {None, NO_USER_TYPE_ID}: + existing = self._user_type_id_to_type_info.get(user_type_id) + if existing is not None and existing.cls is not cls: + raise TypeError(f"user_type_id {user_type_id} already registered for {existing.cls}") if needs_user_type_id(type_id) and user_type_id not in {None, NO_USER_TYPE_ID}: if user_type_id not in self._user_type_id_to_type_info or not internal: self._user_type_id_to_type_info[user_type_id] = typeinfo diff --git a/python/pyfory/tests/test_serializer.py b/python/pyfory/tests/test_serializer.py index 8514d2e020..fe713dff98 100644 --- a/python/pyfory/tests/test_serializer.py +++ b/python/pyfory/tests/test_serializer.py @@ -800,10 +800,6 @@ class RejectedRegistration: pass -class RegistrationAfterFailure: - pass - - @dataclass class FrozenChild: value: int @@ -953,7 +949,7 @@ def test_registry_freezes_at_root(root): lambda: fory.type_resolver.register_serializer(FrozenRegistration, object()), ) for registration in registrations: - with pytest.raises(Exception): + with pytest.raises(RuntimeError): registration() assert fory.type_resolver.get_type_info(RejectedRegistration, create=False) is None assert fory.type_resolver.get_type_info(FrozenRegistration).type_id == TypeId.STRUCT @@ -971,22 +967,9 @@ def factory(type_resolver, cls): assert fory.type_resolver.get_type_info(FrozenRegistration, create=False) is None -def test_failed_factory_keeps_type_id(): - fory = Fory(xlang=True, compatible=False) - first = fory.register_type(FrozenRegistration) - - def factory(_type_resolver, _cls): - raise RuntimeError("serializer construction failed") - - with pytest.raises(RuntimeError): - fory.register_type(RejectedRegistration, serializer=factory) - - following = fory.register_type(RegistrationAfterFailure) - assert following.user_type_id == first.user_type_id + 1 - - def test_inferred_registration_freeze(): fory = Fory(xlang=False, strict=False, compatible=False) + fory.register_type(FrozenRegistration) armed = False class RootDuringMro(type): @@ -1034,22 +1017,6 @@ class Registered(metaclass=RootDuringHash): assert typeinfo.serializer is serializer -def test_id_conflict_no_mutation(): - fory = Fory(xlang=True, compatible=False) - fory.register_type(FrozenRegistration, type_id=701) - resolver = fory.type_resolver - types_info = dict(resolver._types_info) - id_info = dict(resolver._user_type_id_to_type_info) - name_info = dict(resolver._ns_type_to_type_info) - - with pytest.raises(TypeError, match="user_type_id 701"): - fory.register_type(RejectedRegistration, type_id=701) - - assert resolver._types_info == types_info - assert resolver._user_type_id_to_type_info == id_info - assert resolver._ns_type_to_type_info == name_info - - def test_registered_types_build_lazily(): fory = Fory(xlang=True, compatible=True) parent_info = fory.register_type(FrozenParent, name="test.FrozenParent") diff --git a/python/pyfory/tests/test_thread_safe.py b/python/pyfory/tests/test_thread_safe.py index 3f2641d78b..0bb516951d 100644 --- a/python/pyfory/tests/test_thread_safe.py +++ b/python/pyfory/tests/test_thread_safe.py @@ -198,7 +198,7 @@ def test_thread_safe_fory_register_after_use(): fory.register(Address) -def test_thread_safe_serializer_factory_owns_children(): +def test_child_serializer_owners(): fory = ThreadSafeFory(xlang=False, compatible=False) fory.register(Person, serializer=PersonSerializer) @@ -215,7 +215,7 @@ def test_thread_safe_serializer_factory_owns_children(): fory._return_fory(second) -def test_thread_safe_rejects_serializer_instance(): +def test_rejects_serializer_instance(): runtime = pyfory.Fory(xlang=False, compatible=False) serializer = PersonSerializer(runtime.type_resolver, Person) fory = ThreadSafeFory(xlang=False, compatible=False) @@ -224,11 +224,11 @@ def test_thread_safe_rejects_serializer_instance(): fory.register(Person, serializer=serializer) -def test_thread_safe_rejects_foreign_factory_serializer(): +def test_rejects_foreign_factory(): runtime = pyfory.Fory(xlang=False, compatible=False) serializer = PersonSerializer(runtime.type_resolver, Person) fory = ThreadSafeFory(xlang=False, compatible=False) - fory.register(Person, serializer=lambda: serializer) + fory.register(Person, serializer=lambda _resolver: serializer) with pytest.raises(TypeError, match="another resolver"): fory._get_fory() @@ -237,11 +237,11 @@ def test_thread_safe_rejects_foreign_factory_serializer(): @pytest.mark.parametrize( "serializer_factory, error", [ - (lambda: object(), "must return a Fory serializer"), + (lambda _resolver: object(), "must return a Fory serializer"), (lambda resolver: PersonSerializer(resolver, Address), "another type"), ], ) -def test_thread_safe_validates_serializer_factory(serializer_factory, error): +def test_validates_serializer_factory(serializer_factory, error): fory = ThreadSafeFory(xlang=False, compatible=False) fory.register(Person, serializer=serializer_factory) From 753092a84a8776ee5acf5072e5355f346fa48251 Mon Sep 17 00:00:00 2001 From: chaokunyang Date: Sun, 30 Aug 2026 04:05:07 +0800 Subject: [PATCH 116/168] docs(go): keep registry freeze to existing owners --- .agents/languages/go.md | 7 ++-- docs/object-serialization/go/configuration.md | 2 +- docs/object-serialization/go/native.md | 2 +- docs/object-serialization/go/thread-safety.md | 38 ++++++++++++++++--- go/fory/registry_freeze_lifecycle_test.go | 2 +- 5 files changed, 38 insertions(+), 13 deletions(-) diff --git a/.agents/languages/go.md b/.agents/languages/go.md index 64ae88efbc..3b68aa70bf 100644 --- a/.agents/languages/go.md +++ b/.agents/languages/go.md @@ -9,11 +9,10 @@ Load this file when changing `go/fory/` or Go xlang behavior. - The Go implementation focuses on fast serializers. - A Go `Fory` instance has one authoritative registry-frozen flag. The first root serialization or deserialization sets it before codec work and leaves it set after failure. Explicit registration - checks that flag before mutation. Do not add registry finalization or failure states, alternate - identity/preflight machinery, or input normalization beyond the existing registration behavior. + checks that flag before mutation. Do not add another registry lifecycle state or alter existing + registration semantics beyond that boundary check. `threadsafe.Fory` has no registration API or facade registry: configure every pooled child in the - factory passed to `NewWithFactory` before returning it. Do not add prepared runtimes, replay logs, - facade registry state, or callback-reentry lifecycle machinery. + factory passed to `NewWithFactory` before returning it. - Go `ReadContext` intentionally defers codec errors to existing `HasError` or `CheckError` boundaries. After an error, work may continue only while it remains panic- and bounds-safe and cannot cause disproportionate work or allocation, publish state that survives root cleanup, or diff --git a/docs/object-serialization/go/configuration.md b/docs/object-serialization/go/configuration.md index 006244a916..ff1603548d 100644 --- a/docs/object-serialization/go/configuration.md +++ b/docs/object-serialization/go/configuration.md @@ -391,7 +391,7 @@ f := threadsafe.NewWithFactory(func() *fory.Fory { fory.WithXlang(true), fory.WithMaxDepth(30), ) - if err := inner.RegisterStructByName(Request{}, "example.Request"); err != nil { + if err := inner.RegisterStruct(Request{}, 1); err != nil { panic(err) } return inner diff --git a/docs/object-serialization/go/native.md b/docs/object-serialization/go/native.md index 1fb6b8e9f1..798afde830 100644 --- a/docs/object-serialization/go/native.md +++ b/docs/object-serialization/go/native.md @@ -81,7 +81,7 @@ import ( f := threadsafe.NewWithFactory(func() *fory.Fory { inner := fory.New(fory.WithXlang(false), fory.WithTrackRef(true)) - if err := inner.RegisterStructByName(Order{}, "example.Order"); err != nil { + if err := inner.RegisterStruct(Order{}, 100); err != nil { panic(err) } return inner diff --git a/docs/object-serialization/go/thread-safety.md b/docs/object-serialization/go/thread-safety.md index 36bad27a6e..ae58210a02 100644 --- a/docs/object-serialization/go/thread-safety.md +++ b/docs/object-serialization/go/thread-safety.md @@ -66,6 +66,32 @@ go func() { }() ``` +### How It Works + +The thread-safe wrapper uses `sync.Pool`: + +1. **Acquire**: Gets a Fory instance from the pool +2. **Use**: Performs serialization/deserialization +3. **Copy**: Copies result data because the buffer will be reused +4. **Release**: Returns the instance to the pool + +```go +// Simplified implementation +func (f *Fory) Serialize(v any) ([]byte, error) { + inner := f.pool.Get().(*fory.Fory) + defer f.pool.Put(inner) + + data, err := inner.Serialize(v) + if err != nil { + return nil, err + } + + result := make([]byte, len(data)) + copy(result, data) + return result, nil +} +``` + ### API ```go @@ -93,10 +119,10 @@ returning it to the pool: ```go f := threadsafe.NewWithFactory(func() *fory.Fory { inner := fory.New(fory.WithXlang(true)) - if err := inner.RegisterStructByName(User{}, "example.User"); err != nil { + if err := inner.RegisterStruct(User{}, 1); err != nil { panic(err) } - if err := inner.RegisterStructByName(Order{}, "example.Order"); err != nil { + if err := inner.RegisterStruct(Order{}, 2); err != nil { panic(err) } return inner @@ -117,7 +143,7 @@ when the root fails: ```go inner := fory.New(fory.WithXlang(true)) -if err := inner.RegisterStructByName(User{}, "example.User"); err != nil { +if err := inner.RegisterStruct(User{}, 1); err != nil { panic(err) } @@ -184,7 +210,7 @@ func BenchmarkNonThreadSafe(b *testing.B) { func BenchmarkThreadSafe(b *testing.B) { f := threadsafe.NewWithFactory(func() *fory.Fory { inner := fory.New(fory.WithXlang(true)) - if err := inner.RegisterStructByName(User{}, "example.User"); err != nil { + if err := inner.RegisterStruct(User{}, 1); err != nil { panic(err) } return inner @@ -231,7 +257,7 @@ For dynamic goroutine count or simplicity: ```go var f = threadsafe.NewWithFactory(func() *fory.Fory { inner := fory.New(fory.WithXlang(true)) - if err := inner.RegisterStructByName(User{}, "example.User"); err != nil { + if err := inner.RegisterStruct(User{}, 1); err != nil { panic(err) } return inner @@ -249,7 +275,7 @@ func handleRequest(user *User) []byte { ```go var serializer = threadsafe.NewWithFactory(func() *fory.Fory { inner := fory.New(fory.WithXlang(true)) - if err := inner.RegisterStructByName(Response{}, "example.Response"); err != nil { + if err := inner.RegisterStruct(Response{}, 1); err != nil { panic(err) } return inner diff --git a/go/fory/registry_freeze_lifecycle_test.go b/go/fory/registry_freeze_lifecycle_test.go index 37d0997197..295b669958 100644 --- a/go/fory/registry_freeze_lifecycle_test.go +++ b/go/fory/registry_freeze_lifecycle_test.go @@ -106,7 +106,7 @@ func TestRegistryFreezeRegistrations(t *testing.T) { } } -func TestFrozenRegistryAllowsLazySerializer(t *testing.T) { +func TestFrozenAllowsLazySerializer(t *testing.T) { f := New(WithXlang(false), WithCompatible(false)) _, err := f.Serialize(int32(1)) require.NoError(t, err) From 43873ae7dae13bf828ea130f1895d8faedc18ba8 Mon Sep 17 00:00:00 2001 From: chaokunyang Date: Sun, 30 Aug 2026 04:05:49 +0800 Subject: [PATCH 117/168] refactor(swift): keep lazy metadata completion minimal --- .agents/languages/swift.md | 10 +++------- docs/object-serialization/swift/polymorphism.md | 5 ----- swift/Sources/Fory/TypeResolver.swift | 13 +++---------- 3 files changed, 6 insertions(+), 22 deletions(-) diff --git a/.agents/languages/swift.md b/.agents/languages/swift.md index ffdf8f6d46..f06a4750f9 100644 --- a/.agents/languages/swift.md +++ b/.agents/languages/swift.md @@ -37,14 +37,10 @@ Load this file when changing `swift/` or Swift xlang behavior. ignored declaration fields are budget-only and must not enter target access, construction, metadata, or wire code. Omitted large value storage must be declared explicitly and ignored. -- `@ForyStruct` supports protocol conformances but registration rejects every superclass because - macros cannot inspect inherited storage. SwiftSyntax represents both in one inheritance clause, - so keep the minimal `_getSuperclass` check in the existing TypeResolver registration preflight and - out of root hot paths. - Swift registry lifecycle uses one authoritative frozen flag set by the first root serialization or - deserialization. Do not add finalizing, finalized, or failed states or cache a registration - preparation failure as a second lifecycle state. Registered TypeInfo owns lazy TypeMeta completion - after freeze; do not restore an eager whole-registry metadata pass. + deserialization. Do not add another lifecycle state or cache freeze failure separately. Registered + TypeInfo owns lazy TypeMeta completion after freeze; do not add an eager whole-registry metadata + pass. - Direct `Any` and `AnyObject` root overloads remain disfavored forwarding facades over `DynamicSerializer` and `DynamicSerializer`, including their Data-buffer forms. Arbitrary protocol roots explicitly select `DynamicSerializer`. Do not add an unconstrained diff --git a/docs/object-serialization/swift/polymorphism.md b/docs/object-serialization/swift/polymorphism.md index 1350f628be..f862dfd399 100644 --- a/docs/object-serialization/swift/polymorphism.md +++ b/docs/object-serialization/swift/polymorphism.md @@ -169,11 +169,6 @@ var animal: AnimalBase Register every concrete subclass that may appear. -`@ForyStruct` cannot be applied to a class with any superclass because Swift -macros cannot inspect inherited storage. Protocol conformances remain supported. -Use a custom serializer for each concrete subclass, then select it through the -dynamic field serializer. - ## Dynamic `Any` Fields ```swift diff --git a/swift/Sources/Fory/TypeResolver.swift b/swift/Sources/Fory/TypeResolver.swift index b11287cd43..6e9bcecb3a 100644 --- a/swift/Sources/Fory/TypeResolver.swift +++ b/swift/Sources/Fory/TypeResolver.swift @@ -1277,16 +1277,9 @@ final class TypeResolver { throw ForyError.invalidData( "structural serializer \(type) must use STRUCT, ENUM, or UNION type identity") } - if let targetClass = T.Target.self as? AnyClass { - if !T.isRefType { - throw ForyError.invalidData( - "value structural serializer \(type) cannot target class type \(T.Target.self)") - } - if _getSuperclass(targetClass) != nil { - throw ForyError.invalidData( - "@ForyStruct classes cannot inherit from a superclass because macros cannot inspect inherited storage" - ) - } + if !T.isRefType, T.Target.self is AnyObject.Type { + throw ForyError.invalidData( + "value structural serializer \(type) cannot target class type \(T.Target.self)") } return } From 7d37c0c1531170d318743af943b0e7c5f607f40f Mon Sep 17 00:00:00 2001 From: chaokunyang Date: Sun, 30 Aug 2026 04:08:09 +0800 Subject: [PATCH 118/168] fix(jvm): preserve facade serializer ownership --- .agents/languages/java.md | 49 +++++------- .agents/languages/kotlin.md | 25 ++---- .agents/languages/scala.md | 27 ++----- docs/compiler/generated-code/kotlin.md | 3 - docs/compiler/generated-code/scala.md | 10 +-- .../java/custom-serializers.md | 5 ++ .../java/type-registration.md | 15 ++-- .../kotlin/configuration.md | 6 +- .../kotlin/static-generated-serializers.md | 7 +- .../scala/configuration.md | 6 +- .../apache/fory/AbstractThreadSafeFory.java | 13 +++ .../main/java/org/apache/fory/BaseFory.java | 21 ++++- .../java/org/apache/fory/ThreadSafeFory.java | 4 + .../org/apache/fory/ThreadSafeForyTest.java | 79 +++++++++++++++++++ .../serializer/kotlin/KotlinSerializers.java | 12 +-- .../serializer/scala/ScalaSerializers.java | 3 - .../apache/fory/scala/ForySerializer.scala | 2 +- 17 files changed, 177 insertions(+), 110 deletions(-) diff --git a/.agents/languages/java.md b/.agents/languages/java.md index cbd910011c..9289206b3e 100644 --- a/.agents/languages/java.md +++ b/.agents/languages/java.md @@ -84,37 +84,24 @@ Load this file when changing anything under `java/` or when Java drives a cross- work, dynamic stream bytes-read accounting, or stale narrower-scope formulas. - Generated serializers must not retain runtime context fields. `Fory` should stay a root-operation facade rather than accumulating serializer or convenience state. - When the serializer class and constructor shape are known at the call site, prefer direct constructor lambdas or direct instantiation over reflective `Serializers.newSerializer(...)`. -- `FacadeRegistrationGate` owns registration linearization for Java thread-local and pooled - facades. Starting a root closes registration before child or pool access, then finishes every - already-created child before exposing that root. A child created after closure must replay every - accepted registration, finish registration, and only then become visible; a child whose replay - or finalization fails must never be published. A callback registration is one facade transaction across - all children: reject root or registration reentry while it is active and permanently fail the - facade after any callback failure, rather than expose partial child mutation, divergent replay - order, or rollback state. Keep the lock order gate before pool or child storage. -- Registration callbacks must recheck the authoritative freeze owner after returning and before - publishing the entry they prepared. `TypeResolver` owns one construction-local graph for Java - serializer constructors, including self and mutual recursion. The graph separates final - `TypeInfo` owners from unpublished serializer candidates: recursive fields capture the final - owner immediately, while construction owners resolving recursive fields or candidate state use - the construction-local serializer. Ordinary resolver lookups retain their runtime semantics. - When wire and user IDs match, the final owner is the existing canonical `TypeInfo`. After - construction and the lifecycle recheck succeed, the Class/Xtype resolver's normal commit sink - installs the candidate. Static-generated construction starts from an already registered canonical - type, retains that type's identity, and exposes immutable generated descriptors only through the - same construction graph; it must not publish its early-bound serializer candidate. Do not add a - constructor-specific publication path. Reject static-generated serializer classes from the - combined class overload because their construction requires prior canonical type registration. - `Fory.register(ForyModule)` owns module identity, - cycle breaking, and idempotence in one identity set: add the identity before the callback, remove - it on failure, and retain it on success. Do not add separate installing/completed module states. - Direct `Fory` accepts modules before its first root; thread-safe facades accept modules only - through `ForyBuilder.withModule` before construction. -- Every explicit resolver registration or initialization entry must call the authoritative - registration gate as its first executable statement, before argument validation, class loading, - no-op return, callback invocation, or publication. Serializer completion methods used by lazy, - JIT, and generated serializers are internal resolver-owned operations, remain valid after - registration freezes, and must not be treated or repurposed as registration APIs. +- Each natural Java registry or public facade boundary owns one authoritative frozen flag. The + facade flag is not a mirror of a child resolver flag. The first root serialization or + deserialization sets the owning flag before codec work and never clears it, including after + failure. Every explicit type, serializer, module, name, or ID registration checks that flag + before mutation. Do not add another lifecycle state or a parallel registration-commit path. +- Direct and thread-safe facades expose module registration before their first root. Kotlin and + Scala registration extensions target `BaseFory`; do not narrow them to concrete `Fory` or make + builder installation the only thread-safe path. Before the first root, a thread-safe facade + serializes registration, `execute`, and copy through its existing callback monitor and rechecks + the frozen flag after entering it. After freeze, `execute` and copy use the monitor-free path. + Calling `ThreadSafeFory.execute` or copying a value does not freeze registration unless the + callback starts a root serialization or deserialization. +- A serializer instance registered on a thread-safe facade must implement `Shareable`. Resolver- + local serializers use the class, resolver-factory, or module path so every child runtime owns its + instance; never replay one resolver-bound serializer across children. +- Serializer completion used by lazy, JIT, and generated serializers is an internal resolver-owned + operation. It remains valid after registration freezes and must not be treated as explicit + registration. - Registration freeze does not disable native runtime type resolution. When class registration is not required, native roots may discover an unregistered runtime class and materialize its resolver-owned `TypeInfo`, descriptor, serializer, or JIT cache entry after freeze. This runtime diff --git a/.agents/languages/kotlin.md b/.agents/languages/kotlin.md index ebb299ba1a..36cb66ff2f 100644 --- a/.agents/languages/kotlin.md +++ b/.agents/languages/kotlin.md @@ -14,23 +14,14 @@ Load this file when changing `kotlin/` or compiler code that generates Kotlin so Fory. Do not auto-install a new serializer for an existing type-registered Kotlin class unless the wire format matches the previous serializer family and old-payload/new-runtime compatibility is tested. -- Public registration helpers must check the registry freeze before constructing a serializer, - enum serializer, or union serializer. Generated serializer construction must enter the existing - `TypeResolver` construction graph so its candidate remains unpublished until the authoritative - lifecycle recheck and normal resolver commit. -- Combined generated-struct registration must publish the canonical type before constructing its - serializer because generated construction resolves the canonical `TypeInfo`. The serializer-only - helper must reject a missing canonical type rather than auto-register it. Do not move construction - before type registration or add direct replacement, rollback, staging, or a parallel registration - path. -- `Fory.register(ForyModule)` is the only owner of bootstrap identity, cycle breaking, and - idempotence. Kotlin bootstrap code must not add a marker, monitor, or separate reentry policy. - Keep the install body replay-safe until its final non-repeatable publication; publish the single - global Kotlin default-value support owner only after all per-runtime registrations succeed, and - never replace its class-value cache for each runtime. -- Install modules for thread-safe facades through `ForyBuilder.withModule` before building them. - Runtime registration extensions target concrete `Fory` instances and must not recreate a - thread-safe module-registration wrapper. +- Kotlin registration extensions target `BaseFory` so direct and thread-safe facades share the same + pre-root registration API, including module installation. +- Explicit type, serializer, enum, and union registration checks the receiving `BaseFory` facade or + natural registry owner's one frozen flag before mutation. Keep generated serializer construction + on the existing direct resolver path; do not add a parallel registration path or lifecycle state. +- Combined generated structural registration attaches the serializer with `setSerializer` after + registering the canonical STRUCT `TypeInfo`; `registerSerializer` would incorrectly reclassify + that wire identity as EXT. Generated unions use `registerUnion`. - When adding Kotlin gRPC service companions, emit Kotlin source only. Reuse the generated schema module's `ThreadSafeFory` and KSP-generated schema serializers, and keep grpc-java/grpc-kotlin dependencies application-owned instead of adding them as hard `fory-kotlin` dependencies. diff --git a/.agents/languages/scala.md b/.agents/languages/scala.md index 5a4fce815d..afeda059b1 100644 --- a/.agents/languages/scala.md +++ b/.agents/languages/scala.md @@ -9,25 +9,14 @@ Load this file when changing `scala/`. - Scala supports the JVM and GraalVM Native Image, not Android. Do not add Android-specific Scala sources, tests, resources, R8 metadata, compiler plugins, macros, dependencies, or compatibility design. -- Public registration helpers must check the registry freeze before invoking generated serializer - construction or enum discovery. Generated serializer construction must enter the existing - `TypeResolver` construction graph so its candidate remains unpublished until the authoritative - lifecycle recheck and normal resolver commit. Scala enum registration must recheck after - companion-driven value discovery and reuse the values already owned by the serializer. -- Combined generated-struct registration must publish the canonical type before constructing its - serializer because generated construction resolves the canonical `TypeInfo`. The serializer-only - helper must reject a missing canonical type rather than auto-register it. Do not move construction - before type registration or add direct replacement, rollback, staging, or a parallel registration - path. - Union construction is the exception because it does not require canonical registration: finish - its serializer-owned callbacks and recheck the freeze before publishing the union type. -- `Fory.register(ForyModule)` is the only owner of bootstrap identity, cycle breaking, and - idempotence. Scala bootstrap code must not add a marker, monitor, or separate reentry policy. - Keep the install body replay-safe and append its serializer factory only after all repeatable - per-runtime registrations succeed. -- Install modules for thread-safe facades through `ForyBuilder.withModule` before building them. - Runtime registration extensions target concrete `Fory` instances and must not recreate a - thread-safe module-registration wrapper. +- Scala registration extensions target `BaseFory` so direct and thread-safe facades share the same + pre-root registration API, including module installation. +- Explicit type, serializer, enum, and union registration checks the receiving `BaseFory` facade or + natural registry owner's one frozen flag before mutation. Keep generated serializer construction + on the existing direct resolver path; do not add a parallel registration path or lifecycle state. +- Combined generated structural registration attaches the serializer with `setSerializer` after + registering the canonical STRUCT `TypeInfo`; `registerSerializer` would incorrectly reclassify + that wire identity as EXT. Generated unions use `registerUnion`. ## Commands diff --git a/docs/compiler/generated-code/kotlin.md b/docs/compiler/generated-code/kotlin.md index 5ff697d8ff..910ddbf007 100644 --- a/docs/compiler/generated-code/kotlin.md +++ b/docs/compiler/generated-code/kotlin.md @@ -153,9 +153,6 @@ public object AddressbookForyModule : ForyModule { } ``` -Generated modules register all message types before resolving their generated serializers. This -lets circular message schemas use the same module without application-managed registration order. - `registerUnion` discovers the generated `_ForySerializer`; callers do not pass a serializer instance. diff --git a/docs/compiler/generated-code/scala.md b/docs/compiler/generated-code/scala.md index 4c22e711f7..aca455deaf 100644 --- a/docs/compiler/generated-code/scala.md +++ b/docs/compiler/generated-code/scala.md @@ -151,20 +151,14 @@ object AddressbookForyModule extends org.apache.fory.ForyModule { private[addressbook] def getFory: ThreadSafeFory = fory override def install(fory: Fory): Unit = { - ForySerializer.registerType(fory, classOf[Person.PhoneNumber], 102L) - ForySerializer.registerType(fory, classOf[Person], 100L) - ScalaSerializers.registerEnum(fory, classOf[Person.PhoneType], 101L) - ForySerializer.registerSerializer(fory, classOf[Person.PhoneNumber]) - ForySerializer.registerSerializer(fory, classOf[Person]) + ForySerializer.register(fory, classOf[Person.PhoneNumber], 102L) + ForySerializer.register(fory, classOf[Person], 100L) ForySerializer.register(fory, classOf[Animal], 106L) } } ``` -Generated modules register all message types before resolving their generated serializers. This -lets circular message schemas use the same module without application-managed registration order. - ## gRPC Service Companions With `--grpc`, Scala emits one `Grpc.scala` object per local service in the generated models' package. It exposes `SERVICE_NAME`, service and method descriptors, `ImplBase`, and `Client`. See [Scala gRPC](../../grpc/scala.md) for `RpcFuture`, `RpcIterator`, grpc-java variants, and lifecycle guidance. diff --git a/docs/object-serialization/java/custom-serializers.md b/docs/object-serialization/java/custom-serializers.md index efebb98979..1cd483cf9d 100644 --- a/docs/object-serialization/java/custom-serializers.md +++ b/docs/object-serialization/java/custom-serializers.md @@ -212,6 +212,11 @@ fory.registerSerializer( CustomMap.class, resolver -> new CustomMapSerializer<>(resolver, CustomMap.class)); ``` +For `ThreadSafeFory`, pass a serializer class or resolver factory when the serializer is +runtime-local. The facade constructs one instance for each underlying runtime. An instance may be +registered directly only when it implements `Shareable`. Construct a runtime-local union serializer +inside a `ForyModule`, which is installed separately into every underlying runtime. + ## Shareability Implement the `Shareable` marker interface when the serializer can be safely reused across diff --git a/docs/object-serialization/java/type-registration.md b/docs/object-serialization/java/type-registration.md index 46974d4e54..13402529c7 100644 --- a/docs/object-serialization/java/type-registration.md +++ b/docs/object-serialization/java/type-registration.md @@ -44,11 +44,14 @@ Automatically assigned IDs depend on registration order, so readers and writers same classes in the same order. With explicit IDs, the order may differ, but each ID must map to the same class on both sides. -Complete class and serializer registration before the first `serialize`, `deserialize`, or `copy` -call. Starting one of these operations permanently freezes registration even if the operation -fails. Calling `ThreadSafeFory#execute` also freezes registration before the callback runs. The -`Fory` instance passed to that callback is already frozen, including when the callback retains it. -Later registration attempts are rejected. +Complete explicit class and serializer registration before the first `serialize` or `deserialize` +call. Starting either operation permanently freezes registration even if the operation fails. Copy +operations and `ThreadSafeFory#execute` do not freeze registration unless the supplied callback +starts serialization or deserialization. Later explicit registration attempts are rejected. + +In native mode with registration disabled, Fory may still resolve allowed runtime classes and cache +their descriptors or serializers after this boundary. That lazy resolution is not explicit +registration and does not reopen or change the configured registry. `registerSerializer(Foo.class, ...)` is sufficient to use `Foo` when class registration is enabled. Use `registerSerializerAndType(Foo.class, ...)` when you also want Fory to assign a numeric type ID. @@ -100,7 +103,7 @@ Fory fory = Fory.builder().withXlang(false) `STRICT` rejects every class outside the allow list. `WARN` rejects disallowed classes and logs a warning for classes outside the allow list. `DISABLE` skips allow-list checking. -Configure disallow rules before the first `serialize`, `deserialize`, or `copy` call. To use +Configure disallow rules before the first `serialize` or `deserialize` call. To use different disallow rules later, create a new Fory instance. ## Limit Max Deserialization Depth diff --git a/docs/object-serialization/kotlin/configuration.md b/docs/object-serialization/kotlin/configuration.md index 42eb964b81..6a01f5b45e 100644 --- a/docs/object-serialization/kotlin/configuration.md +++ b/docs/object-serialization/kotlin/configuration.md @@ -83,9 +83,9 @@ object ForyHolder { } ``` -Install `ForyModule` instances with `withModule(...)` before calling -`buildThreadSafeFory()`. Runtime module registration and the Kotlin reified registration extension -are available only on a direct `Fory` instance. +`ForyModule` registration and Kotlin reified registration extensions target `BaseFory`, so they are +available on both direct and thread-safe facades. Complete registration before the facade's first +root serialization or deserialization. ### Using Builder Methods diff --git a/docs/object-serialization/kotlin/static-generated-serializers.md b/docs/object-serialization/kotlin/static-generated-serializers.md index bd3a21ccef..1f4a815e14 100644 --- a/docs/object-serialization/kotlin/static-generated-serializers.md +++ b/docs/object-serialization/kotlin/static-generated-serializers.md @@ -273,10 +273,9 @@ fory.register("example.User") `ForyKotlin.builder()` installs the Kotlin serializer bootstrap for the Fory instance. The `fory.register(...)` extension registers your xlang schema type -name and resolves the generated serializer from the target class. This extension -targets a direct `Fory` instance. For a `ThreadSafeFory`, put generated type -registrations in a `ForyModule` and pass it to `withModule(...)` before building -the facade. +name and resolves the generated serializer from the target class. The extension +targets `BaseFory`, so it works with direct and thread-safe facades before their +first root serialization or deserialization. Do not register or reference generated serializer classes in application code. Fory resolves them from the registered target class. diff --git a/docs/object-serialization/scala/configuration.md b/docs/object-serialization/scala/configuration.md index f2b7349b6f..4d6ab03549 100644 --- a/docs/object-serialization/scala/configuration.md +++ b/docs/object-serialization/scala/configuration.md @@ -120,9 +120,9 @@ object ForyHolder { } ``` -Install `ForyModule` instances with `withModule(...)` before calling -`buildThreadSafeFory()`. Runtime module registration and Scala generated-serializer registration -extensions are available only on a direct `Fory` instance. +`ForyModule` registration and Scala generated-serializer registration extensions target +`BaseFory`, so they are available on both direct and thread-safe facades. Complete registration +before the facade's first root serialization or deserialization. ## Configuration diff --git a/java/fory-core/src/main/java/org/apache/fory/AbstractThreadSafeFory.java b/java/fory-core/src/main/java/org/apache/fory/AbstractThreadSafeFory.java index 0098854119..bd7437ab79 100644 --- a/java/fory-core/src/main/java/org/apache/fory/AbstractThreadSafeFory.java +++ b/java/fory-core/src/main/java/org/apache/fory/AbstractThreadSafeFory.java @@ -24,6 +24,7 @@ import org.apache.fory.resolver.TypeResolver; import org.apache.fory.serializer.Serializer; import org.apache.fory.serializer.SerializerFactory; +import org.apache.fory.serializer.Shareable; public abstract class AbstractThreadSafeFory implements ThreadSafeFory { @Override @@ -73,12 +74,14 @@ public void register(ForyModule module) { public void registerUnion( Class cls, int id, org.apache.fory.serializer.Serializer serializer) { + checkShareable(serializer); registerCallback(fory -> fory.registerUnion(cls, id, serializer)); } @Override public void registerUnion( Class cls, String name, org.apache.fory.serializer.Serializer serializer) { + checkShareable(serializer); registerCallback(fory -> fory.registerUnion(cls, name, serializer)); } @@ -87,6 +90,7 @@ public void registerUnion( String namespace, String typeName, org.apache.fory.serializer.Serializer serializer) { + checkShareable(serializer); registerCallback(fory -> fory.registerUnion(cls, namespace, typeName, serializer)); } @@ -97,6 +101,7 @@ public void registerSerializer(Class type, Class se @Override public void registerSerializer(Class type, Serializer serializer) { + checkShareable(serializer); registerCallback(fory -> fory.registerSerializer(type, serializer)); } @@ -114,6 +119,7 @@ public void registerSerializerAndType( @Override public void registerSerializerAndType(Class type, Serializer serializer) { + checkShareable(serializer); registerCallback(fory -> fory.registerSerializerAndType(type, serializer)); } @@ -141,4 +147,11 @@ public void ensureSerializersCompiled() { return null; }); } + + private static void checkShareable(Serializer serializer) { + if (!(serializer instanceof Shareable)) { + throw new IllegalArgumentException( + "Thread-safe Fory requires serializer instances to implement Shareable"); + } + } } diff --git a/java/fory-core/src/main/java/org/apache/fory/BaseFory.java b/java/fory-core/src/main/java/org/apache/fory/BaseFory.java index cc3bfd024b..c52f0dcf53 100644 --- a/java/fory-core/src/main/java/org/apache/fory/BaseFory.java +++ b/java/fory-core/src/main/java/org/apache/fory/BaseFory.java @@ -95,15 +95,24 @@ public interface BaseFory { */ void register(ForyModule module); + /** + * Register a union by ID. A serializer instance passed to {@link ThreadSafeFory} must implement + * {@link org.apache.fory.serializer.Shareable}. + */ void registerUnion(Class cls, int id, Serializer serializer); /** * Register a union with a name used for cross-language serialization. Names with `.` are split by - * the last `.` into namespace and type name. + * the last `.` into namespace and type name. A serializer instance passed to {@link + * ThreadSafeFory} must implement {@link org.apache.fory.serializer.Shareable}. */ void registerUnion(Class cls, String name, Serializer serializer); - /** Register a union with explicit namespace and type name. The type name must not contain `.`. */ + /** + * Register a union with explicit namespace and type name. The type name must not contain `.`. A + * serializer instance passed to {@link ThreadSafeFory} must implement {@link + * org.apache.fory.serializer.Shareable}. + */ void registerUnion(Class cls, String namespace, String typeName, Serializer serializer); /** @@ -125,6 +134,10 @@ public interface BaseFory { * *

NOTE: The registration order is important. If registration order is inconsistent, the * allocated ID will be different, and the deserialization will failed !!! + * + *

A serializer instance passed to {@link ThreadSafeFory} must implement {@link + * org.apache.fory.serializer.Shareable}. Use the class or resolver-factory overload for a + * runtime-local serializer. */ void registerSerializer(Class type, Serializer serializer); @@ -157,6 +170,10 @@ public interface BaseFory { * *

NOTE: The registration order is important. If registration order is inconsistent, the * allocated ID will be different, and the deserialization will failed !!! + * + *

A serializer instance passed to {@link ThreadSafeFory} must implement {@link + * org.apache.fory.serializer.Shareable}. Use the class or resolver-factory overload for a + * runtime-local serializer. */ void registerSerializerAndType(Class type, Serializer serializer); diff --git a/java/fory-core/src/main/java/org/apache/fory/ThreadSafeFory.java b/java/fory-core/src/main/java/org/apache/fory/ThreadSafeFory.java index 1f2c1f3f20..22f4da33a3 100644 --- a/java/fory-core/src/main/java/org/apache/fory/ThreadSafeFory.java +++ b/java/fory-core/src/main/java/org/apache/fory/ThreadSafeFory.java @@ -30,6 +30,10 @@ * *

The runtime class loader is fixed when the thread-safe serializer is built. If you need a * different class loader, build a different {@link ThreadSafeFory} instance. + * + *

Serializer instances registered on this facade must implement {@link + * org.apache.fory.serializer.Shareable}. Use a serializer class or resolver factory for + * runtime-local serializers so every underlying {@link Fory} owns its instance. */ public interface ThreadSafeFory extends BaseFory { diff --git a/java/fory-core/src/test/java/org/apache/fory/ThreadSafeForyTest.java b/java/fory-core/src/test/java/org/apache/fory/ThreadSafeForyTest.java index 39584eef84..2d43f68af7 100644 --- a/java/fory-core/src/test/java/org/apache/fory/ThreadSafeForyTest.java +++ b/java/fory-core/src/test/java/org/apache/fory/ThreadSafeForyTest.java @@ -21,11 +21,13 @@ import static org.testng.Assert.assertEquals; import static org.testng.Assert.assertNotNull; +import static org.testng.Assert.assertNotSame; import static org.testng.Assert.assertNull; import static org.testng.Assert.assertSame; import static org.testng.Assert.assertTrue; import java.nio.ByteBuffer; +import java.util.List; import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.CountDownLatch; import java.util.concurrent.ExecutorService; @@ -33,6 +35,7 @@ import java.util.concurrent.TimeUnit; import java.util.concurrent.atomic.AtomicInteger; import java.util.concurrent.atomic.AtomicReference; +import java.util.function.Consumer; import lombok.Data; import org.apache.fory.context.CopyContext; import org.apache.fory.context.MetaReadContext; @@ -45,6 +48,7 @@ import org.apache.fory.resolver.SharedRegistry; import org.apache.fory.resolver.TypeResolver; import org.apache.fory.serializer.Serializer; +import org.apache.fory.serializer.Shareable; import org.apache.fory.test.bean.BeanA; import org.apache.fory.test.bean.BeanB; import org.testng.Assert; @@ -505,8 +509,29 @@ static class Foo { } public static class FooSerializer extends Serializer { + final TypeResolver typeResolver; + public FooSerializer(TypeResolver typeResolver, Class type) { super(typeResolver.getConfig(), type); + this.typeResolver = typeResolver; + } + + @Override + public void write(WriteContext writeContext, Foo value) { + writeContext.getBuffer().writeInt32(value.f1); + } + + @Override + public Foo read(ReadContext readContext) { + Foo foo = new Foo(); + foo.f1 = readContext.getBuffer().readInt32(); + return foo; + } + } + + public static final class ShareableFooSerializer extends Serializer implements Shareable { + public ShareableFooSerializer(TypeResolver typeResolver) { + super(typeResolver.getConfig(), Foo.class); } @Override @@ -612,6 +637,60 @@ public void testSerializerRegister() { }); } + @Test + public void testSerializerInstanceOwnership() throws InterruptedException { + Fory direct = + Fory.builder() + .withXlang(false) + .requireClassRegistration(false) + .withCompatible(false) + .build(); + FooSerializer local = new FooSerializer(direct.getTypeResolver(), Foo.class); + List> registrations = + List.of( + fory -> fory.registerSerializer(Foo.class, local), + fory -> fory.registerSerializerAndType(Foo.class, local), + fory -> fory.registerUnion(Foo.class, 101, local), + fory -> fory.registerUnion(Foo.class, "test.Foo", local), + fory -> fory.registerUnion(Foo.class, "test", "Foo", local)); + for (Consumer registration : registrations) { + ThreadSafeFory fory = newThreadSafeRuntimes()[0]; + Assert.assertThrows(IllegalArgumentException.class, () -> registration.accept(fory)); + } + + ShareableFooSerializer shareable = new ShareableFooSerializer(direct.getTypeResolver()); + ThreadSafeFory shared = newThreadSafeRuntimes()[0]; + shared.registerSerializer(Foo.class, shareable); + assertSame(shared.execute(fory -> fory.getTypeResolver().getSerializer(Foo.class)), shareable); + + ThreadSafeFory localFactory = newThreadSafeRuntimes()[0]; + localFactory.registerSerializer(Foo.class, FooSerializer.class); + AtomicReference first = new AtomicReference<>(); + AtomicReference second = new AtomicReference<>(); + AtomicReference error = new AtomicReference<>(); + Thread firstThread = new Thread(() -> captureSerializer(localFactory, first, error)); + Thread secondThread = new Thread(() -> captureSerializer(localFactory, second, error)); + firstThread.start(); + secondThread.start(); + firstThread.join(); + secondThread.join(); + assertNull(error.get()); + assertNotSame(first.get(), second.get()); + assertNotSame(first.get().typeResolver, second.get().typeResolver); + } + + private static void captureSerializer( + ThreadSafeFory fory, + AtomicReference serializer, + AtomicReference error) { + try { + serializer.set( + fory.execute(child -> (FooSerializer) child.getTypeResolver().getSerializer(Foo.class))); + } catch (Throwable t) { + error.compareAndSet(null, t); + } + } + @Test public void testRegisterAfterSerializeThrows() { ThreadSafeFory fory = diff --git a/kotlin/fory-kotlin/src/main/java/org/apache/fory/serializer/kotlin/KotlinSerializers.java b/kotlin/fory-kotlin/src/main/java/org/apache/fory/serializer/kotlin/KotlinSerializers.java index c6d93fa31f..204d32f448 100644 --- a/kotlin/fory-kotlin/src/main/java/org/apache/fory/serializer/kotlin/KotlinSerializers.java +++ b/kotlin/fory-kotlin/src/main/java/org/apache/fory/serializer/kotlin/KotlinSerializers.java @@ -209,9 +209,10 @@ public static void registerType(Fory fory, Class cls, String namespace, Strin fory.getTypeResolver().register(cls, namespace, typeName); } + // Combined generated registration preserves the STRUCT TypeInfo created first; + // registerSerializer would reclassify that wire identity as EXT. public static void register(Fory fory, Class cls) { TypeResolver resolver = fory.getTypeResolver(); - resolver.checkRegistrationOpen(); resolver.register(cls); Serializer serializer = newGeneratedSerializer(resolver, cls); resolver.checkRegistrationOpen(); @@ -220,7 +221,6 @@ public static void register(Fory fory, Class cls) { public static void register(Fory fory, Class cls, long typeId) { TypeResolver resolver = fory.getTypeResolver(); - resolver.checkRegistrationOpen(); resolver.register(cls, typeId); Serializer serializer = newGeneratedSerializer(resolver, cls); resolver.checkRegistrationOpen(); @@ -229,7 +229,6 @@ public static void register(Fory fory, Class cls, long typeId) { public static void register(Fory fory, Class cls, String name) { TypeResolver resolver = fory.getTypeResolver(); - resolver.checkRegistrationOpen(); fory.register(cls, name); Serializer serializer = newGeneratedSerializer(resolver, cls); resolver.checkRegistrationOpen(); @@ -239,7 +238,6 @@ public static void register(Fory fory, Class cls, String name) { public static void register(Fory fory, Class cls, String namespace, String typeName) { checkTypeName(typeName); TypeResolver resolver = fory.getTypeResolver(); - resolver.checkRegistrationOpen(); resolver.register(cls, namespace, typeName); Serializer serializer = newGeneratedSerializer(resolver, cls); resolver.checkRegistrationOpen(); @@ -258,7 +256,6 @@ public static void registerEnum(Fory fory, Class cls, long typeId) { TypeResolver resolver = fory.getTypeResolver(); resolver.checkRegistrationOpen(); Serializer serializer = new EnumSerializer(resolver.getConfig(), enumClass(cls)); - resolver.checkRegistrationOpen(); resolver.registerEnum(cls, typeId, serializer); } @@ -267,7 +264,6 @@ public static void registerEnum(Fory fory, Class cls, String namespace, Strin TypeResolver resolver = fory.getTypeResolver(); resolver.checkRegistrationOpen(); Serializer serializer = new EnumSerializer(resolver.getConfig(), enumClass(cls)); - resolver.checkRegistrationOpen(); resolver.registerEnum(cls, namespace, typeName, serializer); } @@ -276,7 +272,6 @@ public static void registerEnum(Fory fory, Class cls, String name) { resolver.checkRegistrationOpen(); String[] parts = splitName(name); Serializer serializer = new EnumSerializer(resolver.getConfig(), enumClass(cls)); - resolver.checkRegistrationOpen(); resolver.registerEnum(cls, parts[0], parts[1], serializer); } @@ -285,7 +280,6 @@ public static void registerUnion(Fory fory, Class cls, long typeId) { resolver.checkRegistrationOpen(); Serializer serializer = newGeneratedSerializer(resolver, cls); Class[] caseClasses = cls.getDeclaredClasses(); - resolver.checkRegistrationOpen(); resolver.registerUnion(cls, typeId, serializer); registerCaseAliases(fory, cls, caseClasses); } @@ -296,7 +290,6 @@ public static void registerUnion(Fory fory, Class cls, String namespace, Stri resolver.checkRegistrationOpen(); Serializer serializer = newGeneratedSerializer(resolver, cls); Class[] caseClasses = cls.getDeclaredClasses(); - resolver.checkRegistrationOpen(); resolver.registerUnion(cls, namespace, typeName, serializer); registerCaseAliases(fory, cls, caseClasses); } @@ -307,7 +300,6 @@ public static void registerUnion(Fory fory, Class cls, String name) { String[] parts = splitName(name); Serializer serializer = newGeneratedSerializer(resolver, cls); Class[] caseClasses = cls.getDeclaredClasses(); - resolver.checkRegistrationOpen(); resolver.registerUnion(cls, parts[0], parts[1], serializer); registerCaseAliases(fory, cls, caseClasses); } diff --git a/scala/fory-scala/src/main/java/org/apache/fory/serializer/scala/ScalaSerializers.java b/scala/fory-scala/src/main/java/org/apache/fory/serializer/scala/ScalaSerializers.java index b5541f496c..e000419fe3 100644 --- a/scala/fory-scala/src/main/java/org/apache/fory/serializer/scala/ScalaSerializers.java +++ b/scala/fory-scala/src/main/java/org/apache/fory/serializer/scala/ScalaSerializers.java @@ -183,7 +183,6 @@ public static void registerEnum(Fory fory, Class cls, long typeId) { TypeResolver resolver = fory.getTypeResolver(); resolver.checkRegistrationOpen(); Serializer serializer = new ScalaEnumSerializer(resolver, cls); - resolver.checkRegistrationOpen(); resolver.registerEnum(cls, typeId, serializer); registerEnumRuntimeAliases(fory, cls); } @@ -215,7 +214,6 @@ public static void registerEnum(Fory fory, Class cls, String name) { resolver.checkRegistrationOpen(); String[] parts = splitName(name); Serializer serializer = new ScalaEnumSerializer(resolver, cls); - resolver.checkRegistrationOpen(); resolver.registerEnum(cls, parts[0], parts[1], serializer); registerEnumRuntimeAliases(fory, cls); } @@ -225,7 +223,6 @@ public static void registerEnum(Fory fory, Class cls, String namespace, Strin TypeResolver resolver = fory.getTypeResolver(); resolver.checkRegistrationOpen(); Serializer serializer = new ScalaEnumSerializer(resolver, cls); - resolver.checkRegistrationOpen(); resolver.registerEnum(cls, namespace, typeName, serializer); registerEnumRuntimeAliases(fory, cls); } diff --git a/scala/fory-scala/src/main/scala-3/org/apache/fory/scala/ForySerializer.scala b/scala/fory-scala/src/main/scala-3/org/apache/fory/scala/ForySerializer.scala index fbc98d9e16..63ce1405cf 100644 --- a/scala/fory-scala/src/main/scala-3/org/apache/fory/scala/ForySerializer.scala +++ b/scala/fory-scala/src/main/scala-3/org/apache/fory/scala/ForySerializer.scala @@ -131,7 +131,6 @@ object ForySerializer { case _ if serializer.isUnion => val runtimeSerializer = serializer.createSerializer(resolver) val runtimeClasses = serializer.handledRuntimeClasses(cls) - resolver.checkRegistrationOpen() if typeId != null then { resolver.registerUnion(cls, typeId.longValue(), runtimeSerializer) } else { @@ -151,6 +150,7 @@ object ForySerializer { registerType(fory, cls, typeId, namespace, typeName) val runtimeSerializer = serializer.createSerializer(resolver) resolver.checkRegistrationOpen() + // Preserve the registered STRUCT TypeInfo; registerSerializer would reclassify it as EXT. resolver.setSerializer(cls, runtimeSerializer) } } From 9f04029b078f4c9b0e8d74f36dad46c657f59aa4 Mon Sep 17 00:00:00 2001 From: chaokunyang Date: Sun, 30 Aug 2026 04:08:38 +0800 Subject: [PATCH 119/168] refactor(javascript): remove registration transaction remnants --- .agents/languages/javascript.md | 35 +++----------- .../javascript/type-registration.md | 27 ----------- javascript/packages/core/lib/context.ts | 45 ++++++++++++++---- javascript/packages/core/lib/fory.ts | 3 +- javascript/packages/core/lib/typeResolver.ts | 14 ++++++ javascript/test/crossLanguage.test.ts | 19 +++----- javascript/test/decimal.test.ts | 15 ++++-- javascript/test/fory.test.ts | 7 +++ javascript/test/map.test.ts | 9 ++-- javascript/test/rootCleanup.test.ts | 24 +++++----- javascript/test/typemeta.test.ts | 47 +++++++++++++++++++ 11 files changed, 149 insertions(+), 96 deletions(-) diff --git a/.agents/languages/javascript.md b/.agents/languages/javascript.md index 9882615ad9..834cc1cb3a 100644 --- a/.agents/languages/javascript.md +++ b/.agents/languages/javascript.md @@ -11,37 +11,16 @@ Load this file when changing `javascript/`. - Preserve generated serializer hot paths that bind writer, reader, ref, resolver, and metadata locals in outer closures; do not replace them with per-call context lookups without a measured reason. - Do not add parallel header-low/header-high slot caches in TypeMeta hot paths to chase benchmark gaps. Header-cache hits must use the concrete checked cache owner directly; if a small hit hint is needed, cache TypeMeta objects themselves and compare `TypeMeta.headerHash`, not separate low/high header fields or benchmark-pattern state. - JavaScript TypeMeta header cache hits should compare the 52-bit TypeMeta header hash directly. The hash is precise in JS `Number` and already includes the low header bits as hash input; do not add extra low-bit fields, sentinel state, nullable accepted headers, or parallel slot arrays around it. -- Root entry releases reference and metadata state left by the previous operation, including a - failed operation, before the context is reused. Do not put full cleanup on the root exit path or - copy Java backing-array retention policies onto native JavaScript arrays. Read-side metadata +- JavaScript `Fory` owns the one authoritative registration-frozen flag. The first root + serialization or deserialization sets it before codec work and leaves it set after failure. + `TypeResolver` owns registration maps, not a second lifecycle flag. +- A failed root releases its operation-local reference and metadata state before the exception + escapes. The next root entry releases state retained by the previous successful operation before + reusing the context. Keep the successful root exit allocation-free and do not copy Java + backing-array retention policies onto native JavaScript arrays. Read-side metadata occurrence arrays use native replacement reset. The MetaString and TypeMeta writer owner tables each have their own logical size: reset active owner IDs and that table's logical size without clearing bounded backing, and replace either backing only after its root has more than 8192 owners. -- Generated registration must build the complete recursive serializer source graph against - generation-local owners before one `TypeResolver` batch publication. The package-internal schema - seal must lock each `TypeInfo` schema pointer before reading or traversing it; schema fields and - occurrence modifiers are immutable afterward, while `dynamicTypeId` remains operation-local - writer state. Seed every complete Struct, enum, and union definition in the sealed graph before - resolving identity-only occurrences, so definition order cannot affect recursive resolution. - One numeric ID or name cannot identify different user-defined type families. Each resolver - identity has one complete schema owner in that graph. Repeated references and clones may share - the same immutable definition containers and settings; reject a second conflicting definition - before code generation without deep schema comparison. Complete anonymous definitions without a - name or user ID do not share registry identity merely because their raw type IDs match. The - definition-free generic enum or union serializer remains the canonical raw-type owner. Each - transaction entry stores the schema/progress facts needed by later code generation. Within one - registration transaction, run all of its application code hooks before instantiating any of its - runtime serializer factories. Then reconcile each same-definition identity with any nested - published winner and instantiate each remaining factory once in dependency completion order, with - fixed direct captures of the final published-or-local owners. Batch-publish only the remaining - local owners. Do not reject a valid late winner, rerun code generation or a hook, rebuild a factory - after publication, or retain a transaction lookup, cell, callback, or wrapper in runtime - serializers. Field occurrence modifiers remain field-owned, and conflicting families or complete - definitions still fail before outer publication. Reject an unresolved nested Struct identity before - resolver publication. An enum without a mapping and a union without cases keep their existing - generic definitions; an extension reference resolves through its registered custom serializer - owner. Never publish generated serializers, descriptors, or cache state before every generated - factory and application code hook succeeds. - Runtime value carriers such as decimal or reduced-precision numeric types belong under the core `types/` ownership boundary, with imports, exports, and codegen externals updated together. - Keep `TypeInfo` as schema metadata. Compatibility-sensitive decisions belong on `TypeResolver` or explicit operations, not as retained resolver state on metadata objects. - Normalize optional boolean config values at config construction; do not carry `null` through runtime paths when it means `false`. diff --git a/docs/object-serialization/javascript/type-registration.md b/docs/object-serialization/javascript/type-registration.md index d8ae4fbe9b..e2b4a3a109 100644 --- a/docs/object-serialization/javascript/type-registration.md +++ b/docs/object-serialization/javascript/type-registration.md @@ -137,33 +137,6 @@ const order = deserialize(bytes); Store and reuse this pair — it is the fast path. -Registration freezes the schema and every nested `TypeInfo`. Set field IDs, nullability, reference -tracking, and other schema options before registration. To use an already registered type as a new -field occurrence with different field options, clone it first: - -```ts -const itemType = Type.struct("example.item", { - value: Type.string(), -}); -fory.register(itemType); - -const wrapperType = Type.struct("example.wrapper", { - item: itemType.clone().setId(1).setNullable(true), -}); -``` - -A nested Struct that supplies only an ID or name must already be registered. Alternatively, include -a complete definition with the same identity anywhere in the recursive `TypeInfo` graph; the -definition may appear before or after the identity-only use. Registration rejects an unresolved -nested identity. Each identity can have only one complete definition in that graph. Repeated uses -may share that definition, but separate conflicting definitions are rejected. One numeric ID or -name also cannot identify different Struct, enum, extension, or union families. - -An anonymous union with declared cases has no registry identity. Keep and use the serializer pair -returned by `fory.register(...)`; registering another anonymous union does not replace or reuse the -earlier union's serializer. A union declared without cases remains an open union with the generic -union encoding and reads each value from the type information carried by that union case. - ## Field Metadata Field nullability, reference tracking, dynamic field behavior, numeric widths, and per-struct diff --git a/javascript/packages/core/lib/context.ts b/javascript/packages/core/lib/context.ts index 7b71c08e6d..1e89e8b7ff 100644 --- a/javascript/packages/core/lib/context.ts +++ b/javascript/packages/core/lib/context.ts @@ -46,6 +46,7 @@ type TypeResolverLike = { getSerializerByData(value: any): Serializer | null | undefined; isCompatible(): boolean; generateReadSerializer(typeInfo: TypeInfo): Serializer; + regenerateReadSerializer(typeInfo: TypeInfo): Serializer; getUnknownStructSerializer(typeMeta?: TypeMeta, wireTypeId?: number): Serializer; }; @@ -277,7 +278,7 @@ export class RefReader { export class MetaStringWriter { private static readonly MAX_RETAINED_META_STRING_OWNERS = 8192; - private disposeMetaStringBytes: MetaStringBytes[] = []; + private metaStringOwners: MetaStringBytes[] = []; private dynamicNameId = 0; private namespaceEncoder = new MetaStringEncoder(".", "_"); private typenameEncoder = new MetaStringEncoder("$", "_"); @@ -289,7 +290,7 @@ export class MetaStringWriter { const index = this.dynamicNameId; bytes.dynamicWriteStringId = index; this.dynamicNameId += 1; - this.disposeMetaStringBytes[index] = bytes; + this.metaStringOwners[index] = bytes; const len = bytes.bytes.getBytes().byteLength; writer.writeVarUInt32(len << 1); if (len !== 0) { @@ -308,7 +309,7 @@ export class MetaStringWriter { } reset() { - const owners = this.disposeMetaStringBytes; + const owners = this.metaStringOwners; const size = this.dynamicNameId; for (let i = 0; i < size; i++) { owners[i].dynamicWriteStringId = -1; @@ -316,7 +317,7 @@ export class MetaStringWriter { // These owners remain serializer-owned. Keep bounded backing without making old entries // protocol-visible, and release only an unusual root's oversized owner table. if (size > MetaStringWriter.MAX_RETAINED_META_STRING_OWNERS) { - this.disposeMetaStringBytes = []; + this.metaStringOwners = []; } this.dynamicNameId = 0; } @@ -371,7 +372,7 @@ export class WriteContext { readonly refWriter: RefWriter; readonly metaStringWriter: MetaStringWriter; - private disposeTypeMetaOwners: Array<{ dynamicTypeId: number }> = []; + private typeMetaOwners: Array<{ dynamicTypeId: number }> = []; private dynamicTypeId = 0; constructor( @@ -387,7 +388,7 @@ export class WriteContext { this.writer.reset(); this.refWriter.reset(); this.metaStringWriter.reset(); - const owners = this.disposeTypeMetaOwners; + const owners = this.typeMetaOwners; const size = this.dynamicTypeId; for (let i = 0; i < size; i++) { owners[i].dynamicTypeId = -1; @@ -395,7 +396,7 @@ export class WriteContext { // The logical size is the current root's visibility boundary. Reuse bounded backing and // release only an unusual root's oversized owner table. if (size > WriteContext.MAX_RETAINED_TYPE_META_OWNERS) { - this.disposeTypeMetaOwners = []; + this.typeMetaOwners = []; } this.dynamicTypeId = 0; } @@ -440,7 +441,7 @@ export class WriteContext { const index = this.dynamicTypeId; owner.dynamicTypeId = index; this.dynamicTypeId += 1; - this.disposeTypeMetaOwners[index] = owner; + this.typeMetaOwners[index] = owner; this.writer.writeVarUInt32(index << 1); this.writer.buffer(bytes); } @@ -1605,6 +1606,34 @@ export class ReadContext { ); } + genSerializerByTypeMetaRuntime( + typeMeta: TypeMeta, + original?: Serializer | TypeInfo, + expectedLocalHash?: number, + ) { + void expectedLocalHash; + const typeId = typeMeta.getTypeId(); + if (!TypeId.structType(typeId)) { + throw new Error("only support reconstructor struct type"); + } + let originalSerializer = original instanceof TypeInfo ? undefined : original; + let originalTypeInfo = original instanceof TypeInfo ? original : original?.getTypeInfo(); + if (originalSerializer === undefined && originalTypeInfo === undefined) { + originalSerializer = this.serializerByTypeMeta(typeMeta); + originalTypeInfo = originalSerializer?.getTypeInfo(); + } + if (originalSerializer === undefined) { + originalTypeInfo ??= TypeId.isNamedType(typeId) + ? Type.struct({ + typeName: typeMeta.getTypeName(), + namespace: typeMeta.getNs(), + }) + : Type.struct(typeMeta.getUserTypeId()); + originalSerializer = this.typeResolver.generateReadSerializer(originalTypeInfo); + } + return this.generateTypeMetaSerializer(typeMeta, originalSerializer); + } + private generateTypeMetaSerializer(typeMeta: TypeMeta, original: Serializer) { const typeId = typeMeta.getTypeId(); if (!TypeId.structType(typeId)) { diff --git a/javascript/packages/core/lib/fory.ts b/javascript/packages/core/lib/fory.ts index 86b85867d8..c72b946ce8 100644 --- a/javascript/packages/core/lib/fory.ts +++ b/javascript/packages/core/lib/fory.ts @@ -42,7 +42,7 @@ const DEFAULT_MAX_GRAPH_MEMORY_BYTES = 128 * 1024 * 1024; const DEFAULT_MAX_UNBACKED_CONTAINER_ITEMS = 8192 as const; const EMPTY_BYTES = new Uint8Array(0); export default class Fory { - readonly typeResolver: TypeResolver; + private readonly typeResolver: TypeResolver; readonly anySerializer: Serializer; readonly config: Config; readonly writeContext: WriteContext; @@ -260,6 +260,7 @@ export default class Fory { } serialize(data: T, serializer: Serializer = this.anySerializer) { + this.registrationFrozen = true; return this.getRootSerializer(serializer)(data); } } diff --git a/javascript/packages/core/lib/typeResolver.ts b/javascript/packages/core/lib/typeResolver.ts index b08844138e..30d27d54c0 100644 --- a/javascript/packages/core/lib/typeResolver.ts +++ b/javascript/packages/core/lib/typeResolver.ts @@ -311,6 +311,20 @@ export default class TypeResolver { return new Gen(this, { creator: typeInfo.options?.creator }).reGenerateSerializer(typeInfo); } + regenerateReadSerializer(typeInfo: TypeInfo) { + const serializer = this.generateReadSerializer(typeInfo); + return this.registerSerializer(typeInfo, { + readDataAlwaysAdvances: serializer.readDataAlwaysAdvances, + getHash: serializer.getHash, + getTypeInfo: serializer.getTypeInfo, + read: serializer.read, + readNoRef: serializer.readNoRef, + readRef: serializer.readRef, + readTypeInfo: serializer.readTypeInfo, + readRefWithoutTypeInfo: serializer.readRefWithoutTypeInfo, + } as any)!; + } + getSerializerByTypeInfo(typeInfo: TypeInfo) { const typeId = this.computeTypeId(typeInfo); if (TypeId.isNamedType(typeId)) { diff --git a/javascript/test/crossLanguage.test.ts b/javascript/test/crossLanguage.test.ts index 6b700c7e99..0409b5fc25 100644 --- a/javascript/test/crossLanguage.test.ts +++ b/javascript/test/crossLanguage.test.ts @@ -261,28 +261,23 @@ describe("bool", () => { } const bfs = []; + const typeResolver = (fory as any).typeResolver; // Serialize each deserialized item back for (let index = 0; index < deserializedData.length; index++) { const item = deserializedData[index]; let serializedData; if (index === 11) { - serializedData = fory.serialize(item, fory.typeResolver.getSerializerById(TypeId.FLOAT32)); + serializedData = fory.serialize(item, typeResolver.getSerializerById(TypeId.FLOAT32)); } else if (index === 12) { - serializedData = fory.serialize(item, fory.typeResolver.getSerializerById(TypeId.FLOAT64)); + serializedData = fory.serialize(item, typeResolver.getSerializerById(TypeId.FLOAT64)); } else if (index === 14) { - serializedData = fory.serialize(item, fory.typeResolver.getSerializerById(TypeId.DATE)); + serializedData = fory.serialize(item, typeResolver.getSerializerById(TypeId.DATE)); } else if (index === 15) { - serializedData = fory.serialize( - item, - fory.typeResolver.getSerializerById(TypeId.TIMESTAMP), - ); + serializedData = fory.serialize(item, typeResolver.getSerializerById(TypeId.TIMESTAMP)); } else if (index === 16) { - serializedData = fory.serialize( - item, - fory.typeResolver.getSerializerById(TypeId.BOOL_ARRAY), - ); + serializedData = fory.serialize(item, typeResolver.getSerializerById(TypeId.BOOL_ARRAY)); } else if (index === 17) { - serializedData = fory.serialize(item, fory.typeResolver.getSerializerById(TypeId.BINARY)); + serializedData = fory.serialize(item, typeResolver.getSerializerById(TypeId.BINARY)); } else if (index === 26) { serializedData = colorSerialize(item); } else { diff --git a/javascript/test/decimal.test.ts b/javascript/test/decimal.test.ts index b6a169cc8e..48ac75260c 100644 --- a/javascript/test/decimal.test.ts +++ b/javascript/test/decimal.test.ts @@ -176,8 +176,15 @@ describe("decimal", () => { const roundTrip = fory.deserialize(fory.serialize(value)) as Decimal; expect(roundTrip.equals(value)).toBe(true); } else { + const writer = (fory as any).writeContext.writer; + const bodyBefore = Array.from( + writer.getPlatformBuffer().subarray(bodyOffset, bodyOffset + 5), + ); expect(() => fory.serialize(value)).toThrow(/Decimal scale/); - expect((fory as any).writeContext.writer.writeGetCursor()).toBe(bodyOffset); + expect(writer.writeGetCursor()).toBe(0); + expect(Array.from(writer.getPlatformBuffer().subarray(bodyOffset, bodyOffset + 5))).toEqual( + bodyBefore, + ); } const payload = decimalPayload(scale); @@ -186,7 +193,7 @@ describe("decimal", () => { expect(decoded.equals(value)).toBe(true); } else { expect(() => fory.deserialize(payload.bytes)).toThrow(/Decimal scale/); - expect((fory as any).readContext.reader.readGetCursor()).toBe(payload.scaleEnd); + expect((fory as any).readContext.reader.readGetCursor()).toBe(0); } } }); @@ -210,7 +217,7 @@ describe("decimal", () => { writer.getPlatformBuffer().subarray(bodyOffset, bodyOffset + 5), ); expect(() => fory.serialize(value)).toThrow(/Decimal magnitude/); - expect(writer.writeGetCursor()).toBe(bodyOffset); + expect(writer.writeGetCursor()).toBe(0); expect(Array.from(writer.getPlatformBuffer().subarray(bodyOffset, bodyOffset + 5))).toEqual( bodyBefore, ); @@ -222,7 +229,7 @@ describe("decimal", () => { expect(decoded.equals(value)).toBe(true); } else { expect(() => fory.deserialize(payload.bytes)).toThrow(/Decimal magnitude length/); - expect((fory as any).readContext.reader.readGetCursor()).toBe(payload.magnitudeOffset); + expect((fory as any).readContext.reader.readGetCursor()).toBe(0); } } }); diff --git a/javascript/test/fory.test.ts b/javascript/test/fory.test.ts index e1ce9f2721..2cda43fc88 100644 --- a/javascript/test/fory.test.ts +++ b/javascript/test/fory.test.ts @@ -96,6 +96,13 @@ describe("fory", () => { expect(() => fory.register(Type.struct(8102, {}))).toThrow(); }); + test("freezes before serializer lookup", () => { + const fory = new Fory({ compatible: false }); + + expect(() => fory.serialize(1, null as any)).toThrow(); + expect(() => fory.register(Type.struct(8104, {}))).toThrow(); + }); + test.each(["serialize", "deserialize"] as const)("freezes after %s", (operation) => { const fory = new Fory({ compatible: false }); diff --git a/javascript/test/map.test.ts b/javascript/test/map.test.ts index ab0fa20cfe..55f8cbf864 100644 --- a/javascript/test/map.test.ts +++ b/javascript/test/map.test.ts @@ -118,7 +118,7 @@ describe("map", () => { test.each([ ["fixed", false, 320], ["evolving", true, 321], - ])("round-trips nullable %s maps", (_, evolving, itemId) => { + ])("round-trips %s map sides beside null", (_, evolving, itemId) => { const fory = new Fory({ compatible: true, ref: true }); const itemType = Type.struct( { typeId: itemId, evolving }, @@ -149,7 +149,7 @@ describe("map", () => { ]); }); - test("keeps compatible map framing", () => { + test("preserves compatible struct map framing", () => { const serializeMap = ( compatible: boolean, evolving: boolean, @@ -186,7 +186,7 @@ describe("map", () => { expect((native.header >> 3) & 0b100).toBe(0b100); }); - test("rejects invalid runtime chunks", () => { + test("rejects invalid runtime chunks before type detection", () => { const fory = new Fory({ compatible: false, ref: true }); const MapAnySerializer = CodegenRegistry.getExternal().MapAnySerializer; const serializer = new MapAnySerializer(fory.writeContext, fory.readContext, null, null); @@ -197,7 +197,7 @@ describe("map", () => { } }); - test("rejects invalid generated chunks", () => { + test("rejects invalid generated chunks and reuses the root", () => { const fory = new Fory({ compatible: false, ref: true }); const serializer = fory.register(Type.map(Type.string(), Type.int32())); const value = new Map([["key", 1]]); @@ -209,6 +209,7 @@ describe("map", () => { malformed[chunkSizeOffset] = chunkSize; expect(() => serializer.deserialize(malformed)).toThrow(); + expect(fory.readContext.depth).toBe(0); expect(serializer.deserialize(valid)).toEqual(value); } }); diff --git a/javascript/test/rootCleanup.test.ts b/javascript/test/rootCleanup.test.ts index 243d17fed5..49e2bd9c51 100644 --- a/javascript/test/rootCleanup.test.ts +++ b/javascript/test/rootCleanup.test.ts @@ -153,12 +153,12 @@ test("reuses root write metastring owners", () => { }; expect(registered.serialize({})).toBeDefined(); - const owners = writeContext.metaStringWriter.disposeMetaStringBytes; + const owners = writeContext.metaStringWriter.metaStringOwners; expect(owners).toHaveLength(1); expect(name.dynamicWriteStringId).toBe(0); expect(registered.serialize({})).toBeDefined(); - expect(writeContext.metaStringWriter.disposeMetaStringBytes).toBe(owners); + expect(writeContext.metaStringWriter.metaStringOwners).toBe(owners); expect(owners).toHaveLength(1); expect(name.dynamicWriteStringId).toBe(0); }); @@ -174,12 +174,12 @@ test("reuses write metadata owners", () => { }; expect(registered.serialize({})).toBeDefined(); - const owners = writeContext.disposeTypeMetaOwners; + const owners = writeContext.typeMetaOwners; expect(owners).toHaveLength(1); expect(typeMeta.dynamicTypeId).toBe(0); expect(registered.serialize({})).toBeDefined(); - expect(writeContext.disposeTypeMetaOwners).toBe(owners); + expect(writeContext.typeMetaOwners).toBe(owners); expect(owners).toHaveLength(1); expect(typeMeta.dynamicTypeId).toBe(0); }); @@ -193,14 +193,14 @@ test.each([8192, 8193])("bounds %s metastring owners", (ownerCount) => { const owner = metaStringWriter.encodeTypeName(`name-${i}`); metaStringWriter.writeBytes(writeContext.writer, owner); } - const owners = metaStringWriter.disposeMetaStringBytes; + const owners = metaStringWriter.metaStringOwners; writeContext.reset(); if (ownerCount === 8192) { - expect(metaStringWriter.disposeMetaStringBytes).toBe(owners); + expect(metaStringWriter.metaStringOwners).toBe(owners); } else { - expect(metaStringWriter.disposeMetaStringBytes).not.toBe(owners); - expect(metaStringWriter.disposeMetaStringBytes).toHaveLength(0); + expect(metaStringWriter.metaStringOwners).not.toBe(owners); + expect(metaStringWriter.metaStringOwners).toHaveLength(0); } const nextOwner = metaStringWriter.encodeTypeName("next-root"); metaStringWriter.writeBytes(writeContext.writer, nextOwner); @@ -216,15 +216,15 @@ test.each([8192, 8193])("bounds %s type metadata owners", (ownerCount) => { for (const owner of typeMetaOwners) { writeContext.writeTypeMeta(owner, bytes); } - const owners = writeContext.disposeTypeMetaOwners; + const owners = writeContext.typeMetaOwners; writeContext.reset(); expect(typeMetaOwners.every((owner) => owner.dynamicTypeId === -1)).toBe(true); if (ownerCount === 8192) { - expect(writeContext.disposeTypeMetaOwners).toBe(owners); + expect(writeContext.typeMetaOwners).toBe(owners); } else { - expect(writeContext.disposeTypeMetaOwners).not.toBe(owners); - expect(writeContext.disposeTypeMetaOwners).toHaveLength(0); + expect(writeContext.typeMetaOwners).not.toBe(owners); + expect(writeContext.typeMetaOwners).toHaveLength(0); } const nextOwner = { dynamicTypeId: -1 }; writeContext.writeTypeMeta(nextOwner, bytes); diff --git a/javascript/test/typemeta.test.ts b/javascript/test/typemeta.test.ts index d399d74314..bd6afe1ac4 100644 --- a/javascript/test/typemeta.test.ts +++ b/javascript/test/typemeta.test.ts @@ -550,6 +550,9 @@ describe("typemeta", () => { generateReadSerializer: () => { throw new Error("unused"); }, + regenerateReadSerializer: () => { + throw new Error("unused"); + }, } as any, config, ); @@ -620,6 +623,29 @@ describe("typemeta", () => { ); }); + test("direct TypeMeta generation keeps its public adapters", () => { + const writerFory = new Fory({ compatible: true }); + const readerFory = new Fory({ compatible: true }); + const remoteTypeMeta = TypeMeta.fromTypeInfo( + Type.struct(7020, { value: Type.string().setId(1) }), + (writerFory as any).typeResolver, + ); + const localTypeInfo = Type.struct(7020, { value: Type.int32().setId(1) }); + const context = (readerFory as any).readContext as ReadContext; + + expect( + context.genSerializerByTypeMetaRuntime(remoteTypeMeta, localTypeInfo, 123), + ).toBeDefined(); + expect(context.genSerializerByTypeMetaRuntime(remoteTypeMeta)).toBeDefined(); + expect((context as any).compatibleReadSerializers.size).toBe(0); + + const localTypeMeta = TypeMeta.fromTypeInfo(localTypeInfo, (readerFory as any).typeResolver); + context.reset(typeMetaRecord(remoteTypeMeta)); + expect( + context.readCompatibleStructSerializer(localTypeMeta.getHash(), localTypeInfo), + ).toBeDefined(); + }); + test("generated named enum validates TypeMeta owner", () => { const colorInfo = Type.enum({ namespace: "example", typeName: "Color" }, { Red: 0 }); const otherInfo = Type.enum({ namespace: "example", typeName: "Other" }, { Blue: 0 }); @@ -735,6 +761,9 @@ describe("typemeta", () => { generateReadSerializer: () => { throw new Error("unused"); }, + regenerateReadSerializer: () => { + throw new Error("unused"); + }, } as any, config, ); @@ -1431,6 +1460,21 @@ describe("typemeta", () => { expect(map.get("second").second).toBe("two"); }); + test("regenerated read serializers keep getTypeInfo", () => { + const fory = new Fory({ compatible: true }); + const serializer = (fory as any).typeResolver.regenerateReadSerializer( + Type.struct( + { namespace: "example", typeName: "repro_struct" }, + { + value: Type.int32(), + }, + ), + ); + + expect(typeof serializer.getTypeInfo).toBe("function"); + expect(serializer.getTypeInfo().named).toBe("example$repro_struct"); + }); + test("caches compatible readers for alternating nested schemas", () => { const stringWriterFory = new Fory({ compatible: true }); const boolWriterFory = new Fory({ compatible: true }); @@ -1514,6 +1558,9 @@ describe("typemeta", () => { generateReadSerializer: () => { throw new Error("unused"); }, + regenerateReadSerializer: () => { + throw new Error("unused"); + }, } as any, config, ); From c300426e230dda49ce370f620b006ced83edc784 Mon Sep 17 00:00:00 2001 From: chaokunyang Date: Sun, 30 Aug 2026 04:09:41 +0800 Subject: [PATCH 120/168] docs: define root lifecycle ownership --- .agents/languages/dart.md | 4 +- AGENTS.md | 81 +- docs/object-serialization/core-concepts.md | 5 +- .../python/basic-serialization.md | 27 +- .../python/configuration.md | 123 +-- .../python/functions-classes-methods.md | 256 ++---- docs/object-serialization/python/index.md | 62 +- docs/object-serialization/python/native.md | 39 +- .../python/numpy-integration.md | 76 +- .../python/out-of-band.md | 106 ++- docs/object-serialization/python/security.md | 22 +- .../python/serialization-hooks.md | 19 +- .../python/troubleshooting.md | 74 +- .../python/type-registration.md | 12 +- docs/security/deserialization.md | 55 +- .../xlang_implementation_guide.md | 123 +-- python/README.md | 852 +++++++++++++++--- 17 files changed, 1139 insertions(+), 797 deletions(-) diff --git a/.agents/languages/dart.md b/.agents/languages/dart.md index a4d76cc700..c09f9670ff 100644 --- a/.agents/languages/dart.md +++ b/.agents/languages/dart.md @@ -98,8 +98,8 @@ Load this file when changing `dart/`. - Do not add parallel header-low/header-high slot caches or multi-slot recent caches in TypeMeta hot paths to chase benchmark gaps. Header-cache hits must use the concrete checked cache owner directly; if a hit hint is needed, cache one TypeInfo/TypeMeta object and compare the protocol-defined top 52 header bits on that object, not separate low/high header fields or benchmark-pattern state. - The top 52 TypeDef/TypeMeta header bits are the schema identity. The full low 12 bits belong only to the current frame and must not participate in hit selection. On a hit, decode the current body size from its low eight bits and any extended-size varuint, prove those bytes readable, and skip exactly that body. Do not validate reserved/compress flags, compare cached or local low bits, parse or rehash the body, repeat schema or policy validation, or grow low-bit sentinels, accepted-header fields, parallel header slots, or benchmark-pattern state. The cold miss path owns low-flag validation. - Dart expected-type TypeDef reads should compare only the top 52 bits of the expected `TypeInfo` object's cached local TypeDef header before consulting the parsed-metadata map. A match is a direct local-schema hit: use the current frame's size encoding only for bounds and skip, add the expected type to the per-read shared type table, and do not validate its low flags, publish to `ParsedTypeMetaCache`, record a remote schema version, or parse/hash the body. -- Dart local TypeDef construction is registration-owned: record registrations - and finalize their dependent TypeDefs and struct serializers before the first +- Dart local TypeDef construction is registration-owned: each explicit registration constructs its + dependent TypeDefs and struct serializers before the first root read or write. The first `serialize`, `serializeTo`, `serializeBuiltin`, `serializeBuiltinTo`, `deserialize`, or `deserializeFrom` call permanently freezes that `Fory` instance's resolver; diff --git a/AGENTS.md b/AGENTS.md index 67fb80a8d5..0086c40611 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -157,34 +157,13 @@ This is the entry point for AI guidance in Apache Fory. Read this file first, th only the size to zero; do not clear retained slots. When the size exceeds 8192, replace the backing array with an eight-slot array. Do not add per-count or benchmark-shape specializations to this path. -- JavaScript root entry releases reference and metadata state left by the previous root, including a - failed root, before the context is reused. Do not add full cleanup to the root exit path or copy - Java backing-array retention policies onto native JavaScript arrays. Read-side metadata +- A failed JavaScript root releases its operation-local reference and metadata state before the + exception escapes. The next root entry releases state retained by the previous successful root. + Keep successful root exit allocation-free, and do not copy Java backing-array retention policies + onto native JavaScript arrays. Read-side metadata occurrence arrays use native replacement reset. The MetaString and TypeMeta writer owner tables each have their own logical size: reset active owner IDs and that table's logical size without clearing bounded backing, and replace either backing only after its root has more than 8192 owners. -- JavaScript generated registration must build the complete recursive serializer source graph - against generation-local owners before one `TypeResolver` batch publication. Each transaction - entry owns the schema and progress facts used by later code generation. Within one registration - transaction, run all of its application code hooks before instantiating any of its runtime - serializer factories. After that transaction's hooks complete, reconcile same-definition - identities with any nested published winner, then instantiate every remaining factory once in - dependency completion order so its fixed captures point directly to the final published-or-local - owners. Batch-publish only the remaining local owners. Do not reject a valid late same-definition - winner, rerun code generation or a hook, rebuild a factory after publication, or retain a - transaction lookup, cell, callback, or wrapper in runtime serializers. Runtime and dynamic lookup - must retain the real resolver. Seal schema definitions before traversal and preseed complete - definitions by identity, - so field order cannot affect recursive resolution. One numeric ID or name cannot identify - different user-defined type families. Each identity has one complete schema owner; repeated clones - are valid only when they share that owner's definition containers and settings. A nested - identity-only Struct must already have a fully initialized registered owner or resolve to an owner - in the current complete recursive schema graph; otherwise registration fails before resolver - publication. Enum without a mapping and union without cases retain the canonical generic owner - for their raw wire type. Complete anonymous definitions without a registry key remain - generation-local and distinct. A conflicting family or complete definition still fails before - outer publication. Do not publish placeholders, nested serializers, descriptors, or cache state - before every generated factory and application code hook succeeds. - Root failure exceptions must not copy or retain the operation reference table or materialized object graph for diagnostics. Root cleanup owns releasing that graph, and failure reporting must remain bounded independently of graph size. @@ -206,8 +185,13 @@ This is the entry point for AI guidance in Apache Fory. Read this file first, th maps, generated descriptors, metadata, serializers, or caches. Do not support post-use registration through cache invalidation, descriptor refresh, serializer rebinding, metadata rebuilding, or other late-registration - machinery. Registration-order finalization before the first root operation - remains registration-owned and must not create a runtime invalidation path. + machinery. Keep one authoritative frozen flag for each natural registration + owner or public facade boundary. A thread-safe facade with its own public + registration surface may own that boundary flag, but must not mirror a child + registry's flag. Do not add another lifecycle state, a registration commit + path, or eager whole-registry preparation. Copy operations and + facade execution callbacks do not freeze registration unless they start a + root serialization or deserialization operation. Registry freeze does not make native runtime type resolution immutable. When registration is not required, Java and Python native modes may still discover an unregistered runtime type and materialize resolver-owned type information, @@ -215,45 +199,10 @@ This is the entry point for AI guidance in Apache Fory. Read this file first, th generated serializer completion for an existing binding is likewise allowed after freeze. These runtime cache operations must not create or change an explicit type, serializer, ID, name, or policy registration. - If serializer construction, factory execution, or another application callback - can reenter a root, complete that callback before publishing the entry it - prepares, then recheck the authoritative per-instance freeze owner immediately - before publication. Kotlin and Scala combined generated-struct registration are - the sole type-first exception: publish the canonical type needed by generated - serializer construction, then enter the existing resolver construction graph. - Keep the candidate unpublished until the authoritative lifecycle recheck and - install it through the normal Class/Xtype commit sink. Do not add direct - replacement, rollback, staging, or a parallel registration path for this exception. - A module installation may perform complete nested - registrations. Java `Fory.register(ForyModule)` owns cycle breaking and - idempotence in one identity set: add the identity before installation, remove - it on failure, and retain it on success; do not add parallel installing and - completed states. Java serializer construction uses one resolver-owned local - graph that separates final `TypeInfo` owners from unpublished serializer - candidates. Recursive fields capture the final owner during construction; when - wire and user IDs match, that owner is the existing canonical `TypeInfo`. - Construction owners that resolve recursive fields or candidate state use the - construction-local serializer; ordinary resolver lookups keep their existing - runtime semantics. The normal Class/Xtype commit sink installs the candidate - only after callback and lifecycle checks. - Do not add a constructor-specific publication path. Java thread-safe callback registration is one facade-owned - transaction across every child. Reject a root or facade registration reentered - by that callback and permanently fail the facade after callback failure rather - than exposing partial child state, divergent replay order, or resolver - rollback. Keep a late thread-local child provisional until replay and - finalization both complete. -- Python `TypeResolver` is the sole registry freeze and finalization owner. Its Cython resolver - companion may cache completion of the Python-owner dispatch needed to populate native tables, - but the `Fory` facade must not mirror that state; Cython roots call the resolver owner directly. - Allocate automatic type IDs only after callback preparation and the final freeze recheck, at the - common registry publication point; do not reserve IDs early or maintain counter rollback state. - `ThreadSafeFory` validates registrations before retaining semantic replay descriptors and never - invokes an application factory or callback under its non-reentrant pool lock. During child replay, - a nested request is a no-op only when it exactly matches the accepted prefix already applied to - that child; reject unknown or different requests before that request mutates the child. Serializer - factories must return carriers bound to the current child resolver and normalized declared type. - Reject root reentry from the active instance-build thread before looking in the pool, including - when another thread returned an instance during the build. + Java `Fory.register(ForyModule)` and the corresponding `BaseFory` operation remain available on + direct and thread-safe facades before the first root. Kotlin and Scala registration extensions + target `BaseFory` so the same API works with `Fory`, `ThreadLocalFory`, and pooled + `ThreadSafeFory` implementations. - Use semantic naming only. Name things after protocol or domain concepts, not history, runtime origin, or workaround style; avoid vague names such as `Internal`, `java_style_*`, `Runtime`, `Session`, `Plan`, `Payload`, or `Binding` when they do not name the real concept. Keep class, method, function, and variable names concise; do not encode the whole scenario or implementation history into one identifier. Never name a class or method with a `Plan` suffix; use the real domain concept instead. For Fory codec/read APIs, do not use generic `payload` naming; name the exact owner and data shape, such as bytes, body, frame, field, string, list, map, compressed bytes, or primitive-array encoding. - Keep one implementation path. Do not keep parallel helpers, serializers, harnesses, wrappers, or registration flows for the same concept; extend the existing owner path instead of inventing another one. - Follow current scope exactly. The latest explicit user instruction overrides earlier plans, and when scope narrows, remove leaked out-of-scope edits immediately. diff --git a/docs/object-serialization/core-concepts.md b/docs/object-serialization/core-concepts.md index e87ba2632f..2506a8e1ce 100644 --- a/docs/object-serialization/core-concepts.md +++ b/docs/object-serialization/core-concepts.md @@ -39,8 +39,9 @@ only values. Use [Row Format](../row-format/index.md) for trusted analytical row A Fory instance owns its mode, schema behavior, reference settings, registered types, custom serializers, and read limits. Configure and register the instance before its first root serialization or deserialization operation, then reuse it. Registration is frozen after the first -root attempt, even when that operation fails, so the same instance always resolves a type in the -same way. +root attempt, even when that operation fails, so its explicit type and serializer mappings stay +fixed. Native modes may still perform documented lazy runtime-type discovery when registration is +not required. Thread-safety differs by Fory implementation. Some implementations provide a thread-safe wrapper or pool; others use one instance per thread or task. Follow the selected language guide instead of sharing an ordinary diff --git a/docs/object-serialization/python/basic-serialization.md b/docs/object-serialization/python/basic-serialization.md index 77b326a7bf..b7245315d6 100644 --- a/docs/object-serialization/python/basic-serialization.md +++ b/docs/object-serialization/python/basic-serialization.md @@ -41,20 +41,6 @@ print(obj) # {'name': 'Alice', 'age': 30, 'scores': [95, 87, 92]} **Note**: `dumps()`/`loads()` are aliases for `serialize()`/`deserialize()`. Both APIs are identical, use whichever feels more intuitive. -## Registration Lifecycle - -Complete every explicit type and serializer registration before the first root serialization or -deserialization attempt. In Python native mode, register the application and native carrier types -whose serializers must be installed before that first root. The first root serialization or -deserialization attempt permanently freezes the instance's registry, even when the operation -fails. `strict=False` does not enable late type or serializer registration; its policy may still -authorize module-global resolution during a trusted native read without mutating the registry. - -If the first operation fails because registration is incomplete or invalid, create a new instance, -register the complete type surface, and retry with that instance. A fully configured instance can -process a later root after a failure while reading input data or serializing a value. See -[Type Registration](type-registration.md) for the complete lifecycle. - ## Custom Class Serialization Use dataclasses and type annotations for stable xlang payloads: @@ -96,14 +82,15 @@ result = f.deserialize(data) assert result[0] is result[1] ``` -For configured Python-native object graphs, local classes, functions, and methods, use +For arbitrary Python object graphs, local classes, functions, and methods, use [Native Serialization](native.md). ## Performance Tips 1. **Disable `ref=True` if not needed**: Reference tracking has overhead -2. **Reuse Fory instances**: Create once, use many times -3. **Enable Cython**: Make sure `ENABLE_FORY_CYTHON_SERIALIZATION=1` +2. **Use type_id instead of name**: Integer IDs are faster than string names +3. **Reuse Fory instances**: Create once, use many times +4. **Enable Cython**: Make sure `ENABLE_FORY_CYTHON_SERIALIZATION=1` ```python # Good: Reuse instance @@ -301,11 +288,7 @@ Fory row-format schemas. ### Differences from Python Native Mode -The binary protocol and API are similar to `pyfory`'s Python native mode, but native mode supports a -configured Python-only type surface that may include global functions, local functions, lambdas, -local classes, and types with custom serialization using `__getstate__`, `__reduce__`, or -`__reduce_ex__`. Register those application and carrier types before the first root attempt. These -Python-specific values are **not allowed** in xlang mode. +The binary protocol and API are similar to `pyfory`'s Python native mode, but Python native mode can serialize any Python object—including global functions, local functions, lambdas, local classes, and types with custom serialization using `__getstate__/__reduce__/__reduce_ex__`, which are **not allowed** in xlang mode. ### Specifications and References diff --git a/docs/object-serialization/python/configuration.md b/docs/object-serialization/python/configuration.md index a099a2f6fe..bf91d74684 100644 --- a/docs/object-serialization/python/configuration.md +++ b/docs/object-serialization/python/configuration.md @@ -54,43 +54,34 @@ Thread-safe serialization interface using a pooled wrapper: ```python class ThreadSafeFory: - def __init__(self, fory_factory=None, **kwargs) + def __init__( + self, fory_factory=None, **kwargs + ) ``` ## Parameters -| Parameter | Type | Default | Description | -| -------------------------------------- | ------------------------------- | ----------- | --------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | -| `xlang` | `bool` | `True` | Use xlang mode. Set `False` for Python native mode. | -| `ref` | `bool` | `False` | Enable reference tracking for shared/circular references. Disable for better performance if your data has no shared references. | -| `strict` | `bool` | `True` | Require registration before loading application classes. Compatible unknown Structs use `UnknownStruct`. `False` permits policy-authorized native module-global resolution but does not replace carrier registration. | -| `compatible` | `bool \| None` | `None` | Schema evolution mode. `None` enables compatible mode in both xlang and native mode. Set `False` only when every reader and writer uses the same schema. | -| `max_depth` | `int` | `50` | Maximum deserialization depth for security, preventing stack overflow attacks. | -| `max_type_fields` | `int` | `512` | Maximum fields accepted in one received remote struct metadata body. | -| `max_type_meta_bytes` | `int` | `4096` | Maximum encoded body bytes accepted for one received TypeDef body, excluding the 8-byte header and any extended-size varint. | -| `max_schema_versions_per_type` | `int` | `10` | Maximum accepted remote metadata versions for one logical type. | -| `max_average_schema_versions_per_type` | `int` | `3` | Average accepted remote metadata versions across accepted remote types. The effective global floor is `8192` schemas. | -| `max_graph_memory_bytes` | `int` | `134217728` | Approximate graph-memory gate for one root deserialization. Explicit non-positive values are rejected. | -| `max_unbacked_container_items` | `int` | `8192` | Maximum collection elements and map entries whose repeated reads are not backed by input progress. Zero is strict. | -| `policy` | `DeserializationPolicy \| None` | `None` | Deserialization policy used for security checks. Strongly recommended when `strict=False`. | -| `field_nullable` | `bool` | `False` | Treat dataclass fields as nullable by default. | -| `meta_compressor` | `Any` | `None` | Optional metadata compressor used for compatible-mode metadata encoding. | -| `fory_factory` | `Callable \| None` | `None` | `ThreadSafeFory` factory hook. When set, `ThreadSafeFory` creates instances via this callback; otherwise it forwards `**kwargs` to `Fory` construction. | +| Parameter | Type | Default | Description | +| -------------------------------------- | ------------------------------- | ----------- | -------------------------------------------------------------------------------------------------------------------------------------------------------- | +| `xlang` | `bool` | `True` | Use xlang mode. Set `False` for Python native mode. | +| `ref` | `bool` | `False` | Enable reference tracking for shared/circular references. Disable for better performance if your data has no shared references. | +| `strict` | `bool` | `True` | Require registration before loading application classes. Compatible unknown Structs use the data-only `UnknownStruct` carrier. | +| `compatible` | `bool \| None` | `None` | Schema evolution mode. `None` enables compatible mode in both xlang and native mode. Set `False` only when every reader and writer uses the same schema. | +| `max_depth` | `int` | `50` | Maximum deserialization depth for security, preventing stack overflow attacks. | +| `max_type_fields` | `int` | `512` | Maximum fields accepted in one received remote struct metadata body. | +| `max_type_meta_bytes` | `int` | `4096` | Maximum encoded body bytes accepted for one received TypeDef body, excluding the 8-byte header and any extended-size varint. | +| `max_schema_versions_per_type` | `int` | `10` | Maximum accepted remote metadata versions for one logical type. | +| `max_average_schema_versions_per_type` | `int` | `3` | Average accepted remote metadata versions across accepted remote types. The effective global floor is `8192` schemas. | +| `max_graph_memory_bytes` | `int` | `134217728` | Approximate graph-memory gate for one root deserialization. Explicit non-positive values are rejected. | +| `max_unbacked_container_items` | `int` | `8192` | Maximum collection elements and map entries whose repeated reads are not backed by input progress. Zero is strict. | +| `policy` | `DeserializationPolicy \| None` | `None` | Deserialization policy used for security checks. Strongly recommended when `strict=False`. | +| `field_nullable` | `bool` | `False` | Treat dataclass fields as nullable by default. | +| `meta_compressor` | `Any` | `None` | Optional metadata compressor used for compatible-mode metadata encoding. | +| `fory_factory` | `Callable \| None` | `None` | `ThreadSafeFory` factory hook. When set, `ThreadSafeFory` creates instances via this callback; otherwise it forwards `**kwargs` to `Fory` construction. | ## Key Methods ```python -# Complete registration before serialization or deserialization. This form is valid in native -# and xlang modes when no explicit stable identity is required. -fory.register(MyClass) - -# Alternatively, use one of these forms when the xlang schema needs an explicit stable identity. -# fory.register(MyClass, type_id=123) -# fory.register(MyClass, name="my.package.MyClass") - -# Direct Fory accepts an instance; ThreadSafeFory requires a serializer class or factory. -# fory.register(MyClass, serializer=MySerializer(fory.type_resolver, MyClass)) - # Serialization (serialize/deserialize are identical to dumps/loads) data: bytes = fory.serialize(obj) obj = fory.deserialize(data) @@ -98,26 +89,30 @@ obj = fory.deserialize(data) # Alternative API (aliases) data: bytes = fory.dumps(obj) obj = fory.loads(data) -``` -Complete registration before the first root serialization or deserialization attempt. That first -attempt permanently freezes registration, even if it fails. If the first operation exposes an -incomplete or invalid registration, create and configure a new instance before retrying. A fully -configured instance can process a later root after a failure while reading input data or serializing -a value. See -[Type Registration](type-registration.md) for the complete lifecycle. +# Direct Fory registration by id; serializer instances belong to that Fory. +fory.register(MyClass, type_id=123) +fory.register(MyClass, type_id=123, serializer=custom_serializer) + +# ThreadSafeFory constructs one serializer per pooled child from a class or factory. +thread_safe_fory.register(MyClass, type_id=123, serializer=CustomSerializer) + +# Direct Fory registration by name +fory.register(MyClass, name="my.package.MyClass") +fory.register(MyClass, name="my.package.MyClass", serializer=custom_serializer) +``` ## Xlang And Native Mode Comparison -| Feature | Native mode (`xlang=False`) | Xlang mode (default) | -| ------------------- | ----------------------------------------------------- | -------------------------------------------------------------------------------- | -| Use case | Python-only applications | Multi-language systems | -| Compatibility | Python only | Java, C++, Go, Rust, JavaScript/TypeScript, C#, Swift, Dart, Scala, Kotlin, etc. | -| Supported types | Configured Python object surface | Cross-language compatible types | -| Functions/lambdas | Require carrier registration and policy authorization | Not allowed | -| Local classes | Require carrier registration and policy authorization | Not allowed | -| Class objects | Require type registration and policy authorization | Not allowed | -| Schema mode default | Compatible | Compatible | +| Feature | Native mode (`xlang=False`) | Xlang mode (default) | +| ------------------- | ---------------------------------------------- | -------------------------------------------------------------------------------- | +| Use case | Python-only applications | Multi-language systems | +| Compatibility | Python only | Java, C++, Go, Rust, JavaScript/TypeScript, C#, Swift, Dart, Scala, Kotlin, etc. | +| Supported types | Python object surface | Cross-language compatible types | +| Functions/lambdas | Supported with trusted dynamic deserialization | Not allowed | +| Local classes | Supported with trusted dynamic deserialization | Not allowed | +| Dynamic classes | Supported with trusted dynamic deserialization | Not allowed | +| Schema mode default | Compatible | Compatible | ## Xlang Mode @@ -136,25 +131,15 @@ Use `compatible=False` for xlang payloads only when every reader and writer alwa ## Native Mode ```python -from dataclasses import dataclass - import pyfory - -@dataclass -class Event: - name: str - - -fory = pyfory.Fory(xlang=False, ref=True, strict=True) -fory.register_type(Event) +fory = pyfory.Fory(xlang=False, ref=True, strict=False) ``` Native mode supports Python-specific object features such as functions, local classes, methods, -`__reduce__`, and `__getstate__` when their application and carrier types are registered before the -first root attempt and the configured policy authorizes their deserialization. Compatible mode is -still enabled by default. Set `compatible=False` only when every reader and writer always uses the -same Python class schema. +`__reduce__`, and `__getstate__`. Compatible mode is still enabled by default. Set +`compatible=False` only when every reader and writer always uses the same Python +class schema and you want faster serialization and smaller size. ## Compatible Mode @@ -181,32 +166,20 @@ fory = pyfory.Fory( fory.register(UserModel, name="example.User") ``` -### Native Mode With Configured Python Types +### Native Mode With Dynamic Types ```python -from dataclasses import dataclass - import pyfory - -@dataclass -class Event: - name: str - - fory = pyfory.Fory( xlang=False, ref=True, - strict=True, + strict=False, + max_depth=1000, ) -fory.register_type(Event) ``` -Use `strict=False` only for trusted data and configure a `policy=` deserialization policy for -functions, local classes, class objects, or other dynamic native values. Register every application -and Python-native carrier type whose serializer must be installed before the first root attempt. -Policy-authorized module-global classes and callables may still be resolved while reading a native -payload; that lookup does not reopen or mutate the frozen registry. +Use `strict=False` only for trusted data, preferably with a `policy=` deserialization policy. ## Security diff --git a/docs/object-serialization/python/functions-classes-methods.md b/docs/object-serialization/python/functions-classes-methods.md index 69907c6a55..c4295b4442 100644 --- a/docs/object-serialization/python/functions-classes-methods.md +++ b/docs/object-serialization/python/functions-classes-methods.md @@ -20,48 +20,24 @@ license: | --- Python native mode serializes Python-specific callable and type values that are outside the xlang -type system. Use `strict=False` only for trusted payloads. Configure a deserialization policy that -authorizes the callable and class references accepted by the application. - -Register every callable carrier and application type whose serializer must be installed before the -first root operation. The first serialization or deserialization attempt permanently freezes the -registry, even when it fails. With `strict=False`, the configured policy may still resolve a -module-global function or class while reading a trusted native payload; that resolution does not -install a type or serializer. +type system. Use `strict=False` only for trusted payloads and apply a deserialization policy when +the accepted dynamic surface must be restricted. ## Serialize Global Functions -Functions imported from a module deserialize to the same function object. Register the function -carrier and authorize the expected module-level function before the first root operation: +Capture and serialize functions defined at module level. Fory deserializes and returns the same +function object: ```python -import statistics -import types - import pyfory -from pyfory import DeserializationPolicy - -class TrustedFunctionPolicy(DeserializationPolicy): - def validate_module(self, module_name, is_local, **kwargs): - if module_name != "statistics": - raise ValueError(f"Blocked module: {module_name}") +fory = pyfory.Fory(xlang=False, ref=True, strict=False) - def validate_function(self, func, is_local, **kwargs): - if func is not statistics.mean or is_local: - raise ValueError(f"Blocked function: {func!r}") +def my_global_function(x): + return 10 * x - -fory = pyfory.Fory( - xlang=False, - ref=True, - strict=False, - policy=TrustedFunctionPolicy(), -) -fory.register_type(types.FunctionType) -restored = fory.loads(fory.dumps(statistics.mean)) -assert restored is statistics.mean -assert restored([10, 20, 30]) == 20 +data = fory.dumps(my_global_function) +print(fory.loads(data)(10)) # 100 ``` ## Serialize Local Functions/Lambdas @@ -70,185 +46,99 @@ Serialize functions with closures and lambda expressions. Fory captures the clos automatically: ```python -import types - import pyfory -from pyfory import DeserializationPolicy - - -class TrustedLocalFunctionPolicy(DeserializationPolicy): - def authorize_instantiation(self, cls, **kwargs): - if cls is not types.FunctionType: - raise ValueError(f"Blocked materialization: {cls!r}") - - def validate_function(self, func, is_local, **kwargs): - if not is_local or func.__name__ not in {"multiply", ""}: - raise ValueError(f"Blocked function: {func!r}") - -fory = pyfory.Fory( - xlang=False, - ref=True, - strict=False, - policy=TrustedLocalFunctionPolicy(), -) -fory.register_type(types.FunctionType) +fory = pyfory.Fory(xlang=False, ref=True, strict=False) -def make_multiplier(factor): - def multiply(value): - return factor * value +# Local functions with closures +def my_function(): + local_var = 10 + def local_func(x): + return x * local_var + return local_func - return multiply +data = fory.dumps(my_function()) +print(fory.loads(data)(10)) # 100 - -restored = fory.loads(fory.dumps(make_multiplier(10))) -assert restored(10) == 100 - -restored_lambda = fory.loads(fory.dumps(lambda x: 10 * x)) -assert restored_lambda(10) == 100 +# Lambdas +data = fory.dumps(lambda x: 10 * x) +print(fory.loads(data)(10)) # 100 ``` -## Serialize Class Objects +## Serialize Global Classes/Methods -Register the `type` carrier before serializing a class object, and authorize class resolution in the -deserialization policy. Register the concrete application class separately when its instances also -appear in the payload: +Serialize class objects, instance methods, class methods, and static methods: ```python -from collections import Counter - +from dataclasses import dataclass import pyfory -from pyfory import DeserializationPolicy +fory = pyfory.Fory(xlang=False, ref=True, strict=False) + +@dataclass +class Person: + name: str + age: int + def f(self, x): + return self.age * x -class TrustedClassPolicy(DeserializationPolicy): - def validate_module(self, module_name, is_local, **kwargs): - if module_name != "collections": - raise ValueError(f"Blocked module: {module_name}") + @classmethod + def g(cls, x): + return 10 * x - def validate_class(self, cls, is_local, **kwargs): - if cls is not Counter or is_local: - raise ValueError(f"Blocked class: {cls!r}") + @staticmethod + def h(x): + return 10 * x + +# Serialize global class +print(fory.loads(fory.dumps(Person))("Bob", 25)) # Person(name='Bob', age=25) +# Serialize instance method +print(fory.loads(fory.dumps(Person("Bob", 20).f))(10)) # 200 -fory = pyfory.Fory( - xlang=False, - ref=True, - strict=False, - policy=TrustedClassPolicy(), -) -fory.register_type(type) +# Serialize class method +print(fory.loads(fory.dumps(Person.g))(10)) # 100 -restored = fory.loads(fory.dumps(Counter)) -assert restored is Counter +# Serialize static method +print(fory.loads(fory.dumps(Person.h))(10)) # 100 ``` -## Serialize Local Classes And Class Methods +## Serialize Local Classes/Methods -Local classes are reconstructed from their definition, so use `ref=True` and a policy that -authorizes construction of the class, its functions, and its bound class methods. Register all -carriers before the first root operation: +Serialize classes defined inside functions along with their methods: ```python -import types - +from dataclasses import dataclass import pyfory -from pyfory import DeserializationPolicy - +fory = pyfory.Fory(xlang=False, ref=True, strict=False) -class TrustedLocalTypePolicy(DeserializationPolicy): - allowed_materialization = {type, types.FunctionType, types.MethodType} - - def authorize_instantiation(self, cls, **kwargs): - if cls not in self.allowed_materialization: - raise ValueError(f"Blocked materialization: {cls!r}") - - def validate_class(self, cls, is_local, **kwargs): - if cls is object and not is_local: - return - if not is_local or cls.__name__ != "LocalMessage": - raise ValueError(f"Blocked class: {cls!r}") - - def validate_function(self, func, is_local, **kwargs): - if not is_local or func.__name__ != "label": - raise ValueError(f"Blocked function: {func!r}") - - def validate_method(self, method, is_local, **kwargs): - if not is_local or method.__name__ != "label": - raise ValueError(f"Blocked method: {method!r}") - - -def make_local_class(): - class LocalMessage: - kind = "local" +def create_local_class(): + class LocalClass: + def f(self, x): + return 10 * x @classmethod - def label(cls, value): - return f"{cls.kind}: {value}" - - return LocalMessage - + def g(cls, x): + return 10 * x -fory = pyfory.Fory( - xlang=False, - ref=True, - strict=False, - policy=TrustedLocalTypePolicy(), -) -for carrier in (type, types.FunctionType, types.MethodType, staticmethod, classmethod): - fory.register_type(carrier) + @staticmethod + def h(x): + return 10 * x + return LocalClass -restored = fory.loads(fory.dumps(make_local_class())) -assert restored.label("hello") == "local: hello" -``` - -## Serialize Methods - -Register the method carriers and receiver class before serializing bound instance methods. A -static method is serialized as its underlying function: - -```python -import types - -import pyfory -from pyfory import DeserializationPolicy +# Serialize local class +data = fory.dumps(create_local_class()) +print(fory.loads(data)().f(10)) # 100 +# Serialize local class instance method +data = fory.dumps(create_local_class()().f) +print(fory.loads(data)(10)) # 100 -class Calculator: - def scale(self, x): - return 3 * x +# Serialize local class method +data = fory.dumps(create_local_class().g) +print(fory.loads(data)(10)) # 100 - @staticmethod - def double(x): - return 2 * x - - -class TrustedMethodPolicy(DeserializationPolicy): - def authorize_instantiation(self, cls, **kwargs): - if cls not in (Calculator, types.FunctionType, types.MethodType): - raise ValueError(f"Blocked materialization: {cls!r}") - - def validate_function(self, func, is_local, **kwargs): - if func is Calculator.double and not is_local: - return - if is_local and func.__qualname__ == "Calculator.double": - return - raise ValueError(f"Blocked function: {func!r}") - - def validate_method(self, method, is_local, **kwargs): - if type(method.__self__) is not Calculator or method.__name__ != "scale": - raise ValueError(f"Blocked method: {method!r}") - - -fory = pyfory.Fory( - xlang=False, - ref=True, - strict=False, - policy=TrustedMethodPolicy(), -) -for carrier in (types.FunctionType, types.MethodType, Calculator): - fory.register_type(carrier) - -assert fory.loads(fory.dumps(Calculator().scale))(10) == 30 -assert fory.loads(fory.dumps(Calculator.double))(10) == 20 +# Serialize local class static method +data = fory.dumps(create_local_class().h) +print(fory.loads(data)(10)) # 100 ``` diff --git a/docs/object-serialization/python/index.md b/docs/object-serialization/python/index.md index 4e51a77b05..d066d46ab5 100644 --- a/docs/object-serialization/python/index.md +++ b/docs/object-serialization/python/index.md @@ -19,8 +19,7 @@ license: | limitations under the License. --- -**Apache Fory™** is a multi-language serialization framework with Python-native and -cross-language object serialization modes. +**Apache Fory™** is a blazing fast multi-language serialization framework powered by **JIT compilation** and **zero-copy** techniques, providing up to **ultra-fast performance** while maintaining ease of use and safety. `pyfory` provides the Python implementation of Apache Fory™, offering xlang mode for cross-language payloads and native mode for Python-only object serialization. @@ -29,28 +28,28 @@ cross-language object serialization modes. ### Flexible Serialization Modes - **Xlang mode**: Default cross-language wire format with compatible schema evolution -- **Python native mode**: Same-language mode for a configured Python type surface +- **Python native mode**: Same-language mode and drop-in replacement for pickle/cloudpickle ### Versatile Serialization Features - **Reference tracking** for shared xlang schema objects and Python native-mode circular graphs -- **Polymorphism support** for registered customized types +- **Polymorphism support** for customized types with automatic type dispatching - **Schema evolution** support for backward/forward compatibility when using dataclasses in xlang mode -- **Out-of-band buffer support** for NumPy ndarrays and `pickle.PickleBuffer` values +- **Out-of-band buffer support** for zero-copy serialization of large data structures like NumPy arrays and Pandas DataFrames, compatible with pickle protocol 5 -### Python Runtime Support +### Blazing Fast Performance -- **Runtime code generation** for registered data models -- **Cython-accelerated** core implementation +- **Extremely fast performance** compared to other serialization frameworks +- **Runtime code generation** and **Cython-accelerated** core implementation for optimal performance ### Compact Data Size -- **Compact object graph protocol** +- **Compact object graph protocol** with minimal space overhead—up to 3× size reduction compared to pickle/cloudpickle - **Meta packing and sharing** to minimize type forward/backward compatibility space overhead ### Security & Safety -- **Strict mode** requires application type registration. +- **Strict mode** prevents deserialization of untrusted types by type registration and checks. - **Reference tracking** for handling circular references safely ## Installation @@ -80,14 +79,13 @@ pip install -e ".[dev]" ## Thread Safety -`pyfory` provides `ThreadSafeFory` for sharing one configured serialization facade across threads: +`pyfory` provides `ThreadSafeFory` for thread-safe serialization using a pooled wrapper: ```python +import pyfory import threading from dataclasses import dataclass -import pyfory - @dataclass class Person: name: str @@ -109,13 +107,14 @@ for t in threads: t.start() for t in threads: t.join() ``` -**Key Behavior:** +**Key Features:** -- **Thread-safe use**: Root serialization and deserialization may be called from multiple threads -- **Shared Configuration**: Complete every registration before the first root attempt -- **Matching Operations**: Exposes the corresponding root and registration methods -- **Registration Safety**: The first root attempt permanently freezes registration, even if the - operation fails +- **Instance Pool**: Maintains a pool of `Fory` instances protected by a lock for thread safety +- **Shared Configuration**: All registrations must be done upfront and are applied to all instances +- **Shared Root API**: Provides the same root serialization and deserialization operations as + `Fory` +- **Registration Safety**: Prevents explicit registration after the first root serialization or + deserialization attempt **When to Use:** @@ -126,9 +125,8 @@ for t in threads: t.join() ## Quick Start ```python -from dataclasses import dataclass - import pyfory +from dataclasses import dataclass @dataclass class Person: @@ -145,29 +143,11 @@ result = fory.deserialize(data) print(result) # Person(name='Alice', age=30) ``` -## Registration Lifecycle - -Register every application type before the first root serialization or deserialization attempt. In -native mode, also register callable, class, method, state, and reduction carrier types that can -appear in the object graph. The first root attempt permanently freezes the instance's registry, -including when that attempt fails. Setting `strict=False` may authorize module-global resolution -during a trusted native read, but it does not permit type or serializer registration after that -point. - -If the first operation exposes an incomplete or invalid registration, create a new instance and -register the complete type surface before retrying. A fully configured instance can process a later -root after a failure while reading input data or serializing a value. See -[Type Registration](type-registration.md) for the complete lifecycle. - ## Xlang Mode And Native Mode Use xlang mode for cross-language payloads and dataclass schemas shared with other Fory implementations. Xlang mode is the default Python wire mode, and Python examples that use it set `xlang=True` explicitly so the mode choice is visible. -Use native mode for Python-only traffic. Native mode is selected with `xlang=False` and supports a -configured surface that may include functions, lambdas, classes, methods, `__reduce__`, -`__getstate__`, NumPy ndarrays, and out-of-band buffers. Register application types and Python-native -carrier types before the first root attempt. Compatible mode is enabled by default. Set -`compatible=False` only when every reader and writer uses the same Python class schema. +Use native mode for Python-only traffic. Native mode is selected with `xlang=False` and owns pickle/cloudpickle-style behavior such as functions, lambdas, classes, methods, `__reduce__`, `__getstate__`, and out-of-band pickle protocol 5 buffers. It is optimized for Python's type system and supports a broader Python object surface than xlang mode, so use it when replacing pickle or cloudpickle. Compatible mode is enabled by default. Set `compatible=False` only when every reader and writer uses the same Python class schema and you want faster serialization and smaller size. See [Native Serialization](native.md) for Python-only serialization details and [Cross-Language Interoperability](basic-serialization.md#cross-language-interoperability) for Python xlang registration and interoperability rules. @@ -178,7 +158,7 @@ See [Native Serialization](native.md) for Python-only serialization details and - [Configuration](configuration.md) - Fory parameters, modes, and security - [Type Registration](type-registration.md) - User-defined type registration - [Custom Serializers](custom-serializers.md) - Extend serialization behavior -- [Row Format](../../row-format/python.md) - Row-format APIs +- [Row Format](../../row-format/python.md) - Zero-copy row format - [gRPC Support](../../grpc/python.md) - Fory payloads over grpcio ## Links diff --git a/docs/object-serialization/python/native.md b/docs/object-serialization/python/native.md index 8ba371bfe9..e77e24bbea 100644 --- a/docs/object-serialization/python/native.md +++ b/docs/object-serialization/python/native.md @@ -50,17 +50,9 @@ import pyfory fory = pyfory.Fory(xlang=False, ref=False, strict=True) ``` -Keep `strict=True` for registered, trusted type surfaces. Use `strict=False` only when the configured -surface includes native carriers such as functions, local classes, or objects reconstructed by -reduction hooks. In either mode, register every application type and native carrier whose serializer -must be installed before the first root serialization or deserialization attempt. The first attempt -permanently freezes the instance's registry even when it fails. With `strict=False`, the configured -policy may still authorize module-global classes and callables resolved while reading a native -payload; that lookup does not add a type or serializer to the frozen registry. - -For an application type whose `__reduce__` or `__reduce_ex__` result contains list-item or -dict-item iterators, register the concrete iterator carrier types before the first root operation. -Fory preserves those carriers without copying their contents into temporary lists. +Keep `strict=True` for registered, trusted type surfaces. Use `strict=False` only when native-mode +payloads need dynamic Python types such as functions, local classes, or objects reconstructed by +reduction hooks. ## Common Usage @@ -69,6 +61,9 @@ import pyfory fory = pyfory.Fory(xlang=False, ref=True, strict=False) +data = fory.dumps({"name": "Alice", "age": 30, "scores": [95, 87, 92]}) +print(fory.loads(data)) + from dataclasses import dataclass @dataclass @@ -76,11 +71,6 @@ class Person: name: str age: int -fory.register_type(Person) - -data = fory.dumps({"name": "Alice", "age": 30, "scores": [95, 87, 92]}) -print(fory.loads(data)) - person = Person("Bob", 25) data = fory.dumps(person) print(fory.loads(data)) # Person(name='Bob', age=25) @@ -96,10 +86,9 @@ deserialization. Treat untrusted native-mode bytes the same way you would treat bytes. - Keep `strict=True` when deserializing data that should contain only registered or built-in types. -- Use `strict=False` only for trusted payloads that require configured Python class or function - carriers. -- Provide a `policy=` deserialization policy when native carriers are required but the accepted - type surface should still be restricted. +- Use `strict=False` only for trusted payloads that require dynamic Python classes or functions. +- Provide a `policy=` deserialization policy when dynamic types are required but the accepted type + surface should still be restricted. - Do not use xlang/native mode choice as a security control. Apply strict mode, policies, registration, and resource limits based on the payload source. @@ -155,9 +144,9 @@ Use this when the payload stays in Python and large buffers should avoid extra c | ------------------------------------------ | ------------------------ | ----------------------- | | Python-only payloads | Yes | Optional | | Non-Python readers or writers | No | Yes | -| Registered functions, methods, and classes | Yes | No | +| Functions, lambdas, local classes | Yes | No | | `__reduce__` / `__getstate__` object hooks | Yes | No | -| Configured pickle-style workloads | Yes | No | +| Pickle/cloudpickle replacement | Yes | No | | Portable type mapping across languages | No | Yes | ## Performance Comparison @@ -183,10 +172,8 @@ on every peer, and avoid Python-only values such as lambdas or local classes. ### A dynamic class or function fails to deserialize -Before the first root operation, register the application type and the native carrier types needed -by the payload. Use `strict=False` for trusted payloads and provide a deserialization `policy=` when -only selected dynamic types should be accepted. Create a new configured `Fory` instance if an -already-used instance needs a different type surface. +Use `strict=False` for trusted payloads and provide a deserialization `policy=` when only selected +dynamic types should be accepted. ### A cycle does not round-trip diff --git a/docs/object-serialization/python/numpy-integration.md b/docs/object-serialization/python/numpy-integration.md index 966cffdeea..8188696ace 100644 --- a/docs/object-serialization/python/numpy-integration.md +++ b/docs/object-serialization/python/numpy-integration.md @@ -1,5 +1,5 @@ --- -title: NumPy +title: NumPy & Pandas sidebar_position: 12 id: numpy-integration license: | @@ -19,61 +19,85 @@ license: | limitations under the License. --- -Python native mode supports NumPy ndarrays as built-in values. +Fory natively supports numpy arrays and pandas DataFrame with optimized serialization. ## NumPy Array Serialization -Serialize and deserialize an ndarray directly: +Large arrays use zero-copy when possible: ```python -import numpy as np import pyfory +import numpy as np -fory = pyfory.Fory(xlang=False) +f = pyfory.Fory(xlang=False) # Numpy arrays are supported natively arrays = { - "matrix": np.arange(12, dtype=np.float64).reshape(3, 4), - "vector": np.arange(10, dtype=np.int64), - "bool_mask": np.array([True, False, True]), + 'matrix': np.random.rand(1000, 1000), + 'vector': np.arange(10000), + 'bool_mask': np.random.choice([True, False], size=5000) } -data = fory.serialize(arrays) -result = fory.deserialize(data) +data = f.serialize(arrays) +result = f.deserialize(data) -assert np.array_equal(arrays["matrix"], result["matrix"]) +# Zero-copy for compatible array types +assert np.array_equal(arrays['matrix'], result['matrix']) ``` -The ndarray carrier itself is available when the instance is created. Register application types -that contain ndarrays, plus every custom type that can appear inside an object-dtype ndarray, before -the first root attempt. The first root attempt permanently freezes registration, including when it -fails. `strict=False` does not permit late type or serializer registration. +## Pandas DataFrames -## Out-of-Band Buffers - -Use a buffer callback to transport ndarray storage separately from the root bytes: +Fory can serialize Pandas DataFrames efficiently: ```python +import pyfory +import pandas as pd import numpy as np + +f = pyfory.Fory(xlang=False, ref=False, strict=False) + +df = pd.DataFrame({ + 'a': np.arange(1000, dtype=np.float64), + 'b': np.arange(1000, dtype=np.int64), + 'c': ['text'] * 1000 +}) + +data = f.serialize(df) +result = f.deserialize(data) + +assert df.equals(result) +``` + +## Zero-Copy with Out-of-Band Buffers + +For maximum performance with large arrays, use out-of-band serialization: + +```python import pyfory +import numpy as np -fory = pyfory.Fory(xlang=False, ref=False) +f = pyfory.Fory(xlang=False, ref=False, strict=False) -array = np.arange(10000, dtype=np.float64).reshape(100, 100) +# Large array +array = np.random.rand(10000, 1000) +# Out-of-band for zero-copy buffer_objects = [] -data = fory.serialize(array, buffer_callback=buffer_objects.append) +data = f.serialize(array, buffer_callback=buffer_objects.append) buffers = [obj.getbuffer() for obj in buffer_objects] -result = fory.deserialize(data, buffers=buffers) +result = f.deserialize(data, buffers=buffers) assert np.array_equal(array, result) ``` -For a contiguous ndarray, `getbuffer()` can expose the existing storage as a `memoryview`. A -non-contiguous ndarray may be copied to create a contiguous transport buffer. The application must -send all collected buffers with the root bytes and provide them to `deserialize` in the same order. +## Supported Array Types + +- `np.ndarray` (all dtypes) +- `np.matrix` +- Structured arrays +- Record arrays ## Related Topics -- [Out-of-Band Serialization](out-of-band.md) - Buffer callback APIs +- [Out-of-Band Serialization](out-of-band.md) - Zero-copy buffers - [Basic Serialization](basic-serialization.md) - Standard usage diff --git a/docs/object-serialization/python/out-of-band.md b/docs/object-serialization/python/out-of-band.md index 263989b0fa..8db222fecb 100644 --- a/docs/object-serialization/python/out-of-band.md +++ b/docs/object-serialization/python/out-of-band.md @@ -19,31 +19,24 @@ license: | limitations under the License. --- -Fory can separate supported binary storage from the main serialized bytes through an out-of-band -buffer callback. Python native mode supports this flow for NumPy ndarrays and -`pickle.PickleBuffer` values. +Fory supports pickle5-compatible out-of-band buffer serialization for efficient zero-copy handling of large data structures. ## Overview -Out-of-band serialization separates the Fory root bytes from selected buffers: +Out-of-band serialization separates metadata from the actual data buffers, allowing for: -- `BufferObject.getbuffer()` exposes a `memoryview`; contiguous NumPy storage can be exposed without - an additional copy. -- The application transports the root bytes and out-of-band buffers together and in order. -- `BufferObject.write_to()` writes a selected buffer to a writable stream. - -`numpy.ndarray` and `pickle.PickleBuffer` are built-in native types. If an application wrapper or -an object-dtype ndarray can contain custom values, register every application and Python-native -carrier type before the first root attempt. That first attempt permanently freezes registration, -including when it fails; `strict=False` does not permit late type or serializer registration. +- **Zero-copy transfers** when sending data over networks or IPC using `memoryview` +- **Improved performance** for large datasets +- **Pickle5 compatibility** using `pickle.PickleBuffer` +- **Flexible stream support** - write to any writable object (files, BytesIO, sockets, etc.) ## Basic Out-of-Band Serialization ```python -import numpy as np import pyfory +import numpy as np -fory = pyfory.Fory(xlang=False, ref=False) +fory = pyfory.Fory(xlang=False, ref=False, strict=False) # Large numpy array array = np.arange(10000, dtype=np.float64) @@ -52,7 +45,9 @@ array = np.arange(10000, dtype=np.float64) buffer_objects = [] serialized_data = fory.serialize(array, buffer_callback=buffer_objects.append) -# Convert collected buffer objects to memoryviews for transport. +# Convert buffer objects to memoryview for zero-copy transmission +# For contiguous buffers (bytes, numpy arrays), this is zero-copy +# For non-contiguous data, a copy may be created to ensure contiguity buffers = [obj.getbuffer() for obj in buffer_objects] # Deserialize with out-of-band buffers (accepts memoryview, bytes, or Buffer) @@ -61,48 +56,73 @@ deserialized_array = fory.deserialize(serialized_data, buffers=buffers) assert np.array_equal(array, deserialized_array) ``` +## Out-of-Band with Pandas DataFrames + +```python +import pyfory +import pandas as pd +import numpy as np + +fory = pyfory.Fory(xlang=False, ref=False, strict=False) + +# Create a DataFrame with numeric columns +df = pd.DataFrame({ + 'a': np.arange(1000, dtype=np.float64), + 'b': np.arange(1000, dtype=np.int64), + 'c': ['text'] * 1000 +}) + +# Serialize with out-of-band buffers +buffer_objects = [] +serialized_data = fory.serialize(df, buffer_callback=buffer_objects.append) +buffers = [obj.getbuffer() for obj in buffer_objects] + +# Deserialize +deserialized_df = fory.deserialize(serialized_data, buffers=buffers) + +assert df.equals(deserialized_df) +``` + ## Selective Out-of-Band Serialization Control which buffers go out-of-band by providing a callback that returns `True` to keep data in-band or `False` to send it out-of-band: ```python -import numpy as np import pyfory +import numpy as np -fory = pyfory.Fory(xlang=False, ref=True) +fory = pyfory.Fory(xlang=False, ref=True, strict=False) arr1 = np.arange(1000, dtype=np.float64) arr2 = np.arange(2000, dtype=np.float64) data = [arr1, arr2] buffer_objects = [] +counter = 0 def selective_callback(buffer_object): - # Send buffers of at least 12,000 bytes out-of-band. - if buffer_object.total_bytes() >= 12_000: + global counter + counter += 1 + # Only send even-numbered buffers out-of-band + if counter % 2 == 0: buffer_objects.append(buffer_object) - return False - return True + return False # Out-of-band + return True # In-band serialized = fory.serialize(data, buffer_callback=selective_callback) buffers = [obj.getbuffer() for obj in buffer_objects] deserialized = fory.deserialize(serialized, buffers=buffers) - -assert np.array_equal(arr1, deserialized[0]) -assert np.array_equal(arr2, deserialized[1]) ``` -## `pickle.PickleBuffer` Values +## Pickle5 Compatibility -Python native mode accepts `pickle.PickleBuffer` as a built-in value. The outer bytes remain Fory -native bytes; they are not Pickle wire data. +Fory's out-of-band serialization is fully compatible with pickle protocol 5: ```python -import pickle - import pyfory +import pickle -fory = pyfory.Fory(xlang=False, ref=False) +fory = pyfory.Fory(xlang=False, ref=False, strict=False) # PickleBuffer objects are automatically supported data = b"Large binary data" @@ -118,17 +138,16 @@ deserialized = fory.deserialize(serialized, buffers=buffers) assert bytes(deserialized.raw()) == data ``` -## Writing A Buffer To A Stream +## Writing Buffers to Different Streams -The `BufferObject.write_to()` method accepts a writable stream object: +The `BufferObject.write_to()` method accepts any writable stream object: ```python -import io - -import numpy as np import pyfory +import numpy as np +import io -fory = pyfory.Fory(xlang=False, ref=False) +fory = pyfory.Fory(xlang=False, ref=False, strict=False) array = np.arange(1000, dtype=np.float64) @@ -136,17 +155,22 @@ array = np.arange(1000, dtype=np.float64) buffer_objects = [] serialized = fory.serialize(array, buffer_callback=buffer_objects.append) -# Write to an in-memory stream and obtain a memoryview. +# Write to different stream types for buffer_obj in buffer_objects: + # Write to BytesIO (in-memory stream) bytes_stream = io.BytesIO() buffer_obj.write_to(bytes_stream) - assert bytes_stream.getvalue() == array.tobytes() + + # Write to file + with open('/tmp/buffer_data.bin', 'wb') as f: + buffer_obj.write_to(f) + + # Get zero-copy memoryview (for contiguous buffers) mv = buffer_obj.getbuffer() assert isinstance(mv, memoryview) ``` -For a contiguous NumPy ndarray, `getbuffer()` can expose the existing storage. A non-contiguous -array may be copied to produce a contiguous transport buffer. +**Note**: For contiguous memory buffers (like bytes, numpy arrays), `getbuffer()` returns a zero-copy `memoryview`. For non-contiguous data, a copy may be created to ensure contiguity. ## Related Topics diff --git a/docs/object-serialization/python/security.md b/docs/object-serialization/python/security.md index 9ee181c76c..f7bfca16d9 100644 --- a/docs/object-serialization/python/security.md +++ b/docs/object-serialization/python/security.md @@ -29,10 +29,10 @@ Before deserialization: - Authenticate the sender and protect message integrity at the transport or storage layer. - Enforce request or file size, timeout, and concurrency limits outside Fory. -- Register only the application types the endpoint accepts and configure the reader before its - first root operation. -- In native mode, register every callable, class, method, state, or reduction carrier that an - accepted graph may contain. +- In strict or xlang mode, register only the application types the endpoint accepts and configure + the reader before its first root operation. +- In non-strict native mode, use a policy to authorize every callable, class, method, state, or + reduction path that an accepted graph may contain. - Validate the deserialized value against application authorization and domain rules before use. ## Built-in safeguards @@ -68,8 +68,6 @@ fory.register(OrderModel, name="example.Order") Use native-mode deserialization with `strict=False` only for trusted Python-only payloads: ```python -import types - import pyfory fory = pyfory.Fory( @@ -78,16 +76,15 @@ fory = pyfory.Fory( strict=False, max_depth=100, ) - -fory.register_type(types.FunctionType) ``` The first root attempt permanently freezes registration, including when that attempt fails. `strict=False` does not permit type or serializer registration after that boundary, but its policy may authorize module-global classes and callables resolved while reading a trusted native payload. -That resolution does not mutate the registry. If the first operation exposes an incomplete or -invalid registration, create and configure a new instance before retrying. A fully configured -reader can process a later root after a malformed-data failure. +That resolution may populate resolver-owned caches but does not add or change an explicit +registration. If a strict-mode failure requires adding a missing registration, create and configure +a new instance. An already configured reader can process a later root after a malformed-data +failure. Received remote metadata is also limited: @@ -159,7 +156,8 @@ unchanged. ### Security Checklist - Keep `strict=True` for untrusted data. -- Register all expected application and Python-native carrier types before the first root attempt. +- Complete any explicit type, name, ID, or custom serializer registration before the first root + attempt. - Use `DeserializationPolicy` when `strict=False` is necessary. - Keep `max_depth` low enough to reject unexpectedly deep payloads. - Keep `max_graph_memory_bytes` at the fixed `128 MiB` default for most inputs, or set a positive diff --git a/docs/object-serialization/python/serialization-hooks.md b/docs/object-serialization/python/serialization-hooks.md index 022476d44f..009f3a41ad 100644 --- a/docs/object-serialization/python/serialization-hooks.md +++ b/docs/object-serialization/python/serialization-hooks.md @@ -20,15 +20,14 @@ license: | --- Python native mode honors Python object customization protocols while writing Fory native bytes. -It does not emit Pickle wire data. Use this page when a class controls its reduction, construction, -or state restoration. +It does not emit Pickle wire data. Use this page when replacing pickle or cloudpickle or when a +class controls its reduction, construction, or state restoration. -## When To Use Native Mode +## Pickle And Cloudpickle Replacement -Native mode supports a configured Python-only type surface that may include Python functions, local -classes, closures, and reduction hooks. Register every application type and Python-native carrier -before the first root attempt. The first attempt permanently freezes registration, even if it -fails, and `strict=False` does not permit late type or serializer registration. +Native mode is the Python mode to choose when the existing boundary uses `pickle` or +`cloudpickle`. It supports richer Python values than JSON and xlang mode, including Python +functions, local classes, closures, and reduction hooks. Use xlang mode instead when the payload crosses language boundaries or the data model should be a portable schema shared with other Fory implementations. @@ -50,15 +49,13 @@ class SessionToken: def __setstate__(self, state): self.value = state["value"] -fory = pyfory.Fory(xlang=False, ref=True, strict=False, compatible=False) -fory.register_type(SessionToken) +fory = pyfory.Fory(xlang=False, strict=False) token = fory.loads(fory.dumps(SessionToken("abc"))) print(token.value) # abc ``` Use these hooks for Python-only payloads. For xlang payloads, model the data as dataclasses with -portable field annotations instead. Complete all registration before the first root operation, as -described in [Type Registration](type-registration.md). +portable field annotations instead. ## Protocol 5 buffers diff --git a/docs/object-serialization/python/troubleshooting.md b/docs/object-serialization/python/troubleshooting.md index e96196e265..221fcfc5c0 100644 --- a/docs/object-serialization/python/troubleshooting.md +++ b/docs/object-serialization/python/troubleshooting.md @@ -62,23 +62,16 @@ object identity or cycles matter: f = pyfory.Fory(ref=True) ``` -For configured Python-native object graphs with circular references, use Python native mode: +For arbitrary Python object graphs with circular references, use Python native mode: ```python -from dataclasses import dataclass -from typing import Optional - -import pyfory - f = pyfory.Fory(xlang=False, ref=True, strict=False) # Example with circular reference -@dataclass class Node: - value: int - next: Optional["Node"] = pyfory.field(ref=True, nullable=True, default=None) - -f.register_type(Node) + def __init__(self, value): + self.value = value + self.next = None node1 = Node(1) node2 = Node(2) @@ -93,18 +86,17 @@ assert result.next.next is result # Circular reference preserved ### Schema Evolution Not Working ```python -from dataclasses import dataclass +# Keep compatible mode enabled. This is the default. +f = pyfory.Fory() -import pyfory - -# Version 1: Original class +# Version 1: Writer schema @dataclass class UserV1: name: str age: pyfory.Int32 writer = pyfory.Fory(xlang=True) -writer.register(UserV1, name="example.User") +writer.register(UserV1, name="User") data = writer.dumps(UserV1("Alice", 30)) # Version 2: Add new field (backward compatible) @@ -114,8 +106,9 @@ class UserV2: age: pyfory.Int32 email: str = "unknown@example.com" # New field with default +# Register the reader schema on a separate instance. reader = pyfory.Fory(xlang=True) -reader.register(UserV2, name="example.User") +reader.register(UserV2, name="User") user = reader.loads(data) print(user.email) # "unknown@example.com" ``` @@ -130,15 +123,14 @@ f = pyfory.Fory(strict=True) f.register(MyClass, type_id=100) f.register(AnotherClass, type_id=101) -# Native mode may use strict=False only for trusted data, but application -# and Python-native carrier types still must be registered before use. -native_fory = pyfory.Fory(xlang=False, strict=False) -native_fory.register_type(MyClass) +# Or disable strict mode (NOT recommended for production) +f = pyfory.Fory(strict=False) # Use only in trusted environments ``` -The first root attempt permanently freezes registration, even when it fails. Do not register a -missing type and retry on that same instance. Create a new instance, register the complete type -surface, and retry with the new instance. +The first root serialization or deserialization attempt permanently freezes explicit registration, +including when that attempt fails. Non-strict native writes may still discover runtime types +lazily, and reads may resolve those authorized by the configured policy, without creating an +explicit registration. ## Debug Mode @@ -160,33 +152,29 @@ import pyfory # Now uses pure Python implementation Handle common serialization errors gracefully: ```python -from dataclasses import dataclass - import pyfory -from pyfory.error import TypeUnregisteredError +from pyfory.error import TypeUnregisteredError, TypeNotCompatibleError -@dataclass -class Message: - text: str +fory = pyfory.Fory(strict=True) -message = Message("hello") -unconfigured = pyfory.Fory(xlang=False, strict=True, compatible=False) try: - unconfigured.dumps(message) + data = fory.dumps(my_object) except TypeUnregisteredError as e: print(f"Type not registered: {e}") - # The failed instance is already frozen. Configure a new one. - fory = pyfory.Fory(xlang=False, strict=True, compatible=False) - fory.register_type(Message, type_id=100) - data = fory.dumps(message) + # A failed root has already frozen this instance. Configure a new one. + fory = pyfory.Fory(strict=True) + fory.register(type(my_object), type_id=100) + data = fory.dumps(my_object) +except Exception as e: + print(f"Serialization failed: {e}") try: - fory.loads(b"") -except Exception: - pass - -# Root cleanup makes the configured instance reusable after the failed read. -assert fory.loads(data) == message + obj = fory.loads(data) +except TypeNotCompatibleError as e: + print(f"Schema mismatch: {e}") + # Handle version mismatch +except Exception as e: + print(f"Deserialization failed: {e}") ``` ## Development Setup diff --git a/docs/object-serialization/python/type-registration.md b/docs/object-serialization/python/type-registration.md index 768ec5b3c1..095d6c89d4 100644 --- a/docs/object-serialization/python/type-registration.md +++ b/docs/object-serialization/python/type-registration.md @@ -84,12 +84,12 @@ payloads, and keep the same registration IDs or names on every peer that shares those payloads. The first root serialization or deserialization attempt permanently closes -registration, including when that attempt fails. `strict=False` permits its -configured policy to resolve module-global classes and callables while reading -trusted native payloads, but that resolution does not reopen the registry or -install a new serializer. Register native carrier types and application types -whose serializers must be installed before the first root operation. -Later registration attempts fail. +registration, including when that attempt fails. `strict=False` permits native +writes to discover runtime classes and callables and permits reads to resolve +those authorized by the configured policy. That lazy resolution does not +reopen the registry or create an explicit binding. Explicit names, IDs, and custom +serializers must be configured before the first root; later registration +attempts fail. Compatible metadata has one data-only exception: when a remote Struct has no local registration, deserialization returns the fixed framework diff --git a/docs/security/deserialization.md b/docs/security/deserialization.md index 1e6afdf3d1..ce9720518f 100644 --- a/docs/security/deserialization.md +++ b/docs/security/deserialization.md @@ -54,7 +54,7 @@ Fory security boundaries include: - Explicit Fory policy checks, such as type, function, method, class, or registration policies that are intended to restrict what may be materialized. - Cleanup boundaries, where state created during a failed read must be released - or reset before the next root operation. + or reset before the root error escapes. Fory security boundaries do not include: @@ -383,9 +383,8 @@ derived from input size, and stream budgeting should not depend on dynamic bytes Graph budget accounting should: -- be initialized in top-level read state, with restoration owned by the runtime's root lifecycle - boundary before that read state is reused; runtimes may restore it in the root `finally` or in - the next root-entry reset according to their established context lifecycle; +- be initialized in top-level read state, with cleanup owned by the top-level deserialization + `finally`; - account only for Fory-created objects or storage that are retained by the returned value graph; temporary helper objects used only during construction are outside the graph budget; @@ -618,20 +617,19 @@ that case, classify the behavior by concrete impact: ## Registry Lifecycle -The first root operation permanently closes type and serializer registration, including when that -operation or registry finalization fails. Permanent freeze and successful finalization are distinct: -a root entered during finalization or after failed finalization must fail before serializer or read -work, and an accelerator must neither cache nor retry incomplete finalization. Failure state must -not retain the exception or traceback graph. Registration that invokes application code must -recheck the authoritative lifecycle before publishing application-derived state. Thread-safe -facades retain only registrations that completed before the freeze. During child -construction, a semantic replay log may reuse only an identical accepted registration that the -child has already applied; unknown or different requests fail before that request mutates the child. -A facade that replays opaque registration callbacks and cannot roll them back must become -permanently unusable when a callback fails rather than expose partially registered children. These -rules prevent a failed or reentrant registration from changing the accepted type surface after -deserialization has begun. -Runtime-specific publication ownership belongs in the implementation guide and language guidance. +The first root serialization or deserialization permanently closes explicit type and serializer +registration, including when that operation fails. Each natural registration owner keeps one +authoritative frozen flag, and every later explicit registration attempt fails before changing +type, serializer, ID, name, metadata, or policy bindings. A thread-safe facade with its own public +registration surface may own its boundary flag, but must not mirror a child registry's flag. +Implementations must use one boolean flag without another lifecycle state, a registration commit +path, or eager whole-registry preparation. + +Registry freeze does not disable native runtime type resolution. When a mode supports unregistered +types, a root may still discover an allowed runtime type and materialize resolver-owned metadata, +serializers, or generated code. Lazy completion of an existing binding is also allowed. These +internal cache operations are not explicit registration and must not create or change an explicit +type or serializer registration, ID, name, or policy binding. ## Metadata And Type Resolution @@ -658,12 +656,16 @@ Metadata readers should: entry so input cannot make the JVM derive an unbounded family of array classes. - Reset or release metadata state at the correct root-operation boundary. -Operation-local metadata occurrences and writer IDs must be reset before the context is reused, -including after a failed root. The reset must make prior-root entries invisible through the current -logical size and release unusual high-water backing without adding allocation or slot-clearing work -to normal roots. Bounded backing may retain inactive slot references when the runtime-specific -retention rule permits it. Runtime-specific thresholds and reset ownership belong in the -implementation guide and language guidance. +Operation-local metadata occurrences and writer IDs from a failed root must reset before its error +escapes. Successful roots may reset before the context is reused. The reset must make prior-root +entries invisible through the current logical size and release unusual high-water backing without +adding allocation or slot-clearing work to normal roots. Bounded backing may retain inactive slot +references when the runtime-specific retention rule permits it. Runtime-specific thresholds and +reset ownership belong in the implementation guide and language guidance. + +One-time warning registries retain their keys beyond the current root. A warning selected by a +remote class or type name must therefore use a fixed key instead of including that untrusted name or +another unbounded remote value in the message arguments. A class-resolution cache reachable from untrusted deserialization may publish an entry only from explicit trusted configuration or after the active class @@ -682,11 +684,6 @@ class-keyed state, or `Class.getName()`. A custom-name registration does not by itself publish the Java class name as an additional alias; ID registration does publish the Java class name. -Read-side warnings selected by remote class or type names must use fixed -one-time-log keys. They must not include remote names or other -untrusted-cardinality values in the message or arguments, because those keys are -retained for the logger lifetime. - Remote metadata that can create persistent read state must be bounded before that state is retained. The check is resource control only: it must not change wire compatibility, type registration, dynamic class loading, unknown-type diff --git a/docs/specification/xlang_implementation_guide.md b/docs/specification/xlang_implementation_guide.md index a22f23ff63..ec27d15932 100644 --- a/docs/specification/xlang_implementation_guide.md +++ b/docs/specification/xlang_implementation_guide.md @@ -79,96 +79,40 @@ not the place where nested serializers do their work. - writing and reading the root xlang header bitmap - delegating nested value encoding to `WriteContext` - delegating nested value decoding to `ReadContext` -- owning registration through `TypeResolver` +- freezing the natural registration owner before root codec work - resetting operation-local context state at the top-level root boundary -Registration preparation may invoke serializer constructors, generated factories, or application -callbacks. Complete the callback before publishing the registry entry it prepares. If the callback -starts a root operation, registration must recheck the authoritative per-instance freeze owner -when the callback returns and reject that publication. Kotlin and Scala combined generated-struct -registration are the type-first exception: publish the canonical type required by generated -serializer construction, then construct the serializer in the resolver's existing construction -graph. The candidate is visible only to that construction and reaches the normal resolver commit -only after the authoritative freeze recheck succeeds. A serializer-only helper rejects a missing -canonical type instead of auto-registering it. Do not use direct serializer replacement or add -rollback, staging, or a parallel registration path for that exception. Module -installation may consist of complete nested registrations. `Fory.register(ForyModule)` alone owns -module identity, cycle breaking, and idempotence; language bootstrap helpers must not add markers, -monitors, or separate reentry policies. Keep a retryable install body replay-safe until its final -non-repeatable publication. - -Java thread-safe facades serialize each registration callback across their children. A root or -facade registration reentered by that callback rejects the in-progress registration, and any -callback failure closes the facade permanently. This fail-closed boundary prevents partially -applied child state or divergent callback order from becoming observable without adding resolver -rollback, snapshots, or a second registration path. A late thread-local child remains provisional -until every accepted callback has replayed and its resolver has finalized; facade access cannot -expose that child sooner. - -C# `ThreadSafeFory` validates registration on its staging runtime and publishes only a successful -replay action. The resolver prepares serializer bindings and encoded names before one map commit; -a failed callback does not require rebuilding or replacing the staging runtime. - -Python `ThreadSafeFory` validates registrations before retaining semantic replay descriptors. A -later child applies the accepted descriptor prefix in order. A nested replay request is a no-op -only when it exactly matches an accepted descriptor already applied to that child; an unknown or -different request fails before that request mutates the child. Retained descriptors may contain a -serializer class or factory, but every result must belong to the current child resolver and -normalized declared type; they must not reuse one resolver-bound serializer instance. Use the -existing `fory_factory` when each child needs a separately configured serializer instance. - -Java `TypeResolver` owns one construction-local graph for serializer constructors, including self -and mutual recursion. The graph separates final `TypeInfo` owners from unpublished serializer -candidates. Recursive fields capture the final owner during construction. Construction owners that -resolve recursive fields or candidate state use the construction-local serializer; ordinary -resolver lookups retain their runtime semantics. When wire and user IDs match, the final owner is -the existing canonical `TypeInfo`, so generated serializers and field metadata never retain -temporary metadata. After construction and the registry lifecycle recheck succeed, the normal -Class/Xtype resolver commit path installs the candidate. Static-generated construction retains the -already registered canonical type identity and exposes its immutable generated descriptors only to -the same construction graph; its early-bound serializer candidate is never published. There is no -constructor-specific publication path. Static-generated serializer classes require an already -registered canonical type and are therefore rejected by the combined class overload. Direct Java -`Fory` instances may install a module before their first root operation; thread-safe facades -install modules only through `ForyBuilder.withModule` during construction. - -JavaScript generated registration seals the complete `TypeInfo` schema graph before code -generation, including nested schemas and field occurrence modifiers. The package-internal seal -locks each schema-owned pointer before reading or traversing it. The writer-owned `dynamicTypeId` -remains mutable because it is reset per root. Code generation seeds every complete Struct, enum, -and union definition by registry identity before resolving identity-only occurrences, so recursive -schema resolution does not depend on field order. One numeric ID or name cannot identify different -user-defined type families. Each resolver identity has one complete schema owner in the graph. -Repeated references and clones may share that owner's immutable definition containers and settings, -while a second conflicting complete definition fails before code generation without a deep -structural comparison. Complete anonymous definitions without a name or user ID remain distinct. -They stay in the current generation graph rather than publishing under their raw wire type ID. An -enum without a mapping and a union without cases use the canonical generic serializer for their raw -wire type; they are definitions, not unresolved schema references. An extension occurrence without -class metadata resolves through its registered custom serializer owner. -Code generation first builds the complete serializer source graph against generation-local owners. -Each transaction entry stores the schema and progress facts used by later code generation, while -field occurrence modifiers remain owned by the containing schema. Within one registration -transaction, all of its application code hooks run before any of its runtime serializer factories -are instantiated. After those hooks complete, a same-definition owner published by nested -registration becomes the final owner for that identity. Each remaining factory is instantiated once -in dependency completion order so its fixed captures point directly to the final published-or-local -owners, and the resolver then batch-publishes only the remaining local owners. This does not rerun -code generation or hooks, rebuild a factory after publication, or leave a transaction lookup, cell, -callback, or wrapper in a runtime serializer. Runtime and dynamic dispatch retain the real -`TypeResolver`. An unresolved nested identity fails registration before resolver publication when -its type family requires a separate definition, such as Struct. An initialized owner published by a -nested registration is authoritative and must not be overwritten. A conflicting family or complete -definition still fails before outer publication. +Explicit type and serializer registration is open only before the first root serialization or +deserialization. Starting either root sets one authoritative frozen flag for the natural +registration owner before codec work and never clears it, including when the root fails. Every +explicit registration path checks that flag before mutation. A thread-safe facade with its own +public registration surface may own its boundary flag, but must not mirror a child registry's flag. +Implementations must use one boolean flag without another lifecycle state, a registration commit +path, or eager whole-registry preparation. + +In JavaScript, `Fory` owns this flag. `TypeResolver` owns the registration maps but must not carry a +second lifecycle flag. + +Registry freeze does not make resolver caches immutable. Native modes that allow unregistered +runtime types may still discover an allowed type and materialize its resolver-owned metadata, +serializer, or generated code. Lazy serializer completion for an existing binding is also allowed. +These operations must not create or change an explicit type, serializer, ID, name, or policy +registration. + +Java module registration remains available through `BaseFory` before the first root. Kotlin and +Scala registration extensions target `BaseFory`, so direct and thread-safe facades share the same +pre-root API. Copy operations and facade execution callbacks do not freeze registration unless they +start a root serialization or deserialization. Nested serializers must not call back into root `serialize(...)` or `deserialize(...)` entry points. ### `WriteContext` and `ReadContext` hold operation-local state -`WriteContext` and `ReadContext` are prepared by `Fory` for one root operation. -`Fory` resets state left by the previous root, including a failed root, before -the context is reused. +`WriteContext` and `ReadContext` are prepared by `Fory` for one root operation. A failed root resets +its operation-local state before propagating the error. Successful roots may retain bounded state +until the next root entry, but must reset it before the context is reused. A failure object must not +retain operation-local state or its materialized object graph. `prepare(...)` should only bind the active buffer and root-operation inputs. `reset()` should clear operation-local mutable state. @@ -186,7 +130,9 @@ slots; a larger table replaces its backing with eight slots. This uniform owner cleanup allocation-free and must not be specialized for particular entry counts or benchmark shapes. JavaScript read-side metadata occurrence arrays use native replacement reset instead. Its MetaString and TypeMeta writer owner tables retain bounded backing through 8192 active owners, reset only -their own logical size after restoring active owner IDs, and release backing above that boundary. +their own logical size after restoring active owner IDs, and release backing above that boundary. A +failed JavaScript root resets this operation-local state before its exception escapes; state from a +successful root resets on the next root entry so the success exit remains allocation-free. That operation-local state includes: @@ -1015,7 +961,8 @@ The current root write flow is: 3. `Fory` calls `writeContext.prepare(...)`. 4. `Fory` writes the root bitmap. 5. `Fory` delegates the root object to `WriteContext`. -6. State left by the write resets before the next root reuses the context. +6. A failed write resets operation-local state before propagating its error. State retained after a + successful write resets before the next root reuses the context. For a non-null root value, `WriteContext.writeRootValue(...)` performs: @@ -1051,7 +998,8 @@ The current root read flow mirrors the write flow: 4. `Fory` validates xlang mode and other root framing requirements. 5. `Fory` calls `readContext.prepare(...)`. 6. `Fory` delegates to `ReadContext`. -7. State left by the read resets before the next root reuses the context. +7. A failed read resets operation-local state before propagating its error. State retained after a + successful read resets before the next root reuses the context. ### `ReadContext` owns ref reservation and payload materialization @@ -1334,9 +1282,8 @@ Important rules: Depth should stay explicit on the contexts rather than relying on the native call stack alone. At the same time, depth cleanup should not depend on nested -`try/finally` blocks throughout serializer code. Top-level context reset must -recover operation-local state before the context is reused after a root -failure. +`try/finally` blocks throughout serializer code. Top-level context reset must be +able to recover operation-local state after failures. ## Struct Compatibility diff --git a/python/README.md b/python/README.md index 6460a5b11a..c93d41c0a1 100644 --- a/python/README.md +++ b/python/README.md @@ -7,38 +7,39 @@ [![Slack Channel](https://img.shields.io/badge/slack-join-3f0e40?logo=slack&style=for-the-badge)](https://join.slack.com/t/fory-project/shared_invite/zt-36g0qouzm-kcQSvV_dtfbtBKHRwT5gsw) [![X](https://img.shields.io/badge/@ApacheFory-follow-blue?logo=x&style=for-the-badge)](https://x.com/ApacheFory) -`pyfory` is the Python implementation of Apache Fory™. It provides Python-native and cross-language -object serialization together with row-format APIs for analytical data. +**Apache Fory™** is a blazing fast multi-language serialization framework powered by **JIT compilation** and **zero-copy** techniques, providing up to **ultra-fast performance** while maintaining ease of use and safety. + +`pyfory` provides the Python implementation of Apache Fory™, offering both high-performance object serialization and advanced row-format capabilities for data processing tasks. ## Key Features ### **Flexible Serialization Modes** - **Xlang mode**: Default cross-language wire format with compatible schema evolution -- **Python native mode**: Same-language mode for configured Python type surfaces -- **Row Format**: Random and partial access to analytical row data +- **Python native mode**: Same-language mode and drop-in replacement for pickle/cloudpickle +- **Row Format**: Zero-copy row format for analytics workloads ### Versatile Serialization Features - **Shared/circular reference support** for complex object graphs in both Python native and xlang modes - **Polymorphism support** for customized types with automatic type dispatching - **Schema evolution** support for backward/forward compatibility when using dataclasses in xlang mode -- **Out-of-band buffer support** for NumPy arrays and `pickle.PickleBuffer` values +- **Out-of-band buffer support** for zero-copy serialization of large data structures like NumPy arrays and Pandas DataFrames, compatible with pickle protocol 5 - **Reduced-precision xlang types** use reserved `pyfory.Float16` and `pyfory.BFloat16` annotations and native Python `float` values; dense array payloads use public wrappers such as `Float16Array` and `BFloat16Array` -### Python Runtime Implementation +### Blazing Fast Performance -- **Runtime code generation** for supported Python classes -- **Cython-accelerated** core implementation +- **Extremely fast performance** compared to other serialization frameworks +- **Runtime code generation** and **Cython-accelerated** core implementation for optimal performance ### Compact Data Size -- **Compact object graph protocol** for Python-native and cross-language payloads +- **Compact object graph protocol** with minimal space overhead—up to 3× size reduction compared to pickle/cloudpickle - **Meta packing and sharing** to minimize type forward/backward compatibility space overhead ### **Security & Safety** -- **Strict mode** limits application class loading to the configured registration surface. +- **Strict mode** prevents deserialization of untrusted types by type registration and checks. - **Reference tracking** for handling circular references safely ## Installation @@ -70,12 +71,11 @@ pip install -e ".[dev,format]" ## Python Native Serialization -`pyfory` provides a Python native mode for configured Python-only payloads, with support for -functions, methods, dataclasses, stateful types, and reduction hooks. +`pyfory` provides a Python native mode for Python-only payloads. It is optimized for Python's type +system and offers the same object surface as pickle/cloudpickle, but with **significantly better +performance, smaller data size, and enhanced security features**. -Register every application type and native carrier before the first root operation. The first -serialization or deserialization attempt permanently freezes that `Fory` instance's registry, -including when the attempt fails. +The binary protocol and API are similar to Fory's xlang mode, but Python native mode can serialize any Python object—including global functions, local functions, lambdas, local classes and types with custom serialization using `__getstate__/__reduce__/__reduce_ex__`, which are not allowed in xlang mode. To use Python native mode, create `Fory` with `xlang=False`. Use this mode when replacing pickle or cloudpickle for pure Python applications: @@ -131,26 +131,18 @@ result = fory.deserialize(data) print(result) # Person(name='Bob', age=25, ...) ``` -## Pickle-Style Python-Native Serialization +## Drop-in Replacement for Pickle/Cloudpickle -`pyfory` can serialize a configured Python type surface with the following options: +`pyfory` can serialize any Python object with the following configuration: - **For circular references**: Set `ref=True` to enable reference tracking -- **For Python-native carriers**: Set `strict=False` only for trusted payloads - -Register every application type and native carrier type before the first root serialization or -deserialization call. The first root operation permanently freezes that `Fory` instance's registry, -even when the operation fails. `strict=False` may authorize module-global resolution while reading -a trusted native payload, but it does not permit late type or serializer registration. If the first -operation exposes incomplete or invalid registration, configure a new instance before retrying. +- **For functions/classes**: Set `strict=False` to allow deserialization of dynamic types -**Security Warning**: Configured native carriers can import modules and construct Python objects -when `strict=False`. Use this mode only with trusted payloads, and provide a -`DeserializationPolicy` through `policy=` when the accepted surface must be restricted. +**Security Warning**: When `strict=False`, Fory will deserialize arbitrary types, which can pose security risks if data comes from untrusted sources. Only use `strict=False` in controlled environments where you trust the data source completely. If you do need to use `strict=False`, please configure a `DeserializationPolicy` when creating fory using `policy=your_policy` to controlling deserialization behavior. ### Common Usage -Built-in containers require no registration. Register custom classes before the first root: +Serialize common Python objects including dicts, lists, and custom classes without any registration: ```python import pyfory @@ -158,6 +150,11 @@ import pyfory # Create Fory instance fory = pyfory.Fory(xlang=False, ref=True, strict=False) +# serialize common Python objects +data = fory.dumps({"name": "Alice", "age": 30, "scores": [95, 87, 92]}) +print(fory.loads(data)) + +# serialize custom objects from dataclasses import dataclass @dataclass @@ -165,44 +162,287 @@ class Person: name: str age: int -fory.register_type(Person) - -# serialize common Python objects -data = fory.dumps({"name": "Alice", "age": 30, "scores": [95, 87, 92]}) -print(fory.loads(data)) - -# serialize custom objects person = Person("Bob", 25) data = fory.dumps(person) print(fory.loads(data)) # Person(name='Bob', age=25) ``` -### Functions, Classes, And Methods +### Serialize Global Functions + +Capture and get functions defined at module level. Fory deserialize and return same function object: + +```python +import pyfory + +# Create Fory instance +fory = pyfory.Fory(xlang=False, ref=True, strict=False) + +# serialize global functions +def my_global_function(x): + return 10 * x + +data = fory.dumps(my_global_function) +print(fory.loads(data)(10)) # 100 +``` + +#### Serialize Local Functions/Lambdas + +Serialize functions with closures and lambda expressions. Fory captures the closure variables automatically: + +```python +import pyfory + +# Create Fory instance +fory = pyfory.Fory(xlang=False, ref=True, strict=False) + +# serialize local functions with closures +def my_function(): + local_var = 10 + def local_func(x): + return x * local_var + return local_func + +data = fory.dumps(my_function()) +print(fory.loads(data)(10)) # 100 + +# serialize lambdas +data = fory.dumps(lambda x: 10 * x) +print(fory.loads(data)(10)) # 100 +``` + +#### Serialize Global Classes/Methods + +Serialize class objects, instance methods, class methods, and static methods. All method types are supported: + +```python +from dataclasses import dataclass +import pyfory +fory = pyfory.Fory(xlang=False, ref=True, strict=False) + +# serialize global class +@dataclass +class Person: + name: str + age: int + + def f(self, x): + return self.age * x + + @classmethod + def g(cls, x): + return 10 * x + + @staticmethod + def h(x): + return 10 * x -Python native mode supports configured global and local functions, lambdas, class objects, and -methods. Register their application and carrier types and authorize them with a deserialization -policy before the first root operation. See -[Functions, Classes, and Methods](https://fory.apache.org/docs/object-serialization/python/functions-classes-methods) -for the supported shapes and complete examples. +print(fory.loads(fory.dumps(Person))("Bob", 25)) # Person(name='Bob', age=25) +# serialize global class instance method +print(fory.loads(fory.dumps(Person("Bob", 20).f))(10)) # 200 +# serialize global class class method +print(fory.loads(fory.dumps(Person.g))(10)) # 100 +# serialize global class static method +print(fory.loads(fory.dumps(Person.h))(10)) # 100 +``` + +#### Serialize Local Classes/Methods + +Serialize classes defined inside functions along with their methods. Useful for dynamic class creation: + +```python +from dataclasses import dataclass +import pyfory +fory = pyfory.Fory(xlang=False, ref=True, strict=False) + +def create_local_class(): + class LocalClass: + def f(self, x): + return 10 * x + + @classmethod + def g(cls, x): + return 10 * x + + @staticmethod + def h(x): + return 10 * x + return LocalClass + +# serialize local class +data = fory.dumps(create_local_class()) +print(fory.loads(data)().f(10)) # 100 + +# serialize local class instance method +data = fory.dumps(create_local_class()().f) +print(fory.loads(data)(10)) # 100 + +# serialize local class method +data = fory.dumps(create_local_class().g) +print(fory.loads(data)(10)) # 100 + +# serialize local class static method +data = fory.dumps(create_local_class().h) +print(fory.loads(data)(10)) # 100 +``` ### Out-of-Band Buffer Serialization -Python native mode can separate supported NumPy ndarray and `pickle.PickleBuffer` storage from the -root bytes through `buffer_callback`. Transport the collected buffers with the root bytes and pass -them to `deserialize` in the same order. For contiguous storage, `BufferObject.getbuffer()` can -expose a `memoryview` without an additional source-side copy; non-contiguous storage may be copied. -This does not promise copy-free transport or decoding. +Fory supports pickle5-compatible out-of-band buffer serialization for efficient zero-copy handling of large data structures. This is particularly useful for NumPy arrays, Pandas DataFrames, and other objects with large memory footprints. -See [Out-of-Band Buffers](https://fory.apache.org/docs/object-serialization/python/out-of-band) for -the callback, transport, and stream APIs. +Out-of-band serialization separates metadata from the actual data buffers, allowing for: + +- **Zero-copy transfers** when sending data over networks or IPC using `memoryview` +- **Improved performance** for large datasets +- **Pickle5 compatibility** using `pickle.PickleBuffer` +- **Flexible stream support** - write to any writable object (files, BytesIO, sockets, etc.) + +#### Basic Out-of-Band Serialization + +```python +import pyfory +import numpy as np + +fory = pyfory.Fory(xlang=False, ref=False, strict=False) + +# Large numpy array +array = np.arange(10000, dtype=np.float64) + +# Serialize with out-of-band buffers +buffer_objects = [] +serialized_data = fory.serialize(array, buffer_callback=buffer_objects.append) + +# Convert buffer objects to memoryview for zero-copy transmission +# For contiguous buffers (bytes, numpy arrays), this is zero-copy +# For non-contiguous data, a copy may be created to ensure contiguity +buffers = [obj.getbuffer() for obj in buffer_objects] + +# Deserialize with out-of-band buffers (accepts memoryview, bytes, or Buffer) +deserialized_array = fory.deserialize(serialized_data, buffers=buffers) + +assert np.array_equal(array, deserialized_array) +``` + +#### Out-of-Band with Pandas DataFrames + +```python +import pyfory +import pandas as pd +import numpy as np + +fory = pyfory.Fory(xlang=False, ref=False, strict=False) + +# Create a DataFrame with numeric columns +df = pd.DataFrame({ + 'a': np.arange(1000, dtype=np.float64), + 'b': np.arange(1000, dtype=np.int64), + 'c': ['text'] * 1000 +}) + +# Serialize with out-of-band buffers +buffer_objects = [] +serialized_data = fory.serialize(df, buffer_callback=buffer_objects.append) +buffers = [obj.getbuffer() for obj in buffer_objects] + +# Deserialize +deserialized_df = fory.deserialize(serialized_data, buffers=buffers) + +assert df.equals(deserialized_df) +``` + +#### Selective Out-of-Band Serialization + +You can control which buffers go out-of-band by providing a callback that returns `True` to keep data in-band or `False` (and appending to a list) to send it out-of-band: + +```python +import pyfory +import numpy as np + +fory = pyfory.Fory(xlang=False, ref=True, strict=False) + +arr1 = np.arange(1000, dtype=np.float64) +arr2 = np.arange(2000, dtype=np.float64) +data = [arr1, arr2] + +buffer_objects = [] +counter = 0 + +def selective_callback(buffer_object): + global counter + counter += 1 + # Only send even-numbered buffers out-of-band + if counter % 2 == 0: + buffer_objects.append(buffer_object) + return False # Out-of-band + return True # In-band + +serialized = fory.serialize(data, buffer_callback=selective_callback) +buffers = [obj.getbuffer() for obj in buffer_objects] +deserialized = fory.deserialize(serialized, buffers=buffers) +``` + +#### Pickle5 Compatibility + +Fory's out-of-band serialization is fully compatible with pickle protocol 5. When objects implement `__reduce_ex__(protocol)`, Fory automatically uses protocol 5 to enable `pickle.PickleBuffer` support: + +```python +import pyfory +import pickle + +fory = pyfory.Fory(xlang=False, ref=False, strict=False) + +# PickleBuffer objects are automatically supported +data = b"Large binary data" +pickle_buffer = pickle.PickleBuffer(data) + +# Serialize with buffer callback for out-of-band handling +buffer_objects = [] +serialized = fory.serialize(pickle_buffer, buffer_callback=buffer_objects.append) +buffers = [obj.getbuffer() for obj in buffer_objects] + +# Deserialize with buffers +deserialized = fory.deserialize(serialized, buffers=buffers) +assert bytes(deserialized.raw()) == data +``` + +#### Writing Buffers to Different Streams + +The `BufferObject.write_to()` method accepts any writable stream object, making it flexible for various use cases: + +```python +import pyfory +import numpy as np +import io + +fory = pyfory.Fory(xlang=False, ref=False, strict=False) + +array = np.arange(1000, dtype=np.float64) + +# Collect out-of-band buffers +buffer_objects = [] +serialized = fory.serialize(array, buffer_callback=buffer_objects.append) + +# Write to different stream types +for buffer_obj in buffer_objects: + # Write to BytesIO (in-memory stream) + bytes_stream = io.BytesIO() + buffer_obj.write_to(bytes_stream) + + # Write to file + with open('/tmp/buffer_data.bin', 'wb') as f: + buffer_obj.write_to(f) + + # Get zero-copy memoryview (for contiguous buffers) + mv = buffer_obj.getbuffer() + assert isinstance(mv, memoryview) +``` + +**Note**: For contiguous memory buffers (like bytes, numpy arrays), `getbuffer()` returns a zero-copy `memoryview`. For non-contiguous data, a copy may be created to ensure contiguity. ## Cross-Language Object Graph Serialization `pyfory` supports cross-language object graph serialization, allowing you to serialize data in Python and deserialize it in Java, Go, Rust, or other supported languages. -The binary protocol and API are similar to `pyfory`'s Python native mode. Python-specific callable, -stateful, and reduction carriers are available only in native mode and must be registered before -the first root operation. +The binary protocol and API are similar to `pyfory`'s Python native mode, but Python native mode can serialize any Python object—including global functions, local functions, lambdas, local classes, and types with custom serialization using `__getstate__/__reduce__/__reduce_ex__`, which are not allowed in xlang mode. Xlang mode is the default. Set `xlang=True` explicitly in cross-language examples so the mode choice is visible: @@ -259,12 +499,9 @@ fory.register(Person.class, "example.Person"); Person person = (Person) fory.deserialize(binaryData); ``` -## Row Format +## Row Format - Zero-Copy Processing -Row Format provides random and partial access to trusted analytical data without reconstructing the -complete object graph. See the -[Python Row Format guide](https://fory.apache.org/docs/row-format/python) for supported types, schema -requirements, and APIs. +Apache Fory™ provides a random-access row format that enables reading nested fields from binary data without full deserialization. This drastically reduces overhead when working with large objects where only partial data access is needed. The format also supports memory-mapped files for ultra-low memory footprint. ### Basic Row Format Usage @@ -304,7 +541,7 @@ foo = Foo( # Encode to row format binary: bytes = encoder.to_row(foo).to_bytes() -# Access selected fields without full deserialization. +# Zero-copy access - no full deserialization needed! foo_row = pyfory.RowData(encoder.schema, binary) print(foo_row.f2[100000]) # Access 100,000th element directly print(foo_row.f4[100000].f1) # Access nested field directly @@ -349,7 +586,7 @@ foo.f4 = bars; // Encode to row format (cross-language compatible with Python) BinaryRow binaryRow = encoder.toRow(foo); -// Random access without full deserialization +// Zero-copy random access without full deserialization BinaryArray f2Array = binaryRow.getArray(1); // Access f2 list BinaryArray f4Array = binaryRow.getArray(3); // Access f4 list BinaryRow bar10 = f4Array.getStruct(10); // Access 11th Bar @@ -407,7 +644,7 @@ fory::row::encoder::RowEncoder encoder; encoder.encode(foo); auto row = encoder.get_writer().to_row(); -// Random access without full deserialization +// Zero-copy random access without full deserialization auto f2_array = row->get_array(1); // Access f2 list auto f4_array = row->get_array(3); // Access f4 list auto bar10 = f4_array->get_struct(10); // Access 11th Bar @@ -417,15 +654,176 @@ std::string str = bar10->get_string(0); // Access bar.f1 ### Key Benefits -- **Random access**: Read nested fields without deserializing the entire object -- **Cross-language layout**: Share Standard Row Format data between supported runtimes -- **Partial deserialization**: Deserialize only the elements the application needs +- **Zero-Copy Access**: Read nested fields without deserializing the entire object +- **Memory Efficiency**: Memory-map large datasets directly from disk +- **Cross-Language**: Binary format is compatible between Python, Java, and other Fory implementations +- **Partial Deserialization**: Deserialize only the specific elements you need +- **High Performance**: Skip unnecessary data parsing for analytics and big data workloads + +## Core API Reference + +### Fory Class + +The main serialization interface: + +```python +class Fory: + def __init__( + self, + xlang: bool = True, + ref: bool = False, + strict: bool = True, + compatible: bool | None = None, + max_depth: int = 50 + ) +``` + +### ThreadSafeFory Class -## API Reference +Thread-safe serialization interface using an instance pool: + +```python +class ThreadSafeFory: + def __init__( + self, + fory_factory=None, + **kwargs + ) +``` -See [Python Configuration](https://fory.apache.org/docs/object-serialization/python/configuration) -for the current `Fory` and `ThreadSafeFory` constructors, mode comparison, registration lifecycle, -configuration options, and root methods. +`ThreadSafeFory` provides thread-safe serialization by maintaining a pool of `Fory` instances protected by a lock. When a thread needs to serialize/deserialize, it gets an instance from the pool, uses it, and returns it. Complete explicit type and serializer registration before the first serialization or deserialization attempt. + +**Thread Safety Example:** + +```python +import pyfory +import threading +from dataclasses import dataclass + +@dataclass +class Person: + name: str + age: int + +# Create thread-safe Fory instance +fory = pyfory.ThreadSafeFory(xlang=False, ref=True) +fory.register(Person) + +# Use in multiple threads safely +def serialize_in_thread(thread_id): + person = Person(name=f"User{thread_id}", age=25 + thread_id) + data = fory.serialize(person) + result = fory.deserialize(data) + print(f"Thread {thread_id}: {result}") + +threads = [threading.Thread(target=serialize_in_thread, args=(i,)) for i in range(10)] +for t in threads: t.start() +for t in threads: t.join() +``` + +**Key Features:** + +- **Instance Pool**: Maintains a pool of `Fory` instances protected by a lock for thread safety +- **Shared Configuration**: All registrations must be done upfront and are applied to all instances +- **Shared Root API**: Provides the same root serialization and deserialization operations as + `Fory` +- **Registration Safety**: Prevents explicit registration after the first root serialization or + deserialization attempt + +**When to Use:** + +- **Multi-threaded Applications**: Web servers, concurrent workers, parallel processing +- **Shared Fory Instances**: When multiple threads need to serialize/deserialize data +- **Thread Pools**: Applications using thread pools or concurrent.futures + +**Parameters:** + +- **`xlang`** (`bool`, default=`True`): Use xlang mode. Set `False` for Python native mode supporting Python-specific objects. +- **`ref`** (`bool`, default=`False`): Enable reference tracking for shared/circular references. Disable for better performance if your data has no shared references. +- **`strict`** (`bool`, default=`True`): Require type registration for security. **Highly recommended** for production. Only disable in trusted environments. +- **`compatible`** (`bool | None`, default `None`): Enable schema evolution. `None` enables compatible mode in both xlang and native mode. Set `False` only when every reader and writer always uses the same Python class schema and you want faster serialization and smaller size. +- **`max_depth`** (`int`, default=`50`): Maximum deserialization depth for security, preventing stack overflow attacks. + +**Key Methods:** + +```python +# Serialization (serialize/deserialize are identical to dumps/loads) +data: bytes = fory.serialize(obj) +obj = fory.deserialize(data) + +# Alternative API (aliases) +data: bytes = fory.dumps(obj) +obj = fory.loads(data) + +# Direct Fory registration by id; serializer instances belong to that Fory. +fory.register(MyClass, type_id=123) +fory.register(MyClass, type_id=123, serializer=custom_serializer) + +# ThreadSafeFory constructs one serializer per pooled child from a class or factory. +thread_safe_fory.register(MyClass, type_id=123, serializer=CustomSerializer) + +# Direct Fory registration by name +fory.register(MyClass, name="my.package.MyClass") +fory.register(MyClass, name="my.package.MyClass", serializer=custom_serializer) +``` + +### Xlang And Native Mode Comparison + +| Feature | Native mode (`xlang=False`) | Xlang mode (default) | +| ------------------- | ---------------------------------------------- | ------------------------------------- | +| Use case | Pure Python applications | Multi-language systems | +| Compatibility | Python only | Java, Go, Rust, C++, JavaScript, etc. | +| Supported types | Python object surface | Cross-language compatible types | +| Functions/lambdas | Supported with trusted dynamic deserialization | Not allowed | +| Local classes | Supported with trusted dynamic deserialization | Not allowed | +| Dynamic classes | Supported with trusted dynamic deserialization | Not allowed | +| Schema mode default | Compatible | Compatible | + +#### Native Mode (`xlang=False`) + +Python native mode supports Python-specific objects including functions, classes, and closures. Use it for Python-only applications: + +```python +import pyfory + +# Python native mode +fory = pyfory.Fory(xlang=False, ref=True, strict=False) + +# Supports ALL Python objects: +data = fory.dumps({ + 'function': lambda x: x * 2, # Functions and lambdas + 'class': type('Dynamic', (), {}), # Dynamic classes + 'method': str.upper, # Methods + 'nested': {'circular_ref': None} # Circular references (when ref=True) +}) + +# Drop-in replacement for pickle/cloudpickle +import pickle +obj = [1, 2, {"nested": [3, 4]}] +assert fory.loads(fory.dumps(obj)) == pickle.loads(pickle.dumps(obj)) + +# Significantly faster and more compact than pickle +import timeit +obj = {f"key{i}": f"value{i}" for i in range(10000)} +print(f"Fory: {timeit.timeit(lambda: fory.dumps(obj), number=1000):.3f}s") +print(f"Pickle: {timeit.timeit(lambda: pickle.dumps(obj), number=1000):.3f}s") +``` + +#### Xlang Mode + +Xlang mode restricts types to those compatible across all Fory implementations. Use it for multi-language systems: + +```python +import pyfory + +f = pyfory.Fory(xlang=True, ref=True) + +# Only supports cross-language compatible types +f.register(MyDataClass, name="com.example.MyDataClass") + +# Data can be read by Java, Go, Rust, etc. +data = f.serialize(MyDataClass(field1="value", field2=42)) +``` ## Advanced Features @@ -434,51 +832,130 @@ configuration options, and root methods. Handle shared references and circular dependencies safely. Set `ref=True` to deduplicate objects: ```python -from dataclasses import dataclass -from typing import Optional - import pyfory f = pyfory.Fory(xlang=False, ref=True) # Enable reference tracking -@dataclass +# Handle circular references safely class Node: - value: str - next: Optional["Node"] = pyfory.field(ref=True, nullable=True, default=None) - -f.register_type(Node) + def __init__(self, value): + self.value = value + self.children = [] + self.parent = None root = Node("root") child = Node("child") -root.next = child -child.next = root # Circular reference +child.parent = root # Circular reference +root.children.append(child) # Serializes without infinite recursion data = f.serialize(root) result = f.deserialize(data) -assert result.next.next is result # Reference preserved +assert result.children[0].parent is result # Reference preserved ``` ### Type Registration -Register the complete application type surface before the first root operation. See -[Type Registration](https://fory.apache.org/docs/object-serialization/python/type-registration) for -registration identity, strict-mode behavior, and the frozen registry lifecycle. See -[Python Security](https://fory.apache.org/docs/object-serialization/python/security) before -accepting untrusted input. +In strict mode, Fory loads and instantiates only registered application types. +Compatible metadata for an unregistered remote Struct returns the fixed +data-only `pyfory.UnknownStruct` carrier; it does not load or generate the +sender-named class. This prevents arbitrary class materialization. + +The first root serialization or deserialization attempt permanently freezes explicit type and +serializer registration for that `Fory` instance, including when the attempt fails. Non-strict +native writes may still discover runtime types lazily, and reads may resolve those authorized by +the configured policy. That discovery does not add an explicit registration. + +```python +import pyfory + +# Strict mode (recommended for production) +f = pyfory.Fory(xlang=False, strict=True) + +class SafeClass: + def __init__(self, data): + self.data = data + +# Must register types in strict mode +f.register(SafeClass, name="com.example.SafeClass") + +# Now serialization works +obj = SafeClass("safe data") +data = f.serialize(obj) +result = f.deserialize(data) + +# Unregistered types will raise an exception +class UnsafeClass: + pass + +# This will fail in strict mode +try: + f.serialize(UnsafeClass()) +except Exception as e: + print("Security protection activated!") +``` ### Custom Serializers -Custom serializers implement the serializer-owned `write` and `read` operations and are registered -before the first root operation. See -[Custom Serializers](https://fory.apache.org/docs/object-serialization/python/custom-serializers) for -the supported constructor and context APIs. +Implement custom serialization logic for specialized types with a single `write/read` API: -### NumPy & Scientific Computing +```python +import pyfory +from pyfory.serializer import Serializer +from dataclasses import dataclass -Python native mode supports NumPy ndarrays, including multidimensional and object-dtype arrays. See -[NumPy Integration](https://fory.apache.org/docs/object-serialization/python/numpy-integration) for -supported behavior and out-of-band transport. +@dataclass +class Foo: + f1: int + f2: str + +class FooSerializer(Serializer): + def __init__(self, type_resolver, cls): + super().__init__(type_resolver, cls) + + def write(self, write_context, obj: Foo): + # Custom serialization logic + write_context.write_varint32(obj.f1) + write_context.write_string(obj.f2) + + def read(self, read_context): + # Custom deserialization logic + f1 = read_context.read_varint32() + f2 = read_context.read_string() + return Foo(f1, f2) + +f = pyfory.Fory(xlang=False) +f.register(Foo, type_id=100, serializer=FooSerializer(f.type_resolver, Foo)) + +# Now Foo uses your custom serializer +data = f.dumps(Foo(42, "hello")) +result = f.loads(data) +print(result) # Foo(f1=42, f2='hello') +``` + +### Numpy & Scientific Computing + +Fory natively supports numpy arrays with optimized serialization. Large arrays use zero-copy when possible: + +```python +import pyfory +import numpy as np + +f = pyfory.Fory(xlang=False) + +# Numpy arrays are supported natively +arrays = { + 'matrix': np.random.rand(1000, 1000), + 'vector': np.arange(10000), + 'bool_mask': np.random.choice([True, False], size=5000) +} + +data = f.serialize(arrays) +result = f.deserialize(data) + +# Zero-copy for compatible array types +assert np.array_equal(arrays['matrix'], result['matrix']) +``` ## Best Practices @@ -505,16 +982,14 @@ fory.register(ProductModel, type_id=102) ### Performance Tips -Use these configuration rules before measuring an application workload: +Optimize serialization speed and memory usage with these guidelines: 1. **Disable `ref=True` if not needed**: Reference tracking has overhead -2. **Reuse configured Fory instances**: Create once, use many times; use `ThreadSafeFory` when an - instance must be shared across threads -3. **Use `compatible=False` only for same-schema data**: Every reader and writer must use the same - Python class schema -4. **Use Row Format for partial reads**: Choose it when applications need random access to trusted - analytical row data instead of object reconstruction; see the - [Python Row Format guide](https://fory.apache.org/docs/row-format/python) +2. **Use type_id instead of name**: Integer IDs are faster than string names +3. **Reuse Fory instances**: Create once, use many times +4. **Use `compatible=False` only for same-schema data**: Disable compatible mode only when every reader and writer always uses the same Python class schema and you want faster serialization and smaller size +5. **Enable Cython**: Make sure `ENABLE_FORY_CYTHON_SERIALIZATION=1`, should be enabled by default +6. **Use row format for large arrays**: Zero-copy access for analytics ```python # Good: Reuse instance @@ -530,17 +1005,54 @@ for obj in objects: ### Type Registration Patterns -Use stable names for shared xlang schemas and numeric IDs for Python-native type identity. See -[Type Registration](https://fory.apache.org/docs/object-serialization/python/type-registration) for -the supported patterns, including custom serializers and batch registration. +Choose the right registration approach for your use case: + +```python +# Pattern 1: Simple registration +fory.register(MyClass, type_id=100) + +# Pattern 2: Cross-language with name +fory.register(MyClass, name="com.example.MyClass") + +# Pattern 3: With custom serializer +fory.register(MyClass, type_id=100, serializer=MySerializer(fory.type_resolver, MyClass)) + +# Pattern 4: Batch registration +type_id = 100 +for model_class in [User, Order, Product, Invoice]: + fory.register(model_class, type_id=type_id) + type_id += 1 +``` ### Error Handling -A failed root never reopens the registry. Create and fully configure a new instance after a missing -or invalid registration failure. A fully configured instance can process another root after a -failure while reading input data or serializing a value. See -[Error Handling](https://fory.apache.org/docs/object-serialization/python/troubleshooting#error-handling) -for a complete example. +Handle common serialization errors gracefully. Catch specific exceptions for better error recovery: + +```python +import pyfory +from pyfory.error import TypeUnregisteredError, TypeNotCompatibleError + +fory = pyfory.Fory(strict=True) + +try: + data = fory.dumps(my_object) +except TypeUnregisteredError as e: + print(f"Type not registered: {e}") + # A failed root has already frozen this instance. Configure a new one. + fory = pyfory.Fory(strict=True) + fory.register(type(my_object), type_id=100) + data = fory.dumps(my_object) +except Exception as e: + print(f"Serialization failed: {e}") + +try: + obj = fory.loads(data) +except TypeNotCompatibleError as e: + print(f"Schema mismatch: {e}") + # Handle version mismatch +except Exception as e: + print(f"Deserialization failed: {e}") +``` ## Security Best Practices @@ -577,11 +1089,9 @@ if os.getenv('ENV') == 'development': fory = pyfory.Fory( xlang=False, ref=True, - strict=False, # Use only with trusted development payloads + strict=False, # Allow any type for development max_depth=1000 # Higher limit for development ) - for model_class in [UserModel, ProductModel, OrderModel]: - fory.register_type(model_class) else: # Production configuration (security hardened) fory = pyfory.Fory( @@ -596,11 +1106,65 @@ else: ### DeserializationPolicy -When `strict=False` is necessary for trusted native-mode payloads, configure a -`DeserializationPolicy` before the first root operation to restrict accepted types and object hooks. -See -[Python Security](https://fory.apache.org/docs/object-serialization/python/security#deserializationpolicy) -for the supported policy hooks and configuration example. +When `strict=False` is necessary (e.g., deserializing functions/lambdas), use `DeserializationPolicy` to implement fine-grained security controls during deserialization. This provides protection similar to `pickle.Unpickler.find_class()` but with more comprehensive hooks. + +**Why use DeserializationPolicy?** + +- Block dangerous classes/modules (e.g., `subprocess.Popen`) +- Intercept and validate `__reduce__` callables before invocation +- Sanitize sensitive data during `__setstate__` +- Replace or reject deserialized objects based on custom rules + +**Example: Blocking Dangerous Classes** + +```python +import pyfory +from pyfory import DeserializationPolicy + +dangerous_modules = {'subprocess', 'os', '__builtin__'} + +class SafeDeserializationPolicy(DeserializationPolicy): + """Block potentially dangerous classes during deserialization.""" + + def validate_class(self, cls, is_local, **kwargs): + # Block dangerous modules + if cls.__module__ in dangerous_modules: + raise ValueError(f"Blocked dangerous class: {cls.__module__}.{cls.__name__}") + + def intercept_reduce_call(self, callable_obj, args, **kwargs): + # Block specific callable invocations during __reduce__ + if getattr(callable_obj, '__name__', "") == 'Popen': + raise ValueError("Blocked attempt to invoke subprocess.Popen") + return None + + def intercept_setstate(self, obj, state, **kwargs): + # Sanitize sensitive data + if isinstance(state, dict) and 'password' in state: + state['password'] = '***REDACTED***' + return None + +# Create Fory with custom security policy +policy = SafeDeserializationPolicy() +fory = pyfory.Fory(xlang=False, ref=True, strict=False, policy=policy) + +# Now deserialization is protected by your custom policy +data = fory.serialize(my_object) +result = fory.deserialize(data) # Policy hooks will be invoked +``` + +**Available Policy Hooks:** + +- Reference validation hooks reject by raising exceptions and otherwise leave deserialized references unchanged. +- `validate_class(cls, is_local)` - Validate/block class types during deserialization +- `validate_module(module_name, is_local)` - Validate/block module imports +- `validate_function(func, is_local)` - Validate/block function references +- `validate_method(method, is_local)` - Validate/block method references +- `intercept_reduce_call(callable_obj, args)` - Intercept `__reduce__` invocations +- `inspect_reduced_object(obj)` - Inspect/replace objects created via `__reduce__` +- `intercept_setstate(obj, state)` - Sanitize state before `__setstate__` +- `authorize_instantiation(cls, args, kwargs)` - Control class instantiation + +**See also:** `pyfory/policy.py` contains detailed documentation and examples for each hook. ## Troubleshooting @@ -650,9 +1214,26 @@ object identity or cycles matter: f = pyfory.Fory(ref=True) ``` -For configured Python object graphs with circular references, use native mode, register every -application type before the first root, and declare reference-tracked recursive fields as shown in -[Reference Tracking & Circular References](#reference-tracking--circular-references). +For arbitrary Python object graphs with circular references, use Python native mode: + +```python +f = pyfory.Fory(xlang=False, ref=True, strict=False) + +# Example with circular reference +class Node: + def __init__(self, value): + self.value = value + self.next = None + +node1 = Node(1) +node2 = Node(2) +node1.next = node2 +node2.next = node1 # Circular reference + +data = f.dumps(node1) +result = f.loads(data) +assert result.next.next is result # Circular reference preserved +``` ### Debug Mode @@ -670,10 +1251,30 @@ import pyfory # Now uses pure Python implementation **Q: Schema evolution not working** -Xlang mode defaults to compatible schema evolution. Configure writer and reader schemas on separate -instances because each instance's registry freezes on its first root operation. See -[Schema Evolution](https://fory.apache.org/docs/object-serialization/python/schema-evolution) for a -complete example. +```python +# A: Xlang mode defaults to compatible schema evolution. +f = pyfory.Fory(xlang=True) + +# Version 1: Original class +@dataclass +class User: + name: str + age: int + +f.register(User, name="User") +data = f.dumps(User("Alice", 30)) + +# Version 2: Add new field (backward compatible) +@dataclass +class User: + name: str + age: int + email: str = "unknown@example.com" # New field with default + +# Can still deserialize old data +user = f.loads(data) +print(user.email) # "unknown@example.com" +``` **Q: Type registration errors in strict mode** @@ -685,9 +1286,8 @@ f = pyfory.Fory(strict=True) f.register(MyClass, type_id=100) f.register(AnotherClass, type_id=101) -# Native carriers still require pre-registration when strict mode is disabled. -f = pyfory.Fory(xlang=False, strict=False) # Use only with trusted payloads -f.register_type(MyClass) +# Or disable strict mode (NOT recommended for production) +f = pyfory.Fory(strict=False) # Use only in trusted environments ``` ## Contributing @@ -707,6 +1307,10 @@ Apache Fory™ is an open-source project under the Apache Software Foundation. W Apache License 2.0. See [LICENSE](https://github.com/apache/fory/blob/main/LICENSE) for details. +--- + +**Apache Fory™** - Blazing fast, secure, and versatile serialization for modern applications. + ## Links - **Documentation**: https://fory.apache.org/docs/object-serialization/python/ From 5eded1c133c2055720758cf1f1ea67b15c458341 Mon Sep 17 00:00:00 2001 From: chaokunyang Date: Sun, 30 Aug 2026 04:12:08 +0800 Subject: [PATCH 121/168] docs(javascript): retain metadata cache ownership note --- javascript/packages/core/lib/context.ts | 3 +++ 1 file changed, 3 insertions(+) diff --git a/javascript/packages/core/lib/context.ts b/javascript/packages/core/lib/context.ts index 1e89e8b7ff..6ffea84063 100644 --- a/javascript/packages/core/lib/context.ts +++ b/javascript/packages/core/lib/context.ts @@ -1631,6 +1631,9 @@ export class ReadContext { : Type.struct(typeMeta.getUserTypeId()); originalSerializer = this.typeResolver.generateReadSerializer(originalTypeInfo); } + // This legacy direct-generation API accepts caller-owned metadata. It must + // not publish into the checked wire cache; only the validated read path can + // do that after binding a concrete local TypeMeta owner. return this.generateTypeMetaSerializer(typeMeta, originalSerializer); } From 2690eef0621ef7e6c9cb00fe7010e7b4767aaba8 Mon Sep 17 00:00:00 2001 From: chaokunyang Date: Sun, 30 Aug 2026 04:17:35 +0800 Subject: [PATCH 122/168] fix(python): preserve compatible enum placeholder --- python/pyfory/registry.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/python/pyfory/registry.py b/python/pyfory/registry.py index f975a62ae8..0ac885cf87 100644 --- a/python/pyfory/registry.py +++ b/python/pyfory/registry.py @@ -891,10 +891,11 @@ def get_type_info(self, cls, create=True): return type_info elif not create: return None - if self.require_registration and not issubclass(cls, Enum): - raise TypeUnregisteredError(f"{cls} not registered") + # Unknown remote enums use this internal placeholder; it is not an explicit registration. if cls is NonExistEnum: return self._get_nonexist_enum_type_info() + if self.require_registration and not issubclass(cls, Enum): + raise TypeUnregisteredError(f"{cls} not registered") logger.info("Type %s not registered", cls) return self._register_inferred_type(cls) From c9df37a6416f5e6c7df626539ba9d143843c0ae1 Mon Sep 17 00:00:00 2001 From: chaokunyang Date: Sun, 30 Aug 2026 04:22:03 +0800 Subject: [PATCH 123/168] test(javascript): use generated registry handles --- .../javascript/test/roundtrip.test.ts | 90 ++++++++----------- 1 file changed, 39 insertions(+), 51 deletions(-) diff --git a/integration_tests/idl_tests/javascript/test/roundtrip.test.ts b/integration_tests/idl_tests/javascript/test/roundtrip.test.ts index 6c1208adb4..5c0378cb46 100644 --- a/integration_tests/idl_tests/javascript/test/roundtrip.test.ts +++ b/integration_tests/idl_tests/javascript/test/roundtrip.test.ts @@ -22,7 +22,7 @@ import Fory, { BoolArray, Decimal, ForyFloat16Array, - Type, + type Serializer, } from "@apache-fory/core"; import { AddressBook, @@ -79,10 +79,12 @@ import { import { Color, Monster, registerMonsterTypes } from "../generated/monster"; import { TreeNode, registerTreeTypes } from "../generated/tree"; -type RegisterFn = (fory: Fory) => unknown; -type RegisteredTypeInfo = - | ReturnType - | ReturnType; +type RegisterFn = (fory: Fory) => Record; + +interface RegisteredFory { + fory: Fory; + serializers: Map; +} const MODES = [ { title: "schema-consistent", compatible: false }, @@ -93,38 +95,28 @@ function buildFory( compatible: boolean, ref: boolean, registerFns: ReadonlyArray, -): Fory { +): RegisteredFory { const fory = new Fory({ compatible, ref }); + const serializers = new Map(); for (const registerFn of registerFns) { - registerFn(fory); - } - return fory; -} - -function getSerializer(fory: Fory, typeInfo: RegisteredTypeInfo) { - const serializer = fory.typeResolver.getSerializerByTypeInfo(typeInfo); - if (!serializer) { - throw new Error(`Missing serializer for type id ${typeInfo.typeId}`); + for (const { serializer } of Object.values(registerFn(fory))) { + serializers.set(serializer.getTypeInfo().userTypeId, serializer); + } } - return serializer; + return { fory, serializers }; } function roundTripValue( - fory: Fory, - typeInfo: RegisteredTypeInfo, + registeredFory: RegisteredFory, + typeId: number, value: T, ): unknown { - const serializer = getSerializer(fory, typeInfo); - const bytes = fory.serialize(value, serializer); - return fory.deserialize(bytes, serializer); -} - -function roundTripStruct(fory: Fory, typeId: number, value: T): unknown { - return roundTripValue(fory, Type.struct(typeId), value); -} - -function roundTripUnion(fory: Fory, typeId: number, value: T): unknown { - return roundTripValue(fory, Type.union(typeId), value); + const serializer = registeredFory.serializers.get(typeId); + if (!serializer) { + throw new Error(`Missing serializer for type id ${typeId}`); + } + const bytes = registeredFory.fory.serialize(value, serializer); + return registeredFory.fory.deserialize(bytes, serializer); } function normalize(value: unknown): unknown { @@ -652,7 +644,7 @@ describe.each(MODES)( expectAcyclicEqual( buildAddressBook(), - roundTripStruct(fory, 103, buildAddressBook()), + roundTripValue(fory, 103, buildAddressBook()), ); const dogAnimal: Animal = { @@ -663,8 +655,8 @@ describe.each(MODES)( case: AnimalCase.CAT, value: buildCat(), }; - expectAcyclicEqual(dogAnimal, roundTripUnion(fory, 106, dogAnimal)); - expectAcyclicEqual(catAnimal, roundTripUnion(fory, 106, catAnimal)); + expectAcyclicEqual(dogAnimal, roundTripValue(fory, 106, dogAnimal)); + expectAcyclicEqual(catAnimal, roundTripValue(fory, 106, catAnimal)); }); test("round-trips auto_id messages and root wrapper unions", () => { @@ -680,14 +672,14 @@ describe.each(MODES)( value: "raw-payload", }; - expectAcyclicEqual(envelope, roundTripStruct(fory, 3022445236, envelope)); + expectAcyclicEqual(envelope, roundTripValue(fory, 3022445236, envelope)); expectAcyclicEqual( wrapperEnvelope, - roundTripUnion(fory, 1471345060, wrapperEnvelope), + roundTripValue(fory, 1471345060, wrapperEnvelope), ); expectAcyclicEqual( wrapperRaw, - roundTripUnion(fory, 1471345060, wrapperRaw), + roundTripValue(fory, 1471345060, wrapperRaw), ); }); @@ -699,23 +691,23 @@ describe.each(MODES)( expectAcyclicEqual( buildPrimitiveTypes(), - roundTripStruct(fory, 200, buildPrimitiveTypes()), + roundTripValue(fory, 200, buildPrimitiveTypes()), ); expectAcyclicEqual( buildNumericCollections(), - roundTripStruct(fory, 210, buildNumericCollections()), + roundTripValue(fory, 210, buildNumericCollections()), ); expectAcyclicEqual( buildNumericCollectionsArray(), - roundTripStruct(fory, 212, buildNumericCollectionsArray()), + roundTripValue(fory, 212, buildNumericCollectionsArray()), ); expectAcyclicEqual( buildNumericCollectionUnion(), - roundTripUnion(fory, 211, buildNumericCollectionUnion()), + roundTripValue(fory, 211, buildNumericCollectionUnion()), ); expectAcyclicEqual( buildNumericCollectionArrayUnion(), - roundTripUnion(fory, 213, buildNumericCollectionArrayUnion()), + roundTripValue(fory, 213, buildNumericCollectionArrayUnion()), ); }); @@ -728,15 +720,15 @@ describe.each(MODES)( expectAcyclicEqual( buildMonster(), - roundTripStruct(flatbufferFory, 438716985, buildMonster()), + roundTripValue(flatbufferFory, 438716985, buildMonster()), ); expectAcyclicEqual( buildContainer(), - roundTripStruct(flatbufferFory, 372413680, buildContainer()), + roundTripValue(flatbufferFory, 372413680, buildContainer()), ); expectAcyclicEqual( buildOptionalHolder(), - roundTripStruct(flatbufferFory, 122, buildOptionalHolder()), + roundTripValue(flatbufferFory, 122, buildOptionalHolder()), ); }); @@ -745,15 +737,11 @@ describe.each(MODES)( expectAcyclicEqual( buildExampleMessage(), - roundTripValue( - fory, - Type.struct({ typeId: 1500, evolving: true }), - buildExampleMessage(), - ), + roundTripValue(fory, 1500, buildExampleMessage()), ); expectAcyclicEqual( buildExampleMessageUnion(), - roundTripUnion(fory, 1501, buildExampleMessageUnion()), + roundTripValue(fory, 1501, buildExampleMessageUnion()), ); }); }, @@ -765,13 +753,13 @@ describe.each(MODES)( test("round-trips tree and preserves shared-node topology", () => { const fory = buildFory(compatible, true, [registerTreeTypes]); const tree = buildTree(); - expectTreeEqual(tree, roundTripStruct(fory, 2251833438, tree)); + expectTreeEqual(tree, roundTripValue(fory, 2251833438, tree)); }); test("round-trips graph and preserves edge/node references", () => { const fory = buildFory(compatible, true, [registerGraphTypes]); const graph = buildGraph(); - expectGraphEqual(graph, roundTripStruct(fory, 2373163777, graph)); + expectGraphEqual(graph, roundTripValue(fory, 2373163777, graph)); }); }, ); From f0b88d2a331bcc436b68d5729d86069dd240ef3f Mon Sep 17 00:00:00 2001 From: chaokunyang Date: Sun, 30 Aug 2026 04:23:18 +0800 Subject: [PATCH 124/168] test(javascript): resolve IDL roots from registrations --- .../idl_tests/javascript/roundtrip.ts | 32 +++++++++++++------ 1 file changed, 22 insertions(+), 10 deletions(-) diff --git a/integration_tests/idl_tests/javascript/roundtrip.ts b/integration_tests/idl_tests/javascript/roundtrip.ts index 62dd418ffe..92a6313f28 100644 --- a/integration_tests/idl_tests/javascript/roundtrip.ts +++ b/integration_tests/idl_tests/javascript/roundtrip.ts @@ -91,9 +91,14 @@ import { import { Monster, Color, registerMonsterTypes } from "./generated/monster"; import { TreeNode, registerTreeTypes } from "./generated/tree"; -type RegisterFn = (fory: Fory) => unknown; +type RegisterFn = (fory: Fory) => Record; type AssertFn = (expected: T, actual: unknown) => void; +interface RegisteredFory { + fory: Fory; + serializers: Map; +} + function resolveCompatibleModes(): boolean[] { const value = process.env.IDL_COMPATIBLE; if (value == null || value.trim() === "") { @@ -113,18 +118,25 @@ function buildFory( compatible: boolean, ref: boolean, registerFns: ReadonlyArray, -): Fory { +): RegisteredFory { const fory = new Fory({ compatible, ref, }); + const serializers = new Map(); for (const registerFn of registerFns) { - registerFn(fory); + for (const { serializer } of Object.values(registerFn(fory))) { + serializers.set(serializer.getTypeInfo().userTypeId, serializer); + } } - return fory; + return { fory, serializers }; } -function resolveRootSerializer(fory: Fory, bytes: Uint8Array): Serializer { +function resolveRootSerializer( + registeredFory: RegisteredFory, + bytes: Uint8Array, +): Serializer { + const { fory, serializers } = registeredFory; fory.readContext.reset(bytes); const reader = fory.readContext.reader; const bitmap = reader.readUint8(); @@ -155,15 +167,14 @@ function resolveRootSerializer(fory: Fory, bytes: Uint8Array): Serializer { // registered serializer when available. const detectedSerializer = AnyHelper.detectSerializer(fory.readContext); const resolvedSerializer = - fory.typeResolver.getSerializerByTypeInfo( - detectedSerializer.getTypeInfo(), - ) ?? detectedSerializer; + serializers.get(detectedSerializer.getTypeInfo().userTypeId) ?? + detectedSerializer; return resolvedSerializer; } function runFileRoundTrip( envVar: string, - fory: Fory, + registeredFory: RegisteredFory, expected: T, assertFn: AssertFn, ): void { @@ -173,7 +184,8 @@ function runFileRoundTrip( } console.log(`Processing ${envVar}: ${filePath}`); const payload = new Uint8Array(fs.readFileSync(filePath)); - const serializer = resolveRootSerializer(fory, payload); + const serializer = resolveRootSerializer(registeredFory, payload); + const { fory } = registeredFory; const decoded = fory.deserialize(payload, serializer); assertFn(expected, decoded); const roundTripBytes = fory.serialize(decoded, serializer); From 3643238ccbde6a79aa26bece7b0b960f1e0155e2 Mon Sep 17 00:00:00 2001 From: chaokunyang Date: Sun, 30 Aug 2026 05:12:00 +0800 Subject: [PATCH 125/168] perf: keep lifecycle slow paths cold --- cpp/fory/serialization/context.cc | 20 +++++++++++-------- go/fory/fory.go | 28 +++++++++++++++------------ python/pyfory/_fory.py | 9 ++++++--- python/pyfory/serialization.pyx | 3 ++- swift/Sources/Fory/ReadContext.swift | 2 +- swift/Sources/Fory/WriteContext.swift | 4 ++-- 6 files changed, 39 insertions(+), 27 deletions(-) diff --git a/cpp/fory/serialization/context.cc b/cpp/fory/serialization/context.cc index 803057cac1..d17939ca18 100644 --- a/cpp/fory/serialization/context.cc +++ b/cpp/fory/serialization/context.cc @@ -88,10 +88,12 @@ WriteContext::write_type_meta(const std::type_index &type_id) { void WriteContext::write_type_meta(const TypeInfo *type_info) { if (first_type_info_ == nullptr) { - auto result = type_resolver_->ensure_type_meta(type_info); - if (FORY_PREDICT_FALSE(!result.ok())) { - set_error(std::move(result).error()); - return; + if (FORY_PREDICT_FALSE(!type_info->type_meta)) { + auto result = type_resolver_->ensure_type_meta(type_info); + if (FORY_PREDICT_FALSE(!result.ok())) { + set_error(std::move(result).error()); + return; + } } first_type_info_ = type_info; buffer_.write_uint8(0); // (index << 1), index=0 @@ -117,10 +119,12 @@ void WriteContext::write_type_meta(const TypeInfo *type_info) { } // New type: index << 1, LSB=0, followed by TypeDef bytes inline - auto result = type_resolver_->ensure_type_meta(type_info); - if (FORY_PREDICT_FALSE(!result.ok())) { - set_error(std::move(result).error()); - return; + if (FORY_PREDICT_FALSE(!type_info->type_meta)) { + auto result = type_resolver_->ensure_type_meta(type_info); + if (FORY_PREDICT_FALSE(!result.ok())) { + set_error(std::move(result).error()); + return; + } } uint32_t index = static_cast(write_type_info_index_map_.size() + 1); uint32_t marker = static_cast(index << 1); diff --git a/go/fory/fory.go b/go/fory/fory.go index 369b55bca1..c5c116b3fa 100644 --- a/go/fory/fory.go +++ b/go/fory/fory.go @@ -672,18 +672,25 @@ func (f *Fory) resetWriteState() { } } +func (f *Fory) restoreWriteBuffer(buffer *ByteBuffer) { + f.writeCtx.buffer = buffer + f.resetWriteState() +} + +func (f *Fory) restoreReadBuffer(buffer *ByteBuffer) { + f.readCtx.buffer = buffer + f.resetReadState() +} + // SerializeTo serializes a value and appends the bytes to the provided buffer. // This is useful when you need to write multiple serialized values to the same buffer. // Returns error if serialization fails. func (f *Fory) SerializeTo(buf *ByteBuffer, value any) error { f.registryFrozen = true origBuffer := f.writeCtx.buffer - defer func() { - // Restore the owned buffer before reset so a serializer panic cannot reset or retain the - // caller-owned buffer. - f.writeCtx.buffer = origBuffer - f.resetWriteState() - }() + // Restore the owned buffer before reset so a serializer panic cannot reset or retain the + // caller-owned buffer. + defer f.restoreWriteBuffer(origBuffer) if !validateRootDecimal(f.writeCtx.Err(), value) { return f.writeCtx.TakeError() } @@ -735,12 +742,9 @@ func (f *Fory) DeserializeFrom(buf *ByteBuffer, v any) error { // Temporarily swap buffer origBuffer := f.readCtx.buffer f.readCtx.buffer = buf - defer func() { - // Restore the owned buffer before root cleanup so an escaping panic cannot - // leave a caller-owned buffer installed for the next operation. - f.readCtx.buffer = origBuffer - f.resetReadState() - }() + // Restore the owned buffer before root cleanup so an escaping panic cannot leave a + // caller-owned buffer installed for the next operation. + defer f.restoreReadBuffer(origBuffer) target := reflect.ValueOf(v).Elem() f.readCtx.remainingGraphMemoryBytes = f.config.MaxGraphMemoryBytes f.readCtx.remainingUnbackedContainerItems = f.config.MaxUnbackedContainerItems diff --git a/python/pyfory/_fory.py b/python/pyfory/_fory.py index 797f071644..7f238306c0 100644 --- a/python/pyfory/_fory.py +++ b/python/pyfory/_fory.py @@ -427,7 +427,8 @@ def dump(self, obj, stream): the passed object (or a view of it) is unsupported. If your sink needs retention, copy bytes inside ``write``. """ - self.type_resolver._freeze_registry() + if not self.type_resolver._registry_frozen: + self.type_resolver._freeze_registry() try: self.buffer.set_writer_index(0) output_stream = Buffer.wrap_output_stream(stream) @@ -483,7 +484,8 @@ def serialize( >>> print(type(data)) """ - self.type_resolver._freeze_registry() + if not self.type_resolver._registry_frozen: + self.type_resolver._freeze_registry() try: write_buffer = self._serialize( obj, @@ -561,7 +563,8 @@ def deserialize( >>> print(obj) {'key': 'value'} """ - self.type_resolver._freeze_registry() + if not self.type_resolver._registry_frozen: + self.type_resolver._freeze_registry() try: return self._deserialize(buffer, buffers, unsupported_objects) finally: diff --git a/python/pyfory/serialization.pyx b/python/pyfory/serialization.pyx index 55720f413b..f76d99c1bd 100644 --- a/python/pyfory/serialization.pyx +++ b/python/pyfory/serialization.pyx @@ -310,7 +310,8 @@ cdef class TypeResolver: self._populate_type_info(typeinfo) cdef inline void _freeze_registry(self): - self.resolver._freeze_registry() + if not self.resolver._registry_frozen: + self.resolver._registry_frozen = True def register_type( self, diff --git a/swift/Sources/Fory/ReadContext.swift b/swift/Sources/Fory/ReadContext.swift index a1dd309495..2885ad3733 100644 --- a/swift/Sources/Fory/ReadContext.swift +++ b/swift/Sources/Fory/ReadContext.swift @@ -184,7 +184,7 @@ public final class ReadContext { return lastTypeInfo } let info = try typeResolver.requireTypeInfo(for: type) - if compatible { + if compatible && info.typeDefBytes == nil { try info.ensureTypeMeta(resolver: typeResolver) } lastTypeInfo = info diff --git a/swift/Sources/Fory/WriteContext.swift b/swift/Sources/Fory/WriteContext.swift index 22ca1203ec..8e2e57a4f0 100644 --- a/swift/Sources/Fory/WriteContext.swift +++ b/swift/Sources/Fory/WriteContext.swift @@ -139,7 +139,7 @@ public final class WriteContext { return lastTypeInfo } let info = try typeResolver.requireTypeInfo(for: type) - if compatible { + if compatible && info.typeDefBytes == nil { try info.ensureTypeMeta(resolver: typeResolver) } lastTypeInfo = info @@ -153,7 +153,7 @@ public final class WriteContext { return lastTargetTypeInfo } let info = try typeResolver.requireTypeInfo(forTarget: type) - if compatible { + if compatible && info.typeDefBytes == nil { try info.ensureTypeMeta(resolver: typeResolver) } lastTargetTypeInfo = info From 9823bafc1cc55e39ed94c9682aae3f66e854c923 Mon Sep 17 00:00:00 2001 From: chaokunyang Date: Sun, 30 Aug 2026 05:12:21 +0800 Subject: [PATCH 126/168] docs: clarify registry freeze ownership --- .agents/languages/cpp.md | 7 ++++--- docs/specification/xlang_implementation_guide.md | 3 ++- 2 files changed, 6 insertions(+), 4 deletions(-) diff --git a/.agents/languages/cpp.md b/.agents/languages/cpp.md index 8a26d10b4a..a0779e45c7 100644 --- a/.agents/languages/cpp.md +++ b/.agents/languages/cpp.md @@ -22,9 +22,10 @@ Load this file when changing `cpp/`, Cython build plumbing, or C++ xlang behavio explicit registration before codec work. Route facade and resolver registration through one authoritative frozen flag for that registry. Do not add another lifecycle flag or a multi-state machine around the existing read/write-context resolver construction. -- Freezing clones the registered resolver tables but must not eagerly complete every registered - `TypeInfo`. Complete metadata only when a read/write context clone first uses that type for - metadata, struct-version, or skip behavior; keep ordinary type lookup free of completion work. +- A root freezes explicit registration before the existing read/write-context construction clones + the registered resolver tables. Context construction must not eagerly complete every registered + `TypeInfo`; complete metadata only when a context first uses that type for metadata, + struct-version, or skip behavior. Keep ordinary type lookup free of completion work. - Keep the registration check out of normal runtime lookup hot paths. - Put private methods last in class definitions, immediately before private fields. - Do not redesign alias-based or low-level public type shapes to add convenience methods unless the user explicitly asks for that API change. diff --git a/docs/specification/xlang_implementation_guide.md b/docs/specification/xlang_implementation_guide.md index ec27d15932..6c98351cbb 100644 --- a/docs/specification/xlang_implementation_guide.md +++ b/docs/specification/xlang_implementation_guide.md @@ -257,7 +257,8 @@ The ownership split is: corresponding readable-byte, policy, and graph-memory checks before allocation - `Fory` owns root framing and operation setup/reset -- `TypeResolver` owns registration and dynamic lookup +- `TypeResolver` owns registration mappings, serializer bindings, and dynamic lookup; the natural + registry or public facade owner owns the registry freeze flag #### C# generated structural serializers From 92c56ad56db6a83d6dbaec711e2b4f0fbb9c9309 Mon Sep 17 00:00:00 2001 From: chaokunyang Date: Sun, 30 Aug 2026 05:48:54 +0800 Subject: [PATCH 127/168] perf: keep registry lifecycle checks on owner paths --- cpp/fory/serialization/context.cc | 17 +++++++---- cpp/fory/serialization/context.h | 2 ++ .../src/main/java/org/apache/fory/Fory.java | 28 ++++++++++++++----- python/pyfory/serialization.pyx | 13 ++++----- swift/Sources/Fory/ReadContext.swift | 7 +++-- swift/Sources/Fory/TypeResolver.swift | 7 +++++ swift/Sources/Fory/WriteContext.swift | 6 ---- 7 files changed, 51 insertions(+), 29 deletions(-) diff --git a/cpp/fory/serialization/context.cc b/cpp/fory/serialization/context.cc index d17939ca18..5b8693e4fa 100644 --- a/cpp/fory/serialization/context.cc +++ b/cpp/fory/serialization/context.cc @@ -89,9 +89,8 @@ WriteContext::write_type_meta(const std::type_index &type_id) { void WriteContext::write_type_meta(const TypeInfo *type_info) { if (first_type_info_ == nullptr) { if (FORY_PREDICT_FALSE(!type_info->type_meta)) { - auto result = type_resolver_->ensure_type_meta(type_info); - if (FORY_PREDICT_FALSE(!result.ok())) { - set_error(std::move(result).error()); + ensure_type_meta(type_info); + if (FORY_PREDICT_FALSE(has_error())) { return; } } @@ -120,9 +119,8 @@ void WriteContext::write_type_meta(const TypeInfo *type_info) { // New type: index << 1, LSB=0, followed by TypeDef bytes inline if (FORY_PREDICT_FALSE(!type_info->type_meta)) { - auto result = type_resolver_->ensure_type_meta(type_info); - if (FORY_PREDICT_FALSE(!result.ok())) { - set_error(std::move(result).error()); + ensure_type_meta(type_info); + if (FORY_PREDICT_FALSE(has_error())) { return; } } @@ -139,6 +137,13 @@ void WriteContext::write_type_meta(const TypeInfo *type_info) { buffer_.write_bytes(type_info->type_def.data(), type_info->type_def.size()); } +void WriteContext::ensure_type_meta(const TypeInfo *type_info) { + auto result = type_resolver_->ensure_type_meta(type_info); + if (FORY_PREDICT_FALSE(!result.ok())) { + set_error(std::move(result).error()); + } +} + /// write pre-encoded meta string to buffer (avoids re-encoding on each write) static void write_encoded_meta_string(Buffer &buffer, const CachedMetaString &encoded) { diff --git a/cpp/fory/serialization/context.h b/cpp/fory/serialization/context.h index f6173f8738..f91c1fa46a 100644 --- a/cpp/fory/serialization/context.h +++ b/cpp/fory/serialization/context.h @@ -345,6 +345,8 @@ class WriteContext { void reset(); private: + FORY_NOINLINE void ensure_type_meta(const TypeInfo *type_info); + // Error state - accumulated during serialization, checked at the end Error error_; diff --git a/java/fory-core/src/main/java/org/apache/fory/Fory.java b/java/fory-core/src/main/java/org/apache/fory/Fory.java index 16a374a1da..ede0f648bb 100644 --- a/java/fory-core/src/main/java/org/apache/fory/Fory.java +++ b/java/fory-core/src/main/java/org/apache/fory/Fory.java @@ -334,7 +334,9 @@ public MemoryBuffer serialize(MemoryBuffer buffer, Object obj) { @Override public MemoryBuffer serialize(MemoryBuffer buffer, Object obj, BufferCallback callback) { - typeResolver.freezeRegistration(); + if (!typeResolver.isRegistrationFrozen()) { + typeResolver.freezeRegistration(); + } writeContext.prepare(buffer, callback); try { byte bitmap = headerBitmap; @@ -420,7 +422,9 @@ public T deserialize(byte[] bytes, Class type) { @Override public T deserialize(MemoryBuffer buffer, Class type) { - typeResolver.freezeRegistration(); + if (!typeResolver.isRegistrationFrozen()) { + typeResolver.freezeRegistration(); + } byte bitmap = buffer.readByte(); if (bitmap != headerBitmap) { checkHeaderBitmapWithoutOutOfBand(bitmap); @@ -445,7 +449,9 @@ public T deserialize(MemoryBuffer buffer, Class type) { @Override public T deserialize(ForyInputStream inputStream, Class type) { - typeResolver.freezeRegistration(); + if (!typeResolver.isRegistrationFrozen()) { + typeResolver.freezeRegistration(); + } try { return deserialize(inputStream.getBuffer(), type); } finally { @@ -455,7 +461,9 @@ public T deserialize(ForyInputStream inputStream, Class type) { @Override public T deserialize(ForyReadableChannel channel, Class type) { - typeResolver.freezeRegistration(); + if (!typeResolver.isRegistrationFrozen()) { + typeResolver.freezeRegistration(); + } try { return deserialize(channel.getBuffer(), type); } finally { @@ -488,7 +496,9 @@ public Object deserialize(MemoryBuffer buffer) { */ @Override public Object deserialize(MemoryBuffer buffer, Iterable outOfBandBuffers) { - typeResolver.freezeRegistration(); + if (!typeResolver.isRegistrationFrozen()) { + typeResolver.freezeRegistration(); + } byte bitmap = buffer.readByte(); boolean peerOutOfBandEnabled = false; if (bitmap != headerBitmap) { @@ -531,7 +541,9 @@ public Object deserialize(ForyInputStream inputStream) { @Override public Object deserialize(ForyInputStream inputStream, Iterable outOfBandBuffers) { - typeResolver.freezeRegistration(); + if (!typeResolver.isRegistrationFrozen()) { + typeResolver.freezeRegistration(); + } try { MemoryBuffer buf = inputStream.getBuffer(); return deserialize(buf, outOfBandBuffers); @@ -547,7 +559,9 @@ public Object deserialize(ForyReadableChannel channel) { @Override public Object deserialize(ForyReadableChannel channel, Iterable outOfBandBuffers) { - typeResolver.freezeRegistration(); + if (!typeResolver.isRegistrationFrozen()) { + typeResolver.freezeRegistration(); + } try { MemoryBuffer buf = channel.getBuffer(); return deserialize(buf, outOfBandBuffers); diff --git a/python/pyfory/serialization.pyx b/python/pyfory/serialization.pyx index f76d99c1bd..f9d33523ff 100644 --- a/python/pyfory/serialization.pyx +++ b/python/pyfory/serialization.pyx @@ -309,10 +309,6 @@ cdef class TypeResolver: for typeinfo in self.resolver._types_info.values(): self._populate_type_info(typeinfo) - cdef inline void _freeze_registry(self): - if not self.resolver._registry_frozen: - self.resolver._registry_frozen = True - def register_type( self, cls, @@ -1227,7 +1223,8 @@ cdef class Fory: ) def dump(self, obj, stream): - self.type_resolver._freeze_registry() + if not self.type_resolver.resolver._registry_frozen: + self.type_resolver.resolver._registry_frozen = True try: self.buffer.set_writer_index(0) self.buffer.bind_output_stream(Buffer.wrap_output_stream(stream)) @@ -1251,7 +1248,8 @@ cdef class Fory: def serialize(self, obj, Buffer buffer=None, buffer_callback=None, unsupported_callback=None): cdef Buffer write_buffer - self.type_resolver._freeze_registry() + if not self.type_resolver.resolver._registry_frozen: + self.type_resolver.resolver._registry_frozen = True try: write_buffer = self._serialize( obj, @@ -1295,7 +1293,8 @@ cdef class Fory: return buffer def deserialize(self, buffer, buffers=None, unsupported_objects=None): - self.type_resolver._freeze_registry() + if not self.type_resolver.resolver._registry_frozen: + self.type_resolver.resolver._registry_frozen = True try: return self._deserialize( buffer, diff --git a/swift/Sources/Fory/ReadContext.swift b/swift/Sources/Fory/ReadContext.swift index 2885ad3733..99f2bc14fb 100644 --- a/swift/Sources/Fory/ReadContext.swift +++ b/swift/Sources/Fory/ReadContext.swift @@ -184,9 +184,6 @@ public final class ReadContext { return lastTypeInfo } let info = try typeResolver.requireTypeInfo(for: type) - if compatible && info.typeDefBytes == nil { - try info.ensureTypeMeta(resolver: typeResolver) - } lastTypeInfo = info return info } @@ -321,6 +318,10 @@ public final class ReadContext { for localTypeInfo: TypeInfo, wireTypeID: TypeId ) throws -> TypeInfo? { + // Generic type lookup must not prepare metadata; this wire owner does so only on a miss. + if localTypeInfo.typeDefBytes == nil { + try localTypeInfo.ensureTypeMeta(resolver: typeResolver) + } let buffer = self.buffer let compatibleTypeDefTypeInfos = self.compatibleTypeDefTypeInfos if !checkClassVersion, diff --git a/swift/Sources/Fory/TypeResolver.swift b/swift/Sources/Fory/TypeResolver.swift index 6e9bcecb3a..1d32eac7dc 100644 --- a/swift/Sources/Fory/TypeResolver.swift +++ b/swift/Sources/Fory/TypeResolver.swift @@ -466,12 +466,19 @@ public final class TypeInfo: @unchecked Sendable { context.writeStaticTypeInfo(wireTypeID) switch wireTypeID { case .compatibleStruct, .namedCompatibleStruct: + // Generic type lookup must not prepare metadata; this wire owner does so only on a miss. + if typeDefBytes == nil { + try ensureTypeMeta(resolver: context.typeResolver) + } guard typeDefBytes != nil else { throw ForyError.invalidData("missing compatible type definition for \(typeID)") } try context.writeTypeMeta(self) case .namedEnum, .namedStruct, .namedExt, .namedUnion: if context.compatible { + if typeDefBytes == nil { + try ensureTypeMeta(resolver: context.typeResolver) + } guard typeDefBytes != nil else { throw ForyError.invalidData("missing compatible type definition for \(typeID)") } diff --git a/swift/Sources/Fory/WriteContext.swift b/swift/Sources/Fory/WriteContext.swift index 8e2e57a4f0..0c4b3d5536 100644 --- a/swift/Sources/Fory/WriteContext.swift +++ b/swift/Sources/Fory/WriteContext.swift @@ -139,9 +139,6 @@ public final class WriteContext { return lastTypeInfo } let info = try typeResolver.requireTypeInfo(for: type) - if compatible && info.typeDefBytes == nil { - try info.ensureTypeMeta(resolver: typeResolver) - } lastTypeInfo = info return info } @@ -153,9 +150,6 @@ public final class WriteContext { return lastTargetTypeInfo } let info = try typeResolver.requireTypeInfo(forTarget: type) - if compatible && info.typeDefBytes == nil { - try info.ensureTypeMeta(resolver: typeResolver) - } lastTargetTypeInfo = info return info } From 3391917c5af897c4115e888c4c684605127fa6bd Mon Sep 17 00:00:00 2001 From: chaokunyang Date: Sun, 30 Aug 2026 05:51:11 +0800 Subject: [PATCH 128/168] perf: keep facade freeze checks direct --- .../src/main/java/org/apache/fory/ThreadLocalFory.java | 4 +++- .../src/main/java/org/apache/fory/pool/ThreadPoolFory.java | 4 +++- 2 files changed, 6 insertions(+), 2 deletions(-) diff --git a/java/fory-core/src/main/java/org/apache/fory/ThreadLocalFory.java b/java/fory-core/src/main/java/org/apache/fory/ThreadLocalFory.java index ec85bccfcb..15caad0254 100644 --- a/java/fory-core/src/main/java/org/apache/fory/ThreadLocalFory.java +++ b/java/fory-core/src/main/java/org/apache/fory/ThreadLocalFory.java @@ -119,7 +119,9 @@ public R execute(Function action) { } } } - fory.getTypeResolver().freezeRegistration(); + if (!fory.getTypeResolver().isRegistrationFrozen()) { + fory.getTypeResolver().freezeRegistration(); + } return action.apply(fory); } diff --git a/java/fory-core/src/main/java/org/apache/fory/pool/ThreadPoolFory.java b/java/fory-core/src/main/java/org/apache/fory/pool/ThreadPoolFory.java index 6ed88d8d71..2affea5438 100644 --- a/java/fory-core/src/main/java/org/apache/fory/pool/ThreadPoolFory.java +++ b/java/fory-core/src/main/java/org/apache/fory/pool/ThreadPoolFory.java @@ -181,7 +181,9 @@ public R execute(Function action) { } PooledEntry entry = acquireEntry(); try { - entry.fory.getTypeResolver().freezeRegistration(); + if (!entry.fory.getTypeResolver().isRegistrationFrozen()) { + entry.fory.getTypeResolver().freezeRegistration(); + } return action.apply(entry.fory); } finally { release(entry); From 5e82d6e6ff0d19f4c0f3ac721942e51c5bc8e380 Mon Sep 17 00:00:00 2001 From: chaokunyang Date: Sun, 30 Aug 2026 07:34:05 +0800 Subject: [PATCH 129/168] perf(cpp): keep lazy metadata off hot paths --- .agents/languages/cpp.md | 3 + cpp/fory/serialization/context.cc | 69 ++++++++++---------- cpp/fory/serialization/context.h | 9 +-- cpp/fory/serialization/fory.h | 6 ++ cpp/fory/serialization/serialization_test.cc | 18 +++++ cpp/fory/serialization/struct_serializer.h | 29 ++++---- cpp/fory/serialization/type_resolver.cc | 1 - cpp/fory/serialization/type_resolver.h | 13 +--- 8 files changed, 86 insertions(+), 62 deletions(-) diff --git a/.agents/languages/cpp.md b/.agents/languages/cpp.md index a0779e45c7..c5689969b0 100644 --- a/.agents/languages/cpp.md +++ b/.agents/languages/cpp.md @@ -27,6 +27,9 @@ Load this file when changing `cpp/`, Cython build plumbing, or C++ xlang behavio `TypeInfo`; complete metadata only when a context first uses that type for metadata, struct-version, or skip behavior. Keep ordinary type lookup free of completion work. - Keep the registration check out of normal runtime lookup hot paths. +- Direct `Fory` and its resolver are creator-thread-owned. Configure `ThreadSafeFory` before + concurrent use and let that facade own first-root synchronization; do not add a resolver mutex to + support concurrent registration that neither facade permits. - Put private methods last in class definitions, immediately before private fields. - Do not redesign alias-based or low-level public type shapes to add convenience methods unless the user explicitly asks for that API change. - For cross-language feature ports, match protocol behavior but use idiomatic C++ ownership and layering instead of mirroring Java structure literally. diff --git a/cpp/fory/serialization/context.cc b/cpp/fory/serialization/context.cc index 5b8693e4fa..0bf36d586e 100644 --- a/cpp/fory/serialization/context.cc +++ b/cpp/fory/serialization/context.cc @@ -79,21 +79,13 @@ WriteContext::write_type_meta(const std::type_index &type_id) { // This ensures consistent indexing when the same type is written via // either type_index or TypeInfo* path FORY_TRY(type_info, type_resolver_->get_type_info(type_id)); + FORY_RETURN_NOT_OK(type_resolver_->ensure_type_meta(type_info)); write_type_meta(type_info); - if (FORY_PREDICT_FALSE(has_error())) { - return Unexpected(error()); - } return Result(); } void WriteContext::write_type_meta(const TypeInfo *type_info) { if (first_type_info_ == nullptr) { - if (FORY_PREDICT_FALSE(!type_info->type_meta)) { - ensure_type_meta(type_info); - if (FORY_PREDICT_FALSE(has_error())) { - return; - } - } first_type_info_ = type_info; buffer_.write_uint8(0); // (index << 1), index=0 buffer_.write_bytes(type_info->type_def.data(), type_info->type_def.size()); @@ -118,12 +110,6 @@ void WriteContext::write_type_meta(const TypeInfo *type_info) { } // New type: index << 1, LSB=0, followed by TypeDef bytes inline - if (FORY_PREDICT_FALSE(!type_info->type_meta)) { - ensure_type_meta(type_info); - if (FORY_PREDICT_FALSE(has_error())) { - return; - } - } uint32_t index = static_cast(write_type_info_index_map_.size() + 1); uint32_t marker = static_cast(index << 1); if (marker < 0x80) { @@ -211,10 +197,13 @@ WriteContext::write_enum_type_info(const TypeInfo *type_info) { } else if (type_id == static_cast(TypeId::NAMED_ENUM)) { if (config_->compatible) { // write type meta inline using streaming protocol - write_type_meta(type_info); - if (FORY_PREDICT_FALSE(has_error())) { - return Unexpected(error()); + if (FORY_PREDICT_FALSE(!type_info->type_meta)) { + ensure_type_meta(type_info); + if (FORY_PREDICT_FALSE(has_error())) { + return Unexpected(error()); + } } + write_type_meta(type_info); } else { // write pre-encoded namespace and type_name if (type_info->encoded_namespace && type_info->encoded_type_name) { @@ -306,10 +295,13 @@ WriteContext::write_any_type_info(const TypeInfo *type_info) { case TypeId::COMPATIBLE_STRUCT: case TypeId::NAMED_COMPATIBLE_STRUCT: // write type meta inline using streaming protocol - write_type_meta(type_info); - if (FORY_PREDICT_FALSE(has_error())) { - return Unexpected(error()); + if (FORY_PREDICT_FALSE(!type_info->type_meta)) { + ensure_type_meta(type_info); + if (FORY_PREDICT_FALSE(has_error())) { + return Unexpected(error()); + } } + write_type_meta(type_info); break; case TypeId::NAMED_ENUM: case TypeId::NAMED_EXT: @@ -317,10 +309,13 @@ WriteContext::write_any_type_info(const TypeInfo *type_info) { case TypeId::NAMED_UNION: if (config_->compatible) { // write type meta inline using streaming protocol - write_type_meta(type_info); - if (FORY_PREDICT_FALSE(has_error())) { - return Unexpected(error()); + if (FORY_PREDICT_FALSE(!type_info->type_meta)) { + ensure_type_meta(type_info); + if (FORY_PREDICT_FALSE(has_error())) { + return Unexpected(error()); + } } + write_type_meta(type_info); } else { // write pre-encoded namespace and type_name if (type_info->encoded_namespace && type_info->encoded_type_name) { @@ -383,8 +378,7 @@ WriteContext::write_struct_type_info(const std::type_index &type_id) { return Result(); } -Result -WriteContext::write_struct_type_info(const TypeInfo *type_info) { +void WriteContext::write_struct_type_info(const TypeInfo *type_info) { uint32_t fory_type_id = type_info->type_id; // write type_id @@ -399,26 +393,33 @@ WriteContext::write_struct_type_info(const TypeInfo *type_info) { case TypeId::COMPATIBLE_STRUCT: case TypeId::NAMED_COMPATIBLE_STRUCT: // write type meta inline using streaming protocol - write_type_meta(type_info); - if (FORY_PREDICT_FALSE(has_error())) { - return Unexpected(error()); + if (FORY_PREDICT_FALSE(!type_info->type_meta)) { + ensure_type_meta(type_info); + if (FORY_PREDICT_FALSE(has_error())) { + return; + } } + write_type_meta(type_info); break; case TypeId::NAMED_STRUCT: if (config_->compatible) { // write type meta inline using streaming protocol - write_type_meta(type_info); - if (FORY_PREDICT_FALSE(has_error())) { - return Unexpected(error()); + if (FORY_PREDICT_FALSE(!type_info->type_meta)) { + ensure_type_meta(type_info); + if (FORY_PREDICT_FALSE(has_error())) { + return; + } } + write_type_meta(type_info); } else { // write pre-encoded namespace and type_name if (type_info->encoded_namespace && type_info->encoded_type_name) { write_encoded_meta_string(buffer_, *type_info->encoded_namespace); write_encoded_meta_string(buffer_, *type_info->encoded_type_name); } else { - return Unexpected( + set_error( Error::invalid("Encoded meta strings not initialized for struct")); + return; } } break; @@ -426,8 +427,6 @@ WriteContext::write_struct_type_info(const TypeInfo *type_info) { // STRUCT type - just writing type_id is sufficient break; } - - return Result(); } void WriteContext::reset() { diff --git a/cpp/fory/serialization/context.h b/cpp/fory/serialization/context.h index f91c1fa46a..afc14f973d 100644 --- a/cpp/fory/serialization/context.h +++ b/cpp/fory/serialization/context.h @@ -262,7 +262,8 @@ class WriteContext { /// Subsequent occurrences: writes (index << 1) | 1 as reference. Result write_type_meta(const std::type_index &type_id); - /// write TypeMeta inline using TypeInfo pointer (fast path). + /// write TypeMeta inline using a TypeInfo whose metadata is ready (fast + /// path). /// First occurrence: writes (index << 1) | 0 followed by TypeDef bytes. /// Subsequent occurrences: writes (index << 1) | 1 as reference. void write_type_meta(const TypeInfo *type_info); @@ -298,9 +299,9 @@ class WriteContext { /// Fastest path for writing struct type info when TypeInfo is already known. /// Avoids type_index creation and lookup overhead. /// - /// @param type_info Pointer to the TypeInfo (must be valid) - /// @return Success or error - Result write_struct_type_info(const TypeInfo *type_info); + /// @param type_info Pointer to a valid TypeInfo whose metadata is ready when + /// the configured wire mode needs it + void write_struct_type_info(const TypeInfo *type_info); /// Fastest path - write struct type_id directly without any lookups. /// Use this when the type_id is already known (e.g., from a cache). diff --git a/cpp/fory/serialization/fory.h b/cpp/fory/serialization/fory.h index cf247e4570..5d8a733865 100644 --- a/cpp/fory/serialization/fory.h +++ b/cpp/fory/serialization/fory.h @@ -900,6 +900,12 @@ class Fory : public BaseFory { if (write_root_type_info_ != nullptr && write_root_type_info_key_ == ctid) { return write_root_type_info_; } + return cache_write_root_type_info(ctid); + } + + template + FORY_NOINLINE Result + cache_write_root_type_info(uint64_t ctid) { FORY_TRY(type_info, write_ctx_->type_resolver().template get_type_info()); write_root_type_info_key_ = ctid; diff --git a/cpp/fory/serialization/serialization_test.cc b/cpp/fory/serialization/serialization_test.cc index 7428fe66a5..50d05a745a 100644 --- a/cpp/fory/serialization/serialization_test.cc +++ b/cpp/fory/serialization/serialization_test.cc @@ -1446,6 +1446,24 @@ TEST(SerializationTest, UnusedTypeMetaStaysLazy) { EXPECT_TRUE(unused.value()->type_def.empty()); } +TEST(SerializationTest, UnneededTypeMetaStaysLazy) { + auto fory = Fory::builder() + .xlang(true) + .compatible(false) + .check_struct_version(false) + .build(); + ASSERT_TRUE(fory.register_struct(1).ok()); + + auto serialized = fory.serialize(SimpleStruct{1, 2}); + ASSERT_TRUE(serialized.ok()) << serialized.error().to_string(); + + auto type_info = + fory.write_context().type_resolver().get_type_info(); + ASSERT_TRUE(type_info.ok()); + EXPECT_EQ(type_info.value()->type_meta, nullptr); + EXPECT_TRUE(type_info.value()->type_def.empty()); +} + TEST(SerializationTest, TypeMetaFailureIsAtomic) { auto fory = Fory::builder().xlang(true).compatible(true).build(); ASSERT_TRUE( diff --git a/cpp/fory/serialization/struct_serializer.h b/cpp/fory/serialization/struct_serializer.h index 643bc6b4e6..cb1cf905f1 100644 --- a/cpp/fory/serialization/struct_serializer.h +++ b/cpp/fory/serialization/struct_serializer.h @@ -4614,9 +4614,9 @@ struct Serializer>> { return; } const TypeInfo *type_info = type_info_res.value(); - auto write_result = ctx.write_struct_type_info(type_info); - if (FORY_PREDICT_FALSE(!write_result.ok())) { - ctx.set_error(std::move(write_result).error()); + ctx.write_struct_type_info(type_info); + if (FORY_PREDICT_FALSE(ctx.has_error())) { + return; } } @@ -4682,9 +4682,8 @@ struct Serializer>> { ctx.write_struct_type_id_direct(tid, type_info->user_type_id); } else { // Complex type (NAMED_STRUCT, COMPATIBLE_STRUCT, etc.) - use TypeInfo* - auto result = ctx.write_struct_type_info(type_info); - if (FORY_PREDICT_FALSE(!result.ok())) { - ctx.set_error(std::move(result).error()); + ctx.write_struct_type_info(type_info); + if (FORY_PREDICT_FALSE(ctx.has_error())) { return; } } @@ -4692,6 +4691,14 @@ struct Serializer>> { write_data_generic(obj, ctx, has_generics); } + static FORY_NOINLINE void ensure_type_meta(WriteContext &ctx, + const TypeInfo *type_info) { + auto result = ctx.type_resolver().ensure_type_meta(type_info); + if (FORY_PREDICT_FALSE(!result.ok())) { + ctx.set_error(std::move(result).error()); + } + } + static void write_data(const T &obj, WriteContext &ctx) { // Only write struct version hash when check_struct_version is enabled, // matching Java's behavior in ObjectSerializer.write(). @@ -4703,9 +4710,8 @@ struct Serializer>> { } const TypeInfo *type_info = type_info_res.value(); if (FORY_PREDICT_FALSE(!type_info->type_meta)) { - auto meta_result = ctx.type_resolver().ensure_type_meta(type_info); - if (FORY_PREDICT_FALSE(!meta_result.ok())) { - ctx.set_error(std::move(meta_result).error()); + ensure_type_meta(ctx, type_info); + if (FORY_PREDICT_FALSE(ctx.has_error())) { return; } } @@ -4737,9 +4743,8 @@ struct Serializer>> { } const TypeInfo *type_info = type_info_res.value(); if (FORY_PREDICT_FALSE(!type_info->type_meta)) { - auto meta_result = ctx.type_resolver().ensure_type_meta(type_info); - if (FORY_PREDICT_FALSE(!meta_result.ok())) { - ctx.set_error(std::move(meta_result).error()); + ensure_type_meta(ctx, type_info); + if (FORY_PREDICT_FALSE(ctx.has_error())) { return; } } diff --git a/cpp/fory/serialization/type_resolver.cc b/cpp/fory/serialization/type_resolver.cc index 8d927b20e0..ca90419d50 100644 --- a/cpp/fory/serialization/type_resolver.cc +++ b/cpp/fory/serialization/type_resolver.cc @@ -1774,7 +1774,6 @@ Result TypeResolver::check_registration() { } std::unique_ptr TypeResolver::build_context_type_resolver() { - std::lock_guard lock(registration_mutex_); registry_frozen_ = true; return clone(); } diff --git a/cpp/fory/serialization/type_resolver.h b/cpp/fory/serialization/type_resolver.h index 0e78a857fb..753102196a 100644 --- a/cpp/fory/serialization/type_resolver.h +++ b/cpp/fory/serialization/type_resolver.h @@ -28,7 +28,6 @@ #include #include #include -#include #include #include #include @@ -1507,7 +1506,6 @@ class TypeResolver { void register_type_internal_runtime(const std::type_index &type_index, TypeInfo *info); - /// Validate registration state while registration_mutex_ is held. Result check_registration(); void register_builtin_types(); @@ -1519,7 +1517,9 @@ class TypeResolver { std::thread::id registration_thread_id_; bool registry_frozen_; - std::mutex registration_mutex_; + // Registration is creator-thread-only. Shared facades are configured before + // concurrent use and own first-root synchronization; the resolver must not + // duplicate that synchronization around its hot lookup state. // Primary storage - owns all TypeInfo objects std::vector> type_infos_; @@ -1736,7 +1736,6 @@ get_type_info_with_resolver(TypeResolver &resolver) { } template Result TypeResolver::register_any_type() { - std::lock_guard lock(registration_mutex_); FORY_RETURN_IF_ERROR(check_registration()); using ChronoTimestamp = std::chrono::time_point; @@ -1775,7 +1774,6 @@ template Result TypeResolver::register_any_type() { template Result TypeResolver::register_by_id(uint32_t type_id) { - std::lock_guard lock(registration_mutex_); FORY_RETURN_IF_ERROR(check_registration()); if (type_id == kInvalidUserTypeId) { return Unexpected(Error::invalid( @@ -1829,7 +1827,6 @@ template Result TypeResolver::register_by_name(const std::string &ns, const std::string &type_name) { - std::lock_guard lock(registration_mutex_); FORY_RETURN_IF_ERROR(check_registration()); if (type_name.empty()) { return Unexpected( @@ -1880,7 +1877,6 @@ TypeResolver::register_by_name(const std::string &ns, template Result TypeResolver::register_ext_type_by_id(uint32_t type_id) { - std::lock_guard lock(registration_mutex_); FORY_RETURN_IF_ERROR(check_registration()); if (type_id == kInvalidUserTypeId) { return Unexpected(Error::invalid("type_id must be in range [0, 0xfffffffe] " @@ -1904,7 +1900,6 @@ template Result TypeResolver::register_ext_type_by_name(const std::string &ns, const std::string &type_name) { - std::lock_guard lock(registration_mutex_); FORY_RETURN_IF_ERROR(check_registration()); if (type_name.empty()) { return Unexpected(Error::invalid( @@ -1928,7 +1923,6 @@ TypeResolver::register_ext_type_by_name(const std::string &ns, template Result TypeResolver::register_union_by_id(uint32_t type_id) { - std::lock_guard lock(registration_mutex_); FORY_RETURN_IF_ERROR(check_registration()); if (type_id == kInvalidUserTypeId) { return Unexpected(Error::invalid( @@ -1951,7 +1945,6 @@ template Result TypeResolver::register_union_by_name(const std::string &ns, const std::string &type_name) { - std::lock_guard lock(registration_mutex_); FORY_RETURN_IF_ERROR(check_registration()); if (type_name.empty()) { return Unexpected(Error::invalid( From becd70ac2abf16563caf2f2cf287aecdc991975e Mon Sep 17 00:00:00 2001 From: chaokunyang Date: Sun, 30 Aug 2026 07:48:59 +0800 Subject: [PATCH 130/168] perf(python): keep registry freeze on compiled owner --- .agents/languages/python.md | 6 ++++-- python/pyfory/registry.py | 9 ++++++++- python/pyfory/serialization.pyx | 14 ++++++++------ 3 files changed, 20 insertions(+), 9 deletions(-) diff --git a/.agents/languages/python.md b/.agents/languages/python.md index e280ee9cdc..2473a42888 100644 --- a/.agents/languages/python.md +++ b/.agents/languages/python.md @@ -11,8 +11,10 @@ Load this file when changing `python/`, Cython serialization, or Python xlang be - Python mode is the pure-Python xlang implementation and is mainly for debugging and testing. - Cython mode is the default high-performance implementation. - Cython mode owns the hot runtime path. Do not duplicate core runtime types between Python and Cython, tunnel Python facade methods into hidden Cython internals, or keep dead shims unless the user explicitly needs a compatibility module path. -- A direct Python `TypeResolver` owns one authoritative `_registry_frozen` flag. Pure Python and - Cython roots set that owner before codec work and never clear it, including after failure. +- A direct Python `TypeResolver` owns one authoritative `_registry_frozen` flag. In Cython mode the + compiled `TypeResolver` is the active owner instead, and the Python resolver delegates explicit + registration checks to that compiled flag; do not mirror the flag between the two resolvers. + Roots set their active owner before codec work and never clear it, including after failure. `ThreadSafeFory` owns its own `_registry_frozen` flag for the public registration boundary over pooled children. Its existing callback list configures newly created children; it is not another lifecycle state. diff --git a/python/pyfory/registry.py b/python/pyfory/registry.py index 0ac885cf87..54e981f498 100644 --- a/python/pyfory/registry.py +++ b/python/pyfory/registry.py @@ -431,7 +431,14 @@ def __init__(self, config, *, shared_registry): self._registry_frozen = False def _check_registry_mutable(self): - if self._registry_frozen: + registry_owner = self._actual_type_resolver + if registry_owner is self: + registry_frozen = self._registry_frozen + else: + # The compiled resolver is the active registry owner. Read its one + # lifecycle flag instead of mirroring that state in both resolvers. + registry_frozen = registry_owner._registry_frozen + if registry_frozen: raise RuntimeError("Cannot register types or serializers after the first root operation has started") def _freeze_registry(self): diff --git a/python/pyfory/serialization.pyx b/python/pyfory/serialization.pyx index f9d33523ff..24f4ce6a89 100644 --- a/python/pyfory/serialization.pyx +++ b/python/pyfory/serialization.pyx @@ -257,6 +257,7 @@ cdef class TypeResolver: cdef readonly bint strict cdef readonly bint compatible cdef readonly bint field_nullable + cdef readonly bint _registry_frozen cdef readonly object policy cdef readonly bint meta_share cdef readonly dict _types_info @@ -291,6 +292,7 @@ cdef class TypeResolver: self.strict = resolver.strict self.compatible = resolver.compatible self.field_nullable = resolver.field_nullable + self._registry_frozen = False self.policy = resolver.policy self.meta_share = resolver.meta_share self._types_info = resolver._types_info @@ -1223,8 +1225,8 @@ cdef class Fory: ) def dump(self, obj, stream): - if not self.type_resolver.resolver._registry_frozen: - self.type_resolver.resolver._registry_frozen = True + if not self.type_resolver._registry_frozen: + self.type_resolver._registry_frozen = True try: self.buffer.set_writer_index(0) self.buffer.bind_output_stream(Buffer.wrap_output_stream(stream)) @@ -1248,8 +1250,8 @@ cdef class Fory: def serialize(self, obj, Buffer buffer=None, buffer_callback=None, unsupported_callback=None): cdef Buffer write_buffer - if not self.type_resolver.resolver._registry_frozen: - self.type_resolver.resolver._registry_frozen = True + if not self.type_resolver._registry_frozen: + self.type_resolver._registry_frozen = True try: write_buffer = self._serialize( obj, @@ -1293,8 +1295,8 @@ cdef class Fory: return buffer def deserialize(self, buffer, buffers=None, unsupported_objects=None): - if not self.type_resolver.resolver._registry_frozen: - self.type_resolver.resolver._registry_frozen = True + if not self.type_resolver._registry_frozen: + self.type_resolver._registry_frozen = True try: return self._deserialize( buffer, From 015a010edb501402651f909d7fcd03fe6bb4298a Mon Sep 17 00:00:00 2001 From: chaokunyang Date: Sun, 30 Aug 2026 07:50:48 +0800 Subject: [PATCH 131/168] fix(java): freeze blocked reads before input access --- .../apache/fory/io/BlockedStreamUtils.java | 2 + .../fory/io/BlockedStreamUtilsTest.java | 55 +++++++++++++++++++ 2 files changed, 57 insertions(+) diff --git a/java/fory-core/src/main/java/org/apache/fory/io/BlockedStreamUtils.java b/java/fory-core/src/main/java/org/apache/fory/io/BlockedStreamUtils.java index 5ac34e23f0..e4157e596e 100644 --- a/java/fory-core/src/main/java/org/apache/fory/io/BlockedStreamUtils.java +++ b/java/fory-core/src/main/java/org/apache/fory/io/BlockedStreamUtils.java @@ -84,6 +84,7 @@ public static T deserialize(Fory fory, ReadableByteChannel channel, Class private static Object readFromChannel( Fory fory, ReadableByteChannel channel, Function action) { + fory.getTypeResolver().freezeRegistration(); try { MemoryBuffer buf = fory.getBuffer(); // resetBuffer may shrink the reusable buffer below the fixed frame header size. @@ -153,6 +154,7 @@ private static void serializeToStream( private static Object deserializeFromStream( Fory fory, InputStream inputStream, Function function) { + fory.getTypeResolver().freezeRegistration(); MemoryBuffer buf = fory.getBuffer(); try { MemoryBuffer frame = readToBufferFromStream(inputStream, buf); diff --git a/java/fory-core/src/test/java/org/apache/fory/io/BlockedStreamUtilsTest.java b/java/fory-core/src/test/java/org/apache/fory/io/BlockedStreamUtilsTest.java index 7e73c6a274..d4466499f1 100644 --- a/java/fory-core/src/test/java/org/apache/fory/io/BlockedStreamUtilsTest.java +++ b/java/fory-core/src/test/java/org/apache/fory/io/BlockedStreamUtilsTest.java @@ -24,12 +24,14 @@ import java.io.ByteArrayInputStream; import java.io.ByteArrayOutputStream; import java.io.IOException; +import java.io.InputStream; import java.nio.ByteBuffer; import java.nio.ByteOrder; import java.nio.channels.ReadableByteChannel; import org.apache.fory.Fory; import org.apache.fory.ForyTestBase; import org.apache.fory.exception.DeserializationException; +import org.apache.fory.exception.ForyException; import org.apache.fory.memory.MemoryBuffer; import org.apache.fory.test.bean.Foo; import org.testng.annotations.Test; @@ -105,6 +107,57 @@ public void testPersistentChannelZeroRead() { } } + @Test + public void testReadFreezesBeforeIo() throws IOException { + Fory streamFory = builder().withCodegen(false).build(); + boolean[] streamRegistrationRejected = {false}; + InputStream inputStream = + new InputStream() { + @Override + public int read() { + return -1; + } + + @Override + public int read(byte[] bytes, int offset, int length) { + expectThrows(ForyException.class, () -> streamFory.register(LateType.class)); + streamRegistrationRejected[0] = true; + return -1; + } + }; + assertThrows( + RuntimeException.class, () -> BlockedStreamUtils.deserialize(streamFory, inputStream)); + assertTrue(streamRegistrationRejected[0]); + + Fory channelFory = builder().withCodegen(false).build(); + boolean[] channelRegistrationRejected = {false}; + try (ReadableByteChannel channel = + new ReadableByteChannel() { + private boolean open = true; + + @Override + public int read(ByteBuffer dst) { + expectThrows(ForyException.class, () -> channelFory.register(LateType.class)); + channelRegistrationRejected[0] = true; + return -1; + } + + @Override + public boolean isOpen() { + return open; + } + + @Override + public void close() { + open = false; + } + }) { + assertThrows( + RuntimeException.class, () -> BlockedStreamUtils.deserialize(channelFory, channel)); + } + assertTrue(channelRegistrationRejected[0]); + } + @Test public void testSmallBufferStreamReuse() { Fory writerFory = builder().withCodegen(false).build(); @@ -162,6 +215,8 @@ private static byte[] frameHeader(int size) { return ByteBuffer.allocate(4).order(ByteOrder.LITTLE_ENDIAN).putInt(size).array(); } + private static final class LateType {} + private static final class ChunkedReadableByteChannel implements ReadableByteChannel { private final byte[] data; private final int chunkSize; From 584caf1b9c55f77ad7227da7d6a3ed5333230b9a Mon Sep 17 00:00:00 2001 From: chaokunyang Date: Sun, 30 Aug 2026 07:52:58 +0800 Subject: [PATCH 132/168] fix(javascript): clean failed serializer lookup state --- javascript/packages/core/lib/fory.ts | 10 +++++++++- javascript/test/rootCleanup.test.ts | 21 +++++++++++++++++++++ 2 files changed, 30 insertions(+), 1 deletion(-) diff --git a/javascript/packages/core/lib/fory.ts b/javascript/packages/core/lib/fory.ts index c72b946ce8..07c6ae3c69 100644 --- a/javascript/packages/core/lib/fory.ts +++ b/javascript/packages/core/lib/fory.ts @@ -261,6 +261,14 @@ export default class Fory { serialize(data: T, serializer: Serializer = this.anySerializer) { this.registrationFrozen = true; - return this.getRootSerializer(serializer)(data); + let rootSerializer; + try { + rootSerializer = this.getRootSerializer(serializer); + } catch (error) { + // Serializer lookup is part of the root attempt and can fail before the cached root owns it. + this.writeContext.reset(); + throw error; + } + return rootSerializer(data); } } diff --git a/javascript/test/rootCleanup.test.ts b/javascript/test/rootCleanup.test.ts index 49e2bd9c51..9590afda64 100644 --- a/javascript/test/rootCleanup.test.ts +++ b/javascript/test/rootCleanup.test.ts @@ -142,6 +142,27 @@ test.each(["success", "failure"] as const)("restores root write state for %s", ( expect(typeMeta.dynamicTypeId).toBe(-1); }); +test("clears write state before serializer lookup", () => { + const fory = new Fory({ compatible: true, ref: true }); + const registered = fory.register(Type.struct(7613, {})); + const writeContext = (fory as any).writeContext; + const typeMeta = TypeMeta.fromTypeInfo(Type.struct(7614, {})); + const name = writeContext.metaStringWriter.encodeTypeName("PreviousRoot"); + const value = {}; + + registered.serializer.writeRef = () => { + writeContext.refWriter.writeRef(value); + writeContext.metaStringWriter.writeBytes(writeContext.writer, name); + writeContext.writeTypeMeta(typeMeta, typeMeta.toBytes()); + }; + expect(registered.serialize(value)).toBeDefined(); + + expect(() => fory.serialize(1, null as any)).toThrow(); + expect(writeContext.refWriter.writeObjects.size).toBe(0); + expect(name.dynamicWriteStringId).toBe(-1); + expect(typeMeta.dynamicTypeId).toBe(-1); +}); + test("reuses root write metastring owners", () => { const fory = new Fory({ compatible: true }); const registered = fory.register(Type.struct(7609, {})); From cfe6bcd6fd2971c2825c6f2c4bfaa696b9b5c5ac Mon Sep 17 00:00:00 2001 From: chaokunyang Date: Sun, 30 Aug 2026 07:55:58 +0800 Subject: [PATCH 133/168] fix(csharp): close failed root cleanup gaps --- .agents/languages/csharp.md | 2 +- csharp/src/Fory/Fory.cs | 10 ++++- csharp/src/Fory/ThreadSafeFory.cs | 16 +++++++ .../tests/Fory.Tests/RuntimeEdgeCaseTests.cs | 45 +++++++------------ 4 files changed, 41 insertions(+), 32 deletions(-) diff --git a/.agents/languages/csharp.md b/.agents/languages/csharp.md index 0c4445b7d3..13e2bbce73 100644 --- a/.agents/languages/csharp.md +++ b/.agents/languages/csharp.md @@ -12,7 +12,7 @@ Load this file when changing `csharp/` or C# xlang behavior. - A direct C# `Fory` registry and the `ThreadSafeFory` public registration boundary each own one authoritative frozen flag. The first root sets the owning flag before codec work and leaves it set after failure. Explicit registration checks that flag before mutation. `ThreadSafeFory` keeps - its existing successful-registration list only to configure newly created child runtimes; do not + its existing registration callbacks only to configure newly created child runtimes; do not turn that list into another registry lifecycle state. - Generated C# gRPC service companions are compiler-owned files that depend on application-provided gRPC packages, not `csharp/src/Fory`. Keep gRPC package references out of the Fory runtime package. - C# generated schema modules are source-file owners. Service companions must use that module's `ThreadSafeFory` and must not introduce namespace-owned aliases or duplicate serializer registration paths. diff --git a/csharp/src/Fory/Fory.cs b/csharp/src/Fory/Fory.cs index d57509dd25..690e38da2d 100644 --- a/csharp/src/Fory/Fory.cs +++ b/csharp/src/Fory/Fory.cs @@ -242,7 +242,15 @@ public T Deserialize(byte[] payload) { _registryFrozen = true; ByteReader reader = _readContext.Reader; - reader.Reset(payload); + try + { + reader.Reset(payload); + } + catch + { + _readContext.Reset(); + throw; + } T value = DeserializeFromReader(reader); if (reader.Remaining != 0) { diff --git a/csharp/src/Fory/ThreadSafeFory.cs b/csharp/src/Fory/ThreadSafeFory.cs index eca4bfecf7..f246b03fb2 100644 --- a/csharp/src/Fory/ThreadSafeFory.cs +++ b/csharp/src/Fory/ThreadSafeFory.cs @@ -67,6 +67,7 @@ public ThreadSafeFory Register(uint typeId) /// Registration has closed because a root operation was attempted. public ThreadSafeFory Register(string name) { + EnsureRegistrationOpen(); _ = TypeResolver.SplitTypeName(name); ApplyRegistration(fory => fory.Register(name)); return this; @@ -83,6 +84,7 @@ public ThreadSafeFory Register(string name) /// Registration has closed because a root operation was attempted. public ThreadSafeFory Register(string typeNamespace, string typeName) { + EnsureRegistrationOpen(); TypeResolver.ValidateSplitTypeName(typeNamespace, typeName); ApplyRegistration(fory => fory.Register(typeNamespace, typeName)); return this; @@ -116,6 +118,7 @@ public ThreadSafeFory Register(uint typeId) public ThreadSafeFory Register(string name) where TSerializer : Serializer, new() { + EnsureRegistrationOpen(); _ = TypeResolver.SplitTypeName(name); ApplyRegistration(fory => fory.Register(name)); return this; @@ -134,6 +137,7 @@ public ThreadSafeFory Register(string name) public ThreadSafeFory Register(string typeNamespace, string typeName) where TSerializer : Serializer, new() { + EnsureRegistrationOpen(); TypeResolver.ValidateSplitTypeName(typeNamespace, typeName); ApplyRegistration(fory => fory.Register(typeNamespace, typeName)); return this; @@ -235,6 +239,18 @@ private void ApplyRegistration(Action registration) } } + private void EnsureRegistrationOpen() + { + lock (_registrationLock) + { + ThrowIfDisposed(); + if (_registryFrozen != 0) + { + ThrowRegistryFrozen(); + } + } + } + [MethodImpl(MethodImplOptions.AggressiveInlining)] private void BeginRoot() { diff --git a/csharp/tests/Fory.Tests/RuntimeEdgeCaseTests.cs b/csharp/tests/Fory.Tests/RuntimeEdgeCaseTests.cs index 0faeb5bf01..6198d1033f 100644 --- a/csharp/tests/Fory.Tests/RuntimeEdgeCaseTests.cs +++ b/csharp/tests/Fory.Tests/RuntimeEdgeCaseTests.cs @@ -872,8 +872,11 @@ public void FrozenRegistryRejectsBeforeMutation() public void FailedRootFreezesRegistry() { ForyRuntime fory = ForyRuntime.Builder().Build(); + ReadContext context = ReadContextFor(fory); + context.AppendReadMetaString(MetaString.Empty('_', '_')); Assert.ThrowsAny(() => fory.Deserialize((byte[])null!)); + Assert.Null(context.GetReadMetaString(0)); Assert.Throws(() => fory.Register(713)); } @@ -936,7 +939,18 @@ public void ThreadSafeFailedRootFreezesRegistry() using ThreadSafeFory fory = ForyRuntime.Builder().BuildThreadSafe(); Assert.ThrowsAny(() => fory.Deserialize(Array.Empty())); - Assert.Throws(() => fory.Register(715)); + Action[] registrations = + [ + () => fory.Register(string.Empty), + () => fory.Register("test", "bad.name"), + () => fory.Register(string.Empty), + () => fory.Register("test", "bad.name"), + ]; + + foreach (Action registration in registrations) + { + Assert.Throws(registration); + } } [Fact] @@ -1008,24 +1022,6 @@ public void TrailingBytesResetReadState(bool useSpan) Assert.Null(context.GetReadMetaString(0)); } - [Fact] - public void RootHeaderFailureKeepsMetaCache() - { - ForyRuntime fory = ForyRuntime.Builder() - .Compatible(false) - .MaxSchemaVersionsPerType(1) - .Build(); - ReadContext context = ReadContextFor(fory); - TypeMeta first = ReadAndStoreTypeMeta(context, RemoteStructTypeMeta(901, "first")); - ulong firstHash = EncodedTypeMetaHash(first); - - Assert.ThrowsAny(() => fory.Deserialize([0])); - - Assert.True(context.TryGetTypeMetaByHash(firstHash, out _)); - Assert.Throws( - () => ReadAndStoreTypeMeta(context, RemoteStructTypeMeta(901, "second"))); - } - [Theory] [InlineData(false)] [InlineData(true)] @@ -1089,17 +1085,6 @@ private static WriteContext WriteContextFor(ForyRuntime fory) return Assert.IsType(field.GetValue(fory)); } - [Fact] - public void DeserializeFromReaderReadsFrames() - { - ForyRuntime fory = ForyRuntime.Builder().Build(); - ByteReader reader = new([.. fory.Serialize(123), .. fory.Serialize(456)]); - - Assert.Equal(123, fory.DeserializeFromReader(reader)); - Assert.Equal(456, fory.DeserializeFromReader(reader)); - Assert.Equal(0, reader.Remaining); - } - [Fact] public void DeserializeRejectsNonXlangBitmap() { From 76eeb597b57f10e287861ea9344b98c35a3f543e Mon Sep 17 00:00:00 2001 From: chaokunyang Date: Sun, 30 Aug 2026 08:04:55 +0800 Subject: [PATCH 134/168] fix(java): close registry setup boundary --- .agents/languages/java.md | 8 ++- .../src/main/java/org/apache/fory/Fory.java | 52 ++++++++++++------- .../java/org/apache/fory/ThreadSafeFory.java | 5 +- .../apache/fory/resolver/TypeResolver.java | 2 + .../serializer/CopyOnlyObjectSerializer.java | 2 +- .../fory/resolver/AllowListCheckerTest.java | 12 +++-- .../apache/fory/serializer/RegisterTest.java | 17 ++++++ 7 files changed, 69 insertions(+), 29 deletions(-) diff --git a/.agents/languages/java.md b/.agents/languages/java.md index 9289206b3e..328b92cd50 100644 --- a/.agents/languages/java.md +++ b/.agents/languages/java.md @@ -87,8 +87,9 @@ Load this file when changing anything under `java/` or when Java drives a cross- - Each natural Java registry or public facade boundary owns one authoritative frozen flag. The facade flag is not a mirror of a child resolver flag. The first root serialization or deserialization sets the owning flag before codec work and never clears it, including after - failure. Every explicit type, serializer, module, name, or ID registration checks that flag - before mutation. Do not add another lifecycle state or a parallel registration-commit path. + failure. Every explicit type, serializer, module, name, ID, or type-checker policy change checks + that flag before mutation. Do not add another lifecycle state or a parallel registration-commit + path. - Direct and thread-safe facades expose module registration before their first root. Kotlin and Scala registration extensions target `BaseFory`; do not narrow them to concrete `Fory` or make builder installation the only thread-safe path. Before the first root, a thread-safe facade @@ -96,6 +97,9 @@ Load this file when changing anything under `java/` or when Java drives a cross- the frozen flag after entering it. After freeze, `execute` and copy use the monitor-free path. Calling `ThreadSafeFory.execute` or copying a value does not freeze registration unless the callback starts a root serialization or deserialization. +- `ThreadSafeFory.execute` exposes one borrowed child only for the callback. Do not retain that + child or register through it; use the facade registration methods so every current and future + child receives the same setup. - A serializer instance registered on a thread-safe facade must implement `Shareable`. Resolver- local serializers use the class, resolver-factory, or module path so every child runtime owns its instance; never replay one resolver-bound serializer across children. diff --git a/java/fory-core/src/main/java/org/apache/fory/Fory.java b/java/fory-core/src/main/java/org/apache/fory/Fory.java index ede0f648bb..f0e595e77f 100644 --- a/java/fory-core/src/main/java/org/apache/fory/Fory.java +++ b/java/fory-core/src/main/java/org/apache/fory/Fory.java @@ -169,13 +169,11 @@ public Fory(ForyBuilder builder, ClassLoader classLoader, SharedRegistry sharedR @Override public void register(Class cls) { - typeResolver.checkRegistrationOpen(); getTypeResolver().register(cls); } @Override public void register(Class cls, int id) { - typeResolver.checkRegistrationOpen(); getTypeResolver().register(cls, Integer.toUnsignedLong(id)); } @@ -185,38 +183,32 @@ public void register(Class cls, int id) { */ @Override public void register(Class cls, String name) { - typeResolver.checkRegistrationOpen(); String[] parts = splitRegistrationName(name); register(cls, parts[0], parts[1]); } public void register(Class cls, String namespace, String typeName) { - typeResolver.checkRegistrationOpen(); getTypeResolver().register(cls, namespace, typeName); } @Override public void register(String className) { - typeResolver.checkRegistrationOpen(); getTypeResolver().register(className); } @Override public void register(String className, int classId) { - typeResolver.checkRegistrationOpen(); getTypeResolver().register(className, Integer.toUnsignedLong(classId)); } @Override public void register(String className, String name) { - typeResolver.checkRegistrationOpen(); String[] parts = splitRegistrationName(name); getTypeResolver().register(className, parts[0], parts[1]); } @Override public void register(String className, String namespace, String typeName) { - typeResolver.checkRegistrationOpen(); getTypeResolver().register(className, namespace, typeName); } @@ -239,13 +231,11 @@ public void register(ForyModule module) { @Override public void registerUnion(Class cls, int id, Serializer serializer) { - typeResolver.checkRegistrationOpen(); getTypeResolver().registerUnion(cls, Integer.toUnsignedLong(id), serializer); } @Override public void registerUnion(Class cls, String name, Serializer serializer) { - typeResolver.checkRegistrationOpen(); String[] parts = splitRegistrationName(name); getTypeResolver().registerUnion(cls, parts[0], parts[1], serializer); } @@ -253,19 +243,16 @@ public void registerUnion(Class cls, String name, Serializer serializer) { @Override public void registerUnion( Class cls, String namespace, String typeName, Serializer serializer) { - typeResolver.checkRegistrationOpen(); getTypeResolver().registerUnion(cls, namespace, typeName, serializer); } @Override public void registerSerializer(Class type, Class serializerClass) { - typeResolver.checkRegistrationOpen(); getTypeResolver().registerSerializer(type, serializerClass); } @Override public void registerSerializer(Class type, Serializer serializer) { - typeResolver.checkRegistrationOpen(); getTypeResolver().registerSerializer(type, serializer); } @@ -279,13 +266,11 @@ public void registerSerializer( @Override public void registerSerializerAndType( Class type, Class serializerClass) { - typeResolver.checkRegistrationOpen(); getTypeResolver().registerSerializerAndType(type, serializerClass); } @Override public void registerSerializerAndType(Class type, Serializer serializer) { - typeResolver.checkRegistrationOpen(); getTypeResolver().registerSerializerAndType(type, serializer); } @@ -298,7 +283,6 @@ public void registerSerializerAndType( @Override public void registerSerializerFactory(SerializerFactory serializerFactory) { - typeResolver.checkRegistrationOpen(); typeResolver.registerSerializerFactory(serializerFactory); } @@ -407,17 +391,38 @@ private ForyException processCopyError(Throwable e) { @Override public Object deserialize(byte[] bytes) { - return deserialize(MemoryUtils.wrap(bytes), (Iterable) null); + MemoryBuffer buffer; + try { + buffer = MemoryUtils.wrap(bytes); + } catch (RuntimeException | Error e) { + typeResolver.freezeRegistration(); + throw e; + } + return deserialize(buffer, (Iterable) null); } @Override public Object deserialize(ByteBuffer byteBuffer) { - return deserialize(MemoryUtils.wrap(byteBuffer)); + MemoryBuffer buffer; + try { + buffer = MemoryUtils.wrap(byteBuffer); + } catch (RuntimeException | Error e) { + typeResolver.freezeRegistration(); + throw e; + } + return deserialize(buffer); } @Override public T deserialize(byte[] bytes, Class type) { - return deserialize(MemoryUtils.wrap(bytes), type); + MemoryBuffer buffer; + try { + buffer = MemoryUtils.wrap(bytes); + } catch (RuntimeException | Error e) { + typeResolver.freezeRegistration(); + throw e; + } + return deserialize(buffer, type); } @Override @@ -473,7 +478,14 @@ public T deserialize(ForyReadableChannel channel, Class type) { @Override public Object deserialize(byte[] bytes, Iterable outOfBandBuffers) { - return deserialize(MemoryUtils.wrap(bytes), outOfBandBuffers); + MemoryBuffer buffer; + try { + buffer = MemoryUtils.wrap(bytes); + } catch (RuntimeException | Error e) { + typeResolver.freezeRegistration(); + throw e; + } + return deserialize(buffer, outOfBandBuffers); } @Override diff --git a/java/fory-core/src/main/java/org/apache/fory/ThreadSafeFory.java b/java/fory-core/src/main/java/org/apache/fory/ThreadSafeFory.java index 22f4da33a3..ae6c76e90c 100644 --- a/java/fory-core/src/main/java/org/apache/fory/ThreadSafeFory.java +++ b/java/fory-core/src/main/java/org/apache/fory/ThreadSafeFory.java @@ -39,12 +39,13 @@ public interface ThreadSafeFory extends BaseFory { /** * Provide a context to execution operations on {@link Fory} directly and return the executed - * result. + * result. The action must not retain the runtime or register through it; use this facade's + * registration methods so every underlying runtime receives the same registration. */ R execute(Function action); /** - * Set TypeChecker of serializer for current thread only. + * Set the TypeChecker for all current and future underlying runtimes before the first root. * * @param typeChecker {@link TypeChecker} for type checking */ diff --git a/java/fory-core/src/main/java/org/apache/fory/resolver/TypeResolver.java b/java/fory-core/src/main/java/org/apache/fory/resolver/TypeResolver.java index efb6812072..e42cd0f45c 100644 --- a/java/fory-core/src/main/java/org/apache/fory/resolver/TypeResolver.java +++ b/java/fory-core/src/main/java/org/apache/fory/resolver/TypeResolver.java @@ -2310,7 +2310,9 @@ private void buildGenericMap(Map map, GenericType genericTy } } + /** Sets the deserialization type policy before the first root operation. */ public void setTypeChecker(TypeChecker typeChecker) { + checkRegistrationOpen(); TypeChecker newChecker = typeChecker == null ? DEFAULT_TYPE_CHECKER : typeChecker; if (newChecker instanceof AllowListChecker) { ((AllowListChecker) newChecker).addListener(this); diff --git a/java/fory-core/src/main/java/org/apache/fory/serializer/CopyOnlyObjectSerializer.java b/java/fory-core/src/main/java/org/apache/fory/serializer/CopyOnlyObjectSerializer.java index d686be20eb..8f3f007d5f 100644 --- a/java/fory-core/src/main/java/org/apache/fory/serializer/CopyOnlyObjectSerializer.java +++ b/java/fory-core/src/main/java/org/apache/fory/serializer/CopyOnlyObjectSerializer.java @@ -25,7 +25,7 @@ import org.apache.fory.resolver.TypeResolver; /** - * Serializer used only for copy after registration has been frozen. + * Serializer used to copy an unregistered object type that is not allowed on wire read or write. * *

Read/write keep the same security failure semantics as the normal insecure path, while copy * reuses {@link AbstractObjectSerializer}'s field-copy implementation. diff --git a/java/fory-core/src/test/java/org/apache/fory/resolver/AllowListCheckerTest.java b/java/fory-core/src/test/java/org/apache/fory/resolver/AllowListCheckerTest.java index a2d5aecbf0..c1bde12559 100644 --- a/java/fory-core/src/test/java/org/apache/fory/resolver/AllowListCheckerTest.java +++ b/java/fory-core/src/test/java/org/apache/fory/resolver/AllowListCheckerTest.java @@ -28,6 +28,7 @@ import org.apache.fory.Fory; import org.apache.fory.ThreadSafeFory; import org.apache.fory.config.Language; +import org.apache.fory.exception.ForyException; import org.apache.fory.exception.InsecureException; import org.apache.fory.logging.LogLevel; import org.apache.fory.logging.LoggerFactory; @@ -193,8 +194,7 @@ public void testDisallowSetupOnly() { AllowListChecker lateChecker = new AllowListChecker(AllowListChecker.CheckLevel.WARN); lateChecker.disallowClass("org.apache.fory.missing.Type"); - assertThrows( - IllegalStateException.class, () -> fory.getTypeResolver().setTypeChecker(lateChecker)); + assertThrows(ForyException.class, () -> fory.getTypeResolver().setTypeChecker(lateChecker)); AllowListChecker disabledChecker = new AllowListChecker(AllowListChecker.CheckLevel.DISABLE); disabledChecker.disallowClass("org.apache.fory.missing.Type"); @@ -230,9 +230,13 @@ public void testCheckerReplacement() { .requireClassRegistration(false) .withTypeChecker(oldChecker) .build(); - fory.serialize("value"); - fory.getTypeResolver().setTypeChecker(new AllowListChecker(AllowListChecker.CheckLevel.WARN)); + fory.serialize("value"); + assertThrows( + ForyException.class, + () -> + fory.getTypeResolver() + .setTypeChecker(new AllowListChecker(AllowListChecker.CheckLevel.WARN))); oldChecker.disallowClass(missingClass); assertThrows(InsecureException.class, () -> oldChecker.checkType(null, missingClass)); } diff --git a/java/fory-core/src/test/java/org/apache/fory/serializer/RegisterTest.java b/java/fory-core/src/test/java/org/apache/fory/serializer/RegisterTest.java index b0d8adbc86..91cd6fef74 100644 --- a/java/fory-core/src/test/java/org/apache/fory/serializer/RegisterTest.java +++ b/java/fory-core/src/test/java/org/apache/fory/serializer/RegisterTest.java @@ -20,7 +20,10 @@ package org.apache.fory.serializer; import java.io.ByteArrayInputStream; +import java.nio.ByteBuffer; +import java.util.Arrays; import java.util.concurrent.atomic.AtomicBoolean; +import java.util.function.Consumer; import org.apache.fory.Fory; import org.apache.fory.ForyModule; import org.apache.fory.ForyTestBase; @@ -222,6 +225,20 @@ public MemoryBuffer getBuffer() { Assert.assertTrue(registrationRejected.get()); } + @Test + public void testInvalidRootFreezes() { + for (Consumer root : + Arrays.>asList( + fory -> fory.deserialize((byte[]) null), + fory -> fory.deserialize((ByteBuffer) null), + fory -> fory.deserialize((byte[]) null, Object.class), + fory -> fory.deserialize((byte[]) null, (Iterable) null))) { + Fory fory = Fory.builder().build(); + Assert.assertThrows(NullPointerException.class, () -> root.accept(fory)); + Assert.assertThrows(ForyException.class, () -> fory.register(MyExt.class)); + } + } + public static class MyExtSerializer extends Serializer { public MyExtSerializer(TypeResolver typeResolver) { super(typeResolver.getConfig(), MyExt.class); From 7284ab4d85242f47ed050594f8188faacf527f55 Mon Sep 17 00:00:00 2001 From: chaokunyang Date: Sun, 30 Aug 2026 08:09:58 +0800 Subject: [PATCH 135/168] docs: align lifecycle guidance with registry owners --- cpp/fory/serialization/context.h | 4 ++-- docs/object-serialization/go/native.md | 13 +++--------- docs/object-serialization/go/thread-safety.md | 19 ++++++++---------- .../java/type-registration.md | 5 ++--- .../python/configuration.md | 10 +++++++--- docs/object-serialization/python/security.md | 7 +++---- .../python/troubleshooting.md | 3 --- .../python/type-registration.md | 7 +++---- docs/security/deserialization.md | 10 ++++++---- .../xlang_implementation_guide.md | 14 +++++++------ python/README.md | 20 +++++++++---------- python/pyfory/_fory.py | 10 +++------- python/pyfory/tests/test_serializer.py | 2 +- 13 files changed, 56 insertions(+), 68 deletions(-) diff --git a/cpp/fory/serialization/context.h b/cpp/fory/serialization/context.h index afc14f973d..27877c0e5b 100644 --- a/cpp/fory/serialization/context.h +++ b/cpp/fory/serialization/context.h @@ -299,8 +299,8 @@ class WriteContext { /// Fastest path for writing struct type info when TypeInfo is already known. /// Avoids type_index creation and lookup overhead. /// - /// @param type_info Pointer to a valid TypeInfo whose metadata is ready when - /// the configured wire mode needs it + /// @param type_info Pointer to a valid TypeInfo. This method completes its + /// metadata lazily when the configured wire mode needs it. void write_struct_type_info(const TypeInfo *type_info); /// Fastest path - write struct type_id directly without any lookups. diff --git a/docs/object-serialization/go/native.md b/docs/object-serialization/go/native.md index 798afde830..445f75e10c 100644 --- a/docs/object-serialization/go/native.md +++ b/docs/object-serialization/go/native.md @@ -109,21 +109,14 @@ every reader and writer always uses the same Go struct schema. Register structs before serializing them. Prefer explicit numeric IDs for long-lived payloads: ```go -f := fory.New(fory.WithXlang(false)) -if err := f.RegisterStruct(Order{}, 100); err != nil { - panic(err) -} -if err := f.RegisterStruct(LineItem{}, 101); err != nil { - panic(err) -} +_ = f.RegisterStruct(Order{}, 100) +_ = f.RegisterStruct(LineItem{}, 101) ``` Name-based registration is useful when ID coordination is harder: ```go -if err := f.RegisterStructByName(Order{}, "example.Order"); err != nil { - panic(err) -} +_ = f.RegisterStructByName(Order{}, "example.Order") ``` If you register without stable IDs, every writer and reader must make the same registration choices. diff --git a/docs/object-serialization/go/thread-safety.md b/docs/object-serialization/go/thread-safety.md index ae58210a02..8a2bcca3c3 100644 --- a/docs/object-serialization/go/thread-safety.md +++ b/docs/object-serialization/go/thread-safety.md @@ -72,20 +72,21 @@ The thread-safe wrapper uses `sync.Pool`: 1. **Acquire**: Gets a Fory instance from the pool 2. **Use**: Performs serialization/deserialization -3. **Copy**: Copies result data because the buffer will be reused -4. **Release**: Returns the instance to the pool +3. **Copy**: Copies result data (buffer will be reused) +4. **Release**: Returns instance to pool ```go // Simplified implementation func (f *Fory) Serialize(v any) ([]byte, error) { - inner := f.pool.Get().(*fory.Fory) - defer f.pool.Put(inner) + fory := f.pool.Get().(*fory.Fory) + defer f.pool.Put(fory) - data, err := inner.Serialize(v) + data, err := fory.Serialize(v) if err != nil { return nil, err } + // Copy because underlying buffer will be reused result := make([]byte, len(data)) copy(result, data) return result, nil @@ -196,9 +197,7 @@ This is safer but has allocation overhead. ```go func BenchmarkNonThreadSafe(b *testing.B) { f := fory.New(fory.WithXlang(true)) - if err := f.RegisterStruct(User{}, 1); err != nil { - b.Fatal(err) - } + f.RegisterStruct(User{}, 1) user := &User{ID: 1, Name: "Alice"} for i := 0; i < b.N; i++ { @@ -234,9 +233,7 @@ For maximum performance with known goroutine count: func worker(id int) { // Each worker has its own Fory instance f := fory.New(fory.WithXlang(true)) - if err := f.RegisterStruct(User{}, 1); err != nil { - panic(err) - } + f.RegisterStruct(User{}, 1) for task := range tasks { data, _ := f.Serialize(task) diff --git a/docs/object-serialization/java/type-registration.md b/docs/object-serialization/java/type-registration.md index 13402529c7..e4f0995ea5 100644 --- a/docs/object-serialization/java/type-registration.md +++ b/docs/object-serialization/java/type-registration.md @@ -49,9 +49,8 @@ call. Starting either operation permanently freezes registration even if the ope operations and `ThreadSafeFory#execute` do not freeze registration unless the supplied callback starts serialization or deserialization. Later explicit registration attempts are rejected. -In native mode with registration disabled, Fory may still resolve allowed runtime classes and cache -their descriptors or serializers after this boundary. That lazy resolution is not explicit -registration and does not reopen or change the configured registry. +In native mode with registration disabled, allowed unregistered runtime classes may still be used +after this boundary. That behavior does not reopen or change explicit registration. `registerSerializer(Foo.class, ...)` is sufficient to use `Foo` when class registration is enabled. Use `registerSerializerAndType(Foo.class, ...)` when you also want Fory to assign a numeric type ID. diff --git a/docs/object-serialization/python/configuration.md b/docs/object-serialization/python/configuration.md index bf91d74684..d5801f2e4c 100644 --- a/docs/object-serialization/python/configuration.md +++ b/docs/object-serialization/python/configuration.md @@ -54,11 +54,12 @@ Thread-safe serialization interface using a pooled wrapper: ```python class ThreadSafeFory: - def __init__( - self, fory_factory=None, **kwargs - ) + def __init__(self, fory_factory=None, **kwargs) ``` +Pass either a no-argument `fory_factory` that returns a configured `Fory` instance, or pass normal +`Fory` construction options through `**kwargs`. + ## Parameters | Parameter | Type | Default | Description | @@ -82,6 +83,9 @@ class ThreadSafeFory: ## Key Methods ```python +fory = pyfory.Fory(xlang=True) +thread_safe_fory = pyfory.ThreadSafeFory(xlang=True) + # Serialization (serialize/deserialize are identical to dumps/loads) data: bytes = fory.serialize(obj) obj = fory.deserialize(data) diff --git a/docs/object-serialization/python/security.md b/docs/object-serialization/python/security.md index f7bfca16d9..5ece2a3607 100644 --- a/docs/object-serialization/python/security.md +++ b/docs/object-serialization/python/security.md @@ -81,10 +81,9 @@ fory = pyfory.Fory( The first root attempt permanently freezes registration, including when that attempt fails. `strict=False` does not permit type or serializer registration after that boundary, but its policy may authorize module-global classes and callables resolved while reading a trusted native payload. -That resolution may populate resolver-owned caches but does not add or change an explicit -registration. If a strict-mode failure requires adding a missing registration, create and configure -a new instance. An already configured reader can process a later root after a malformed-data -failure. +That resolution does not add or change an explicit registration. If a strict-mode failure requires +adding a missing registration, create and configure a new instance. An already configured reader +can process a later root after a malformed-data failure. Received remote metadata is also limited: diff --git a/docs/object-serialization/python/troubleshooting.md b/docs/object-serialization/python/troubleshooting.md index 221fcfc5c0..0e7eb59ec2 100644 --- a/docs/object-serialization/python/troubleshooting.md +++ b/docs/object-serialization/python/troubleshooting.md @@ -86,9 +86,6 @@ assert result.next.next is result # Circular reference preserved ### Schema Evolution Not Working ```python -# Keep compatible mode enabled. This is the default. -f = pyfory.Fory() - # Version 1: Writer schema @dataclass class UserV1: diff --git a/docs/object-serialization/python/type-registration.md b/docs/object-serialization/python/type-registration.md index 095d6c89d4..f952173e8e 100644 --- a/docs/object-serialization/python/type-registration.md +++ b/docs/object-serialization/python/type-registration.md @@ -86,10 +86,9 @@ those payloads. The first root serialization or deserialization attempt permanently closes registration, including when that attempt fails. `strict=False` permits native writes to discover runtime classes and callables and permits reads to resolve -those authorized by the configured policy. That lazy resolution does not -reopen the registry or create an explicit binding. Explicit names, IDs, and custom -serializers must be configured before the first root; later registration -attempts fail. +those authorized by the configured policy. That lazy resolution does not register the type. +Explicit names, IDs, and custom serializers must be configured before the first root; later +registration attempts fail. Compatible metadata has one data-only exception: when a remote Struct has no local registration, deserialization returns the fixed framework diff --git a/docs/security/deserialization.md b/docs/security/deserialization.md index ce9720518f..ba4a9ff077 100644 --- a/docs/security/deserialization.md +++ b/docs/security/deserialization.md @@ -619,11 +619,13 @@ that case, classify the behavior by concrete impact: The first root serialization or deserialization permanently closes explicit type and serializer registration, including when that operation fails. Each natural registration owner keeps one -authoritative frozen flag, and every later explicit registration attempt fails before changing +authoritative lifecycle fact, and every later explicit registration attempt fails before changing type, serializer, ID, name, metadata, or policy bindings. A thread-safe facade with its own public -registration surface may own its boundary flag, but must not mirror a child registry's flag. -Implementations must use one boolean flag without another lifecycle state, a registration commit -path, or eager whole-registry preparation. +registration surface may own its boundary fact, but must not mirror a child registry's lifecycle. +Each natural owner must keep exactly one authoritative lifecycle fact. That fact may be an +owner-native flag or the presence of an immutable registry snapshot. Do not add another lifecycle +state, a registration commit or rollback path, or eager whole-registry preparation solely to +implement freeze. Registry freeze does not disable native runtime type resolution. When a mode supports unregistered types, a root may still discover an allowed runtime type and materialize resolver-owned metadata, diff --git a/docs/specification/xlang_implementation_guide.md b/docs/specification/xlang_implementation_guide.md index 6c98351cbb..2496b5f333 100644 --- a/docs/specification/xlang_implementation_guide.md +++ b/docs/specification/xlang_implementation_guide.md @@ -83,12 +83,14 @@ not the place where nested serializers do their work. - resetting operation-local context state at the top-level root boundary Explicit type and serializer registration is open only before the first root serialization or -deserialization. Starting either root sets one authoritative frozen flag for the natural -registration owner before codec work and never clears it, including when the root fails. Every -explicit registration path checks that flag before mutation. A thread-safe facade with its own -public registration surface may own its boundary flag, but must not mirror a child registry's flag. -Implementations must use one boolean flag without another lifecycle state, a registration commit -path, or eager whole-registry preparation. +deserialization. Starting either root establishes one authoritative frozen lifecycle fact for the +natural registration owner before codec work and never clears it, including when the root fails. +Every explicit registration path checks that fact before mutation. A thread-safe facade with its own +public registration surface may own its boundary fact, but must not mirror a child registry's +lifecycle. Each natural owner must keep exactly one authoritative lifecycle fact. That fact may be +an owner-native flag or the presence of an immutable registry snapshot. Do not add another +lifecycle state, a registration commit or rollback path, or eager whole-registry preparation solely +to implement freeze. In JavaScript, `Fory` owns this flag. `TypeResolver` owns the registration maps but must not carry a second lifecycle flag. diff --git a/python/README.md b/python/README.md index c93d41c0a1..8ab6b77bc7 100644 --- a/python/README.md +++ b/python/README.md @@ -684,15 +684,14 @@ Thread-safe serialization interface using an instance pool: ```python class ThreadSafeFory: - def __init__( - self, - fory_factory=None, - **kwargs - ) + def __init__(self, fory_factory=None, **kwargs) ``` `ThreadSafeFory` provides thread-safe serialization by maintaining a pool of `Fory` instances protected by a lock. When a thread needs to serialize/deserialize, it gets an instance from the pool, uses it, and returns it. Complete explicit type and serializer registration before the first serialization or deserialization attempt. +Pass either a no-argument `fory_factory` that returns a configured `Fory` instance, or pass normal +`Fory` construction options through `**kwargs`. + **Thread Safety Example:** ```python @@ -738,15 +737,16 @@ for t in threads: t.join() **Parameters:** -- **`xlang`** (`bool`, default=`True`): Use xlang mode. Set `False` for Python native mode supporting Python-specific objects. -- **`ref`** (`bool`, default=`False`): Enable reference tracking for shared/circular references. Disable for better performance if your data has no shared references. -- **`strict`** (`bool`, default=`True`): Require type registration for security. **Highly recommended** for production. Only disable in trusted environments. -- **`compatible`** (`bool | None`, default `None`): Enable schema evolution. `None` enables compatible mode in both xlang and native mode. Set `False` only when every reader and writer always uses the same Python class schema and you want faster serialization and smaller size. -- **`max_depth`** (`int`, default=`50`): Maximum deserialization depth for security, preventing stack overflow attacks. +- **`fory_factory`** (`Callable | None`, default=`None`): No-argument factory for configured `Fory` + instances. +- **`**kwargs`**: Normal `Fory` construction options, used when `fory_factory` is not supplied. **Key Methods:** ```python +fory = pyfory.Fory(xlang=True) +thread_safe_fory = pyfory.ThreadSafeFory(xlang=True) + # Serialization (serialize/deserialize are identical to dumps/loads) data: bytes = fory.serialize(obj) obj = fory.deserialize(data) diff --git a/python/pyfory/_fory.py b/python/pyfory/_fory.py index 7f238306c0..c7ee992791 100644 --- a/python/pyfory/_fory.py +++ b/python/pyfory/_fory.py @@ -643,13 +643,9 @@ class ThreadSafeFory: remains closed even when that first operation fails. Args: - xlang (bool): Whether to enable xlang mode. Defaults to True. - ref (bool): Whether to enable reference tracking. Defaults to False. - strict (bool): Whether to require type registration. Defaults to True. - compatible (bool): Whether to enable compatible mode. Defaults to compatible mode - in both xlang and Python native mode. Set False only when every reader and - writer always uses the same Python class schema and smaller payloads matter. - max_depth (int): Maximum depth for deserialization. Defaults to 50. + fory_factory: Optional no-argument factory that returns a configured Fory + instance. + **kwargs: Fory construction options used when fory_factory is not supplied. Example: >>> import pyfory >>> import threading diff --git a/python/pyfory/tests/test_serializer.py b/python/pyfory/tests/test_serializer.py index fe713dff98..490ab8a7e5 100644 --- a/python/pyfory/tests/test_serializer.py +++ b/python/pyfory/tests/test_serializer.py @@ -967,7 +967,7 @@ def factory(type_resolver, cls): assert fory.type_resolver.get_type_info(FrozenRegistration, create=False) is None -def test_inferred_registration_freeze(): +def test_type_setup_freeze(): fory = Fory(xlang=False, strict=False, compatible=False) fory.register_type(FrozenRegistration) armed = False From af3d9996c21385b3bbc9b97503a5a7052e80939c Mon Sep 17 00:00:00 2001 From: chaokunyang Date: Sun, 30 Aug 2026 08:17:07 +0800 Subject: [PATCH 136/168] refactor(cpp): remove resolver lifecycle tag --- AGENTS.md | 12 +++++++----- cpp/fory/serialization/fory.h | 24 ++++++++++++------------ 2 files changed, 19 insertions(+), 17 deletions(-) diff --git a/AGENTS.md b/AGENTS.md index 0086c40611..56cd2c50eb 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -185,11 +185,13 @@ This is the entry point for AI guidance in Apache Fory. Read this file first, th maps, generated descriptors, metadata, serializers, or caches. Do not support post-use registration through cache invalidation, descriptor refresh, serializer rebinding, metadata rebuilding, or other late-registration - machinery. Keep one authoritative frozen flag for each natural registration - owner or public facade boundary. A thread-safe facade with its own public - registration surface may own that boundary flag, but must not mirror a child - registry's flag. Do not add another lifecycle state, a registration commit - path, or eager whole-registry preparation. Copy operations and + machinery. Keep one authoritative lifecycle fact for each natural registration + owner or public facade boundary. That fact may be an owner-native flag or the + presence of an immutable registry snapshot. A thread-safe facade with its own + public registration surface may own that boundary fact, but must not mirror a + child registry's lifecycle. Do not add another lifecycle state, a registration + commit or rollback path, or eager whole-registry preparation solely to implement + freeze. Copy operations and facade execution callbacks do not freeze registration unless they start a root serialization or deserialization operation. Registry freeze does not make native runtime type resolution immutable. When diff --git a/cpp/fory/serialization/fory.h b/cpp/fory/serialization/fory.h index 5d8a733865..068742ec0f 100644 --- a/cpp/fory/serialization/fory.h +++ b/cpp/fory/serialization/fory.h @@ -705,14 +705,15 @@ class Fory : public BaseFory { : BaseFory(config, std::move(resolver)), precomputed_header_(compute_header(config.xlang)) {} - /// Constructor for ThreadSafeFory pool - registration is already frozen. - struct FrozenResolver {}; - explicit Fory(const Config &config, std::shared_ptr resolver, - FrozenResolver) - : BaseFory(config, std::move(resolver)), - precomputed_header_(compute_header(config.xlang)) { - write_ctx_.emplace(config_, type_resolver_->clone()); - read_ctx_.emplace(config_, type_resolver_->clone()); + /// Create a runtime whose operation contexts clone an already prepared + /// resolver. + static std::unique_ptr + create_with_contexts(const Config &config, + std::shared_ptr resolver) { + auto fory = std::unique_ptr(new Fory(config, std::move(resolver))); + fory->write_ctx_.emplace(config, fory->type_resolver_->clone()); + fory->read_ctx_.emplace(config, fory->type_resolver_->clone()); + return fory; } /// Freeze registration and initialize operation contexts. @@ -1019,10 +1020,9 @@ class ThreadSafeFory : public BaseFory { std::shared_ptr resolver) : BaseFory(config, std::move(resolver)), shared_resolver_(), resolver_once_flag_(), fory_pool_([this]() { - // Every public root freezes the resolver before pool acquisition. - // The pooled Fory constructor owns its context clones. - return std::unique_ptr( - new Fory(config_, shared_resolver_, Fory::FrozenResolver{})); + // Every public root prepares the shared resolver before pool + // acquisition. Fory owns the per-runtime context clones. + return Fory::create_with_contexts(config_, shared_resolver_); }) {} void ensure_resolver_initialized() const { From 934ef17563ed437ddf7fe485b4ba33b32483d996 Mon Sep 17 00:00:00 2001 From: chaokunyang Date: Sun, 30 Aug 2026 08:21:38 +0800 Subject: [PATCH 137/168] refactor: remove redundant lifecycle drift --- .agents/languages/java.md | 6 +- .../python/type-registration.md | 6 +- .../xlang_implementation_guide.md | 2 +- .../fory/resolver/AllowListChecker.java | 5 -- python/pyfory/registry.py | 64 +++++++++---------- 5 files changed, 39 insertions(+), 44 deletions(-) diff --git a/.agents/languages/java.md b/.agents/languages/java.md index 328b92cd50..555f43e30c 100644 --- a/.agents/languages/java.md +++ b/.agents/languages/java.md @@ -87,9 +87,9 @@ Load this file when changing anything under `java/` or when Java drives a cross- - Each natural Java registry or public facade boundary owns one authoritative frozen flag. The facade flag is not a mirror of a child resolver flag. The first root serialization or deserialization sets the owning flag before codec work and never clears it, including after - failure. Every explicit type, serializer, module, name, ID, or type-checker policy change checks - that flag before mutation. Do not add another lifecycle state or a parallel registration-commit - path. + failure. Every explicit type, serializer, module, name, ID, or type-checker binding checks that + flag before mutation. Disallow-list changes use the bound checker's resolver listeners. Do not + add another lifecycle state or a parallel registration-commit path. - Direct and thread-safe facades expose module registration before their first root. Kotlin and Scala registration extensions target `BaseFory`; do not narrow them to concrete `Fory` or make builder installation the only thread-safe path. Before the first root, a thread-safe facade diff --git a/docs/object-serialization/python/type-registration.md b/docs/object-serialization/python/type-registration.md index f952173e8e..89b68e9e15 100644 --- a/docs/object-serialization/python/type-registration.md +++ b/docs/object-serialization/python/type-registration.md @@ -86,9 +86,9 @@ those payloads. The first root serialization or deserialization attempt permanently closes registration, including when that attempt fails. `strict=False` permits native writes to discover runtime classes and callables and permits reads to resolve -those authorized by the configured policy. That lazy resolution does not register the type. -Explicit names, IDs, and custom serializers must be configured before the first root; later -registration attempts fail. +those authorized by the configured policy. That lazy resolution does not add an explicit +registration. Explicit names, IDs, and custom serializers must be configured before the first +root; later registration attempts fail. Compatible metadata has one data-only exception: when a remote Struct has no local registration, deserialization returns the fixed framework diff --git a/docs/specification/xlang_implementation_guide.md b/docs/specification/xlang_implementation_guide.md index 2496b5f333..b22b5e75fe 100644 --- a/docs/specification/xlang_implementation_guide.md +++ b/docs/specification/xlang_implementation_guide.md @@ -260,7 +260,7 @@ The ownership split is: allocation - `Fory` owns root framing and operation setup/reset - `TypeResolver` owns registration mappings, serializer bindings, and dynamic lookup; the natural - registry or public facade owner owns the registry freeze flag + registry or public facade owner owns the authoritative registry lifecycle fact #### C# generated structural serializers diff --git a/java/fory-core/src/main/java/org/apache/fory/resolver/AllowListChecker.java b/java/fory-core/src/main/java/org/apache/fory/resolver/AllowListChecker.java index d21546de27..d2e645b5c5 100644 --- a/java/fory-core/src/main/java/org/apache/fory/resolver/AllowListChecker.java +++ b/java/fory-core/src/main/java/org/apache/fory/resolver/AllowListChecker.java @@ -250,11 +250,6 @@ private void disallow(String classNameOrPrefix) { void addListener(TypeResolver resolver) { try { lock.writeLock().lock(); - if ((!disallowList.isEmpty() || !disallowListPrefix.isEmpty()) - && resolver.isRegistrationFrozen()) { - throw new IllegalStateException( - "A checker with disallow entries cannot be installed after registration."); - } listeners.put(resolver, true); } finally { lock.writeLock().unlock(); diff --git a/python/pyfory/registry.py b/python/pyfory/registry.py index 54e981f498..95106daca2 100644 --- a/python/pyfory/registry.py +++ b/python/pyfory/registry.py @@ -263,13 +263,13 @@ def _split_registration_name(name: str): class TypeInfo: __slots__ = ( "cls", - "dynamic_type", - "namespace_bytes", - "serializer", - "type_def", "type_id", - "typename_bytes", "user_type_id", + "serializer", + "namespace_bytes", + "typename_bytes", + "dynamic_type", + "type_def", ) def __init__( @@ -307,7 +307,7 @@ def decode_typename(self) -> str: class SharedRegistry: - __slots__ = ("_encoded_metastrings", "_metastr_to_bytes") + __slots__ = ("_metastr_to_bytes", "_encoded_metastrings") def __init__(self): self._metastr_to_bytes = {} @@ -356,38 +356,38 @@ def get_or_create_encoded_meta_string(self, data: bytes, hashcode: int) -> Encod class TypeResolver: __slots__ = ( - "_actual_type_resolver", + "xlang", + "track_ref", + "strict", + "compatible", + "field_nullable", + "policy", + "config", + "shared_registry", + "_type_id_counter", + "_types_info", + "_python_name_to_type", + "_metastr_to_type", "_hash_to_type_info", - "_internal_py_serializer_map", + "_ns_type_to_type_info", + "_named_type_to_type_info", + "namespace_encoder", + "namespace_decoder", + "typename_encoder", + "typename_decoder", + "meta_compressor", + "require_registration", + "_type_id_to_type_info", + "_user_type_id_to_type_info", + "_used_user_type_ids", "_local_type_info_by_hash", "_meta_shared_type_info", - "_metastr_to_type", - "_named_type_to_type_info", - "_ns_type_to_type_info", - "_python_name_to_type", - "_registry_frozen", "_remote_schema_versions_by_type", "_total_accepted_schema_versions", - "_type_id_counter", - "_type_id_to_type_info", - "_types_info", - "_used_user_type_ids", - "_user_type_id_to_type_info", - "compatible", - "config", - "field_nullable", - "meta_compressor", "meta_share", - "namespace_decoder", - "namespace_encoder", - "policy", - "require_registration", - "shared_registry", - "strict", - "track_ref", - "typename_decoder", - "typename_encoder", - "xlang", + "_internal_py_serializer_map", + "_actual_type_resolver", + "_registry_frozen", ) def __init__(self, config, *, shared_registry): From 1358f4e7a7b00153f5b5cd94e642a92d6ce481e1 Mon Sep 17 00:00:00 2001 From: chaokunyang Date: Sun, 30 Aug 2026 08:22:28 +0800 Subject: [PATCH 138/168] docs: remove stale lifecycle examples --- .../go/type-registration.md | 3 ++- python/README.md | 20 +++++++++---------- 2 files changed, 12 insertions(+), 11 deletions(-) diff --git a/docs/object-serialization/go/type-registration.md b/docs/object-serialization/go/type-registration.md index ed4577d05c..9a6005f9db 100644 --- a/docs/object-serialization/go/type-registration.md +++ b/docs/object-serialization/go/type-registration.md @@ -234,7 +234,8 @@ fory.register_by_name::("example.User")?; ## Best Practices -1. **Register early**: Register all types at application startup before any serialization +1. **Register early**: Register all types at application startup before any serialization or + deserialization 2. **Be consistent**: Use the same ID or name across all languages and all instances 3. **Register all types**: Include nested struct types, not just top-level types 4. **Prefer IDs for performance**: Numeric IDs have lower serialization overhead than names diff --git a/python/README.md b/python/README.md index 8ab6b77bc7..7ed784ef12 100644 --- a/python/README.md +++ b/python/README.md @@ -1253,26 +1253,26 @@ import pyfory # Now uses pure Python implementation ```python # A: Xlang mode defaults to compatible schema evolution. -f = pyfory.Fory(xlang=True) - -# Version 1: Original class +# Version 1: Writer schema @dataclass -class User: +class UserV1: name: str age: int -f.register(User, name="User") -data = f.dumps(User("Alice", 30)) +writer = pyfory.Fory(xlang=True) +writer.register(UserV1, name="User") +data = writer.dumps(UserV1("Alice", 30)) -# Version 2: Add new field (backward compatible) +# Version 2: Reader schema with a new field @dataclass -class User: +class UserV2: name: str age: int email: str = "unknown@example.com" # New field with default -# Can still deserialize old data -user = f.loads(data) +reader = pyfory.Fory(xlang=True) +reader.register(UserV2, name="User") +user = reader.loads(data) print(user.email) # "unknown@example.com" ``` From 9075e30a5961c3e9a3f13f2275143847addeb439 Mon Sep 17 00:00:00 2001 From: chaokunyang Date: Sun, 30 Aug 2026 08:25:41 +0800 Subject: [PATCH 139/168] test: exercise failed lookup cleanup directly --- csharp/src/Fory/ThreadSafeFory.cs | 12 ++++++------ csharp/tests/Fory.Tests/RuntimeEdgeCaseTests.cs | 17 ++++++++++------- go/fory/threadsafe/fory.go | 3 +-- 3 files changed, 17 insertions(+), 15 deletions(-) diff --git a/csharp/src/Fory/ThreadSafeFory.cs b/csharp/src/Fory/ThreadSafeFory.cs index f246b03fb2..eb8d673aae 100644 --- a/csharp/src/Fory/ThreadSafeFory.cs +++ b/csharp/src/Fory/ThreadSafeFory.cs @@ -44,7 +44,7 @@ internal ThreadSafeFory(Config config) public Config Config => _config; ///

- /// Registers a user type by numeric type identifier for all current and future thread-local runtimes. + /// Registers a user type by numeric type identifier on this thread-safe runtime. /// /// Type to register. /// Numeric type identifier used on the wire. @@ -58,7 +58,7 @@ public ThreadSafeFory Register(uint typeId) } /// - /// Registers a user type by name for all current and future thread-local runtimes. + /// Registers a user type by name on this thread-safe runtime. /// /// Type to register. /// Name used on the wire. A dotted name is split at the last dot. @@ -74,7 +74,7 @@ public ThreadSafeFory Register(string name) } /// - /// Registers a user type by namespace and name for all current and future thread-local runtimes. + /// Registers a user type by namespace and name on this thread-safe runtime. /// /// Type to register. /// Namespace used on the wire. @@ -91,7 +91,7 @@ public ThreadSafeFory Register(string typeNamespace, string typeName) } /// - /// Registers a user type by numeric type identifier with a custom serializer for all thread-local runtimes. + /// Registers a user type by numeric type identifier with a custom serializer on this thread-safe runtime. /// /// Type to register. /// Serializer implementation used for . @@ -107,7 +107,7 @@ public ThreadSafeFory Register(uint typeId) } /// - /// Registers a user type by name with a custom serializer for all thread-local runtimes. + /// Registers a user type by name with a custom serializer on this thread-safe runtime. /// /// Type to register. /// Serializer implementation used for . @@ -125,7 +125,7 @@ public ThreadSafeFory Register(string name) } /// - /// Registers a user type by namespace and name with a custom serializer for all thread-local runtimes. + /// Registers a user type by namespace and name with a custom serializer on this thread-safe runtime. /// /// Type to register. /// Serializer implementation used for . diff --git a/csharp/tests/Fory.Tests/RuntimeEdgeCaseTests.cs b/csharp/tests/Fory.Tests/RuntimeEdgeCaseTests.cs index 6198d1033f..b5f9bb1de2 100644 --- a/csharp/tests/Fory.Tests/RuntimeEdgeCaseTests.cs +++ b/csharp/tests/Fory.Tests/RuntimeEdgeCaseTests.cs @@ -900,10 +900,10 @@ public void FailedLookupRestoresWriteState() { TypeResolver.RegisterGenerated(); ForyRuntime fory = ForyRuntime.Builder().TrackRef(true).Build(); - fory.Register(736); - - Assert.Throws( - () => fory.Serialize(new FailingWritePayload { Value = 1 })); + WriteContext context = WriteContextFor(fory); + _ = context.RefWriter.ReserveRefId(); + Assert.True(context.AssignTypeMetaIndexIfAbsent(typeof(FrozenPayload)).IsNew); + Assert.True(context.AssignMetaStringIndexIfAbsent(MetaString.Empty('_', '_')).IsNew); bool lookupStarted = false; LookupFailureSerializer.ConstructionAction = () => { @@ -920,7 +920,9 @@ public void FailedLookupRestoresWriteState() LookupFailureSerializer.ConstructionAction = null; } - Assert.Equal(0u, WriteContextFor(fory).RefWriter.ReserveRefId()); + Assert.Equal(0u, context.RefWriter.ReserveRefId()); + Assert.True(context.AssignTypeMetaIndexIfAbsent(typeof(FrozenPayload)).IsNew); + Assert.True(context.AssignMetaStringIndexIfAbsent(MetaString.Empty('_', '_')).IsNew); } [Fact] @@ -1015,8 +1017,9 @@ public void TrailingBytesResetReadState(bool useSpan) byte[] invalidPayload = [.. payload, 0x7F]; _ = useSpan - ? Assert.ThrowsAny(() => DeserializeSpan(reader, invalidPayload)) - : Assert.ThrowsAny(() => reader.Deserialize(invalidPayload)); + ? Assert.Throws(() => DeserializeSpan(reader, invalidPayload)) + : Assert.Throws( + () => reader.Deserialize(invalidPayload)); ReadContext context = ReadContextFor(reader); Assert.Null(context.GetTypeMetaRef(0)); Assert.Null(context.GetReadMetaString(0)); diff --git a/go/fory/threadsafe/fory.go b/go/fory/threadsafe/fory.go index 4cd3fb2bbd..8aac021d60 100644 --- a/go/fory/threadsafe/fory.go +++ b/go/fory/threadsafe/fory.go @@ -24,8 +24,7 @@ import ( "github.com/apache/fory/go/fory" ) -// Fory is a thread-safe wrapper around fory.Fory using sync.Pool. -// It provides the same API as fory.Fory but is safe for concurrent use. +// Fory is a thread-safe serialization wrapper using a pool of fory.Fory instances. type Fory struct { pool sync.Pool } From f4314ca54a945e3e82e7e45286eeb93d66adc874 Mon Sep 17 00:00:00 2001 From: chaokunyang Date: Sun, 30 Aug 2026 10:42:24 +0800 Subject: [PATCH 140/168] docs: clarify dynamic type registration --- docs/object-serialization/core-concepts.md | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/docs/object-serialization/core-concepts.md b/docs/object-serialization/core-concepts.md index 2506a8e1ce..328bc859af 100644 --- a/docs/object-serialization/core-concepts.md +++ b/docs/object-serialization/core-concepts.md @@ -55,8 +55,9 @@ model should read this value_; a field schema describes _what data that model co A statically known field can use its declared type directly. A dynamic field also carries the concrete type needed for interfaces, abstract classes, trait objects, broad object types, -or heterogeneous values. Dynamic typing is more flexible but requires every possible concrete type -to be registered and supported by the selected mode. +or heterogeneous values. Every concrete type must be supported by the selected mode and explicitly +registered unless that language's native-mode guide documents lazy discovery. Xlang mode and +registration-required configurations require explicit registration. In xlang mode, peers must coordinate the same portable type identity and mapping. Native mode may use implementation-specific identities and language-specific types. See From 4203ec7bd0454f9498d6275c805281a1ade6ce01 Mon Sep 17 00:00:00 2001 From: chaokunyang Date: Sun, 30 Aug 2026 11:06:29 +0800 Subject: [PATCH 141/168] fix(java): clean failed write roots --- .../src/main/java/org/apache/fory/Fory.java | 50 +++++++++++-------- .../apache/fory/io/BlockedStreamUtils.java | 6 ++- .../test/java/org/apache/fory/ForyTest.java | 40 +++++++++++++++ 3 files changed, 74 insertions(+), 22 deletions(-) diff --git a/java/fory-core/src/main/java/org/apache/fory/Fory.java b/java/fory-core/src/main/java/org/apache/fory/Fory.java index f0e595e77f..75bbdf6b35 100644 --- a/java/fory-core/src/main/java/org/apache/fory/Fory.java +++ b/java/fory-core/src/main/java/org/apache/fory/Fory.java @@ -293,22 +293,28 @@ public Serializer getSerializer(Class cls) { @Override public byte[] serialize(Object obj) { - MemoryBuffer buf = getBuffer(); - buf.writerIndex(0); - serialize(buf, obj, null); - byte[] bytes = buf.getBytes(0, buf.writerIndex()); - resetBuffer(); - return bytes; + typeResolver.freezeRegistration(); + try { + MemoryBuffer buf = getBuffer(); + buf.writerIndex(0); + serializeRoot(buf, obj, null); + return buf.getBytes(0, buf.writerIndex()); + } finally { + resetBuffer(); + } } @Override public byte[] serialize(Object obj, BufferCallback callback) { - MemoryBuffer buf = getBuffer(); - buf.writerIndex(0); - serialize(buf, obj, callback); - byte[] bytes = buf.getBytes(0, buf.writerIndex()); - resetBuffer(); - return bytes; + typeResolver.freezeRegistration(); + try { + MemoryBuffer buf = getBuffer(); + buf.writerIndex(0); + serializeRoot(buf, obj, callback); + return buf.getBytes(0, buf.writerIndex()); + } finally { + resetBuffer(); + } } @Override @@ -318,9 +324,11 @@ public MemoryBuffer serialize(MemoryBuffer buffer, Object obj) { @Override public MemoryBuffer serialize(MemoryBuffer buffer, Object obj, BufferCallback callback) { - if (!typeResolver.isRegistrationFrozen()) { - typeResolver.freezeRegistration(); - } + typeResolver.freezeRegistration(); + return serializeRoot(buffer, obj, callback); + } + + private MemoryBuffer serializeRoot(MemoryBuffer buffer, Object obj, BufferCallback callback) { writeContext.prepare(buffer, callback); try { byte bitmap = headerBitmap; @@ -347,12 +355,14 @@ public MemoryBuffer serialize(MemoryBuffer buffer, Object obj, BufferCallback ca @Override public void serialize(OutputStream outputStream, Object obj) { - serializeToStream(outputStream, buf -> serialize(buf, obj, null)); + typeResolver.freezeRegistration(); + serializeToStream(outputStream, buf -> serializeRoot(buf, obj, null)); } @Override public void serialize(OutputStream outputStream, Object obj, BufferCallback callback) { - serializeToStream(outputStream, buf -> serialize(buf, obj, callback)); + typeResolver.freezeRegistration(); + serializeToStream(outputStream, buf -> serializeRoot(buf, obj, callback)); } private ForyException processSerializationError(Throwable e) { @@ -635,10 +645,10 @@ public T copy(T obj) { } private void serializeToStream(OutputStream outputStream, Consumer function) { - MemoryBuffer buf = getBuffer(); - buf.writerIndex(0); - function.accept(buf); try { + MemoryBuffer buf = getBuffer(); + buf.writerIndex(0); + function.accept(buf); byte[] bytes = buf.getHeapMemory(); if (bytes != null) { outputStream.write(bytes, 0, buf.writerIndex()); diff --git a/java/fory-core/src/main/java/org/apache/fory/io/BlockedStreamUtils.java b/java/fory-core/src/main/java/org/apache/fory/io/BlockedStreamUtils.java index e4157e596e..e6d7db15b6 100644 --- a/java/fory-core/src/main/java/org/apache/fory/io/BlockedStreamUtils.java +++ b/java/fory-core/src/main/java/org/apache/fory/io/BlockedStreamUtils.java @@ -47,11 +47,13 @@ public class BlockedStreamUtils { private static final int MAX_CONSECUTIVE_ZERO_READS = 100; public static void serialize(Fory fory, OutputStream outputStream, Object obj) { + fory.getTypeResolver().freezeRegistration(); serializeToStream(fory, outputStream, buf -> fory.serialize(buf, obj, null)); } public static void serialize( Fory fory, OutputStream outputStream, Object obj, BufferCallback callback) { + fory.getTypeResolver().freezeRegistration(); serializeToStream(fory, outputStream, buf -> fory.serialize(buf, obj, callback)); } @@ -132,9 +134,9 @@ private static void readByteBuffer(ReadableByteChannel channel, ByteBuffer buffe private static void serializeToStream( Fory fory, OutputStream outputStream, Consumer function) { - MemoryBuffer buf = fory.getBuffer(); - buf.writerIndex(0); try { + MemoryBuffer buf = fory.getBuffer(); + buf.writerIndex(0); buf.writeInt32(-1); function.accept(buf); buf.putInt32(0, buf.writerIndex() - 4); diff --git a/java/fory-core/src/test/java/org/apache/fory/ForyTest.java b/java/fory-core/src/test/java/org/apache/fory/ForyTest.java index 4b4db16174..53dc6bcada 100644 --- a/java/fory-core/src/test/java/org/apache/fory/ForyTest.java +++ b/java/fory-core/src/test/java/org/apache/fory/ForyTest.java @@ -27,6 +27,7 @@ import com.google.common.collect.HashBasedTable; import com.google.common.collect.ImmutableList; import com.google.common.collect.ImmutableMap; +import java.io.ByteArrayOutputStream; import java.io.Serializable; import java.lang.invoke.MethodHandles; import java.math.BigDecimal; @@ -811,6 +812,45 @@ public void testResetBufferToSizeLimit() { assertEquals(getDefaultWriteBuffer(fory).size(), limitInBytes); } + @Test + public void testFailedWriteReleasesBuffer() { + int limitInBytes = 128; + Fory fory = + Fory.builder() + .withXlang(false) + .requireClassRegistration(false) + .withBufferSizeLimitBytes(limitInBytes) + .build(); + fory.registerSerializer(FailingWrite.class, new FailingWriteSerializer(fory.getTypeResolver())); + + assertThrows(SerializationException.class, () -> fory.serialize(new FailingWrite())); + assertEquals(getDefaultWriteBuffer(fory).size(), limitInBytes); + + assertThrows( + SerializationException.class, + () -> fory.serialize(new ByteArrayOutputStream(), new FailingWrite())); + assertEquals(getDefaultWriteBuffer(fory).size(), limitInBytes); + } + + private static final class FailingWrite {} + + private static final class FailingWriteSerializer extends Serializer { + private FailingWriteSerializer(TypeResolver typeResolver) { + super(typeResolver.getConfig(), FailingWrite.class); + } + + @Override + public void write(WriteContext writeContext, FailingWrite value) { + writeContext.getBuffer().ensure(1024); + throw new SerializationException("expected failure"); + } + + @Override + public FailingWrite read(ReadContext readContext) { + throw new UnsupportedOperationException("unused"); + } + } + private static MemoryBuffer getDefaultWriteBuffer(Fory fory) { return (MemoryBuffer) ReflectionUtils.getObjectFieldValue(fory, "buffer"); } From c3254bb7d25cbc1d4bd5b9cc6520c3113267c782 Mon Sep 17 00:00:00 2001 From: chaokunyang Date: Sun, 30 Aug 2026 11:39:40 +0800 Subject: [PATCH 142/168] style(java): group serialize overloads --- .../src/main/java/org/apache/fory/Fory.java | 24 +++++++++---------- 1 file changed, 12 insertions(+), 12 deletions(-) diff --git a/java/fory-core/src/main/java/org/apache/fory/Fory.java b/java/fory-core/src/main/java/org/apache/fory/Fory.java index 75bbdf6b35..909ce61fa0 100644 --- a/java/fory-core/src/main/java/org/apache/fory/Fory.java +++ b/java/fory-core/src/main/java/org/apache/fory/Fory.java @@ -328,6 +328,18 @@ public MemoryBuffer serialize(MemoryBuffer buffer, Object obj, BufferCallback ca return serializeRoot(buffer, obj, callback); } + @Override + public void serialize(OutputStream outputStream, Object obj) { + typeResolver.freezeRegistration(); + serializeToStream(outputStream, buf -> serializeRoot(buf, obj, null)); + } + + @Override + public void serialize(OutputStream outputStream, Object obj, BufferCallback callback) { + typeResolver.freezeRegistration(); + serializeToStream(outputStream, buf -> serializeRoot(buf, obj, callback)); + } + private MemoryBuffer serializeRoot(MemoryBuffer buffer, Object obj, BufferCallback callback) { writeContext.prepare(buffer, callback); try { @@ -353,18 +365,6 @@ private MemoryBuffer serializeRoot(MemoryBuffer buffer, Object obj, BufferCallba } } - @Override - public void serialize(OutputStream outputStream, Object obj) { - typeResolver.freezeRegistration(); - serializeToStream(outputStream, buf -> serializeRoot(buf, obj, null)); - } - - @Override - public void serialize(OutputStream outputStream, Object obj, BufferCallback callback) { - typeResolver.freezeRegistration(); - serializeToStream(outputStream, buf -> serializeRoot(buf, obj, callback)); - } - private ForyException processSerializationError(Throwable e) { if (!config.trackingRef()) { String msg = From 56dba800e683590fda8ad3fbce40443c19f84011 Mon Sep 17 00:00:00 2001 From: chaokunyang Date: Sun, 30 Aug 2026 11:54:11 +0800 Subject: [PATCH 143/168] test(java): keep registry tests on JDK 8 --- .../src/test/java/org/apache/fory/ThreadSafeForyTest.java | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/java/fory-core/src/test/java/org/apache/fory/ThreadSafeForyTest.java b/java/fory-core/src/test/java/org/apache/fory/ThreadSafeForyTest.java index 2d43f68af7..0553bad14b 100644 --- a/java/fory-core/src/test/java/org/apache/fory/ThreadSafeForyTest.java +++ b/java/fory-core/src/test/java/org/apache/fory/ThreadSafeForyTest.java @@ -27,6 +27,7 @@ import static org.testng.Assert.assertTrue; import java.nio.ByteBuffer; +import java.util.Arrays; import java.util.List; import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.CountDownLatch; @@ -647,7 +648,7 @@ public void testSerializerInstanceOwnership() throws InterruptedException { .build(); FooSerializer local = new FooSerializer(direct.getTypeResolver(), Foo.class); List> registrations = - List.of( + Arrays.asList( fory -> fory.registerSerializer(Foo.class, local), fory -> fory.registerSerializerAndType(Foo.class, local), fory -> fory.registerUnion(Foo.class, 101, local), From f61f31ebc3b063b45d41a9cd2178f9fd5f22d2e6 Mon Sep 17 00:00:00 2001 From: chaokunyang Date: Sun, 30 Aug 2026 12:46:47 +0800 Subject: [PATCH 144/168] fix(java): simplify thread-safe registry freeze --- .agents/languages/java.md | 26 ++- .agents/languages/kotlin.md | 3 +- .agents/languages/scala.md | 3 +- .../java/type-registration.md | 6 +- .../kotlin/configuration.md | 2 +- .../scala/configuration.md | 3 +- docs/security/deserialization.md | 4 - .../src/main/java/org/apache/fory/Fory.java | 175 +++++++--------- .../java/org/apache/fory/ThreadLocalFory.java | 57 +---- .../java/org/apache/fory/ThreadSafeFory.java | 5 +- .../org/apache/fory/pool/ThreadPoolFory.java | 61 +----- .../apache/fory/resolver/SharedRegistry.java | 25 +++ .../apache/fory/resolver/TypeResolver.java | 12 +- .../apache/fory/resolver/XtypeResolver.java | 3 +- .../org/apache/fory/ThreadSafeForyTest.java | 196 +++++++++--------- .../fory/resolver/ClassResolverTest.java | 51 +++-- 16 files changed, 272 insertions(+), 360 deletions(-) diff --git a/.agents/languages/java.md b/.agents/languages/java.md index 555f43e30c..83617a5fa0 100644 --- a/.agents/languages/java.md +++ b/.agents/languages/java.md @@ -84,19 +84,20 @@ Load this file when changing anything under `java/` or when Java drives a cross- work, dynamic stream bytes-read accounting, or stale narrower-scope formulas. - Generated serializers must not retain runtime context fields. `Fory` should stay a root-operation facade rather than accumulating serializer or convenience state. - When the serializer class and constructor shape are known at the call site, prefer direct constructor lambdas or direct instantiation over reflective `Serializers.newSerializer(...)`. -- Each natural Java registry or public facade boundary owns one authoritative frozen flag. The - facade flag is not a mirror of a child resolver flag. The first root serialization or - deserialization sets the owning flag before codec work and never clears it, including after - failure. Every explicit type, serializer, module, name, ID, or type-checker binding checks that - flag before mutation. Disallow-list changes use the bound checker's resolver listeners. Do not - add another lifecycle state or a parallel registration-commit path. +- Each natural Java registry or public facade boundary owns one authoritative lifecycle fact. A + concrete resolver uses its shared registration snapshot; a thread-safe facade uses the shared + registry's one frozen flag. The first root serialization or deserialization establishes the + owning fact before codec work and never clears it, including after failure. Every explicit type, + serializer, module, name, ID, or type-checker binding checks that fact before mutation. + Disallow-list changes use the bound checker's resolver listeners. Do not add another lifecycle + state or a parallel registration-commit path. - Direct and thread-safe facades expose module registration before their first root. Kotlin and Scala registration extensions target `BaseFory`; do not narrow them to concrete `Fory` or make - builder installation the only thread-safe path. Before the first root, a thread-safe facade - serializes registration, `execute`, and copy through its existing callback monitor and rechecks - the frozen flag after entering it. After freeze, `execute` and copy use the monitor-free path. - Calling `ThreadSafeFory.execute` or copying a value does not freeze registration unless the - callback starts a root serialization or deserialization. + builder installation the only thread-safe path. A thread-safe facade's shared registry owns its + registration boundary so a root started by any child closes the facade before registration can + mutate another child. Non-root `execute` and copy operations remain concurrent and do not freeze + registration. Complete facade registration before concurrent serialization, deserialization, + copy, or `execute` calls begin; do not serialize those operations behind a registration lock. - `ThreadSafeFory.execute` exposes one borrowed child only for the callback. Do not retain that child or register through it; use the facade registration methods so every current and future child receives the same setup. @@ -160,9 +161,6 @@ Load this file when changing anything under `java/` or when Java drives a cross- unmatched named sender layers as data-only metadata, and does not route layer names through `ClassResolver.readClassInternal`. Inverse registration must not turn a missed input name into an accepted class. -- `warnOnce` keys live for the logger lifetime. Read-side resolver warnings selected by remote - names must use a fixed message and must not include a remote name or another - untrusted-cardinality value in the message arguments. - Keep JDK interface names that do not require explicit registration in `DefaultJdkClassAllowList`. `TypeResolver.loadClass` and `ClassResolver.isSecure` must both use this single owner. Keep custom `TypeChecker` and fixed disallowed-list checks on their existing diff --git a/.agents/languages/kotlin.md b/.agents/languages/kotlin.md index 36cb66ff2f..82dbcc7485 100644 --- a/.agents/languages/kotlin.md +++ b/.agents/languages/kotlin.md @@ -15,7 +15,8 @@ Load this file when changing `kotlin/` or compiler code that generates Kotlin so wire format matches the previous serializer family and old-payload/new-runtime compatibility is tested. - Kotlin registration extensions target `BaseFory` so direct and thread-safe facades share the same - pre-root registration API, including module installation. + pre-root registration API, including module installation. Complete thread-safe facade + registration before concurrent serialization, deserialization, copy, or execution begins. - Explicit type, serializer, enum, and union registration checks the receiving `BaseFory` facade or natural registry owner's one frozen flag before mutation. Keep generated serializer construction on the existing direct resolver path; do not add a parallel registration path or lifecycle state. diff --git a/.agents/languages/scala.md b/.agents/languages/scala.md index afeda059b1..aaa38d0c4c 100644 --- a/.agents/languages/scala.md +++ b/.agents/languages/scala.md @@ -10,7 +10,8 @@ Load this file when changing `scala/`. sources, tests, resources, R8 metadata, compiler plugins, macros, dependencies, or compatibility design. - Scala registration extensions target `BaseFory` so direct and thread-safe facades share the same - pre-root registration API, including module installation. + pre-root registration API, including module installation. Complete thread-safe facade + registration before concurrent serialization, deserialization, copy, or execution begins. - Explicit type, serializer, enum, and union registration checks the receiving `BaseFory` facade or natural registry owner's one frozen flag before mutation. Keep generated serializer construction on the existing direct resolver path; do not add a parallel registration path or lifecycle state. diff --git a/docs/object-serialization/java/type-registration.md b/docs/object-serialization/java/type-registration.md index e4f0995ea5..c17d5aa676 100644 --- a/docs/object-serialization/java/type-registration.md +++ b/docs/object-serialization/java/type-registration.md @@ -49,6 +49,9 @@ call. Starting either operation permanently freezes registration even if the ope operations and `ThreadSafeFory#execute` do not freeze registration unless the supplied callback starts serialization or deserialization. Later explicit registration attempts are rejected. +For a thread-safe facade, complete registration before concurrent serialization, deserialization, +copy, or `execute` calls begin. + In native mode with registration disabled, allowed unregistered runtime classes may still be used after this boundary. That behavior does not reopen or change explicit registration. @@ -122,7 +125,8 @@ Fory fory = Fory.builder() 1. Keep class registration enabled for untrusted input. 2. Prefer explicit numeric IDs when readers and writers can share a stable ID mapping. 3. Use the same registration order on both sides when IDs are assigned automatically. -4. Configure all classes, serializers, and disallow rules before the first operation. +4. Configure all classes, serializers, and disallow rules before the first root serialization or + deserialization operation. 5. Configure `AllowListChecker` when class registration is disabled. ## Related Topics diff --git a/docs/object-serialization/kotlin/configuration.md b/docs/object-serialization/kotlin/configuration.md index 6a01f5b45e..384fcb95ca 100644 --- a/docs/object-serialization/kotlin/configuration.md +++ b/docs/object-serialization/kotlin/configuration.md @@ -85,7 +85,7 @@ object ForyHolder { `ForyModule` registration and Kotlin reified registration extensions target `BaseFory`, so they are available on both direct and thread-safe facades. Complete registration before the facade's first -root serialization or deserialization. +root serialization or deserialization, and before concurrent use of a thread-safe facade begins. ### Using Builder Methods diff --git a/docs/object-serialization/scala/configuration.md b/docs/object-serialization/scala/configuration.md index 4d6ab03549..90f284eb4b 100644 --- a/docs/object-serialization/scala/configuration.md +++ b/docs/object-serialization/scala/configuration.md @@ -122,7 +122,8 @@ object ForyHolder { `ForyModule` registration and Scala generated-serializer registration extensions target `BaseFory`, so they are available on both direct and thread-safe facades. Complete registration -before the facade's first root serialization or deserialization. +before the facade's first root serialization or deserialization, and before concurrent use of a +thread-safe facade begins. ## Configuration diff --git a/docs/security/deserialization.md b/docs/security/deserialization.md index ba4a9ff077..56b3fe3215 100644 --- a/docs/security/deserialization.md +++ b/docs/security/deserialization.md @@ -665,10 +665,6 @@ adding allocation or slot-clearing work to normal roots. Bounded backing may ret references when the runtime-specific retention rule permits it. Runtime-specific thresholds and reset ownership belong in the implementation guide and language guidance. -One-time warning registries retain their keys beyond the current root. A warning selected by a -remote class or type name must therefore use a fixed key instead of including that untrusted name or -another unbounded remote value in the message arguments. - A class-resolution cache reachable from untrusted deserialization may publish an entry only from explicit trusted configuration or after the active class policy has accepted the resolved class. A cache hit therefore represents an diff --git a/java/fory-core/src/main/java/org/apache/fory/Fory.java b/java/fory-core/src/main/java/org/apache/fory/Fory.java index 909ce61fa0..601f39fe28 100644 --- a/java/fory-core/src/main/java/org/apache/fory/Fory.java +++ b/java/fory-core/src/main/java/org/apache/fory/Fory.java @@ -401,74 +401,33 @@ private ForyException processCopyError(Throwable e) { @Override public Object deserialize(byte[] bytes) { - MemoryBuffer buffer; - try { - buffer = MemoryUtils.wrap(bytes); - } catch (RuntimeException | Error e) { - typeResolver.freezeRegistration(); - throw e; - } - return deserialize(buffer, (Iterable) null); + typeResolver.freezeRegistration(); + return deserializeRoot(MemoryUtils.wrap(bytes), (Iterable) null); } @Override public Object deserialize(ByteBuffer byteBuffer) { - MemoryBuffer buffer; - try { - buffer = MemoryUtils.wrap(byteBuffer); - } catch (RuntimeException | Error e) { - typeResolver.freezeRegistration(); - throw e; - } - return deserialize(buffer); + typeResolver.freezeRegistration(); + return deserializeRoot(MemoryUtils.wrap(byteBuffer), (Iterable) null); } @Override public T deserialize(byte[] bytes, Class type) { - MemoryBuffer buffer; - try { - buffer = MemoryUtils.wrap(bytes); - } catch (RuntimeException | Error e) { - typeResolver.freezeRegistration(); - throw e; - } - return deserialize(buffer, type); + typeResolver.freezeRegistration(); + return deserializeRoot(MemoryUtils.wrap(bytes), type); } @Override public T deserialize(MemoryBuffer buffer, Class type) { - if (!typeResolver.isRegistrationFrozen()) { - typeResolver.freezeRegistration(); - } - byte bitmap = buffer.readByte(); - if (bitmap != headerBitmap) { - checkHeaderBitmapWithoutOutOfBand(bitmap); - } - readContext.prepare(buffer, null, false); - try { - try { - jitContext.lock(); - if (readContext.getDepth() > 0) { - throwDepthDeserializationException(); - } - return deserializeByType(buffer, type); - } finally { - jitContext.unlock(); - } - } catch (Throwable t) { - throw ExceptionUtils.handleReadFailed(t); - } finally { - readContext.reset(); - } + typeResolver.freezeRegistration(); + return deserializeRoot(buffer, type); } @Override public T deserialize(ForyInputStream inputStream, Class type) { - if (!typeResolver.isRegistrationFrozen()) { - typeResolver.freezeRegistration(); - } + typeResolver.freezeRegistration(); try { - return deserialize(inputStream.getBuffer(), type); + return deserializeRoot(inputStream.getBuffer(), type); } finally { inputStream.shrinkBuffer(); } @@ -476,11 +435,9 @@ public T deserialize(ForyInputStream inputStream, Class type) { @Override public T deserialize(ForyReadableChannel channel, Class type) { - if (!typeResolver.isRegistrationFrozen()) { - typeResolver.freezeRegistration(); - } + typeResolver.freezeRegistration(); try { - return deserialize(channel.getBuffer(), type); + return deserializeRoot(channel.getBuffer(), type); } finally { channel.compactBuffer(); } @@ -488,19 +445,14 @@ public T deserialize(ForyReadableChannel channel, Class type) { @Override public Object deserialize(byte[] bytes, Iterable outOfBandBuffers) { - MemoryBuffer buffer; - try { - buffer = MemoryUtils.wrap(bytes); - } catch (RuntimeException | Error e) { - typeResolver.freezeRegistration(); - throw e; - } - return deserialize(buffer, outOfBandBuffers); + typeResolver.freezeRegistration(); + return deserializeRoot(MemoryUtils.wrap(bytes), outOfBandBuffers); } @Override public Object deserialize(MemoryBuffer buffer) { - return deserialize(buffer, (Iterable) null); + typeResolver.freezeRegistration(); + return deserializeRoot(buffer, (Iterable) null); } /** @@ -518,9 +470,64 @@ public Object deserialize(MemoryBuffer buffer) { */ @Override public Object deserialize(MemoryBuffer buffer, Iterable outOfBandBuffers) { - if (!typeResolver.isRegistrationFrozen()) { - typeResolver.freezeRegistration(); + typeResolver.freezeRegistration(); + return deserializeRoot(buffer, outOfBandBuffers); + } + + @Override + public Object deserialize(ForyInputStream inputStream) { + return deserialize(inputStream, (Iterable) null); + } + + @Override + public Object deserialize(ForyInputStream inputStream, Iterable outOfBandBuffers) { + typeResolver.freezeRegistration(); + try { + return deserializeRoot(inputStream.getBuffer(), outOfBandBuffers); + } finally { + inputStream.shrinkBuffer(); } + } + + @Override + public Object deserialize(ForyReadableChannel channel) { + return deserialize(channel, (Iterable) null); + } + + @Override + public Object deserialize(ForyReadableChannel channel, Iterable outOfBandBuffers) { + typeResolver.freezeRegistration(); + try { + return deserializeRoot(channel.getBuffer(), outOfBandBuffers); + } finally { + channel.compactBuffer(); + } + } + + private T deserializeRoot(MemoryBuffer buffer, Class type) { + byte bitmap = buffer.readByte(); + if (bitmap != headerBitmap) { + checkHeaderBitmapWithoutOutOfBand(bitmap); + } + readContext.prepare(buffer, null, false); + try { + try { + jitContext.lock(); + if (readContext.getDepth() > 0) { + throwDepthDeserializationException(); + } + return deserializeByType(buffer, type); + } finally { + jitContext.unlock(); + } + } catch (Throwable t) { + throw ExceptionUtils.handleReadFailed(t); + } finally { + readContext.reset(); + } + } + + private Object deserializeRoot(MemoryBuffer buffer, Iterable outOfBandBuffers) { byte bitmap = buffer.readByte(); boolean peerOutOfBandEnabled = false; if (bitmap != headerBitmap) { @@ -556,42 +563,6 @@ public Object deserialize(MemoryBuffer buffer, Iterable outOfBandB } } - @Override - public Object deserialize(ForyInputStream inputStream) { - return deserialize(inputStream, (Iterable) null); - } - - @Override - public Object deserialize(ForyInputStream inputStream, Iterable outOfBandBuffers) { - if (!typeResolver.isRegistrationFrozen()) { - typeResolver.freezeRegistration(); - } - try { - MemoryBuffer buf = inputStream.getBuffer(); - return deserialize(buf, outOfBandBuffers); - } finally { - inputStream.shrinkBuffer(); - } - } - - @Override - public Object deserialize(ForyReadableChannel channel) { - return deserialize(channel, (Iterable) null); - } - - @Override - public Object deserialize(ForyReadableChannel channel, Iterable outOfBandBuffers) { - if (!typeResolver.isRegistrationFrozen()) { - typeResolver.freezeRegistration(); - } - try { - MemoryBuffer buf = channel.getBuffer(); - return deserialize(buf, outOfBandBuffers); - } finally { - channel.compactBuffer(); - } - } - @SuppressWarnings("unchecked") private T deserializeByType(MemoryBuffer buffer, Class type) { // The outer root operation resets generic state after failure; balance this push here only diff --git a/java/fory-core/src/main/java/org/apache/fory/ThreadLocalFory.java b/java/fory-core/src/main/java/org/apache/fory/ThreadLocalFory.java index 15caad0254..60f99131fe 100644 --- a/java/fory-core/src/main/java/org/apache/fory/ThreadLocalFory.java +++ b/java/fory-core/src/main/java/org/apache/fory/ThreadLocalFory.java @@ -30,7 +30,6 @@ import javax.annotation.concurrent.ThreadSafe; import org.apache.fory.annotation.Internal; import org.apache.fory.config.ForyBuilder; -import org.apache.fory.exception.ForyException; import org.apache.fory.io.ForyInputStream; import org.apache.fory.io.ForyReadableChannel; import org.apache.fory.memory.MemoryBuffer; @@ -48,11 +47,10 @@ public class ThreadLocalFory extends AbstractThreadSafeFory { private final ThreadLocal foryThreadLocal; private Consumer factoryCallback; private final Map allFory; - private final Object callbackLock = new Object(); - private volatile boolean registrationFrozen; + private final SharedRegistry sharedRegistry; public ThreadLocalFory(Function factory) { - SharedRegistry sharedRegistry = new SharedRegistry(); + sharedRegistry = new SharedRegistry(); foryFactory = () -> factory.apply(Fory.builder().withSharedRegistry(sharedRegistry)); factoryCallback = f -> {}; allFory = Collections.synchronizedMap(new WeakHashMap<>()); @@ -65,10 +63,10 @@ public ThreadLocalFory(Function factory) { } private Fory newFory() { - synchronized (callbackLock) { + synchronized (sharedRegistry) { Fory fory = foryFactory.get(); factoryCallback.accept(fory); - if (registrationFrozen) { + if (sharedRegistry.isRegistrationFrozen()) { fory.getTypeResolver().freezeRegistration(); } allFory.put(fory, null); @@ -77,25 +75,15 @@ private Fory newFory() { } private Fory currentFory() { - freezeRegistration(); + sharedRegistry.freezeRegistration(); return foryThreadLocal.get(); } - private void freezeRegistration() { - if (!registrationFrozen) { - synchronized (callbackLock) { - if (!registrationFrozen) { - registrationFrozen = true; - } - } - } - } - @Internal @Override public void registerCallback(Consumer callback) { - synchronized (callbackLock) { - checkRegistrationOpen(); + synchronized (sharedRegistry) { + sharedRegistry.checkRegistrationOpen(); synchronized (allFory) { allFory.keySet().forEach(callback); } @@ -106,34 +94,12 @@ public void registerCallback(Consumer callback) { @Override public R execute(Function action) { Fory fory = foryThreadLocal.get(); - if (!registrationFrozen) { - synchronized (callbackLock) { - if (!registrationFrozen) { - try { - return action.apply(fory); - } finally { - if (fory.getTypeResolver().isRegistrationFrozen()) { - registrationFrozen = true; - } - } - } - } - } - if (!fory.getTypeResolver().isRegistrationFrozen()) { + if (sharedRegistry.isRegistrationFrozen() && !fory.getTypeResolver().isRegistrationFrozen()) { fory.getTypeResolver().freezeRegistration(); } return action.apply(fory); } - private void checkRegistrationOpen() { - if (registrationFrozen) { - throw new ForyException( - "Cannot register class/serializer after registration has been frozen. Please register " - + "all classes before invoking top-level `serialize/deserialize` methods of " - + "ThreadSafeFory."); - } - } - @Override public byte[] serialize(Object obj) { return currentFory().serialize(obj); @@ -231,13 +197,6 @@ public Object deserialize(ForyReadableChannel channel, Iterable ou @Override public T copy(T obj) { - if (!registrationFrozen) { - synchronized (callbackLock) { - if (!registrationFrozen) { - return foryThreadLocal.get().copy(obj); - } - } - } return foryThreadLocal.get().copy(obj); } } diff --git a/java/fory-core/src/main/java/org/apache/fory/ThreadSafeFory.java b/java/fory-core/src/main/java/org/apache/fory/ThreadSafeFory.java index ae6c76e90c..cb6605c228 100644 --- a/java/fory-core/src/main/java/org/apache/fory/ThreadSafeFory.java +++ b/java/fory-core/src/main/java/org/apache/fory/ThreadSafeFory.java @@ -34,13 +34,16 @@ *

Serializer instances registered on this facade must implement {@link * org.apache.fory.serializer.Shareable}. Use a serializer class or resolver factory for * runtime-local serializers so every underlying {@link Fory} owns its instance. + * + *

Complete facade registration before concurrent serialization, deserialization, copy, or {@link + * #execute} calls begin. */ public interface ThreadSafeFory extends BaseFory { /** * Provide a context to execution operations on {@link Fory} directly and return the executed * result. The action must not retain the runtime or register through it; use this facade's - * registration methods so every underlying runtime receives the same registration. + * registration methods during setup so every underlying runtime receives the same registration. */ R execute(Function action); diff --git a/java/fory-core/src/main/java/org/apache/fory/pool/ThreadPoolFory.java b/java/fory-core/src/main/java/org/apache/fory/pool/ThreadPoolFory.java index 2affea5438..198c299117 100644 --- a/java/fory-core/src/main/java/org/apache/fory/pool/ThreadPoolFory.java +++ b/java/fory-core/src/main/java/org/apache/fory/pool/ThreadPoolFory.java @@ -32,7 +32,6 @@ import org.apache.fory.Fory; import org.apache.fory.annotation.Internal; import org.apache.fory.config.ForyBuilder; -import org.apache.fory.exception.ForyException; import org.apache.fory.io.ForyInputStream; import org.apache.fory.io.ForyReadableChannel; import org.apache.fory.memory.MemoryBuffer; @@ -53,15 +52,14 @@ public class ThreadPoolFory extends AbstractThreadSafeFory { private final Fory[] pooledFory; private final Semaphore waiterSignal = new Semaphore(0); private final AtomicInteger waitingBorrowers = new AtomicInteger(); - private final Object callbackLock = new Object(); - private volatile boolean registrationFrozen; + private final SharedRegistry sharedRegistry; public ThreadPoolFory(Function foryFactory, int poolSize) { if (poolSize <= 0) { throw new IllegalArgumentException( String.format("thread safe fory pool size error, please check it, size:[%s]", poolSize)); } - SharedRegistry sharedRegistry = new SharedRegistry(); + sharedRegistry = new SharedRegistry(); Supplier factory = () -> foryFactory.apply(Fory.builder().withSharedRegistry(sharedRegistry)); this.poolSize = poolSize; @@ -75,7 +73,7 @@ public ThreadPoolFory(Function foryFactory, int poolSize) { } private PooledEntry acquire() { - freezeRegistration(); + sharedRegistry.freezeRegistration(); return acquireEntry(); } @@ -154,8 +152,8 @@ private static int spread(int hash) { @Internal @Override public void registerCallback(Consumer callback) { - synchronized (callbackLock) { - checkRegistrationOpen(); + synchronized (sharedRegistry) { + sharedRegistry.checkRegistrationOpen(); for (Fory fory : pooledFory) { callback.accept(fory); } @@ -164,24 +162,10 @@ public void registerCallback(Consumer callback) { @Override public R execute(Function action) { - if (!registrationFrozen) { - synchronized (callbackLock) { - if (!registrationFrozen) { - PooledEntry entry = acquireEntry(); - try { - return action.apply(entry.fory); - } finally { - if (entry.fory.getTypeResolver().isRegistrationFrozen()) { - registrationFrozen = true; - } - release(entry); - } - } - } - } PooledEntry entry = acquireEntry(); try { - if (!entry.fory.getTypeResolver().isRegistrationFrozen()) { + if (sharedRegistry.isRegistrationFrozen() + && !entry.fory.getTypeResolver().isRegistrationFrozen()) { entry.fory.getTypeResolver().freezeRegistration(); } return action.apply(entry.fory); @@ -190,25 +174,6 @@ public R execute(Function action) { } } - private void freezeRegistration() { - if (!registrationFrozen) { - synchronized (callbackLock) { - if (!registrationFrozen) { - registrationFrozen = true; - } - } - } - } - - private void checkRegistrationOpen() { - if (registrationFrozen) { - throw new ForyException( - "Cannot register class/serializer after registration has been frozen. Please register " - + "all classes before invoking top-level `serialize/deserialize` methods of " - + "ThreadSafeFory."); - } - } - @Override public byte[] serialize(Object obj) { PooledEntry entry = acquire(); @@ -401,18 +366,6 @@ public Object deserialize(ForyReadableChannel channel, Iterable ou @Override public T copy(T obj) { - if (!registrationFrozen) { - synchronized (callbackLock) { - if (!registrationFrozen) { - PooledEntry entry = acquireEntry(); - try { - return entry.fory.copy(obj); - } finally { - release(entry); - } - } - } - } PooledEntry entry = acquireEntry(); try { return entry.fory.copy(obj); diff --git a/java/fory-core/src/main/java/org/apache/fory/resolver/SharedRegistry.java b/java/fory-core/src/main/java/org/apache/fory/resolver/SharedRegistry.java index 1a95d73529..e4def43948 100644 --- a/java/fory-core/src/main/java/org/apache/fory/resolver/SharedRegistry.java +++ b/java/fory-core/src/main/java/org/apache/fory/resolver/SharedRegistry.java @@ -102,6 +102,10 @@ public final class SharedRegistry { final StaticGeneratedSerializerRegistry staticGeneratedSerializerRegistry = new StaticGeneratedSerializerRegistry(); private final Object metaStringCacheLock = new Object(); + // Thread-safe facades share this boundary across their children. A new thread-local child first + // receives the facade's fixed setup, then switches to the shared registration snapshot before it + // is exposed. + private volatile boolean registrationFrozen; private volatile int maxSchemaVersionsPerType = -1; private volatile int maxAverageSchemaVersionsPerType = -1; private final HashMap remoteTypeDefVersionsByType = new HashMap<>(); @@ -111,6 +115,27 @@ public final class SharedRegistry { public SharedRegistry() {} + public boolean isRegistrationFrozen() { + return registrationFrozen; + } + + public void freezeRegistration() { + if (!registrationFrozen) { + synchronized (this) { + registrationFrozen = true; + } + } + } + + public void checkRegistrationOpen() { + if (registrationFrozen) { + throw new ForyException( + "Cannot register class/serializer after registration has been frozen. Please register " + + "all classes before invoking top-level `serialize/deserialize` methods of " + + "ThreadSafeFory."); + } + } + public synchronized void setRemoteSchemaLimits( int maxSchemaVersionsPerType, int maxAverageSchemaVersionsPerType) { if (maxSchemaVersionsPerType <= 0) { diff --git a/java/fory-core/src/main/java/org/apache/fory/resolver/TypeResolver.java b/java/fory-core/src/main/java/org/apache/fory/resolver/TypeResolver.java index e42cd0f45c..a6b76cf9f2 100644 --- a/java/fory-core/src/main/java/org/apache/fory/resolver/TypeResolver.java +++ b/java/fory-core/src/main/java/org/apache/fory/resolver/TypeResolver.java @@ -155,7 +155,6 @@ private static final class TransformedTypeInfo { // Caches for readTypeInfo(ReadContext) - persist between calls to avoid reloading // dynamically created classes that can't be found by Class.forName private final TypeInfo[] typeInfoCache; - private boolean registrationFrozen; protected TypeResolver( Config config, @@ -195,7 +194,8 @@ public final JITContext getJITContext() { @Internal public final boolean isRegistrationFrozen() { - return registrationFrozen; + IdentityHashMap, Integer> registeredClassIdMap = sharedRegistry.registeredClassIdMap; + return registeredClassIdMap != null && extRegistry.registeredClassIdMap == registeredClassIdMap; } public final boolean isCrossLanguage() { @@ -228,7 +228,7 @@ public final Class getDefaultJDKStreamSerializerType() { @Internal public final void checkRegistrationOpen() { - if (registrationFrozen) { + if (isRegistrationFrozen()) { throw new ForyException( "Cannot register class/serializer after registration has been frozen. Please register " + "all classes before invoking top-level `serialize/deserialize` methods of Fory."); @@ -398,10 +398,12 @@ public abstract void registerSerializer( * points can call it defensively. */ public final void freezeRegistration() { - if (registrationFrozen) { + if (isRegistrationFrozen()) { return; } - registrationFrozen = true; + // A root may start through a borrowed child, so the child must close the shared facade + // boundary before publishing or adopting the registration snapshot. + sharedRegistry.freezeRegistration(); sharedRegistry.setRegistrationIfAbsent( extRegistry.registeredClassIdMap, extRegistry.registeredClasses); extRegistry.freezeRegistration( diff --git a/java/fory-core/src/main/java/org/apache/fory/resolver/XtypeResolver.java b/java/fory-core/src/main/java/org/apache/fory/resolver/XtypeResolver.java index 66efe50a9f..1546adc968 100644 --- a/java/fory-core/src/main/java/org/apache/fory/resolver/XtypeResolver.java +++ b/java/fory-core/src/main/java/org/apache/fory/resolver/XtypeResolver.java @@ -1345,10 +1345,11 @@ private TypeInfo populateBytesToTypeInfo( compositeClassNameBytes2TypeInfo.put(typeNameBytes, typeInfo); return typeInfo; } + String msg = String.format("Class %s not registered", qualifiedName); Class type = null; if (config.deserializeUnknownClass()) { if (!config.suppressClassRegistrationWarnings()) { - LOG.warnOnce("A named type is not registered and will be read as an unknown class."); + LOG.warnOnce(msg); } switch (typeId) { case Types.NAMED_ENUM: diff --git a/java/fory-core/src/test/java/org/apache/fory/ThreadSafeForyTest.java b/java/fory-core/src/test/java/org/apache/fory/ThreadSafeForyTest.java index 0553bad14b..a66ef2e680 100644 --- a/java/fory-core/src/test/java/org/apache/fory/ThreadSafeForyTest.java +++ b/java/fory-core/src/test/java/org/apache/fory/ThreadSafeForyTest.java @@ -154,7 +154,6 @@ public void testThreadSafeRuntimesShareRegistry() throws Exception { AtomicReference threadPoolRegistry1 = new AtomicReference<>(); AtomicReference threadPoolRegistry2 = new AtomicReference<>(); AtomicReference error = new AtomicReference<>(); - threadPool.serialize("warm"); Thread poolThread1 = new Thread( () -> { @@ -692,6 +691,103 @@ private static void captureSerializer( } } + @Test + public void testExecuteConcurrency() throws InterruptedException { + for (ThreadSafeFory fory : newThreadSafeRuntimes()) { + CountDownLatch entered = new CountDownLatch(2); + CountDownLatch release = new CountDownLatch(1); + AtomicReference error = new AtomicReference<>(); + Thread first = new Thread(() -> runBlockingExecute(fory, entered, release, error)); + Thread second = new Thread(() -> runBlockingExecute(fory, entered, release, error)); + + first.start(); + second.start(); + boolean concurrent = entered.await(10, TimeUnit.SECONDS); + release.countDown(); + first.join(); + second.join(); + + assertTrue(concurrent); + assertNull(error.get()); + fory.register(BeanA.class); + } + } + + private static void runBlockingExecute( + ThreadSafeFory fory, + CountDownLatch entered, + CountDownLatch release, + AtomicReference error) { + try { + fory.execute( + child -> { + entered.countDown(); + awaitUnchecked(release); + return null; + }); + } catch (Throwable t) { + error.compareAndSet(null, t); + } + } + + @Test + public void testCopyConcurrency() throws InterruptedException { + for (ThreadSafeFory fory : newThreadSafeRuntimes()) { + CountDownLatch entered = new CountDownLatch(2); + CountDownLatch release = new CountDownLatch(1); + AtomicReference error = new AtomicReference<>(); + fory.registerSerializer( + BlockingCopyValue.class, + resolver -> new BlockingCopySerializer(resolver, entered, release)); + Thread first = new Thread(() -> runBlockingCopy(fory, error)); + Thread second = new Thread(() -> runBlockingCopy(fory, error)); + + first.start(); + second.start(); + boolean concurrent = entered.await(10, TimeUnit.SECONDS); + release.countDown(); + first.join(); + second.join(); + + assertTrue(concurrent); + assertNull(error.get()); + fory.register(BeanA.class); + } + } + + private static void runBlockingCopy(ThreadSafeFory fory, AtomicReference error) { + try { + fory.copy(new BlockingCopyValue()); + } catch (Throwable t) { + error.compareAndSet(null, t); + } + } + + @Test + public void testFrozenThreadLocalCreatesChild() throws InterruptedException { + ThreadSafeFory fory = + Fory.builder() + .withXlang(false) + .requireClassRegistration(true) + .withCompatible(false) + .buildThreadLocalFory(); + fory.register(Foo.class); + fory.serialize("freeze"); + AtomicReference error = new AtomicReference<>(); + Thread thread = + new Thread( + () -> { + try { + fory.serialize(new Foo()); + } catch (Throwable t) { + error.set(t); + } + }); + thread.start(); + thread.join(); + assertNull(error.get()); + } + @Test public void testRegisterAfterSerializeThrows() { ThreadSafeFory fory = @@ -800,104 +896,6 @@ public void testExecuteRootRegistrationRace() throws InterruptedException { } } - @Test - public void testExecuteChildRegisterRace() throws InterruptedException { - for (ThreadSafeFory fory : newThreadSafeRuntimes()) { - CountDownLatch rootStarted = new CountDownLatch(1); - CountDownLatch finishRoot = new CountDownLatch(1); - AtomicReference rootError = new AtomicReference<>(); - AtomicReference registrationError = new AtomicReference<>(); - Thread rootThread = - new Thread( - () -> { - try { - fory.execute( - child -> { - child.serialize("value"); - rootStarted.countDown(); - awaitUnchecked(finishRoot); - return null; - }); - } catch (Throwable t) { - rootError.set(t); - } - }); - Thread registrationThread = - new Thread( - () -> { - try { - fory.execute( - child -> { - child.register(BeanB.class); - return null; - }); - } catch (Throwable t) { - registrationError.set(t); - } - }); - - rootThread.start(); - assertTrue(rootStarted.await(30, TimeUnit.SECONDS)); - registrationThread.start(); - finishRoot.countDown(); - rootThread.join(); - registrationThread.join(); - - assertNull(rootError.get()); - assertTrue(registrationError.get() instanceof ForyException); - } - } - - @Test - public void testCopyRegistrationRace() throws InterruptedException { - for (ThreadSafeFory fory : newThreadSafeRuntimes()) { - CountDownLatch copyStarted = new CountDownLatch(1); - CountDownLatch finishCopy = new CountDownLatch(1); - CountDownLatch registrationStarted = new CountDownLatch(1); - CountDownLatch registrationDone = new CountDownLatch(1); - AtomicInteger callbacks = new AtomicInteger(); - AtomicReference copyError = new AtomicReference<>(); - AtomicReference registrationError = new AtomicReference<>(); - fory.registerSerializer( - BlockingCopyValue.class, - resolver -> new BlockingCopySerializer(resolver, copyStarted, finishCopy)); - Thread copyThread = - new Thread( - () -> { - try { - fory.copy(new BlockingCopyValue()); - } catch (Throwable t) { - copyError.set(t); - } - }); - Thread registrationThread = - new Thread( - () -> { - registrationStarted.countDown(); - try { - fory.registerCallback(child -> callbacks.incrementAndGet()); - } catch (Throwable t) { - registrationError.set(t); - } finally { - registrationDone.countDown(); - } - }); - - copyThread.start(); - assertTrue(copyStarted.await(30, TimeUnit.SECONDS)); - registrationThread.start(); - assertTrue(registrationStarted.await(30, TimeUnit.SECONDS)); - Assert.assertFalse(registrationDone.await(100, TimeUnit.MILLISECONDS)); - finishCopy.countDown(); - copyThread.join(); - registrationThread.join(); - - assertNull(copyError.get()); - assertNull(registrationError.get()); - assertTrue(callbacks.get() > 0); - } - } - @Test public void testNonRootKeepsRegistrationOpen() { Fory direct = diff --git a/java/fory-core/src/test/java/org/apache/fory/resolver/ClassResolverTest.java b/java/fory-core/src/test/java/org/apache/fory/resolver/ClassResolverTest.java index 748b79d7b3..edbb5f4723 100644 --- a/java/fory-core/src/test/java/org/apache/fory/resolver/ClassResolverTest.java +++ b/java/fory-core/src/test/java/org/apache/fory/resolver/ClassResolverTest.java @@ -306,21 +306,27 @@ public void testGetSerializerClass() throws ClassNotFoundException { public void testSuppressXtypeWarnings() throws Exception { String suppressed = captureOutput( - () -> { - Fory fory = newUnknownClassFory(true); - resolveMissingXtype(fory, "FirstType"); - resolveMissingXtype(fory, "SecondType"); - }); - assertEquals(count(suppressed, " WARN XtypeResolver:"), 0); + () -> + resolveMissingXtype( + Fory.builder() + .withXlang(true) + .withMetaShare(true) + .withDeserializeUnknownClass(true) + .suppressClassRegistrationWarnings(true) + .build())); + assertEquals(count(suppressed, "Class missing.pkg.MissingType not registered"), 0); String unsuppressed = captureOutput( - () -> { - Fory fory = newUnknownClassFory(false); - resolveMissingXtype(fory, "ThirdType"); - resolveMissingXtype(fory, "FourthType"); - }); - assertEquals(count(unsuppressed, " WARN XtypeResolver:"), 1); + () -> + resolveMissingXtype( + Fory.builder() + .withXlang(true) + .withMetaShare(true) + .withDeserializeUnknownClass(true) + .suppressClassRegistrationWarnings(false) + .build())); + assertEquals(count(unsuppressed, "Class missing.pkg.MissingType not registered"), 1); } @Test @@ -962,7 +968,7 @@ public void testIdRegistrationAcceptsJavaName() { } @Test - public void testFinishRegisterPublishesAndAdoptsSharedRegistration() { + public void testFreezePublishesRegistration() { ForyBuilder builder = Fory.builder().withXlang(false).requireClassRegistration(true).withCompatible(false); finishBuilder(builder); @@ -1936,15 +1942,6 @@ public int hashCode() { } } - private static Fory newUnknownClassFory(boolean suppressWarnings) { - return Fory.builder() - .withXlang(true) - .withMetaShare(true) - .withDeserializeUnknownClass(true) - .suppressClassRegistrationWarnings(suppressWarnings) - .build(); - } - private static String captureOutput(Runnable action) throws Exception { int previousLogLevel = LoggerFactory.getLogLevel(); PrintStream previousOut = System.out; @@ -1960,7 +1957,7 @@ private static String captureOutput(Runnable action) throws Exception { return out.toString(StandardCharsets.UTF_8.name()); } - private static void resolveMissingXtype(Fory fory, String typeName) { + private static void resolveMissingXtype(Fory fory) { try { Method method = XtypeResolver.class.getDeclaredMethod( @@ -1973,10 +1970,12 @@ private static void resolveMissingXtype(Fory fory, String typeName) { fory.getTypeResolver(), Types.NAMED_STRUCT, Encoders.PACKAGE_ENCODER.encodeBinary("missing.pkg"), - Encoders.TYPE_NAME_ENCODER.encodeBinary(typeName)); + Encoders.TYPE_NAME_ENCODER.encodeBinary("MissingType")); } catch (InvocationTargetException e) { - if (!(e.getCause() instanceof IllegalStateException)) { - throw new AssertionError(e.getCause()); + Throwable cause = e.getCause(); + if (!(cause instanceof IllegalStateException) + || !cause.getMessage().contains("missing.pkg.MissingType")) { + throw new AssertionError(e); } } catch (ReflectiveOperationException e) { throw new AssertionError(e); From 3f2cc3acb9a54c3e03da93a3d027561b53452743 Mon Sep 17 00:00:00 2001 From: chaokunyang Date: Sun, 30 Aug 2026 12:46:47 +0800 Subject: [PATCH 145/168] fix(python): keep explicit registration atomic --- python/pyfory/_fory.py | 6 +++-- python/pyfory/registry.py | 34 +++++++++---------------- python/pyfory/serialization.pyx | 5 +++- python/pyfory/tests/test_function.py | 2 -- python/pyfory/tests/test_serializer.py | 21 +++++++++++++++ python/pyfory/tests/test_thread_safe.py | 5 ++++ 6 files changed, 46 insertions(+), 27 deletions(-) diff --git a/python/pyfory/_fory.py b/python/pyfory/_fory.py index c7ee992791..548e0d8444 100644 --- a/python/pyfory/_fory.py +++ b/python/pyfory/_fory.py @@ -681,6 +681,8 @@ class ThreadSafeFory: def __init__(self, fory_factory=None, **kwargs): import threading + if fory_factory is not None and kwargs: + raise TypeError("fory_factory and Fory construction options are mutually exclusive") self._config = kwargs self._fory_factory = fory_factory self._callbacks = [] @@ -723,9 +725,9 @@ def _register_callback(self, callback): def _check_serializer_factory(serializer): if serializer is None: return - from pyfory.registry import CythonSerializer, Serializer + from pyfory.registry import Serializer - if isinstance(serializer, (Serializer, CythonSerializer)) or not callable(serializer): + if isinstance(serializer, Serializer) or not callable(serializer): raise TypeError("ThreadSafeFory requires a serializer class or factory") def register( diff --git a/python/pyfory/registry.py b/python/pyfory/registry.py index 95106daca2..609efb802d 100644 --- a/python/pyfory/registry.py +++ b/python/pyfory/registry.py @@ -89,9 +89,6 @@ fory_array_serializer_type, ) from pyfory.policy import DEFAULT_POLICY -from pyfory.serialization import ( - Serializer as CythonSerializer, -) from pyfory.annotation import ( BFloat16Array, Float32, @@ -196,7 +193,7 @@ def _accepts_n_positional_args(factory, nargs: int) -> bool: signature = inspect.signature(factory.__init__) parameters = tuple(signature.parameters.values())[1:] except (AttributeError, TypeError, ValueError): - if inspect.isclass(factory) and issubclass(factory, (Serializer, CythonSerializer)): + if inspect.isclass(factory) and issubclass(factory, Serializer): return nargs == 2 raise TypeError(f"Unable to inspect serializer constructor for {factory!r}") min_args = 0 @@ -226,7 +223,7 @@ def _construct_serializer(serializer_factory, type_resolver, cls): break else: raise TypeError(f"Unsupported serializer constructor for {serializer_factory!r}; expected `(type_resolver, cls)` or `(type_resolver)`.") - if not isinstance(serializer, (Serializer, CythonSerializer)): + if not isinstance(serializer, Serializer): raise TypeError("Serializer factory must return a Fory serializer") if serializer.type_resolver is not type_resolver: raise TypeError("Serializer factory returned a serializer for another resolver") @@ -234,7 +231,7 @@ def _construct_serializer(serializer_factory, type_resolver, cls): def _check_serializer_owner(serializer, type_resolver, cls): - if not isinstance(serializer, (Serializer, CythonSerializer)): + if not isinstance(serializer, Serializer): raise TypeError("Expected a Fory serializer") if serializer.type_resolver is not type_resolver: raise TypeError("Serializer belongs to another resolver") @@ -431,14 +428,7 @@ def __init__(self, config, *, shared_registry): self._registry_frozen = False def _check_registry_mutable(self): - registry_owner = self._actual_type_resolver - if registry_owner is self: - registry_frozen = self._registry_frozen - else: - # The compiled resolver is the active registry owner. Read its one - # lifecycle flag instead of mirroring that state in both resolvers. - registry_frozen = registry_owner._registry_frozen - if registry_frozen: + if self._actual_type_resolver._registry_frozen: raise RuntimeError("Cannot register types or serializers after the first root operation has started") def _freeze_registry(self): @@ -631,7 +621,7 @@ def register_union( namespace, typename = _split_registration_name(name) if serializer is None: raise TypeError("register_union requires a serializer") - if serializer is not None and not isinstance(serializer, (Serializer, CythonSerializer)): + if serializer is not None and not isinstance(serializer, Serializer): serializer = _construct_serializer( serializer, self._actual_type_resolver, @@ -679,7 +669,7 @@ def _register_type( else: if user_type_id not in {None, NO_USER_TYPE_ID} and (user_type_id < 0 or user_type_id > 0xFFFFFFFE): raise ValueError(f"user_type_id must be in range [0, 0xfffffffe], got {user_type_id}") - if serializer is not None and not isinstance(serializer, (Serializer, CythonSerializer)): + if serializer is not None and not isinstance(serializer, Serializer): serializer = _construct_serializer( serializer, self._actual_type_resolver, @@ -785,6 +775,10 @@ def __register_type( internal: bool = False, ): dynamic_type = type_id is not None and type_id < 0 + if type_id is not None and type_id != 0 and needs_user_type_id(type_id) and user_type_id not in {None, NO_USER_TYPE_ID}: + existing = self._user_type_id_to_type_info.get(user_type_id) + if existing is not None and existing.cls is not cls: + raise TypeError(f"user_type_id {user_type_id} already registered for {existing.cls}") # In metashare mode, for struct types, we want to keep serializer=None # so that _set_type_info will be called to create the TypeDef-based serializer # This applies to both types registered by name and by ID @@ -817,10 +811,6 @@ def __register_type( self._ns_type_to_type_info[(ns_meta_bytes, type_meta_bytes)] = typeinfo self._types_info[cls] = typeinfo if type_id is not None and type_id != 0: - if needs_user_type_id(type_id) and user_type_id not in {None, NO_USER_TYPE_ID}: - existing = self._user_type_id_to_type_info.get(user_type_id) - if existing is not None and existing.cls is not cls: - raise TypeError(f"user_type_id {user_type_id} already registered for {existing.cls}") if needs_user_type_id(type_id) and user_type_id not in {None, NO_USER_TYPE_ID}: if user_type_id not in self._user_type_id_to_type_info or not internal: self._user_type_id_to_type_info[user_type_id] = typeinfo @@ -904,9 +894,9 @@ def get_type_info(self, cls, create=True): if self.require_registration and not issubclass(cls, Enum): raise TypeUnregisteredError(f"{cls} not registered") logger.info("Type %s not registered", cls) - return self._register_inferred_type(cls) + return self._create_inferred_type_info(cls) - def _register_inferred_type(self, cls): + def _create_inferred_type_info(self, cls): serializer = self._create_serializer(cls) native_registration = self._internal_py_serializer_map.get(type(serializer)) if native_registration is not None: diff --git a/python/pyfory/serialization.pyx b/python/pyfory/serialization.pyx index 24f4ce6a89..85f701111e 100644 --- a/python/pyfory/serialization.pyx +++ b/python/pyfory/serialization.pyx @@ -44,6 +44,7 @@ from pyfory._fory import ( from pyfory.meta.typedef_decoder import decode_typedef from pyfory.meta.typedef import is_struct_typedef_kind from pyfory.meta.metastring import MetaStringDecoder +from pyfory.error import TypeUnregisteredError from pyfory.policy import DEFAULT_POLICY from pyfory.resolver import NULL_FLAG, NOT_NULL_VALUE_FLAG from pyfory.type_util import normalize_fory_type @@ -350,7 +351,9 @@ cdef class TypeResolver: cdef uint8_t previous_type_id cdef uint32_t previous_user_type_id self.resolver._check_registry_mutable() - typeinfo = self.resolver.get_type_info(cls) + typeinfo = self.resolver.get_type_info(cls, create=False) + if typeinfo is None: + raise TypeUnregisteredError(f"{cls} not registered") previous_type_id = typeinfo.type_id previous_user_type_id = typeinfo.user_type_id self.resolver.register_serializer(cls, serializer) diff --git a/python/pyfory/tests/test_function.py b/python/pyfory/tests/test_function.py index ebb5de638c..21e84cc1ae 100644 --- a/python/pyfory/tests/test_function.py +++ b/python/pyfory/tests/test_function.py @@ -137,8 +137,6 @@ def complex_function(a, b, c=10): deserialized = fory.deserialize(serialized) assert add_one(test_input) == deserialized(test_input) - # dict is already registered by default with MapSerializer - # Test complex function serialized = fory.serialize(complex_function) deserialized = fory.deserialize(serialized) diff --git a/python/pyfory/tests/test_serializer.py b/python/pyfory/tests/test_serializer.py index 490ab8a7e5..f8b5e82ddb 100644 --- a/python/pyfory/tests/test_serializer.py +++ b/python/pyfory/tests/test_serializer.py @@ -37,6 +37,7 @@ import pyfory from pyfory.serialization import Buffer, _bfloat16_from_bits, _bfloat16_to_bits, _float16_from_bits, _float16_to_bits from pyfory import Fory, EnumSerializer +from pyfory.error import TypeUnregisteredError from pyfory.serializer import ( DecimalSerializer, TimestampSerializer, @@ -918,6 +919,26 @@ def test_register_type_name_exclusive(): fory.register_type(A, type_id=100, name="example.A") +def test_duplicate_id_keeps_registry_clean(): + fory = Fory(xlang=True, compatible=False) + fory.register_type(A, type_id=100) + + with pytest.raises(TypeError, match="user_type_id 100"): + fory.register_type(RejectedRegistration, type_id=100) + + assert fory.type_resolver.get_type_info(RejectedRegistration, create=False) is None + + +def test_serializer_requires_registered_type(): + fory = Fory(xlang=False, strict=False, compatible=False) + serializer = BarSerializer(fory.type_resolver, RejectedRegistration) + + with pytest.raises(TypeUnregisteredError): + fory.register_serializer(RejectedRegistration, serializer) + + assert fory.type_resolver.get_type_info(RejectedRegistration, create=False) is None + + @pytest.mark.parametrize("root", ["serialize", "deserialize", "dump"]) def test_registry_freezes_at_root(root): fory = Fory(xlang=True, compatible=False) diff --git a/python/pyfory/tests/test_thread_safe.py b/python/pyfory/tests/test_thread_safe.py index 0bb516951d..876fa89e80 100644 --- a/python/pyfory/tests/test_thread_safe.py +++ b/python/pyfory/tests/test_thread_safe.py @@ -40,6 +40,11 @@ class PersonSerializer(pyfory.Serializer): pass +def test_factory_rejects_options(): + with pytest.raises(TypeError, match="mutually exclusive"): + ThreadSafeFory(lambda: pyfory.Fory(), xlang=False) + + def test_thread_safe_fory_basic_serialization(): fory = ThreadSafeFory( xlang=False, From 29ae5ee6840b629b8382f6c917eef96f2c8bdcb7 Mon Sep 17 00:00:00 2001 From: chaokunyang Date: Sun, 30 Aug 2026 12:46:47 +0800 Subject: [PATCH 146/168] test(javascript): keep root cleanup coverage causal --- javascript/test/depthLimit.test.ts | 2 + javascript/test/rootCleanup.test.ts | 63 +++++++++-------------------- 2 files changed, 20 insertions(+), 45 deletions(-) diff --git a/javascript/test/depthLimit.test.ts b/javascript/test/depthLimit.test.ts index d75ad025f3..7a2ce28039 100644 --- a/javascript/test/depthLimit.test.ts +++ b/javascript/test/depthLimit.test.ts @@ -291,6 +291,7 @@ describe("depth-limit", () => { expect(() => reader.deserialize(malformedDepth)).toThrow( "Deserialization depth limit exceeded", ); + expect(readerFory.readContext.depth).toBe(0); expect(shallowReader.deserialize(shallowWriter.serialize({ value: 10 }))).toEqual({ value: 10, @@ -431,6 +432,7 @@ describe("depth-limit", () => { for (const readRoot of rootReaders) { expect(() => readRoot(serialized.subarray(0, serialized.length - 1))).toThrow(); + expect(fory.readContext.depth).toBe(0); expect(readRoot(serialized)).toEqual(value); expect(fory.readContext.depth).toBe(0); diff --git a/javascript/test/rootCleanup.test.ts b/javascript/test/rootCleanup.test.ts index 9590afda64..cc95883208 100644 --- a/javascript/test/rootCleanup.test.ts +++ b/javascript/test/rootCleanup.test.ts @@ -44,7 +44,7 @@ describe.each([ registered.deserialize(bytes), }, ])("$name root cleanup", ({ invoke }) => { - test.each(["success", "failure"] as const)("restores generated root state for %s", (outcome) => { + test("restores generated root state after failure", () => { const writerFory = new Fory({ compatible: true, ref: true }); const readerFory = new Fory({ compatible: true, ref: true }); const writer = writerFory.register( @@ -58,23 +58,14 @@ describe.each([ }), ); const bytes = writer.serialize({ value: 7 }); - const input = outcome === "failure" ? bytes.subarray(0, bytes.length - 1) : bytes; - const read = () => invoke(readerFory, reader, input); - - if (outcome === "failure") { - expect(read).toThrow(); - expectRootStateCleared(readerFory.readContext); - expect(invoke(readerFory, reader, bytes)).toEqual({ value: 7 }); - } else { - const first = read(); - const second = read(); - expect(first).toEqual({ value: 7 }); - expect(second).toEqual({ value: 7 }); - expect(second).not.toBe(first); - } + const input = bytes.subarray(0, bytes.length - 1); + + expect(() => invoke(readerFory, reader, input)).toThrow(); + expectRootStateCleared(readerFory.readContext); + expect(invoke(readerFory, reader, bytes)).toEqual({ value: 7 }); }); - test.each(["success", "failure"] as const)("restores logical tables for %s", (outcome) => { + test("restores logical tables after failure", () => { const fory = new Fory({ compatible: true, ref: true }); const registered = fory.register(Type.struct(7602, {})); const readContext = (fory as any).readContext; @@ -86,31 +77,23 @@ describe.each([ registered.serializer.readRef = () => { expectRootStateCleared(readContext); populateLogicalTables(readContext, typeMeta); - if (outcome === "failure") { - throw new Error("root read failed"); - } - return 7; + throw new Error("root read failed"); }; const read = () => invoke(fory, registered, new Uint8Array([1])); - if (outcome === "failure") { - expect(read).toThrow(); + expect(read).toThrow(); + expectRootStateCleared(readContext); + registered.serializer.readRef = () => { expectRootStateCleared(readContext); - registered.serializer.readRef = () => { - expectRootStateCleared(readContext); - return 7; - }; - expect(read()).toBe(7); - } else { - expect(read()).toBe(7); - expect(read()).toBe(7); - } + return 7; + }; + expect(read()).toBe(7); expect(readContext.typeMetaCache.get(headerHash)).toBe(typeMeta); }); }); -test.each(["success", "failure"] as const)("restores root write state for %s", (outcome) => { +test("restores root write state after failure", () => { const fory = new Fory({ compatible: true, ref: true }); const registered = fory.register(Type.struct(7606, {})); const writeContext = (fory as any).writeContext; @@ -122,24 +105,14 @@ test.each(["success", "failure"] as const)("restores root write state for %s", ( writeContext.refWriter.writeRef(value); writeContext.metaStringWriter.writeBytes(writeContext.writer, name); writeContext.writeTypeMeta(typeMeta, typeMeta.toBytes()); - if (outcome === "failure") { - throw new Error("root write failed"); - } + throw new Error("root write failed"); }; - if (outcome === "failure") { - expect(() => registered.serialize(value)).toThrow(); - expect(writeContext.refWriter.writeObjects.size).toBe(0); - expect(name.dynamicWriteStringId).toBe(-1); - expect(typeMeta.dynamicTypeId).toBe(-1); - expect(fory.serialize(7)).toBeDefined(); - } else { - expect(registered.serialize(value)).toBeDefined(); - expect(fory.serialize(7)).toBeDefined(); - } + expect(() => registered.serialize(value)).toThrow(); expect(writeContext.refWriter.writeObjects.size).toBe(0); expect(name.dynamicWriteStringId).toBe(-1); expect(typeMeta.dynamicTypeId).toBe(-1); + expect(fory.serialize(7)).toBeDefined(); }); test("clears write state before serializer lookup", () => { From 1aa49bd293f98460323003651ef61a98e47407ee Mon Sep 17 00:00:00 2001 From: chaokunyang Date: Sun, 30 Aug 2026 12:52:46 +0800 Subject: [PATCH 147/168] test(python): keep registry checks concise --- python/pyfory/tests/test_serializer.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/python/pyfory/tests/test_serializer.py b/python/pyfory/tests/test_serializer.py index f8b5e82ddb..61def96a85 100644 --- a/python/pyfory/tests/test_serializer.py +++ b/python/pyfory/tests/test_serializer.py @@ -919,7 +919,7 @@ def test_register_type_name_exclusive(): fory.register_type(A, type_id=100, name="example.A") -def test_duplicate_id_keeps_registry_clean(): +def test_duplicate_id_is_atomic(): fory = Fory(xlang=True, compatible=False) fory.register_type(A, type_id=100) @@ -929,7 +929,7 @@ def test_duplicate_id_keeps_registry_clean(): assert fory.type_resolver.get_type_info(RejectedRegistration, create=False) is None -def test_serializer_requires_registered_type(): +def test_serializer_type_is_registered(): fory = Fory(xlang=False, strict=False, compatible=False) serializer = BarSerializer(fory.type_resolver, RejectedRegistration) From 2d8e8567f0eaee6f4cdf88ebb51e8cbc5454a7ae Mon Sep 17 00:00:00 2001 From: chaokunyang Date: Sun, 30 Aug 2026 13:08:04 +0800 Subject: [PATCH 148/168] fix(java): publish read context atomically --- .../java/org/apache/fory/context/ReadContext.java | 6 +++++- .../src/test/java/org/apache/fory/ForyTest.java | 14 ++++++++++++++ 2 files changed, 19 insertions(+), 1 deletion(-) diff --git a/java/fory-core/src/main/java/org/apache/fory/context/ReadContext.java b/java/fory-core/src/main/java/org/apache/fory/context/ReadContext.java index bf2624a88f..ab10f74fdf 100644 --- a/java/fory-core/src/main/java/org/apache/fory/context/ReadContext.java +++ b/java/fory-core/src/main/java/org/apache/fory/context/ReadContext.java @@ -111,9 +111,13 @@ public ReadContext( */ public void prepare( MemoryBuffer buffer, Iterable outOfBandBuffers, boolean peerOutOfBandEnabled) { + // Resolve user code before publishing root state so a failing iterator cannot leave a + // partially prepared context outside the root cleanup boundary. + Iterator outOfBandIterator = + outOfBandBuffers == null ? null : outOfBandBuffers.iterator(); this.buffer = buffer; this.peerOutOfBandEnabled = peerOutOfBandEnabled; - this.outOfBandBuffers = outOfBandBuffers == null ? null : outOfBandBuffers.iterator(); + this.outOfBandBuffers = outOfBandIterator; remainingGraphMemoryBytes = config.maxGraphMemoryBytes(); remainingUnbackedContainerItems = config.maxUnbackedContainerItems(); } diff --git a/java/fory-core/src/test/java/org/apache/fory/ForyTest.java b/java/fory-core/src/test/java/org/apache/fory/ForyTest.java index 53dc6bcada..5383c73a47 100644 --- a/java/fory-core/src/test/java/org/apache/fory/ForyTest.java +++ b/java/fory-core/src/test/java/org/apache/fory/ForyTest.java @@ -131,6 +131,20 @@ public void testRegistrationFreezesOnUse() { assertRegistrationFrozen(reader); } + @Test + public void testOutOfBandSetupFailureCleanup() { + Fory fory = newNativeFory(); + byte[] bytes = fory.serialize(7, bufferObject -> true); + Iterable failingBuffers = + () -> { + throw new IllegalStateException("iterator failed"); + }; + + assertThrows(IllegalStateException.class, () -> fory.deserialize(bytes, failingBuffers)); + assertNull(fory.getReadContext().getBuffer()); + assertEquals(fory.deserialize(fory.serialize(8)), 8); + } + private static Fory newNativeFory() { return Fory.builder() .withXlang(false) From 54d9af95b9d3c1d3dc5a6b665da5fd6879eac3a2 Mon Sep 17 00:00:00 2001 From: chaokunyang Date: Sun, 30 Aug 2026 13:23:53 +0800 Subject: [PATCH 149/168] fix(python): preserve registry owners --- .agents/languages/python.md | 4 + AGENTS.md | 4 + .../python/custom-serializers.md | 3 + docs/security/deserialization.md | 5 + .../xlang_implementation_guide.md | 5 + python/README.md | 3 + python/pyfory/registry.py | 147 +++++++++++------- python/pyfory/tests/test_serializer.py | 97 +++++++++++- 8 files changed, 209 insertions(+), 59 deletions(-) diff --git a/.agents/languages/python.md b/.agents/languages/python.md index 2473a42888..0f61a3b10a 100644 --- a/.agents/languages/python.md +++ b/.agents/languages/python.md @@ -21,6 +21,10 @@ Load this file when changing `python/`, Cython serialization, or Python xlang be - Explicit type, serializer, name, and ID registration checks the frozen flag before mutation. Automatic IDs remain registration-owned and must not turn native runtime discovery into explicit registration. +- A Python wire name or user ID identifies one `TypeInfo`. Reject explicit or native-discovery + collisions before publishing resolver maps. Lazy TypeDef completion preserves a configured + serializer, does not retain partial state, and restores the prior serializer and TypeDef after + failed completion without adding a lifecycle state. - `ThreadSafeFory` accepts serializer classes or factories and constructs a serializer for each child resolver. It must reject resolver-bound serializer instances instead of replaying one instance across pooled children. diff --git a/AGENTS.md b/AGENTS.md index 56cd2c50eb..55adb80721 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -201,6 +201,10 @@ This is the entry point for AI guidance in Apache Fory. Read this file first, th generated serializer completion for an existing binding is likewise allowed after freeze. These runtime cache operations must not create or change an explicit type, serializer, ID, name, or policy registration. + Each explicit name and ID has one type-information owner. Native discovery + must reject an identity collision instead of replacing that owner. Lazy + metadata or serializer completion preserves the selected explicit serializer + and does not retain partial state; failure leaves the prior valid state. Java `Fory.register(ForyModule)` and the corresponding `BaseFory` operation remain available on direct and thread-safe facades before the first root. Kotlin and Scala registration extensions target `BaseFory` so the same API works with `Fory`, `ThreadLocalFory`, and pooled diff --git a/docs/object-serialization/python/custom-serializers.md b/docs/object-serialization/python/custom-serializers.md index a641bd02f8..6a516a9731 100644 --- a/docs/object-serialization/python/custom-serializers.md +++ b/docs/object-serialization/python/custom-serializers.md @@ -121,6 +121,9 @@ value = buffer.read_bool() ## Registering Custom Serializers +Attach custom serializers to application types registered by name or user ID. Serializers for +built-in framework types cannot be replaced. + ```python fory = pyfory.Fory(xlang=False) diff --git a/docs/security/deserialization.md b/docs/security/deserialization.md index 56b3fe3215..53d9097b49 100644 --- a/docs/security/deserialization.md +++ b/docs/security/deserialization.md @@ -633,6 +633,11 @@ serializers, or generated code. Lazy completion of an existing binding is also a internal cache operations are not explicit registration and must not create or change an explicit type or serializer registration, ID, name, or policy binding. +Each explicit name and ID identifies one type-information owner. Native discovery must reject an +identity collision instead of replacing that owner. Lazy metadata or serializer completion must +preserve an explicitly selected serializer and must not retain partial state; failure leaves the +previous valid state. + ## Metadata And Type Resolution Metadata parsing is security-sensitive when it affects retained read-side state, diff --git a/docs/specification/xlang_implementation_guide.md b/docs/specification/xlang_implementation_guide.md index b22b5e75fe..63a67afe73 100644 --- a/docs/specification/xlang_implementation_guide.md +++ b/docs/specification/xlang_implementation_guide.md @@ -101,6 +101,11 @@ serializer, or generated code. Lazy serializer completion for an existing bindin These operations must not create or change an explicit type, serializer, ID, name, or policy registration. +Each explicit name and ID identifies one type-information owner. Native discovery must reject an +identity collision instead of replacing that owner. Lazy metadata or serializer completion must +preserve an explicitly selected serializer and must not retain partial state; failure leaves the +previous valid state. + Java module registration remains available through `BaseFory` before the first root. Kotlin and Scala registration extensions target `BaseFory`, so direct and thread-safe facades share the same pre-root API. Copy operations and facade execution callbacks do not freeze registration unless they diff --git a/python/README.md b/python/README.md index 7ed784ef12..de2a9686e1 100644 --- a/python/README.md +++ b/python/README.md @@ -899,6 +899,9 @@ except Exception as e: Implement custom serialization logic for specialized types with a single `write/read` API: +Attach custom serializers to application types registered by name or user ID. Serializers for +built-in framework types cannot be replaced. + ```python import pyfory from pyfory.serializer import Serializer diff --git a/python/pyfory/registry.py b/python/pyfory/registry.py index 609efb802d..8ed99e3dfc 100644 --- a/python/pyfory/registry.py +++ b/python/pyfory/registry.py @@ -618,7 +618,15 @@ def register_union( serializer=None, ): self._check_registry_mutable() + cls = normalize_fory_type(cls) namespace, typename = _split_registration_name(name) + registration_id = type_id if type_id not in {0, None} else NO_USER_TYPE_ID + self._check_registration_identity( + cls, + namespace=namespace, + typename=typename, + user_type_id=registration_id, + ) if serializer is None: raise TypeError("register_union requires a serializer") if serializer is not None and not isinstance(serializer, Serializer): @@ -689,12 +697,17 @@ def _register_type( ): return self._types_info[cls] n_params = len({typename, type_id, None}) - 1 - if n_params == 0 and typename is None: - type_id = self._next_type_id() if n_params == 2: raise TypeError(f"type name {typename} and id {type_id} should not be set at the same time") - if cls in self._types_info: - raise TypeError(f"{cls} registered already") + registration_id = type_id if not internal and type_id is not None else user_type_id + self._check_registration_identity( + cls, + namespace=namespace, + typename=typename, + user_type_id=registration_id, + ) + if n_params == 0 and typename is None: + type_id = self._next_type_id() return self._register_xtype( cls, type_id=type_id, @@ -775,10 +788,23 @@ def __register_type( internal: bool = False, ): dynamic_type = type_id is not None and type_id < 0 - if type_id is not None and type_id != 0 and needs_user_type_id(type_id) and user_type_id not in {None, NO_USER_TYPE_ID}: - existing = self._user_type_id_to_type_info.get(user_type_id) - if existing is not None and existing.cls is not cls: - raise TypeError(f"user_type_id {user_type_id} already registered for {existing.cls}") + if typename is not None: + if namespace is None: + splits = typename.rsplit(".", 1) + if len(splits) == 2: + namespace, typename = splits + else: + namespace = "" + else: + namespace = namespace or "" + if not typename: + raise ValueError("type name must not be empty") + self._check_registration_identity( + cls, + namespace=namespace, + typename=typename, + user_type_id=user_type_id, + ) # In metashare mode, for struct types, we want to keep serializer=None # so that _set_type_info will be called to create the TypeDef-based serializer # This applies to both types registered by name and by ID @@ -794,14 +820,6 @@ def __register_type( if typename is None: typeinfo = TypeInfo(cls, type_id, user_type_id, serializer, None, None, dynamic_type) else: - if namespace is None: - splits = typename.rsplit(".", 1) - if len(splits) == 2: - namespace, typename = splits - else: - namespace = "" # Use empty string for consistency with lookup - if not typename: - raise ValueError("type name must not be empty") ns_metastr = self.namespace_encoder.encode(namespace or "") ns_meta_bytes = self.shared_registry.get_encoded_meta_string(ns_metastr) type_metastr = self.typename_encoder.encode(typename) @@ -821,6 +839,25 @@ def __register_type( self._index_python_type(cls) return typeinfo + def _check_registration_identity( + self, + cls, + *, + namespace, + typename, + user_type_id, + ): + if cls in self._types_info: + raise TypeError(f"{cls} registered already") + if typename is not None: + existing = self._named_type_to_type_info.get((namespace, typename)) + if existing is not None and existing.cls is not cls: + raise TypeError(f"type name {(namespace, typename)!r} already registered for {existing.cls}") + if user_type_id not in {None, NO_USER_TYPE_ID}: + existing = self._user_type_id_to_type_info.get(user_type_id) + if existing is not None and existing.cls is not cls: + raise TypeError(f"user_type_id {user_type_id} already registered for {existing.cls}") + def _index_python_type(self, cls): if self._python_name_to_type is None or not isinstance(cls, type): return @@ -850,13 +887,11 @@ def register_serializer(self, cls, serializer): if cls not in self._types_info: raise TypeUnregisteredError(f"{cls} not registered") typeinfo = self._types_info[cls] - prev_type_id = typeinfo.type_id - prev_user_type_id = typeinfo.user_type_id + # Framework type IDs may be shared by multiple Python carriers; only a namespaced or + # user-ID TypeInfo has an independent wire owner that can change serializers. + if typeinfo.typename_bytes is None and typeinfo.user_type_id in {None, NO_USER_TYPE_ID}: + raise TypeError("Cannot replace serializers for framework types") self._check_registry_mutable() - if needs_user_type_id(prev_type_id) and prev_user_type_id not in {None, NO_USER_TYPE_ID}: - self._user_type_id_to_type_info.pop(prev_user_type_id, None) - else: - self._type_id_to_type_info.pop(prev_type_id, None) if typeinfo.serializer is not serializer: if typeinfo.typename_bytes is not None: typeinfo.type_id = TypeId.NAMED_EXT @@ -864,10 +899,7 @@ def register_serializer(self, cls, serializer): else: typeinfo.type_id = TypeId.EXT typeinfo.serializer = serializer - if needs_user_type_id(typeinfo.type_id) and typeinfo.user_type_id not in {None, NO_USER_TYPE_ID}: - self._user_type_id_to_type_info[typeinfo.user_type_id] = typeinfo - else: - self._type_id_to_type_info[typeinfo.type_id] = typeinfo + typeinfo.type_def = None def get_serializer(self, cls: type): """ @@ -934,34 +966,43 @@ def _create_inferred_type_info(self, cls): def _set_type_info(self, typeinfo): serializer_type_resolver = self._actual_type_resolver type_id = typeinfo.type_id - if is_struct_type(type_id): - from pyfory.struct import DataClassSerializer, DataClassStubSerializer - - # Set a stub serializer FIRST to break recursion for self-referencing types. - # get_type_info() only calls _set_type_info when serializer is None, - # so setting stub first prevents re-entry for circular type references. - typeinfo.serializer = DataClassStubSerializer(serializer_type_resolver, typeinfo.cls) - - if self.meta_share: - type_def = encode_typedef(serializer_type_resolver, typeinfo.cls) - if type_def is not None: - typeinfo.serializer = type_def.create_serializer(serializer_type_resolver) - typeinfo.type_def = type_def - else: - typeinfo.serializer = DataClassSerializer(serializer_type_resolver, typeinfo.cls) + previous_serializer = typeinfo.serializer + previous_type_def = typeinfo.type_def + try: + if is_struct_type(type_id): + from pyfory.struct import DataClassSerializer, DataClassStubSerializer + + if typeinfo.serializer is None or isinstance(typeinfo.serializer, DataClassStubSerializer): + # Recursive construction needs the stub to be temporarily visible. Restore the + # prior owner if later TypeDef or serializer construction fails. + typeinfo.serializer = DataClassStubSerializer(serializer_type_resolver, typeinfo.cls) + if self.meta_share: + type_def = encode_typedef(serializer_type_resolver, typeinfo.cls) + if type_def is not None: + typeinfo.serializer = type_def.create_serializer(serializer_type_resolver) + typeinfo.type_def = type_def + else: + typeinfo.serializer = DataClassSerializer(serializer_type_resolver, typeinfo.cls) + else: + typeinfo.serializer = DataClassSerializer(serializer_type_resolver, typeinfo.cls) + elif self.meta_share and typeinfo.type_def is None and TypeId.is_type_share_meta(type_id): + typeinfo.type_def = encode_typedef(serializer_type_resolver, typeinfo.cls) else: - typeinfo.serializer = DataClassSerializer(serializer_type_resolver, typeinfo.cls) - else: - typeinfo.serializer = self._create_serializer(typeinfo.cls) - if ( - self.meta_share - and typeinfo.type_def is None - and ( - TypeId.is_namespaced_type(type_id) - or (needs_user_type_id(type_id) and typeinfo.user_type_id is not None and typeinfo.user_type_id != NO_USER_TYPE_ID) - ) - ): - typeinfo.type_def = encode_typedef(serializer_type_resolver, typeinfo.cls) + if typeinfo.serializer is None: + typeinfo.serializer = self._create_serializer(typeinfo.cls) + if ( + self.meta_share + and typeinfo.type_def is None + and ( + TypeId.is_namespaced_type(type_id) + or (needs_user_type_id(type_id) and typeinfo.user_type_id is not None and typeinfo.user_type_id != NO_USER_TYPE_ID) + ) + ): + typeinfo.type_def = encode_typedef(serializer_type_resolver, typeinfo.cls) + except Exception: + typeinfo.serializer = previous_serializer + typeinfo.type_def = previous_type_def + raise return typeinfo diff --git a/python/pyfory/tests/test_serializer.py b/python/pyfory/tests/test_serializer.py index 61def96a85..bec6eed939 100644 --- a/python/pyfory/tests/test_serializer.py +++ b/python/pyfory/tests/test_serializer.py @@ -845,17 +845,27 @@ def read(self, read_context): self.read_count += 1 return Value(read_context.read_int32() - 17) - fory = Fory(xlang=True, ref=False, compatible=False) + fory = Fory(xlang=True, ref=False, compatible=True) if registration == "id": fory.register_type(Value, type_id=701) else: fory.register_type(Value, name="test.ReplacedValue") + assert fory.type_resolver.get_serializer(Value) is not None + assert fory.type_resolver.get_type_info(Value).type_def is not None replacement = ReplacementSerializer(fory.type_resolver) fory.register_serializer(Value, replacement) - assert fory.type_resolver.get_serializer(Value) is replacement + type_info = fory.type_resolver.get_type_info(Value) + assert type_info.serializer is replacement + assert type_info.type_def is None + assert TypeId.NAMED_EXT not in fory.type_resolver._type_id_to_type_info + if registration == "id": + assert fory.type_resolver._user_type_id_to_type_info[701] is type_info + else: + assert fory.type_resolver.get_type_info_by_name("test", "ReplacedValue") is type_info assert fory.deserialize(fory.serialize(Value(25))) == Value(25) + assert type_info.serializer is replacement assert (replacement.write_count, replacement.read_count) == (1, 1) @@ -919,14 +929,55 @@ def test_register_type_name_exclusive(): fory.register_type(A, type_id=100, name="example.A") -def test_duplicate_id_is_atomic(): +@pytest.mark.parametrize("identity", ["id", "name"]) +def test_registration_identity_atomic(identity): fory = Fory(xlang=True, compatible=False) - fory.register_type(A, type_id=100) + options = {"type_id": 100} if identity == "id" else {"name": "test.SharedName"} + type_info = fory.register_type(A, **options) - with pytest.raises(TypeError, match="user_type_id 100"): - fory.register_type(RejectedRegistration, type_id=100) + with pytest.raises(TypeError): + fory.register_type(RejectedRegistration, **options) assert fory.type_resolver.get_type_info(RejectedRegistration, create=False) is None + if identity == "id": + assert fory.type_resolver.get_type_info_by_id(type_info.type_id, 100) is type_info + else: + assert fory.type_resolver.get_type_info_by_name("test", "SharedName") is type_info + + +def test_union_registration_atomic(): + fory = Fory(xlang=True, compatible=True) + serializer = BarSerializer(fory.type_resolver, RejectedRegistration) + type_info = fory.register_union( + RejectedRegistration, + name="test.FirstUnion", + serializer=serializer, + ) + + with pytest.raises(TypeError, match="registered already"): + fory.register_union( + RejectedRegistration, + name="test.SecondUnion", + serializer=serializer, + ) + + assert fory.type_resolver.get_type_info(RejectedRegistration) is type_info + assert fory.type_resolver.get_type_info_by_name("test", "FirstUnion") is type_info + assert fory.type_resolver.get_type_info_by_name("test", "SecondUnion") is None + + +def test_lazy_type_keeps_explicit_name(): + explicit_type = type("SharedType", (), {"__module__": "registry_owner"}) + lazy_type = type("SharedType", (), {"__module__": "registry_owner"}) + fory = Fory(xlang=False, strict=False, compatible=False) + type_info = fory.register_type(explicit_type, name="registry_owner.SharedType") + fory.serialize(None) + + with pytest.raises(TypeError, match="type name"): + fory.serialize(lazy_type()) + + assert fory.type_resolver.get_type_info(lazy_type, create=False) is None + assert fory.type_resolver.get_type_info_by_name("registry_owner", "SharedType") is type_info def test_serializer_type_is_registered(): @@ -939,6 +990,19 @@ def test_serializer_type_is_registered(): assert fory.type_resolver.get_type_info(RejectedRegistration, create=False) is None +def test_builtin_serializer_owner(): + fory = Fory(xlang=True, compatible=False) + resolver = fory.type_resolver + type_info = resolver.get_type_info(int) + wire_owner = resolver._type_id_to_type_info[type_info.type_id] + + with pytest.raises(TypeError, match="framework types"): + fory.register_serializer(int, BarSerializer(resolver, int)) + + assert resolver.get_type_info(int) is type_info + assert resolver._type_id_to_type_info[type_info.type_id] is wire_owner + + @pytest.mark.parametrize("root", ["serialize", "deserialize", "dump"]) def test_registry_freezes_at_root(root): fory = Fory(xlang=True, compatible=False) @@ -1055,6 +1119,27 @@ def test_registered_types_build_lazily(): assert fory.deserialize(data) == value +def test_lazy_completion_is_atomic(monkeypatch): + from pyfory import registry + + fory = Fory(xlang=True, compatible=True) + type_info = fory.register_type(FrozenChild, name="test.AtomicChild") + encode_typedef = registry.encode_typedef + + def fail_typedef(*_args, **_kwargs): + raise ValueError("TypeDef failure") + + monkeypatch.setattr(registry, "encode_typedef", fail_typedef) + with pytest.raises(ValueError, match="TypeDef failure"): + fory.serialize(FrozenChild(7)) + assert type_info.serializer is None + assert type_info.type_def is None + + monkeypatch.setattr(registry, "encode_typedef", encode_typedef) + value = FrozenChild(7) + assert fory.deserialize(fory.serialize(value)) == value + + def test_lazy_dataclass_serializer(): from pyfory.struct import DataClassStubSerializer From 64172a1d69b26b1723a9ceb397dfca5321396ea4 Mon Sep 17 00:00:00 2001 From: chaokunyang Date: Sun, 30 Aug 2026 13:52:53 +0800 Subject: [PATCH 150/168] fix(java): keep registry freeze owner-local --- .agents/languages/java.md | 6 +++--- .../main/java/org/apache/fory/resolver/TypeResolver.java | 7 ++++--- 2 files changed, 7 insertions(+), 6 deletions(-) diff --git a/.agents/languages/java.md b/.agents/languages/java.md index 83617a5fa0..e7720da07b 100644 --- a/.agents/languages/java.md +++ b/.agents/languages/java.md @@ -84,9 +84,9 @@ Load this file when changing anything under `java/` or when Java drives a cross- work, dynamic stream bytes-read accounting, or stale narrower-scope formulas. - Generated serializers must not retain runtime context fields. `Fory` should stay a root-operation facade rather than accumulating serializer or convenience state. - When the serializer class and constructor shape are known at the call site, prefer direct constructor lambdas or direct instantiation over reflective `Serializers.newSerializer(...)`. -- Each natural Java registry or public facade boundary owns one authoritative lifecycle fact. A - concrete resolver uses its shared registration snapshot; a thread-safe facade uses the shared - registry's one frozen flag. The first root serialization or deserialization establishes the +- Each natural Java registry or public facade boundary owns one authoritative lifecycle flag. A + concrete resolver owns its local frozen flag; a thread-safe facade uses the shared registry's + frozen flag for the facade-wide boundary. The first root serialization or deserialization sets the owning fact before codec work and never clears it, including after failure. Every explicit type, serializer, module, name, ID, or type-checker binding checks that fact before mutation. Disallow-list changes use the bound checker's resolver listeners. Do not add another lifecycle diff --git a/java/fory-core/src/main/java/org/apache/fory/resolver/TypeResolver.java b/java/fory-core/src/main/java/org/apache/fory/resolver/TypeResolver.java index a6b76cf9f2..a70edc6a2c 100644 --- a/java/fory-core/src/main/java/org/apache/fory/resolver/TypeResolver.java +++ b/java/fory-core/src/main/java/org/apache/fory/resolver/TypeResolver.java @@ -155,6 +155,7 @@ private static final class TransformedTypeInfo { // Caches for readTypeInfo(ReadContext) - persist between calls to avoid reloading // dynamically created classes that can't be found by Class.forName private final TypeInfo[] typeInfoCache; + private boolean registrationFrozen; protected TypeResolver( Config config, @@ -194,8 +195,7 @@ public final JITContext getJITContext() { @Internal public final boolean isRegistrationFrozen() { - IdentityHashMap, Integer> registeredClassIdMap = sharedRegistry.registeredClassIdMap; - return registeredClassIdMap != null && extRegistry.registeredClassIdMap == registeredClassIdMap; + return registrationFrozen; } public final boolean isCrossLanguage() { @@ -398,9 +398,10 @@ public abstract void registerSerializer( * points can call it defensively. */ public final void freezeRegistration() { - if (isRegistrationFrozen()) { + if (registrationFrozen) { return; } + registrationFrozen = true; // A root may start through a borrowed child, so the child must close the shared facade // boundary before publishing or adopting the registration snapshot. sharedRegistry.freezeRegistration(); From 37088f4a29c39acbdff0078decc73aff9aa4e9f9 Mon Sep 17 00:00:00 2001 From: chaokunyang Date: Sun, 30 Aug 2026 14:11:55 +0800 Subject: [PATCH 151/168] perf(java): isolate registry snapshot publication --- .../main/java/org/apache/fory/resolver/TypeResolver.java | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/java/fory-core/src/main/java/org/apache/fory/resolver/TypeResolver.java b/java/fory-core/src/main/java/org/apache/fory/resolver/TypeResolver.java index a70edc6a2c..4e47f4e106 100644 --- a/java/fory-core/src/main/java/org/apache/fory/resolver/TypeResolver.java +++ b/java/fory-core/src/main/java/org/apache/fory/resolver/TypeResolver.java @@ -398,9 +398,12 @@ public abstract void registerSerializer( * points can call it defensively. */ public final void freezeRegistration() { - if (registrationFrozen) { - return; + if (!registrationFrozen) { + publishRegistrationSnapshot(); } + } + + private void publishRegistrationSnapshot() { registrationFrozen = true; // A root may start through a borrowed child, so the child must close the shared facade // boundary before publishing or adopting the registration snapshot. From 62e98ce562513e088eb905ebc2c57e1c0a317a0f Mon Sep 17 00:00:00 2001 From: chaokunyang Date: Sun, 30 Aug 2026 14:17:15 +0800 Subject: [PATCH 152/168] fix(java): serialize facade setup before child freeze --- .../apache/fory/resolver/TypeResolver.java | 5 +- .../org/apache/fory/ThreadSafeForyTest.java | 72 +++++++++++++++++++ 2 files changed, 75 insertions(+), 2 deletions(-) diff --git a/java/fory-core/src/main/java/org/apache/fory/resolver/TypeResolver.java b/java/fory-core/src/main/java/org/apache/fory/resolver/TypeResolver.java index 4e47f4e106..c540456489 100644 --- a/java/fory-core/src/main/java/org/apache/fory/resolver/TypeResolver.java +++ b/java/fory-core/src/main/java/org/apache/fory/resolver/TypeResolver.java @@ -404,10 +404,11 @@ public final void freezeRegistration() { } private void publishRegistrationSnapshot() { - registrationFrozen = true; // A root may start through a borrowed child, so the child must close the shared facade - // boundary before publishing or adopting the registration snapshot. + // boundary before freezing itself or publishing/adopting the registration snapshot. Otherwise + // a facade registration already holding that boundary could fail partway through its children. sharedRegistry.freezeRegistration(); + registrationFrozen = true; sharedRegistry.setRegistrationIfAbsent( extRegistry.registeredClassIdMap, extRegistry.registeredClasses); extRegistry.freezeRegistration( diff --git a/java/fory-core/src/test/java/org/apache/fory/ThreadSafeForyTest.java b/java/fory-core/src/test/java/org/apache/fory/ThreadSafeForyTest.java index a66ef2e680..1a4f333bae 100644 --- a/java/fory-core/src/test/java/org/apache/fory/ThreadSafeForyTest.java +++ b/java/fory-core/src/test/java/org/apache/fory/ThreadSafeForyTest.java @@ -896,6 +896,78 @@ public void testExecuteRootRegistrationRace() throws InterruptedException { } } + @Test + public void testRegistryFreezeWaitsForSetup() throws InterruptedException { + ThreadSafeFory fory = + Fory.builder() + .withXlang(false) + .requireClassRegistration(true) + .withCompatible(false) + .buildThreadLocalFory(); + CountDownLatch childReady = new CountDownLatch(1); + CountDownLatch setupOwnsBoundary = new CountDownLatch(1); + CountDownLatch startRoot = new CountDownLatch(1); + CountDownLatch finishSetup = new CountDownLatch(1); + AtomicReference rootChild = new AtomicReference<>(); + AtomicReference rootError = new AtomicReference<>(); + AtomicReference setupError = new AtomicReference<>(); + Thread rootThread = + new Thread( + () -> { + try { + fory.execute( + child -> { + rootChild.set(child); + childReady.countDown(); + awaitUnchecked(startRoot); + child.serialize("value"); + return null; + }); + } catch (Throwable t) { + rootError.set(t); + } + }); + rootThread.start(); + assertTrue(childReady.await(30, TimeUnit.SECONDS)); + + Thread setupThread = + new Thread( + () -> { + try { + fory.registerCallback( + child -> { + if (child == rootChild.get()) { + setupOwnsBoundary.countDown(); + awaitUnchecked(finishSetup); + } + child.register(BeanA.class); + }); + } catch (Throwable t) { + setupError.set(t); + } + }); + setupThread.start(); + assertTrue(setupOwnsBoundary.await(30, TimeUnit.SECONDS)); + startRoot.countDown(); + awaitThreadBlocked(rootThread); + Assert.assertFalse(rootChild.get().getTypeResolver().isRegistrationFrozen()); + + finishSetup.countDown(); + rootThread.join(); + setupThread.join(); + assertNull(rootError.get()); + assertNull(setupError.get()); + assertTrue(fory.execute(child -> child.getTypeResolver().isRegistered(BeanA.class))); + } + + private static void awaitThreadBlocked(Thread thread) throws InterruptedException { + long deadline = System.nanoTime() + TimeUnit.SECONDS.toNanos(30); + while (thread.getState() != Thread.State.BLOCKED && System.nanoTime() < deadline) { + Thread.sleep(1); + } + assertEquals(thread.getState(), Thread.State.BLOCKED); + } + @Test public void testNonRootKeepsRegistrationOpen() { Fory direct = From 2d50b90a070c8dcad4ee58c985d06db89435fef0 Mon Sep 17 00:00:00 2001 From: chaokunyang Date: Sun, 30 Aug 2026 14:18:14 +0800 Subject: [PATCH 153/168] test(java): keep facade freeze race focused --- .../org/apache/fory/ThreadSafeForyTest.java | 57 ++++--------------- 1 file changed, 12 insertions(+), 45 deletions(-) diff --git a/java/fory-core/src/test/java/org/apache/fory/ThreadSafeForyTest.java b/java/fory-core/src/test/java/org/apache/fory/ThreadSafeForyTest.java index 1a4f333bae..c80add1827 100644 --- a/java/fory-core/src/test/java/org/apache/fory/ThreadSafeForyTest.java +++ b/java/fory-core/src/test/java/org/apache/fory/ThreadSafeForyTest.java @@ -897,67 +897,34 @@ public void testExecuteRootRegistrationRace() throws InterruptedException { } @Test - public void testRegistryFreezeWaitsForSetup() throws InterruptedException { - ThreadSafeFory fory = + public void testChildFreezeWaitsForFacade() throws InterruptedException { + SharedRegistry sharedRegistry = new SharedRegistry(); + Fory child = Fory.builder() .withXlang(false) .requireClassRegistration(true) .withCompatible(false) - .buildThreadLocalFory(); - CountDownLatch childReady = new CountDownLatch(1); - CountDownLatch setupOwnsBoundary = new CountDownLatch(1); - CountDownLatch startRoot = new CountDownLatch(1); - CountDownLatch finishSetup = new CountDownLatch(1); - AtomicReference rootChild = new AtomicReference<>(); + .withSharedRegistry(sharedRegistry) + .build(); AtomicReference rootError = new AtomicReference<>(); - AtomicReference setupError = new AtomicReference<>(); Thread rootThread = new Thread( () -> { try { - fory.execute( - child -> { - rootChild.set(child); - childReady.countDown(); - awaitUnchecked(startRoot); - child.serialize("value"); - return null; - }); + child.serialize("value"); } catch (Throwable t) { rootError.set(t); } }); - rootThread.start(); - assertTrue(childReady.await(30, TimeUnit.SECONDS)); - Thread setupThread = - new Thread( - () -> { - try { - fory.registerCallback( - child -> { - if (child == rootChild.get()) { - setupOwnsBoundary.countDown(); - awaitUnchecked(finishSetup); - } - child.register(BeanA.class); - }); - } catch (Throwable t) { - setupError.set(t); - } - }); - setupThread.start(); - assertTrue(setupOwnsBoundary.await(30, TimeUnit.SECONDS)); - startRoot.countDown(); - awaitThreadBlocked(rootThread); - Assert.assertFalse(rootChild.get().getTypeResolver().isRegistrationFrozen()); - - finishSetup.countDown(); + synchronized (sharedRegistry) { + rootThread.start(); + awaitThreadBlocked(rootThread); + Assert.assertFalse(child.getTypeResolver().isRegistrationFrozen()); + } rootThread.join(); - setupThread.join(); assertNull(rootError.get()); - assertNull(setupError.get()); - assertTrue(fory.execute(child -> child.getTypeResolver().isRegistered(BeanA.class))); + assertTrue(child.getTypeResolver().isRegistrationFrozen()); } private static void awaitThreadBlocked(Thread thread) throws InterruptedException { From b2ab95502ba7fd7add068f6a670f7d4dd74efe36 Mon Sep 17 00:00:00 2001 From: chaokunyang Date: Sun, 30 Aug 2026 14:47:15 +0800 Subject: [PATCH 154/168] perf(python): freeze compiled roots without a branch --- python/pyfory/serialization.pyx | 9 +++------ 1 file changed, 3 insertions(+), 6 deletions(-) diff --git a/python/pyfory/serialization.pyx b/python/pyfory/serialization.pyx index 85f701111e..3572f64d1f 100644 --- a/python/pyfory/serialization.pyx +++ b/python/pyfory/serialization.pyx @@ -1228,8 +1228,7 @@ cdef class Fory: ) def dump(self, obj, stream): - if not self.type_resolver._registry_frozen: - self.type_resolver._registry_frozen = True + self.type_resolver._registry_frozen = True try: self.buffer.set_writer_index(0) self.buffer.bind_output_stream(Buffer.wrap_output_stream(stream)) @@ -1253,8 +1252,7 @@ cdef class Fory: def serialize(self, obj, Buffer buffer=None, buffer_callback=None, unsupported_callback=None): cdef Buffer write_buffer - if not self.type_resolver._registry_frozen: - self.type_resolver._registry_frozen = True + self.type_resolver._registry_frozen = True try: write_buffer = self._serialize( obj, @@ -1298,8 +1296,7 @@ cdef class Fory: return buffer def deserialize(self, buffer, buffers=None, unsupported_objects=None): - if not self.type_resolver._registry_frozen: - self.type_resolver._registry_frozen = True + self.type_resolver._registry_frozen = True try: return self._deserialize( buffer, From ef96201935ed478fe905305ffb7273aee0311298 Mon Sep 17 00:00:00 2001 From: chaokunyang Date: Sun, 30 Aug 2026 15:23:21 +0800 Subject: [PATCH 155/168] fix(javascript): clean up failed input binding --- javascript/packages/core/lib/fory.ts | 4 ++-- javascript/test/rootCleanup.test.ts | 24 ++++++++++++++++++++++++ 2 files changed, 26 insertions(+), 2 deletions(-) diff --git a/javascript/packages/core/lib/fory.ts b/javascript/packages/core/lib/fory.ts index 07c6ae3c69..c87f1ac485 100644 --- a/javascript/packages/core/lib/fory.ts +++ b/javascript/packages/core/lib/fory.ts @@ -179,8 +179,8 @@ export default class Fory { deserialize(bytes: Uint8Array, serializer: Serializer = this.anySerializer): T | null { this.registrationFrozen = true; - this.readContext.reset(bytes); try { + this.readContext.reset(bytes); const reader = this.readContext.reader; const bitmap = reader.readUint8(); if (bitmap !== ConfigFlags.isCrossLanguageFlag) { @@ -243,8 +243,8 @@ export default class Fory { const rootHeader = ConfigFlags.isCrossLanguageFlag; rootDeserializer = (bytes: Uint8Array) => { this.registrationFrozen = true; - readContext.reset(bytes); try { + readContext.reset(bytes); const bitmap = reader.readUint8(); if (bitmap !== rootHeader) { this.throwInvalidRootHeader(bitmap); diff --git a/javascript/test/rootCleanup.test.ts b/javascript/test/rootCleanup.test.ts index cc95883208..2d293ad67b 100644 --- a/javascript/test/rootCleanup.test.ts +++ b/javascript/test/rootCleanup.test.ts @@ -91,6 +91,30 @@ describe.each([ expect(readContext.typeMetaCache.get(headerHash)).toBe(typeMeta); }); + + test("clears retained state when input binding fails", () => { + const fory = new Fory({ compatible: true, ref: true }); + const registered = fory.register(Type.struct(7615, {})); + const readContext = (fory as any).readContext; + const typeMeta = TypeMeta.fromTypeInfo(Type.struct(7616, {})); + + registered.serializer.readRef = () => { + readContext.refReader.reference({}); + populateLogicalTables(readContext, typeMeta); + return 7; + }; + expect(invoke(fory, registered, new Uint8Array([1]))).toBe(7); + + const invalidInput = new Uint8Array([1]); + Object.defineProperty(invalidInput, "buffer", { + get() { + throw new Error("input binding failed"); + }, + }); + expect(() => invoke(fory, registered, invalidInput)).toThrow("input binding failed"); + expect(readContext.reader.platformBuffer).toHaveLength(0); + expectRootStateCleared(readContext); + }); }); test("restores root write state after failure", () => { From 3744b4096f54b834753e5f9885852e7df4b16bf9 Mon Sep 17 00:00:00 2001 From: chaokunyang Date: Sun, 30 Aug 2026 18:49:29 +0800 Subject: [PATCH 156/168] fix(java): keep builder module dedupe --- .../org/apache/fory/config/ForyBuilder.java | 17 ++++++++++++++--- 1 file changed, 14 insertions(+), 3 deletions(-) diff --git a/java/fory-core/src/main/java/org/apache/fory/config/ForyBuilder.java b/java/fory-core/src/main/java/org/apache/fory/config/ForyBuilder.java index e66b0b7df8..b1887ce7aa 100644 --- a/java/fory-core/src/main/java/org/apache/fory/config/ForyBuilder.java +++ b/java/fory-core/src/main/java/org/apache/fory/config/ForyBuilder.java @@ -423,12 +423,14 @@ public ForyBuilder withSerializerFactory(SerializerFactory serializerFactory) { /** * Installs a runtime module into every Fory instance created by this builder. * - *

Each created Fory instance ignores repeated registration of the same module object. Dedupe - * uses identity, not {@link Object#equals(Object)}, so distinct module instances are installed - * independently. + *

Repeated registration of the same module object is ignored. Dedupe uses identity, not {@link + * Object#equals(Object)}, so distinct module instances are installed independently. */ public ForyBuilder withModule(ForyModule module) { ForyModule checkedModule = Objects.requireNonNull(module); + if (containsModule(checkedModule)) { + return this; + } modules.add(checkedModule); recordAction(b -> b.withModule(checkedModule)); return this; @@ -641,6 +643,15 @@ private void recordAction(Consumer action) { } } + private boolean containsModule(ForyModule module) { + for (int i = 0; i < modules.size(); i++) { + if (modules.get(i) == module) { + return true; + } + } + return false; + } + private void install(Fory fory) { for (int i = 0; i < serializerFactories.size(); i++) { fory.registerSerializerFactory(serializerFactories.get(i)); From b85458e38c1551157b8fbe0178ac4c81ae0b2888 Mon Sep 17 00:00:00 2001 From: chaokunyang Date: Sun, 30 Aug 2026 19:31:21 +0800 Subject: [PATCH 157/168] fix: remove noncausal registry policies --- .agents/languages/java.md | 3 - .agents/languages/python.md | 6 -- .../serialization/collection_serializer.h | 48 +++-------- cpp/fory/serialization/serialization_test.cc | 44 ---------- cpp/fory/serialization/struct_serializer.h | 3 - csharp/src/Fory/Fory.cs | 4 +- .../ExternalTypeSerializationTests.cs | 4 +- .../tests/Fory.Tests/RuntimeEdgeCaseTests.cs | 14 +--- .../java/custom-serializers.md | 5 -- .../python/configuration.md | 12 +-- .../python/custom-serializers.md | 16 ---- .../python/type-registration.md | 4 - docs/security/deserialization.md | 10 +-- .../xlang_implementation_guide.md | 10 +-- .../apache/fory/AbstractThreadSafeFory.java | 13 --- .../main/java/org/apache/fory/BaseFory.java | 22 +---- .../java/org/apache/fory/ThreadSafeFory.java | 4 - .../org/apache/fory/context/MapRefReader.java | 21 +++-- .../apache/fory/resolver/TypeResolver.java | 1 - .../org/apache/fory/ThreadSafeForyTest.java | 80 ------------------- .../apache/fory/context/MapRefReaderTest.java | 49 ------------ javascript/packages/core/lib/fory.ts | 6 ++ javascript/packages/core/lib/writer/index.ts | 12 +-- javascript/test/fory.test.ts | 19 +++++ .../serializer/kotlin/KotlinSerializers.java | 65 ++++++--------- .../kotlin/KotlinDefaultValueSupport.kt | 2 +- .../serializer/kotlin/DefaultValueTest.kt | 2 +- python/README.md | 15 +--- python/pyfory/_fory.py | 50 +++--------- python/pyfory/context.pxi | 14 ++-- python/pyfory/registry.py | 59 +++++++------- python/pyfory/serialization.pyx | 17 ++-- python/pyfory/serializer.py | 9 +-- python/pyfory/tests/test_function.py | 77 +----------------- .../pyfory/tests/test_metastring_resolver.py | 13 +-- python/pyfory/tests/test_policy.py | 28 +++---- python/pyfory/tests/test_reduce_serializer.py | 20 ++++- python/pyfory/tests/test_serializer.py | 25 ------ python/pyfory/tests/test_thread_safe.py | 61 -------------- .../serializer/scala/ScalaSerializers.java | 6 +- .../apache/fory/scala/ForySerializer.scala | 5 ++ swift/Sources/Fory/ReadContext.swift | 15 +++- .../ForyTests/CollectionSerializerTests.swift | 1 - swift/Tests/ForyTests/DecoderStateTests.swift | 1 - swift/Tests/ForyTests/ForySwiftTests.swift | 7 -- .../ForyTests/GraphMemoryBudgetTests.swift | 1 - .../Tests/ForyTests/TypeMetaDepthTests.swift | 3 - 47 files changed, 230 insertions(+), 676 deletions(-) delete mode 100644 java/fory-core/src/test/java/org/apache/fory/context/MapRefReaderTest.java diff --git a/.agents/languages/java.md b/.agents/languages/java.md index e7720da07b..c1e4d19c85 100644 --- a/.agents/languages/java.md +++ b/.agents/languages/java.md @@ -101,9 +101,6 @@ Load this file when changing anything under `java/` or when Java drives a cross- - `ThreadSafeFory.execute` exposes one borrowed child only for the callback. Do not retain that child or register through it; use the facade registration methods so every current and future child receives the same setup. -- A serializer instance registered on a thread-safe facade must implement `Shareable`. Resolver- - local serializers use the class, resolver-factory, or module path so every child runtime owns its - instance; never replay one resolver-bound serializer across children. - Serializer completion used by lazy, JIT, and generated serializers is an internal resolver-owned operation. It remains valid after registration freezes and must not be treated as explicit registration. diff --git a/.agents/languages/python.md b/.agents/languages/python.md index 0f61a3b10a..c5707ba2c6 100644 --- a/.agents/languages/python.md +++ b/.agents/languages/python.md @@ -25,18 +25,12 @@ Load this file when changing `python/`, Cython serialization, or Python xlang be collisions before publishing resolver maps. Lazy TypeDef completion preserves a configured serializer, does not retain partial state, and restores the prior serializer and TypeDef after failed completion without adding a lifecycle state. -- `ThreadSafeFory` accepts serializer classes or factories and constructs a serializer for each - child resolver. It must reject resolver-bound serializer instances instead of replaying one - instance across pooled children. - Registry freeze prohibits explicit type and serializer registration after the first root; it does not prohibit native runtime type resolution. Non-strict native writes may discover runtime classes or callables, and reads may resolve those authorized by the deserialization policy. Both paths may materialize resolver-owned type information or serializer cache entries without creating or changing an explicit type, serializer, ID, name, or policy registration. Do not describe these operations as late registration. -- Function serialization writes captured globals as a data-only exact `dict`. Keep the reader's - exact-type check before sizing or merging the namespace; a dict subclass or other mapping must not - introduce runtime behavior into function reconstruction. - Use explicit Cython fields and methods for fixed hot-path shapes. Avoid `__getattr__`, generic `object` fields, public bridge internals, or `Fory` backreferences where ownership can stay explicit. - Keep Python and Cython context/ref-tracking branch conditions and stack mutations semantically aligned unless a documented intentional difference exists. - Root deserialization graph memory budget state belongs to pure-Python and Cython `ReadContext`. diff --git a/cpp/fory/serialization/collection_serializer.h b/cpp/fory/serialization/collection_serializer.h index f6e23ad933..bafe03a46e 100644 --- a/cpp/fory/serialization/collection_serializer.h +++ b/cpp/fory/serialization/collection_serializer.h @@ -562,9 +562,6 @@ inline bool read_declared_same_type_collection(Container &result, auto elem = Serializer::read(ctx, RefMode::None, false); collection_insert(result, std::move(elem)); } - if (FORY_PREDICT_FALSE(ctx.has_error())) { - return false; - } return true; } @@ -586,15 +583,9 @@ inline bool read_declared_same_type_collection(Container &result, checkpoint_byte = ctx.buffer().logical_reader_index(); } } - if (FORY_PREDICT_FALSE(ctx.has_error())) { - return false; - } - if (checkpoint_item != length && - FORY_PREDICT_FALSE(!detail::settle_unbacked_container_items( - ctx, length - checkpoint_item, checkpoint_byte))) { - return false; - } - return true; + return checkpoint_item == length || + detail::settle_unbacked_container_items(ctx, length - checkpoint_item, + checkpoint_byte); } template @@ -625,15 +616,10 @@ read_declared_same_type_collection(std::forward_list &result, } } } - if (FORY_PREDICT_FALSE(ctx.has_error())) { - return false; - } if constexpr (!read_data_always_advances_v) { - if (checkpoint_item != length && - FORY_PREDICT_FALSE(!detail::settle_unbacked_container_items( - ctx, length - checkpoint_item, checkpoint_byte))) { - return false; - } + return checkpoint_item == length || + detail::settle_unbacked_container_items( + ctx, length - checkpoint_item, checkpoint_byte); } return true; } @@ -679,15 +665,10 @@ read_same_type_info_collection_body(Container &result, ReadContext &ctx, } } } - if (FORY_PREDICT_FALSE(ctx.has_error())) { - return false; - } if constexpr (MeasureProgress) { - if (checkpoint_item != length && - FORY_PREDICT_FALSE(!detail::settle_unbacked_container_items( - ctx, length - checkpoint_item, checkpoint_byte))) { - return false; - } + return checkpoint_item == length || + detail::settle_unbacked_container_items( + ctx, length - checkpoint_item, checkpoint_byte); } return true; } @@ -720,15 +701,10 @@ inline bool read_same_type_info_collection_body( } } } - if (FORY_PREDICT_FALSE(ctx.has_error())) { - return false; - } if constexpr (MeasureProgress) { - if (checkpoint_item != length && - FORY_PREDICT_FALSE(!detail::settle_unbacked_container_items( - ctx, length - checkpoint_item, checkpoint_byte))) { - return false; - } + return checkpoint_item == length || + detail::settle_unbacked_container_items( + ctx, length - checkpoint_item, checkpoint_byte); } return true; } diff --git a/cpp/fory/serialization/serialization_test.cc b/cpp/fory/serialization/serialization_test.cc index 50d05a745a..dc1a054cf9 100644 --- a/cpp/fory/serialization/serialization_test.cc +++ b/cpp/fory/serialization/serialization_test.cc @@ -29,7 +29,6 @@ #include #include #include -#include #include #include #include @@ -984,49 +983,6 @@ TEST(SerializationTest, SkipNoneListConsumesBudget) { ASSERT_TRUE(ctx.has_error()); } -TEST(SerializationTest, LastElementErrorSafepoints) { - Config config; - - std::vector declared_bytes{2}; - Buffer declared_buffer(declared_bytes); - ReadContext declared_ctx(config, std::make_unique()); - declared_ctx.attach(declared_buffer); - std::vector declared_values; - EXPECT_FALSE(read_declared_same_type_collection(declared_values, - declared_ctx, 2)); - EXPECT_TRUE(declared_ctx.has_error()); - - std::vector forward_bytes{2}; - Buffer forward_buffer(forward_bytes); - ReadContext forward_ctx(config, std::make_unique()); - forward_ctx.attach(forward_buffer); - std::forward_list forward_values; - EXPECT_FALSE(read_declared_same_type_collection(forward_values, - forward_ctx, 2)); - EXPECT_TRUE(forward_ctx.has_error()); - - std::vector type_info_bytes{2}; - Buffer type_info_buffer(type_info_bytes); - ReadContext type_info_ctx(config, std::make_unique()); - type_info_ctx.attach(type_info_buffer); - TypeInfo type_info; - type_info.harness.read_data_always_advances = true; - std::vector type_info_values; - EXPECT_FALSE(read_same_type_info_collection( - type_info_values, type_info_ctx, 2, type_info)); - EXPECT_TRUE(type_info_ctx.has_error()); - - std::vector measured_bytes{2}; - Buffer measured_buffer(measured_bytes); - ReadContext measured_ctx(config, std::make_unique()); - measured_ctx.attach(measured_buffer); - type_info.harness.read_data_always_advances = false; - std::vector measured_values; - EXPECT_FALSE(read_same_type_info_collection( - measured_values, measured_ctx, 2, type_info)); - EXPECT_TRUE(measured_ctx.has_error()); -} - // ============================================================================ // Character Type Tests (C++ native only) // ============================================================================ diff --git a/cpp/fory/serialization/struct_serializer.h b/cpp/fory/serialization/struct_serializer.h index cb1cf905f1..521e8f6a72 100644 --- a/cpp/fory/serialization/struct_serializer.h +++ b/cpp/fory/serialization/struct_serializer.h @@ -4615,9 +4615,6 @@ struct Serializer>> { } const TypeInfo *type_info = type_info_res.value(); ctx.write_struct_type_info(type_info); - if (FORY_PREDICT_FALSE(ctx.has_error())) { - return; - } } /// Read and validate type info. diff --git a/csharp/src/Fory/Fory.cs b/csharp/src/Fory/Fory.cs index 690e38da2d..169f8fdc53 100644 --- a/csharp/src/Fory/Fory.cs +++ b/csharp/src/Fory/Fory.cs @@ -177,8 +177,8 @@ public byte[] Serialize(in T value) _registryFrozen = true; ByteWriter writer = _writeContext.Writer; writer.Reset(); - // A previous failed root may leave references behind. Reset before serializer lookup, - // because generated or application serializer factories can fail during that lookup. + // Serializer lookup is part of the root and may fail before codec entry, so establish the + // root's clean context before invoking generated or application serializer factories. _writeContext.ResetFor(writer); Serializer serializer = _typeResolver.GetSerializer(); WriteHead(writer); diff --git a/csharp/tests/Fory.Tests/ExternalTypeSerializationTests.cs b/csharp/tests/Fory.Tests/ExternalTypeSerializationTests.cs index 2839269a81..66d8b23cc2 100644 --- a/csharp/tests/Fory.Tests/ExternalTypeSerializationTests.cs +++ b/csharp/tests/Fory.Tests/ExternalTypeSerializationTests.cs @@ -401,6 +401,8 @@ public void CustomSerializerReplacesGenerated() ExternalFields value = new() { Count = 19, Name = "custom" }; ForyRuntime generated = ForyRuntime.Builder().Build(); generated.Register(6106); + Assert.Throws( + () => generated.Register(6107)); byte[] generatedBytes = generated.Serialize(value); ForyRuntime custom = ForyRuntime.Builder().Build(); @@ -411,8 +413,6 @@ public void CustomSerializerReplacesGenerated() Assert.NotEqual(generatedBytes, customBytes); Assert.Equal(value.Count, decoded.Count); Assert.Equal(value.Name, decoded.Name); - Assert.Throws( - () => generated.Register(6107)); } [Fact] diff --git a/csharp/tests/Fory.Tests/RuntimeEdgeCaseTests.cs b/csharp/tests/Fory.Tests/RuntimeEdgeCaseTests.cs index b5f9bb1de2..b959f3889c 100644 --- a/csharp/tests/Fory.Tests/RuntimeEdgeCaseTests.cs +++ b/csharp/tests/Fory.Tests/RuntimeEdgeCaseTests.cs @@ -137,7 +137,6 @@ public override LookupFailureValue ReadData(ReadContext context) } } -[ForyStruct] public sealed class FailingWritePayload { public int Value { get; set; } @@ -834,15 +833,6 @@ public void ThreadSafeDottedNameRoundTrip() Assert.Equal("custom", decoded.Marker); } - [Fact] - public void RegistryFreezesAfterSuccessfulRoot() - { - ForyRuntime fory = ForyRuntime.Builder().Build(); - Assert.Equal(1, fory.Deserialize(fory.Serialize(1))); - - Assert.Throws(() => fory.Register(710)); - } - [Fact] public void FrozenRegistryRejectsBeforeMutation() { @@ -1041,11 +1031,11 @@ public void TrailingFailureKeepsTypeMetaCache(bool useSpan) if (useSpan) { - Assert.ThrowsAny(() => DeserializeIntSpan(fory, invalidPayload)); + Assert.Throws(() => DeserializeIntSpan(fory, invalidPayload)); } else { - Assert.ThrowsAny(() => fory.Deserialize(invalidPayload)); + Assert.Throws(() => fory.Deserialize(invalidPayload)); } Assert.True(context.TryGetTypeMetaByHash(firstHash, out _)); diff --git a/docs/object-serialization/java/custom-serializers.md b/docs/object-serialization/java/custom-serializers.md index 1cd483cf9d..efebb98979 100644 --- a/docs/object-serialization/java/custom-serializers.md +++ b/docs/object-serialization/java/custom-serializers.md @@ -212,11 +212,6 @@ fory.registerSerializer( CustomMap.class, resolver -> new CustomMapSerializer<>(resolver, CustomMap.class)); ``` -For `ThreadSafeFory`, pass a serializer class or resolver factory when the serializer is -runtime-local. The facade constructs one instance for each underlying runtime. An instance may be -registered directly only when it implements `Shareable`. Construct a runtime-local union serializer -inside a `ForyModule`, which is installed separately into every underlying runtime. - ## Shareability Implement the `Shareable` marker interface when the serializer can be safely reused across diff --git a/docs/object-serialization/python/configuration.md b/docs/object-serialization/python/configuration.md index d5801f2e4c..e984e0f4a8 100644 --- a/docs/object-serialization/python/configuration.md +++ b/docs/object-serialization/python/configuration.md @@ -57,8 +57,8 @@ class ThreadSafeFory: def __init__(self, fory_factory=None, **kwargs) ``` -Pass either a no-argument `fory_factory` that returns a configured `Fory` instance, or pass normal -`Fory` construction options through `**kwargs`. +When supplied, `fory_factory` creates each pooled `Fory` instance. Otherwise, `**kwargs` are passed +to the normal `Fory` constructor. ## Parameters @@ -84,7 +84,6 @@ Pass either a no-argument `fory_factory` that returns a configured `Fory` instan ```python fory = pyfory.Fory(xlang=True) -thread_safe_fory = pyfory.ThreadSafeFory(xlang=True) # Serialization (serialize/deserialize are identical to dumps/loads) data: bytes = fory.serialize(obj) @@ -94,14 +93,11 @@ obj = fory.deserialize(data) data: bytes = fory.dumps(obj) obj = fory.loads(data) -# Direct Fory registration by id; serializer instances belong to that Fory. +# Type registration by id fory.register(MyClass, type_id=123) fory.register(MyClass, type_id=123, serializer=custom_serializer) -# ThreadSafeFory constructs one serializer per pooled child from a class or factory. -thread_safe_fory.register(MyClass, type_id=123, serializer=CustomSerializer) - -# Direct Fory registration by name +# Type registration by name fory.register(MyClass, name="my.package.MyClass") fory.register(MyClass, name="my.package.MyClass", serializer=custom_serializer) ``` diff --git a/docs/object-serialization/python/custom-serializers.md b/docs/object-serialization/python/custom-serializers.md index 6a516a9731..da907d4302 100644 --- a/docs/object-serialization/python/custom-serializers.md +++ b/docs/object-serialization/python/custom-serializers.md @@ -121,9 +121,6 @@ value = buffer.read_bool() ## Registering Custom Serializers -Attach custom serializers to application types registered by name or user ID. Serializers for -built-in framework types cannot be replaced. - ```python fory = pyfory.Fory(xlang=False) @@ -134,19 +131,6 @@ fory.register(MyClass, type_id=100, serializer=MySerializer(fory.type_resolver, fory.register(MyClass, name="com.example.MyClass", serializer=MySerializer(fory.type_resolver, MyClass)) ``` -### Thread-safe registration - -`ThreadSafeFory` accepts a serializer class or factory rather than an already constructed -serializer: - -```python -thread_safe = pyfory.ThreadSafeFory(xlang=False) -thread_safe.register(Foo, type_id=100, serializer=FooSerializer) -``` - -A serializer factory passed to `ThreadSafeFory.register` must accept `(resolver, type)` or -`(resolver)` and return a serializer for that resolver and registered type. - ## Related Topics - [Type Registration](type-registration.md) - Registration patterns diff --git a/docs/object-serialization/python/type-registration.md b/docs/object-serialization/python/type-registration.md index 89b68e9e15..f0d4322173 100644 --- a/docs/object-serialization/python/type-registration.md +++ b/docs/object-serialization/python/type-registration.md @@ -72,10 +72,6 @@ for model_class in [User, Order, Product, Invoice]: type_id += 1 ``` -A direct `Fory` may receive a serializer instance. `ThreadSafeFory` accepts a serializer class or -factory. The factory must accept `(resolver, type)` or `(resolver)` and return a serializer for that -resolver and registered type. - ## Strict Mode Relationship With `strict=True`, Fory loads and instantiates only registered application diff --git a/docs/security/deserialization.md b/docs/security/deserialization.md index 53d9097b49..ccc78fb0d3 100644 --- a/docs/security/deserialization.md +++ b/docs/security/deserialization.md @@ -618,14 +618,14 @@ that case, classify the behavior by concrete impact: ## Registry Lifecycle The first root serialization or deserialization permanently closes explicit type and serializer -registration, including when that operation fails. Each natural registration owner keeps one +registration, including when that operation fails. Each natural registration owner keeps exactly +one authoritative lifecycle fact, and every later explicit registration attempt fails before changing type, serializer, ID, name, metadata, or policy bindings. A thread-safe facade with its own public registration surface may own its boundary fact, but must not mirror a child registry's lifecycle. -Each natural owner must keep exactly one authoritative lifecycle fact. That fact may be an -owner-native flag or the presence of an immutable registry snapshot. Do not add another lifecycle -state, a registration commit or rollback path, or eager whole-registry preparation solely to -implement freeze. +That fact may be an owner-native flag or the presence of an immutable registry snapshot. Do not add +another lifecycle state, a registration commit or rollback path, or eager whole-registry +preparation solely to implement freeze. Registry freeze does not disable native runtime type resolution. When a mode supports unregistered types, a root may still discover an allowed runtime type and materialize resolver-owned metadata, diff --git a/docs/specification/xlang_implementation_guide.md b/docs/specification/xlang_implementation_guide.md index 63a67afe73..d312a6d9bf 100644 --- a/docs/specification/xlang_implementation_guide.md +++ b/docs/specification/xlang_implementation_guide.md @@ -83,14 +83,14 @@ not the place where nested serializers do their work. - resetting operation-local context state at the top-level root boundary Explicit type and serializer registration is open only before the first root serialization or -deserialization. Starting either root establishes one authoritative frozen lifecycle fact for the +deserialization. Starting either root establishes exactly one authoritative frozen lifecycle fact +for the natural registration owner before codec work and never clears it, including when the root fails. Every explicit registration path checks that fact before mutation. A thread-safe facade with its own public registration surface may own its boundary fact, but must not mirror a child registry's -lifecycle. Each natural owner must keep exactly one authoritative lifecycle fact. That fact may be -an owner-native flag or the presence of an immutable registry snapshot. Do not add another -lifecycle state, a registration commit or rollback path, or eager whole-registry preparation solely -to implement freeze. +lifecycle. That fact may be an owner-native flag or the presence of an immutable registry snapshot. +Do not add another lifecycle state, a registration commit or rollback path, or eager whole-registry +preparation solely to implement freeze. In JavaScript, `Fory` owns this flag. `TypeResolver` owns the registration maps but must not carry a second lifecycle flag. diff --git a/java/fory-core/src/main/java/org/apache/fory/AbstractThreadSafeFory.java b/java/fory-core/src/main/java/org/apache/fory/AbstractThreadSafeFory.java index bd7437ab79..0098854119 100644 --- a/java/fory-core/src/main/java/org/apache/fory/AbstractThreadSafeFory.java +++ b/java/fory-core/src/main/java/org/apache/fory/AbstractThreadSafeFory.java @@ -24,7 +24,6 @@ import org.apache.fory.resolver.TypeResolver; import org.apache.fory.serializer.Serializer; import org.apache.fory.serializer.SerializerFactory; -import org.apache.fory.serializer.Shareable; public abstract class AbstractThreadSafeFory implements ThreadSafeFory { @Override @@ -74,14 +73,12 @@ public void register(ForyModule module) { public void registerUnion( Class cls, int id, org.apache.fory.serializer.Serializer serializer) { - checkShareable(serializer); registerCallback(fory -> fory.registerUnion(cls, id, serializer)); } @Override public void registerUnion( Class cls, String name, org.apache.fory.serializer.Serializer serializer) { - checkShareable(serializer); registerCallback(fory -> fory.registerUnion(cls, name, serializer)); } @@ -90,7 +87,6 @@ public void registerUnion( String namespace, String typeName, org.apache.fory.serializer.Serializer serializer) { - checkShareable(serializer); registerCallback(fory -> fory.registerUnion(cls, namespace, typeName, serializer)); } @@ -101,7 +97,6 @@ public void registerSerializer(Class type, Class se @Override public void registerSerializer(Class type, Serializer serializer) { - checkShareable(serializer); registerCallback(fory -> fory.registerSerializer(type, serializer)); } @@ -119,7 +114,6 @@ public void registerSerializerAndType( @Override public void registerSerializerAndType(Class type, Serializer serializer) { - checkShareable(serializer); registerCallback(fory -> fory.registerSerializerAndType(type, serializer)); } @@ -147,11 +141,4 @@ public void ensureSerializersCompiled() { return null; }); } - - private static void checkShareable(Serializer serializer) { - if (!(serializer instanceof Shareable)) { - throw new IllegalArgumentException( - "Thread-safe Fory requires serializer instances to implement Shareable"); - } - } } diff --git a/java/fory-core/src/main/java/org/apache/fory/BaseFory.java b/java/fory-core/src/main/java/org/apache/fory/BaseFory.java index c52f0dcf53..a792d37263 100644 --- a/java/fory-core/src/main/java/org/apache/fory/BaseFory.java +++ b/java/fory-core/src/main/java/org/apache/fory/BaseFory.java @@ -95,24 +95,16 @@ public interface BaseFory { */ void register(ForyModule module); - /** - * Register a union by ID. A serializer instance passed to {@link ThreadSafeFory} must implement - * {@link org.apache.fory.serializer.Shareable}. - */ + /** Register a union by ID. */ void registerUnion(Class cls, int id, Serializer serializer); /** * Register a union with a name used for cross-language serialization. Names with `.` are split by - * the last `.` into namespace and type name. A serializer instance passed to {@link - * ThreadSafeFory} must implement {@link org.apache.fory.serializer.Shareable}. + * the last `.` into namespace and type name. */ void registerUnion(Class cls, String name, Serializer serializer); - /** - * Register a union with explicit namespace and type name. The type name must not contain `.`. A - * serializer instance passed to {@link ThreadSafeFory} must implement {@link - * org.apache.fory.serializer.Shareable}. - */ + /** Register a union with explicit namespace and type name. The type name must not contain `.`. */ void registerUnion(Class cls, String namespace, String typeName, Serializer serializer); /** @@ -134,10 +126,6 @@ public interface BaseFory { * *

NOTE: The registration order is important. If registration order is inconsistent, the * allocated ID will be different, and the deserialization will failed !!! - * - *

A serializer instance passed to {@link ThreadSafeFory} must implement {@link - * org.apache.fory.serializer.Shareable}. Use the class or resolver-factory overload for a - * runtime-local serializer. */ void registerSerializer(Class type, Serializer serializer); @@ -170,10 +158,6 @@ public interface BaseFory { * *

NOTE: The registration order is important. If registration order is inconsistent, the * allocated ID will be different, and the deserialization will failed !!! - * - *

A serializer instance passed to {@link ThreadSafeFory} must implement {@link - * org.apache.fory.serializer.Shareable}. Use the class or resolver-factory overload for a - * runtime-local serializer. */ void registerSerializerAndType(Class type, Serializer serializer); diff --git a/java/fory-core/src/main/java/org/apache/fory/ThreadSafeFory.java b/java/fory-core/src/main/java/org/apache/fory/ThreadSafeFory.java index cb6605c228..45dc8ae769 100644 --- a/java/fory-core/src/main/java/org/apache/fory/ThreadSafeFory.java +++ b/java/fory-core/src/main/java/org/apache/fory/ThreadSafeFory.java @@ -31,10 +31,6 @@ *

The runtime class loader is fixed when the thread-safe serializer is built. If you need a * different class loader, build a different {@link ThreadSafeFory} instance. * - *

Serializer instances registered on this facade must implement {@link - * org.apache.fory.serializer.Shareable}. Use a serializer class or resolver factory for - * runtime-local serializers so every underlying {@link Fory} owns its instance. - * *

Complete facade registration before concurrent serialization, deserialization, copy, or {@link * #execute} calls begin. */ diff --git a/java/fory-core/src/main/java/org/apache/fory/context/MapRefReader.java b/java/fory-core/src/main/java/org/apache/fory/context/MapRefReader.java index 3fe0b66b87..bc4a1a3ef8 100644 --- a/java/fory-core/src/main/java/org/apache/fory/context/MapRefReader.java +++ b/java/fory-core/src/main/java/org/apache/fory/context/MapRefReader.java @@ -34,6 +34,8 @@ public final class MapRefReader implements RefReader { private static final int DEFAULT_ARRAY_CAPACITY = 3; + private long readCounter; + private long readTotalObjectSize = 0; private final ObjectArray readObjects = new ObjectArray(DEFAULT_ARRAY_CAPACITY); private final IntArray readRefIds = new IntArray(DEFAULT_ARRAY_CAPACITY); private Object readObject; @@ -122,15 +124,22 @@ public void setReadRef(int id, Object object) { } } - /** Clears the current read state and keeps capacity based on the most recent operation. */ + /** Clears the current read state and keeps an approximate capacity for the next operation. */ @Override public void reset() { - int nextCapacity = Math.max(readObjects.size(), DEFAULT_ARRAY_CAPACITY); - if (readObjects.objects.length > (long) nextCapacity * 4) { - readObjects.clearApproximate(nextCapacity); - } else { - readObjects.clear(); + long totalObjectSize = this.readTotalObjectSize + readObjects.size(); + long counter = this.readCounter + 1; + if (counter < 0 || totalObjectSize < 0) { + counter = 1; + totalObjectSize = readObjects.size(); + } + this.readCounter = counter; + this.readTotalObjectSize = totalObjectSize; + int avg = (int) (totalObjectSize / counter); + if (avg <= DEFAULT_ARRAY_CAPACITY) { + avg = DEFAULT_ARRAY_CAPACITY; } + readObjects.clearApproximate(avg); readRefIds.clear(); readObject = null; } diff --git a/java/fory-core/src/main/java/org/apache/fory/resolver/TypeResolver.java b/java/fory-core/src/main/java/org/apache/fory/resolver/TypeResolver.java index c540456489..b42cb2e75e 100644 --- a/java/fory-core/src/main/java/org/apache/fory/resolver/TypeResolver.java +++ b/java/fory-core/src/main/java/org/apache/fory/resolver/TypeResolver.java @@ -385,7 +385,6 @@ public abstract void registerSerializer( * @param type the class to register * @param serializer the serializer to use */ - @Internal public abstract void registerInternalSerializer(Class type, Serializer serializer); /** diff --git a/java/fory-core/src/test/java/org/apache/fory/ThreadSafeForyTest.java b/java/fory-core/src/test/java/org/apache/fory/ThreadSafeForyTest.java index c80add1827..f4506e0eab 100644 --- a/java/fory-core/src/test/java/org/apache/fory/ThreadSafeForyTest.java +++ b/java/fory-core/src/test/java/org/apache/fory/ThreadSafeForyTest.java @@ -21,14 +21,11 @@ import static org.testng.Assert.assertEquals; import static org.testng.Assert.assertNotNull; -import static org.testng.Assert.assertNotSame; import static org.testng.Assert.assertNull; import static org.testng.Assert.assertSame; import static org.testng.Assert.assertTrue; import java.nio.ByteBuffer; -import java.util.Arrays; -import java.util.List; import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.CountDownLatch; import java.util.concurrent.ExecutorService; @@ -36,7 +33,6 @@ import java.util.concurrent.TimeUnit; import java.util.concurrent.atomic.AtomicInteger; import java.util.concurrent.atomic.AtomicReference; -import java.util.function.Consumer; import lombok.Data; import org.apache.fory.context.CopyContext; import org.apache.fory.context.MetaReadContext; @@ -49,7 +45,6 @@ import org.apache.fory.resolver.SharedRegistry; import org.apache.fory.resolver.TypeResolver; import org.apache.fory.serializer.Serializer; -import org.apache.fory.serializer.Shareable; import org.apache.fory.test.bean.BeanA; import org.apache.fory.test.bean.BeanB; import org.testng.Assert; @@ -509,29 +504,8 @@ static class Foo { } public static class FooSerializer extends Serializer { - final TypeResolver typeResolver; - public FooSerializer(TypeResolver typeResolver, Class type) { super(typeResolver.getConfig(), type); - this.typeResolver = typeResolver; - } - - @Override - public void write(WriteContext writeContext, Foo value) { - writeContext.getBuffer().writeInt32(value.f1); - } - - @Override - public Foo read(ReadContext readContext) { - Foo foo = new Foo(); - foo.f1 = readContext.getBuffer().readInt32(); - return foo; - } - } - - public static final class ShareableFooSerializer extends Serializer implements Shareable { - public ShareableFooSerializer(TypeResolver typeResolver) { - super(typeResolver.getConfig(), Foo.class); } @Override @@ -637,60 +611,6 @@ public void testSerializerRegister() { }); } - @Test - public void testSerializerInstanceOwnership() throws InterruptedException { - Fory direct = - Fory.builder() - .withXlang(false) - .requireClassRegistration(false) - .withCompatible(false) - .build(); - FooSerializer local = new FooSerializer(direct.getTypeResolver(), Foo.class); - List> registrations = - Arrays.asList( - fory -> fory.registerSerializer(Foo.class, local), - fory -> fory.registerSerializerAndType(Foo.class, local), - fory -> fory.registerUnion(Foo.class, 101, local), - fory -> fory.registerUnion(Foo.class, "test.Foo", local), - fory -> fory.registerUnion(Foo.class, "test", "Foo", local)); - for (Consumer registration : registrations) { - ThreadSafeFory fory = newThreadSafeRuntimes()[0]; - Assert.assertThrows(IllegalArgumentException.class, () -> registration.accept(fory)); - } - - ShareableFooSerializer shareable = new ShareableFooSerializer(direct.getTypeResolver()); - ThreadSafeFory shared = newThreadSafeRuntimes()[0]; - shared.registerSerializer(Foo.class, shareable); - assertSame(shared.execute(fory -> fory.getTypeResolver().getSerializer(Foo.class)), shareable); - - ThreadSafeFory localFactory = newThreadSafeRuntimes()[0]; - localFactory.registerSerializer(Foo.class, FooSerializer.class); - AtomicReference first = new AtomicReference<>(); - AtomicReference second = new AtomicReference<>(); - AtomicReference error = new AtomicReference<>(); - Thread firstThread = new Thread(() -> captureSerializer(localFactory, first, error)); - Thread secondThread = new Thread(() -> captureSerializer(localFactory, second, error)); - firstThread.start(); - secondThread.start(); - firstThread.join(); - secondThread.join(); - assertNull(error.get()); - assertNotSame(first.get(), second.get()); - assertNotSame(first.get().typeResolver, second.get().typeResolver); - } - - private static void captureSerializer( - ThreadSafeFory fory, - AtomicReference serializer, - AtomicReference error) { - try { - serializer.set( - fory.execute(child -> (FooSerializer) child.getTypeResolver().getSerializer(Foo.class))); - } catch (Throwable t) { - error.compareAndSet(null, t); - } - } - @Test public void testExecuteConcurrency() throws InterruptedException { for (ThreadSafeFory fory : newThreadSafeRuntimes()) { diff --git a/java/fory-core/src/test/java/org/apache/fory/context/MapRefReaderTest.java b/java/fory-core/src/test/java/org/apache/fory/context/MapRefReaderTest.java deleted file mode 100644 index 7852402b06..0000000000 --- a/java/fory-core/src/test/java/org/apache/fory/context/MapRefReaderTest.java +++ /dev/null @@ -1,49 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -package org.apache.fory.context; - -import org.apache.fory.TestUtils; -import org.apache.fory.collection.ObjectArray; -import org.testng.Assert; -import org.testng.annotations.Test; - -public class MapRefReaderTest { - @Test - public void testResetUsesRecentSize() { - MapRefReader reader = new MapRefReader(); - for (int i = 0; i < 100; i++) { - reader.preserveRefId(); - reader.reference(i); - } - ObjectArray readObjects = TestUtils.getFieldValue(reader, "readObjects"); - Object[] proportionateTable = readObjects.objects; - reader.reset(); - Assert.assertSame(readObjects.objects, proportionateTable); - for (int i = 0; i < 100; i++) { - Assert.assertNull(proportionateTable[i]); - } - - reader.preserveRefId(); - reader.reference("small"); - reader.reset(); - Assert.assertEquals(readObjects.objects.length, 3); - Assert.assertNull(readObjects.objects[0]); - } -} diff --git a/javascript/packages/core/lib/fory.ts b/javascript/packages/core/lib/fory.ts index c87f1ac485..8ca9f46a85 100644 --- a/javascript/packages/core/lib/fory.ts +++ b/javascript/packages/core/lib/fory.ts @@ -161,6 +161,9 @@ export default class Fory { creator: constructor, customSerializer, }).generateSerializer(typeInfo); + if (this.registrationFrozen) { + throw new Error("types and serializers must be registered before the first root operation"); + } this.typeResolver.registerSerializer(typeInfo, serializer); } else { const typeInfo = constructor; @@ -168,6 +171,9 @@ export default class Fory { serializer = new Gen(this.typeResolver, { customSerializer, }).generateSerializer(typeInfo); + if (this.registrationFrozen) { + throw new Error("types and serializers must be registered before the first root operation"); + } this.typeResolver.registerSerializer(typeInfo, serializer); } return { diff --git a/javascript/packages/core/lib/writer/index.ts b/javascript/packages/core/lib/writer/index.ts index 03d8fa505a..7a4b18b339 100644 --- a/javascript/packages/core/lib/writer/index.ts +++ b/javascript/packages/core/lib/writer/index.ts @@ -63,7 +63,7 @@ export class BinaryWriter { hps?: Hps; } = {}, ) { - this.initPool(); + this.initPoll(); this.config = config; this.hpsEnable = Boolean(config?.hps); this.internalStringDetector = getInternalStringDetector(); @@ -76,7 +76,7 @@ export class BinaryWriter { } } - private initPool() { + private initPoll() { this.byteLength = 1024 * 100; this.platformBuffer = alloc(this.byteLength); this.dataView = new DataView(this.platformBuffer.buffer, this.platformBuffer.byteOffset); @@ -100,7 +100,7 @@ export class BinaryWriter { this.cursor = 0; this.reserved = 0; // Successful dumps already release a large buffer; this also covers aborted roots. - this.releaseLargeBuffer(); + this.tryFreePool(); } bool(bool: boolean) { @@ -498,16 +498,16 @@ export class BinaryWriter { this.platformBuffer[this.cursor++] = Number(val & 255n); } - private releaseLargeBuffer() { + tryFreePool() { if (this.byteLength > MAX_POOL_SIZE) { - this.initPool(); + this.initPoll(); } } dump() { const result = alloc(this.cursor); this.platformBuffer.copy(result, 0, 0, this.cursor); - this.releaseLargeBuffer(); + this.tryFreePool(); return result; } diff --git a/javascript/test/fory.test.ts b/javascript/test/fory.test.ts index 2cda43fc88..c3e5073a18 100644 --- a/javascript/test/fory.test.ts +++ b/javascript/test/fory.test.ts @@ -103,6 +103,25 @@ describe("fory", () => { expect(() => fory.register(Type.struct(8104, {}))).toThrow(); }); + test("freezes during serializer generation", () => { + let armed = false; + let fory: Fory; + fory = new Fory({ + compatible: false, + hooks: { + afterCodeGenerated(code) { + if (armed) { + fory.serialize(null); + } + return code; + }, + }, + }); + armed = true; + + expect(() => fory.register(Type.struct(8105, {}))).toThrow(); + }); + test.each(["serialize", "deserialize"] as const)("freezes after %s", (operation) => { const fory = new Fory({ compatible: false }); diff --git a/kotlin/fory-kotlin/src/main/java/org/apache/fory/serializer/kotlin/KotlinSerializers.java b/kotlin/fory-kotlin/src/main/java/org/apache/fory/serializer/kotlin/KotlinSerializers.java index 204d32f448..663c72f5f7 100644 --- a/kotlin/fory-kotlin/src/main/java/org/apache/fory/serializer/kotlin/KotlinSerializers.java +++ b/kotlin/fory-kotlin/src/main/java/org/apache/fory/serializer/kotlin/KotlinSerializers.java @@ -20,17 +20,12 @@ package org.apache.fory.serializer.kotlin; import java.util.Objects; -import kotlin.Result; +import kotlin.*; import kotlin.UByteArray; import kotlin.UIntArray; import kotlin.ULongArray; import kotlin.UShortArray; -import kotlin.text.CharCategory; -import kotlin.text.CharDirectionality; -import kotlin.text.HexFormat; -import kotlin.text.MatchGroup; -import kotlin.text.Regex; -import kotlin.text.RegexOption; +import kotlin.text.*; import kotlin.time.Duration; import kotlin.time.DurationUnit; import kotlin.time.TimedValue; @@ -64,7 +59,7 @@ public static void registerSerializers(Fory fory) { @Internal public static void installSerializers(Fory fory) { - DefaultValueUtils.setKotlinDefaultValueSupport(KotlinDefaultValueSupport.INSTANCE); + DefaultValueUtils.setKotlinDefaultValueSupport(new KotlinDefaultValueSupport()); TypeResolver resolver = fory.getTypeResolver(); if (resolver.isCrossLanguage()) { return; @@ -209,39 +204,26 @@ public static void registerType(Fory fory, Class cls, String namespace, Strin fory.getTypeResolver().register(cls, namespace, typeName); } - // Combined generated registration preserves the STRUCT TypeInfo created first; - // registerSerializer would reclassify that wire identity as EXT. public static void register(Fory fory, Class cls) { - TypeResolver resolver = fory.getTypeResolver(); - resolver.register(cls); - Serializer serializer = newGeneratedSerializer(resolver, cls); - resolver.checkRegistrationOpen(); - resolver.setSerializer(cls, serializer); + // Generated construction resolves the registered STRUCT TypeInfo. Publish that identity first; + // registerSerializer rechecks freeze after construction before installing the serializer. + fory.register(cls); + registerSerializer(fory, cls); } public static void register(Fory fory, Class cls, long typeId) { - TypeResolver resolver = fory.getTypeResolver(); - resolver.register(cls, typeId); - Serializer serializer = newGeneratedSerializer(resolver, cls); - resolver.checkRegistrationOpen(); - resolver.setSerializer(cls, serializer); + registerType(fory, cls, typeId); + registerSerializer(fory, cls); } public static void register(Fory fory, Class cls, String name) { - TypeResolver resolver = fory.getTypeResolver(); - fory.register(cls, name); - Serializer serializer = newGeneratedSerializer(resolver, cls); - resolver.checkRegistrationOpen(); - resolver.setSerializer(cls, serializer); + registerType(fory, cls, name); + registerSerializer(fory, cls); } public static void register(Fory fory, Class cls, String namespace, String typeName) { - checkTypeName(typeName); - TypeResolver resolver = fory.getTypeResolver(); - resolver.register(cls, namespace, typeName); - Serializer serializer = newGeneratedSerializer(resolver, cls); - resolver.checkRegistrationOpen(); - resolver.setSerializer(cls, serializer); + registerType(fory, cls, namespace, typeName); + registerSerializer(fory, cls); } public static void registerSerializer(Fory fory, Class cls) { @@ -249,7 +231,12 @@ public static void registerSerializer(Fory fory, Class cls) { resolver.checkRegistrationOpen(); Serializer serializer = newGeneratedSerializer(resolver, cls); resolver.checkRegistrationOpen(); - resolver.setSerializer(cls, serializer); + if (resolver.isRegistered(cls)) { + // Preserve the registered STRUCT TypeInfo; registerSerializer would reclassify it as EXT. + resolver.setSerializer(cls, serializer); + } else { + resolver.registerSerializer(cls, serializer); + } } public static void registerEnum(Fory fory, Class cls, long typeId) { @@ -279,9 +266,8 @@ public static void registerUnion(Fory fory, Class cls, long typeId) { TypeResolver resolver = fory.getTypeResolver(); resolver.checkRegistrationOpen(); Serializer serializer = newGeneratedSerializer(resolver, cls); - Class[] caseClasses = cls.getDeclaredClasses(); resolver.registerUnion(cls, typeId, serializer); - registerCaseAliases(fory, cls, caseClasses); + registerCaseAliases(fory, cls); } public static void registerUnion(Fory fory, Class cls, String namespace, String typeName) { @@ -289,9 +275,8 @@ public static void registerUnion(Fory fory, Class cls, String namespace, Stri TypeResolver resolver = fory.getTypeResolver(); resolver.checkRegistrationOpen(); Serializer serializer = newGeneratedSerializer(resolver, cls); - Class[] caseClasses = cls.getDeclaredClasses(); resolver.registerUnion(cls, namespace, typeName, serializer); - registerCaseAliases(fory, cls, caseClasses); + registerCaseAliases(fory, cls); } public static void registerUnion(Fory fory, Class cls, String name) { @@ -299,9 +284,8 @@ public static void registerUnion(Fory fory, Class cls, String name) { resolver.checkRegistrationOpen(); String[] parts = splitName(name); Serializer serializer = newGeneratedSerializer(resolver, cls); - Class[] caseClasses = cls.getDeclaredClasses(); resolver.registerUnion(cls, parts[0], parts[1], serializer); - registerCaseAliases(fory, cls, caseClasses); + registerCaseAliases(fory, cls); } private static Serializer newGeneratedSerializer(TypeResolver resolver, Class cls) { @@ -348,9 +332,8 @@ private static Class enumClass(Class cls) { return (Class) cls; } - private static void registerCaseAliases( - Fory fory, Class canonicalClass, Class[] caseClasses) { - for (Class nestedClass : caseClasses) { + private static void registerCaseAliases(Fory fory, Class canonicalClass) { + for (Class nestedClass : canonicalClass.getDeclaredClasses()) { if (canonicalClass.isAssignableFrom(nestedClass)) { fory.getTypeResolver().registerRuntimeTypeAlias(nestedClass, canonicalClass); } diff --git a/kotlin/fory-kotlin/src/main/kotlin/org/apache/fory/serializer/kotlin/KotlinDefaultValueSupport.kt b/kotlin/fory-kotlin/src/main/kotlin/org/apache/fory/serializer/kotlin/KotlinDefaultValueSupport.kt index 6ac8f23960..dff2ebce21 100644 --- a/kotlin/fory-kotlin/src/main/kotlin/org/apache/fory/serializer/kotlin/KotlinDefaultValueSupport.kt +++ b/kotlin/fory-kotlin/src/main/kotlin/org/apache/fory/serializer/kotlin/KotlinDefaultValueSupport.kt @@ -41,7 +41,7 @@ import org.apache.fory.util.DefaultValueUtils * This class uses Kotlin native reflection to analyze data classes and extract default values from * their primary constructor parameters. */ -internal object KotlinDefaultValueSupport : DefaultValueUtils.DefaultValueSupport() { +internal class KotlinDefaultValueSupport : DefaultValueUtils.DefaultValueSupport() { private val LOG: Logger = LoggerFactory.getLogger(KotlinDefaultValueSupport::class.java) private val cachedKotlinDataClassDefaultValues = ClassValueCache.newClassKeyCache>(32) diff --git a/kotlin/fory-kotlin/src/test/kotlin/org/apache/fory/serializer/kotlin/DefaultValueTest.kt b/kotlin/fory-kotlin/src/test/kotlin/org/apache/fory/serializer/kotlin/DefaultValueTest.kt index 771c6446f4..67c76a1ab2 100644 --- a/kotlin/fory-kotlin/src/test/kotlin/org/apache/fory/serializer/kotlin/DefaultValueTest.kt +++ b/kotlin/fory-kotlin/src/test/kotlin/org/apache/fory/serializer/kotlin/DefaultValueTest.kt @@ -41,7 +41,7 @@ data class ClassObjectRequiredWithDefaults(val v: RegularClass, val x: Int = 3) class DefaultValueTest { - private val support = KotlinDefaultValueSupport + private val support = KotlinDefaultValueSupport() @Test fun testHasDefaultValues() { diff --git a/python/README.md b/python/README.md index de2a9686e1..86d175d750 100644 --- a/python/README.md +++ b/python/README.md @@ -689,8 +689,8 @@ class ThreadSafeFory: `ThreadSafeFory` provides thread-safe serialization by maintaining a pool of `Fory` instances protected by a lock. When a thread needs to serialize/deserialize, it gets an instance from the pool, uses it, and returns it. Complete explicit type and serializer registration before the first serialization or deserialization attempt. -Pass either a no-argument `fory_factory` that returns a configured `Fory` instance, or pass normal -`Fory` construction options through `**kwargs`. +When supplied, `fory_factory` creates each pooled `Fory` instance. Otherwise, `**kwargs` are passed +to the normal `Fory` constructor. **Thread Safety Example:** @@ -745,7 +745,6 @@ for t in threads: t.join() ```python fory = pyfory.Fory(xlang=True) -thread_safe_fory = pyfory.ThreadSafeFory(xlang=True) # Serialization (serialize/deserialize are identical to dumps/loads) data: bytes = fory.serialize(obj) @@ -755,14 +754,11 @@ obj = fory.deserialize(data) data: bytes = fory.dumps(obj) obj = fory.loads(data) -# Direct Fory registration by id; serializer instances belong to that Fory. +# Type registration by id fory.register(MyClass, type_id=123) fory.register(MyClass, type_id=123, serializer=custom_serializer) -# ThreadSafeFory constructs one serializer per pooled child from a class or factory. -thread_safe_fory.register(MyClass, type_id=123, serializer=CustomSerializer) - -# Direct Fory registration by name +# Type registration by name fory.register(MyClass, name="my.package.MyClass") fory.register(MyClass, name="my.package.MyClass", serializer=custom_serializer) ``` @@ -899,9 +895,6 @@ except Exception as e: Implement custom serialization logic for specialized types with a single `write/read` API: -Attach custom serializers to application types registered by name or user ID. Serializers for -built-in framework types cannot be replaced. - ```python import pyfory from pyfory.serializer import Serializer diff --git a/python/pyfory/_fory.py b/python/pyfory/_fory.py index 548e0d8444..a0c25ffa63 100644 --- a/python/pyfory/_fory.py +++ b/python/pyfory/_fory.py @@ -24,9 +24,9 @@ "true", } -from pyfory.policy import DEFAULT_POLICY, DeserializationPolicy from pyfory.resolver import NOT_NULL_VALUE_FLAG from pyfory.types import TypeId +from pyfory.policy import DeserializationPolicy, DEFAULT_POLICY DYNAMIC_TYPE_ID = -1 # preserve 0 as flag for type id not set in TypeInfo` @@ -442,7 +442,7 @@ def dump(self, obj, stream): self.force_flush() finally: self.buffer.bind_output_stream(None) - self.write_context.reset() + self.reset_write() def loads( self, @@ -499,7 +499,7 @@ def serialize( return write_buffer return write_buffer.to_bytes(0, write_buffer.get_writer_index()) finally: - self.write_context.reset() + self.reset_write() def _serialize( self, @@ -568,7 +568,7 @@ def deserialize( try: return self._deserialize(buffer, buffers, unsupported_objects) finally: - self.read_context.reset() + self.reset_read() def _deserialize( self, @@ -681,8 +681,6 @@ class ThreadSafeFory: def __init__(self, fory_factory=None, **kwargs): import threading - if fory_factory is not None and kwargs: - raise TypeError("fory_factory and Fory construction options are mutually exclusive") self._config = kwargs self._fory_factory = fory_factory self._callbacks = [] @@ -703,13 +701,13 @@ def _get_fory(self): self._registry_frozen = True if self._pool: return self._pool.pop() - if self._fory_factory is not None: - fory = self._fory_factory() - else: - fory = self._fory_class(**self._config) - for callback in self._callbacks: - callback(fory) - return fory + if self._fory_factory is not None: + fory = self._fory_factory() + else: + fory = self._fory_class(**self._config) + for callback in self._callbacks: + callback(fory) + return fory def _return_fory(self, fory): with self._lock: @@ -721,15 +719,6 @@ def _register_callback(self, callback): raise RuntimeError("Cannot register types after the first root serialization or deserialization operation has started.") self._callbacks.append(callback) - @staticmethod - def _check_serializer_factory(serializer): - if serializer is None: - return - from pyfory.registry import Serializer - - if isinstance(serializer, Serializer) or not callable(serializer): - raise TypeError("ThreadSafeFory requires a serializer class or factory") - def register( self, cls, @@ -738,7 +727,6 @@ def register( name: str = None, serializer=None, ): - self._check_serializer_factory(serializer) self._register_callback(lambda f: f.register(cls, type_id=type_id, name=name, serializer=serializer)) def register_type( @@ -749,7 +737,6 @@ def register_type( name: str = None, serializer=None, ): - self._check_serializer_factory(serializer) self._register_callback(lambda f: f.register_type(cls, type_id=type_id, name=name, serializer=serializer)) def register_union( @@ -760,21 +747,10 @@ def register_union( name: str = None, serializer=None, ): - self._check_serializer_factory(serializer) self._register_callback(lambda f: f.register_union(cls, type_id=type_id, name=name, serializer=serializer)) - def register_serializer(self, cls: type, serializer_factory): - self._check_serializer_factory(serializer_factory) - if serializer_factory is None: - raise TypeError("ThreadSafeFory requires a serializer class or factory") - - def register(fory): - from pyfory.registry import _construct_serializer - - serializer = _construct_serializer(serializer_factory, fory.type_resolver, cls) - fory.register_serializer(cls, serializer) - - self._register_callback(register) + def register_serializer(self, cls: type, serializer): + self._register_callback(lambda f: f.register_serializer(cls, serializer)) def serialize( self, diff --git a/python/pyfory/context.pxi b/python/pyfory/context.pxi index 8609fd5b5d..b701185bcd 100644 --- a/python/pyfory/context.pxi +++ b/python/pyfory/context.pxi @@ -126,7 +126,7 @@ cdef class RefWriter: return True return False - cpdef inline void reset(self) noexcept: + cpdef inline reset(self): cdef PyObject *item if not self.track_ref: return @@ -328,7 +328,7 @@ cdef class MetaStringWriter: return buffer.write_var_uint32(((deref(entry).second + 1) << 1) | 1) - cpdef inline void reset(self) noexcept: + cpdef inline reset(self): cdef PyObject *item self._written_encoded_meta_strings.clear() for item in self._written_objects: @@ -504,7 +504,7 @@ cdef class MetaStringReader: cdef class MetaShareWriteContext: cdef flat_hash_map[uint64_t, int32_t] class_map - cpdef inline void reset(self) noexcept: + cpdef inline reset(self): self.class_map.clear() @@ -565,7 +565,7 @@ cdef class WriteContext: self.buffer_callback = buffer_callback self.unsupported_callback = unsupported_callback - cpdef inline void reset(self) noexcept: + cpdef inline reset(self): self.ref_writer.reset() self.meta_string_writer.reset() if self.meta_share_context is not None: @@ -574,10 +574,8 @@ cdef class WriteContext: self.context_objects.clear() self.buffer = None self.c_buffer = NULL - if self.buffer_callback is not None: - self.buffer_callback = None - if self.unsupported_callback is not None: - self.unsupported_callback = None + self.buffer_callback = None + self.unsupported_callback = None cpdef inline add_context_object(self, key, obj): self.context_objects[id(key)] = obj diff --git a/python/pyfory/registry.py b/python/pyfory/registry.py index 8ed99e3dfc..1c6391807c 100644 --- a/python/pyfory/registry.py +++ b/python/pyfory/registry.py @@ -89,6 +89,9 @@ fory_array_serializer_type, ) from pyfory.policy import DEFAULT_POLICY +from pyfory.serialization import ( + Serializer as CythonSerializer, +) from pyfory.annotation import ( BFloat16Array, Float32, @@ -193,7 +196,7 @@ def _accepts_n_positional_args(factory, nargs: int) -> bool: signature = inspect.signature(factory.__init__) parameters = tuple(signature.parameters.values())[1:] except (AttributeError, TypeError, ValueError): - if inspect.isclass(factory) and issubclass(factory, Serializer): + if inspect.isclass(factory) and issubclass(factory, (Serializer, CythonSerializer)): return nargs == 2 raise TypeError(f"Unable to inspect serializer constructor for {factory!r}") min_args = 0 @@ -217,26 +220,11 @@ def _construct_serializer(serializer_factory, type_resolver, cls): for nargs, args in ( (2, (type_resolver, cls)), (1, (type_resolver,)), + (0, ()), ): if _accepts_n_positional_args(serializer_factory, nargs): - serializer = serializer_factory(*args) - break - else: - raise TypeError(f"Unsupported serializer constructor for {serializer_factory!r}; expected `(type_resolver, cls)` or `(type_resolver)`.") - if not isinstance(serializer, Serializer): - raise TypeError("Serializer factory must return a Fory serializer") - if serializer.type_resolver is not type_resolver: - raise TypeError("Serializer factory returned a serializer for another resolver") - return serializer - - -def _check_serializer_owner(serializer, type_resolver, cls): - if not isinstance(serializer, Serializer): - raise TypeError("Expected a Fory serializer") - if serializer.type_resolver is not type_resolver: - raise TypeError("Serializer belongs to another resolver") - if normalize_fory_type(serializer.type_) != normalize_fory_type(cls): - raise TypeError("Serializer belongs to another type") + return serializer_factory(*args) + raise TypeError(f"Unsupported serializer constructor for {serializer_factory!r}; expected `(type_resolver, cls)`, `(type_resolver)`, or `()`.") def _split_registration_name(name: str): @@ -636,7 +624,6 @@ def register_union( cls, ) self._check_registry_mutable() - _check_serializer_owner(serializer, self._actual_type_resolver, cls) if typename is not None and type_id is not None: raise TypeError(f"type name {typename} and id {type_id} should not be set at the same time") if typename is None and type_id is None: @@ -685,8 +672,6 @@ def _register_type( ) if not internal: self._check_registry_mutable() - if serializer is not None and not internal: - _check_serializer_owner(serializer, self._actual_type_resolver, cls) if ( cls in self._types_info and type_id is None @@ -883,15 +868,16 @@ def register_serializer(self, cls, serializer): self._check_registry_mutable() cls = normalize_fory_type(cls) assert isinstance(cls, type) or type(cls) is int, cls - _check_serializer_owner(serializer, self._actual_type_resolver, cls) if cls not in self._types_info: raise TypeUnregisteredError(f"{cls} not registered") typeinfo = self._types_info[cls] - # Framework type IDs may be shared by multiple Python carriers; only a namespaced or - # user-ID TypeInfo has an independent wire owner that can change serializers. - if typeinfo.typename_bytes is None and typeinfo.user_type_id in {None, NO_USER_TYPE_ID}: - raise TypeError("Cannot replace serializers for framework types") self._check_registry_mutable() + prev_type_id = typeinfo.type_id + prev_user_type_id = typeinfo.user_type_id + if needs_user_type_id(prev_type_id) and prev_user_type_id not in {None, NO_USER_TYPE_ID}: + self._user_type_id_to_type_info.pop(prev_user_type_id, None) + else: + self._type_id_to_type_info.pop(prev_type_id, None) if typeinfo.serializer is not serializer: if typeinfo.typename_bytes is not None: typeinfo.type_id = TypeId.NAMED_EXT @@ -900,6 +886,10 @@ def register_serializer(self, cls, serializer): typeinfo.type_id = TypeId.EXT typeinfo.serializer = serializer typeinfo.type_def = None + if needs_user_type_id(typeinfo.type_id) and typeinfo.user_type_id not in {None, NO_USER_TYPE_ID}: + self._user_type_id_to_type_info[typeinfo.user_type_id] = typeinfo + else: + self._type_id_to_type_info[typeinfo.type_id] = typeinfo def get_serializer(self, cls: type): """ @@ -1128,12 +1118,27 @@ def _load_metabytes_to_type_info(self, ns_metabytes, type_metabytes): typename = type_metabytes.decode(self.typename_decoder) # the hash computed between languages may be different. typeinfo = self._named_type_to_type_info.get((ns, typename)) + if typeinfo is None and typename and not self.strict: + alt_typename = typename[0].upper() + typename[1:] + typeinfo = self._named_type_to_type_info.get((ns, alt_typename)) if typeinfo is not None: self._cache_wire_type_info(ns_metabytes, type_metabytes, typeinfo) return typeinfo if self.strict: name = ns + "." + typename if ns else typename raise TypeUnregisteredError(f"{name} not registered") + if not ns and "." in typename: + split_ns, split_typename = typename.rsplit(".", 1) + typeinfo = self._named_type_to_type_info.get((split_ns, split_typename)) + if typeinfo is not None: + self._cache_wire_type_info(ns_metabytes, type_metabytes, typeinfo) + return typeinfo + typename = split_typename + ns = split_ns + if typename: + matches = [info for (reg_ns, reg_typename), info in self._named_type_to_type_info.items() if reg_typename == typename] + if len(matches) == 1: + return matches[0] cls = load_class(ns + "#" + typename, policy=self.policy) typeinfo = self.get_type_info(cls) self._cache_wire_type_info(ns_metabytes, type_metabytes, typeinfo) diff --git a/python/pyfory/serialization.pyx b/python/pyfory/serialization.pyx index 3572f64d1f..07efc5511d 100644 --- a/python/pyfory/serialization.pyx +++ b/python/pyfory/serialization.pyx @@ -44,7 +44,6 @@ from pyfory._fory import ( from pyfory.meta.typedef_decoder import decode_typedef from pyfory.meta.typedef import is_struct_typedef_kind from pyfory.meta.metastring import MetaStringDecoder -from pyfory.error import TypeUnregisteredError from pyfory.policy import DEFAULT_POLICY from pyfory.resolver import NULL_FLAG, NOT_NULL_VALUE_FLAG from pyfory.type_util import normalize_fory_type @@ -351,9 +350,7 @@ cdef class TypeResolver: cdef uint8_t previous_type_id cdef uint32_t previous_user_type_id self.resolver._check_registry_mutable() - typeinfo = self.resolver.get_type_info(cls, create=False) - if typeinfo is None: - raise TypeUnregisteredError(f"{cls} not registered") + typeinfo = self.resolver.get_type_info(cls) previous_type_id = typeinfo.type_id previous_user_type_id = typeinfo.user_type_id self.resolver.register_serializer(cls, serializer) @@ -1241,7 +1238,7 @@ cdef class Fory: self.force_flush() finally: self.buffer.bind_output_stream(None) - self.write_context.reset() + self.reset_write() def loads(self, buffer, buffers=None, unsupported_objects=None): return self.deserialize( @@ -1266,7 +1263,7 @@ cdef class Fory: return write_buffer return write_buffer.to_bytes(0, write_buffer.get_writer_index()) finally: - self.write_context.reset() + self.reset_write() cdef Buffer _serialize(self, obj, Buffer buffer=None, buffer_callback=None, unsupported_callback=None): cdef WriteContext write_context = self.write_context @@ -1279,12 +1276,8 @@ cdef class Fory: # so it should not pay an extra method call just to bind the active buffer. write_context.buffer = buffer write_context.c_buffer = buffer.c_buffer - # Root cleanup clears prior callbacks, so the common None path needs no - # object assignment or reference-count traffic here. - if buffer_callback is not None: - write_context.buffer_callback = buffer_callback - if unsupported_callback is not None: - write_context.unsupported_callback = unsupported_callback + write_context.buffer_callback = buffer_callback + write_context.unsupported_callback = unsupported_callback mask_index = buffer.get_writer_index() buffer.grow(1) buffer.set_writer_index(mask_index + 1) diff --git a/python/pyfory/serializer.py b/python/pyfory/serializer.py index 9360761492..cf025ab283 100644 --- a/python/pyfory/serializer.py +++ b/python/pyfory/serializer.py @@ -1799,15 +1799,12 @@ def _deserialize_function(self, read_context): freevars.append(read_context.read_string()) globals_dict = read_context.read_ref() - # The writer defines this as a data-only namespace snapshot. Reject subclasses and other - # mappings so their runtime behavior cannot become part of function reconstruction. - if type(globals_dict) is not dict: - raise ValueError("function globals must be a dict") # Create a globals dictionary with module's globals as the base func_global_entries = len(mod.__dict__) if mod else 0 - func_global_entries = max(func_global_entries, len(globals_dict)) - has_builtins = (mod is not None and "__builtins__" in mod.__dict__) or "__builtins__" in globals_dict + if isinstance(globals_dict, dict): + func_global_entries = max(func_global_entries, len(globals_dict)) + has_builtins = (mod is not None and "__builtins__" in mod.__dict__) or (isinstance(globals_dict, dict) and "__builtins__" in globals_dict) if not has_builtins: func_global_entries += 1 read_context.reserve_graph_memory(_DICT_OWNER_BYTES + func_global_entries * 2 * _REFERENCE_BYTES) diff --git a/python/pyfory/tests/test_function.py b/python/pyfory/tests/test_function.py index 21e84cc1ae..57934cfe7d 100644 --- a/python/pyfory/tests/test_function.py +++ b/python/pyfory/tests/test_function.py @@ -15,82 +15,7 @@ # specific language governing permissions and limitations # under the License. -import marshal -import types - -import pytest - import pyfory -from pyfory.policy import DEFAULT_POLICY -from pyfory.serialization import Buffer -from pyfory.serializer import FunctionSerializer - - -def test_function_globals_carrier(): - def local_func(): - return None - - writer = pyfory.Fory(xlang=False) - buffer = Buffer.allocate(256) - try: - writer.write_context.prepare(buffer) - buffer.write_int8(2) - buffer.write_string(local_func.__module__) - buffer.write_string(local_func.__qualname__) - buffer.write_bytes_and_size(marshal.dumps(local_func.__code__)) - buffer.write_bool(False) - buffer.write_bool(False) - buffer.write_var_uint32(0) - buffer.write_var_uint32(0) - writer.write_context.write_ref([]) - writer.write_context.write_ref({}) - data = buffer.to_bytes(0, buffer.get_writer_index()) - finally: - writer.reset_write() - - reader = pyfory.Fory(xlang=False, strict=False) - serializer = FunctionSerializer(reader.type_resolver, types.FunctionType) - try: - reader.read_context.prepare(Buffer(data)) - with pytest.raises(Exception): - reader.read_context.read_non_ref(serializer) - finally: - reader.reset_read() - - class DictSubclass(dict): - def __len__(self): - raise AssertionError("dict subclass operations must not run") - - class FunctionReadContext: - policy = DEFAULT_POLICY - - def __init__(self): - self._strings = iter((local_func.__module__, local_func.__qualname__)) - - def read_int8(self): - return 2 - - def read_string(self): - return next(self._strings) - - def read_bytes_and_size(self): - return marshal.dumps(local_func.__code__) - - def reserve_graph_memory(self, _size): - pass - - def read_bool(self): - return False - - def read_var_uint32(self): - return 0 - - def read_ref(self): - return DictSubclass() - - with pytest.raises(Exception) as failure: - serializer._deserialize_function(FunctionReadContext()) - assert not isinstance(failure.value, AssertionError) def test_lambda_functions_serialization(): @@ -116,7 +41,7 @@ def test_lambda_functions_serialization(): assert closure_lambda(test_input) == deserialized(test_input) -def test_regular_function_roundtrip(): +def test_regular_functions_serialization(): """Tests serialization of regular functions.""" fory = pyfory.Fory( xlang=False, diff --git a/python/pyfory/tests/test_metastring_resolver.py b/python/pyfory/tests/test_metastring_resolver.py index 87b7008b4e..0e49cd53da 100644 --- a/python/pyfory/tests/test_metastring_resolver.py +++ b/python/pyfory/tests/test_metastring_resolver.py @@ -276,7 +276,7 @@ def validate_module(self, module_name, *, is_local, **kwargs): ENABLE_FORY_CYTHON_SERIALIZATION, reason="pure TypeResolver regression", ) -def test_namespace_alias_rejected(): +def test_namespace_alias_not_cached(): config = Fory(xlang=True, compatible=False, strict=False).config resolver = TypeResolver(config, shared_registry=SharedRegistry()) resolver.initialize() @@ -297,8 +297,7 @@ def test_namespace_alias_rejected(): meta_string_reader=MetaStringReader(resolver.shared_registry), ) - with pytest.raises(Exception): - resolver.read_type_info(read_context) + assert resolver.read_type_info(read_context) is typeinfo assert (namespace, typename) not in resolver._ns_type_to_type_info assert ( typeinfo.namespace_bytes, @@ -306,13 +305,16 @@ def test_namespace_alias_rejected(): ) in resolver._ns_type_to_type_info -def test_wire_type_alias_rejected(): +def test_wire_type_alias_cache_is_bounded(): fory = Fory(xlang=True, compatible=False, strict=False) resolver = fory.type_resolver typeinfo = resolver.register_type( NamespaceAliasType, name="trusted.NamespaceAliasType", ) + for i in range(MAX_CACHED_ENCODED_META_STRINGS): + resolver._ns_type_to_type_info[(i, i)] = typeinfo + namespace = resolver.shared_registry.get_encoded_meta_string(MetaStringEncoder(".", "_").encode("trusted")) typename = resolver.shared_registry.get_encoded_meta_string(MetaStringEncoder("$", "_").encode("namespaceAliasType")) buffer = Buffer.allocate(128) @@ -323,8 +325,7 @@ def test_wire_type_alias_rejected(): buffer.set_reader_index(0) try: fory.read_context.prepare(buffer) - with pytest.raises(Exception): - resolver.read_type_info(fory.read_context) + assert resolver.read_type_info(fory.read_context) is typeinfo assert (namespace, typename) not in resolver._ns_type_to_type_info finally: fory.reset_read() diff --git a/python/pyfory/tests/test_policy.py b/python/pyfory/tests/test_policy.py index d8312c070f..9db9cff94d 100644 --- a/python/pyfory/tests/test_policy.py +++ b/python/pyfory/tests/test_policy.py @@ -256,7 +256,7 @@ def intercept_setstate(self, obj, state, **kwargs): return None -def test_block_class_deserialization(): +def test_block_class_type_deserialization(): """Test blocking class type (not instance) deserialization.""" class SafeClass: @@ -372,7 +372,7 @@ def __setstate__(self, state): assert result.password == "***REDACTED***" -def test_falsey_state_hook_before_bool(): +def test_stateful_intercepts_falsey_state_before_bool(): """Test stateful path calls intercept_setstate without evaluating state truthiness.""" class BlockSetStatePolicy(DeserializationPolicy): @@ -536,7 +536,7 @@ def __reduce__(self): fory.deserialize(data) -def test_stateful_instantiation_policy(): +def test_stateful_authorizes_instantiation(): """Test authorize_instantiation policy hook for stateful deserialization.""" class StatefulPayload: @@ -566,7 +566,7 @@ def authorize_instantiation(self, cls, **kwargs): assert policy.authorize_instantiation_calls == 1 -def test_reduce_class_instantiation(): +def test_reduce_class_callable_authorizes_instantiation(): """Test authorize_instantiation policy hook for reduce class callables.""" class ReduceTarget: @@ -814,7 +814,7 @@ def validate_class(self, cls, is_local, **kwargs): assert SafeClass.run() == "safe" -def test_type_module_policy(): +def test_type_deserialization_validates_module(): """Test validate_module policy hook for global class deserialization.""" import subprocess @@ -838,7 +838,7 @@ def validate_module(self, module_name, is_local, **kwargs): assert policy.is_local_values == [False] -def test_native_method_policy_dispatch(): +def test_native_bound_method_uses_validate_method(): """Test bound native methods are checked by method policy, not function policy.""" class BlockMethodPolicy(DeserializationPolicy): @@ -863,7 +863,7 @@ def validate_function(self, func, is_local, **kwargs): assert policy.validate_function_calls == 0 -def test_bound_method_policy_order(): +def test_bound_method_policy_runs_before_getattribute_side_effect(): """Test bound method deserialization validates before dynamic attribute lookup.""" class GuardedMethod: @@ -1603,7 +1603,7 @@ def validate_function(self, func, is_local, **kwargs): policy_global_function.__module__ = original_module -def test_global_function_module_policy(): +def test_global_function_deserialization_validates_module(): """Test validate_module policy hook for global function deserialization.""" class BlockModulePolicy(DeserializationPolicy): @@ -1626,7 +1626,7 @@ def validate_module(self, module_name, is_local, **kwargs): assert policy.is_local_values == [False] -def test_local_function_module_policy(): +def test_local_function_deserialization_validates_module(): """Test local function code does not reclassify its module owner.""" def local_function(): @@ -1677,7 +1677,7 @@ def authorize_instantiation(self, cls, **kwargs): assert policy.instantiation_calls == [] -def test_native_function_module_policy(): +def test_native_function_deserialization_validates_module(): """Test validate_module policy hook for native function deserialization.""" import time @@ -1760,7 +1760,7 @@ def validate_class(self, cls, is_local, **kwargs): assert policy.validate_class_calls == 1 -def test_reduce_global_module_policy(): +def test_reduce_global_name_validates_module(): """Test validate_module policy hook for reduce global-name deserialization.""" class GlobalNamePayload: @@ -1787,7 +1787,7 @@ def validate_module(self, module_name, is_local, **kwargs): assert policy.is_local_values == [False] -def test_reduce_global_class_policy(): +def test_reduce_global_name_validates_class(): """Test validate_class policy hook for reduce global-name deserialization.""" class GlobalNamePayload: @@ -1818,7 +1818,7 @@ def validate_class(self, cls, is_local, **kwargs): assert policy.validate_class_calls == 1 -def test_reduce_global_function_policy(): +def test_reduce_global_name_validates_function(): """Test validate_function policy hook for reduce builtins-name deserialization.""" class GlobalNamePayload: @@ -1849,7 +1849,7 @@ def validate_function(self, func, is_local, **kwargs): assert policy.validate_function_calls == 1 -def test_reduce_global_method_policy(): +def test_reduce_global_method_resolution_uses_validate_method(): """Test reduce global-name method deserialization uses validate_method.""" class GlobalNamePayload: diff --git a/python/pyfory/tests/test_reduce_serializer.py b/python/pyfory/tests/test_reduce_serializer.py index 29ed458c91..8442fcf5b9 100644 --- a/python/pyfory/tests/test_reduce_serializer.py +++ b/python/pyfory/tests/test_reduce_serializer.py @@ -549,7 +549,7 @@ def test_reduce_with_dict_items(): assert deserialized.name == "dict_obj" -def test_reduce_precedes_stateful(): +def test_reduce_precedence_over_stateful(): """Test that ReduceSerializer has higher precedence than StatefulSerializer""" fory = Fory(xlang=False, ref=True, strict=False, compatible=False) @@ -604,3 +604,21 @@ def test_nested_reduce_objects(): assert deserialized.data["inner"] == inner assert deserialized.data["inner"].value == 10 assert deserialized.data["inner"].multiplier == 2 + + +def test_cross_language_compatibility(): + """Test cross-language compatibility""" + fory = Fory(xlang=False, ref=True, strict=False, compatible=False) + + obj = BasicReduceObject(123, 4) + + # Serialize with Python + serialized = fory.serialize(obj) + + # Should be able to deserialize (basic test) + deserialized = fory.deserialize(serialized) + assert deserialized == obj + + # The serialized data should use Fory's native format, not pickle + # This is verified by the fact that we're using write_ref/read_ref + # in the ReduceSerializer implementation diff --git a/python/pyfory/tests/test_serializer.py b/python/pyfory/tests/test_serializer.py index bec6eed939..22113c556a 100644 --- a/python/pyfory/tests/test_serializer.py +++ b/python/pyfory/tests/test_serializer.py @@ -37,7 +37,6 @@ import pyfory from pyfory.serialization import Buffer, _bfloat16_from_bits, _bfloat16_to_bits, _float16_from_bits, _float16_to_bits from pyfory import Fory, EnumSerializer -from pyfory.error import TypeUnregisteredError from pyfory.serializer import ( DecimalSerializer, TimestampSerializer, @@ -859,7 +858,6 @@ def read(self, read_context): type_info = fory.type_resolver.get_type_info(Value) assert type_info.serializer is replacement assert type_info.type_def is None - assert TypeId.NAMED_EXT not in fory.type_resolver._type_id_to_type_info if registration == "id": assert fory.type_resolver._user_type_id_to_type_info[701] is type_info else: @@ -980,29 +978,6 @@ def test_lazy_type_keeps_explicit_name(): assert fory.type_resolver.get_type_info_by_name("registry_owner", "SharedType") is type_info -def test_serializer_type_is_registered(): - fory = Fory(xlang=False, strict=False, compatible=False) - serializer = BarSerializer(fory.type_resolver, RejectedRegistration) - - with pytest.raises(TypeUnregisteredError): - fory.register_serializer(RejectedRegistration, serializer) - - assert fory.type_resolver.get_type_info(RejectedRegistration, create=False) is None - - -def test_builtin_serializer_owner(): - fory = Fory(xlang=True, compatible=False) - resolver = fory.type_resolver - type_info = resolver.get_type_info(int) - wire_owner = resolver._type_id_to_type_info[type_info.type_id] - - with pytest.raises(TypeError, match="framework types"): - fory.register_serializer(int, BarSerializer(resolver, int)) - - assert resolver.get_type_info(int) is type_info - assert resolver._type_id_to_type_info[type_info.type_id] is wire_owner - - @pytest.mark.parametrize("root", ["serialize", "deserialize", "dump"]) def test_registry_freezes_at_root(root): fory = Fory(xlang=True, compatible=False) diff --git a/python/pyfory/tests/test_thread_safe.py b/python/pyfory/tests/test_thread_safe.py index 876fa89e80..d7111283cd 100644 --- a/python/pyfory/tests/test_thread_safe.py +++ b/python/pyfory/tests/test_thread_safe.py @@ -20,7 +20,6 @@ import pytest -import pyfory from pyfory import ThreadSafeFory @@ -36,15 +35,6 @@ class Address: country: str -class PersonSerializer(pyfory.Serializer): - pass - - -def test_factory_rejects_options(): - with pytest.raises(TypeError, match="mutually exclusive"): - ThreadSafeFory(lambda: pyfory.Fory(), xlang=False) - - def test_thread_safe_fory_basic_serialization(): fory = ThreadSafeFory( xlang=False, @@ -201,54 +191,3 @@ def test_thread_safe_fory_register_after_use(): with pytest.raises(RuntimeError): fory.register(Address) - - -def test_child_serializer_owners(): - fory = ThreadSafeFory(xlang=False, compatible=False) - fory.register(Person, serializer=PersonSerializer) - - first = fory._get_fory() - second = fory._get_fory() - try: - first_serializer = first.type_resolver.get_serializer(Person) - second_serializer = second.type_resolver.get_serializer(Person) - assert first_serializer is not second_serializer - assert first_serializer.type_resolver is first.type_resolver - assert second_serializer.type_resolver is second.type_resolver - finally: - fory._return_fory(first) - fory._return_fory(second) - - -def test_rejects_serializer_instance(): - runtime = pyfory.Fory(xlang=False, compatible=False) - serializer = PersonSerializer(runtime.type_resolver, Person) - fory = ThreadSafeFory(xlang=False, compatible=False) - - with pytest.raises(TypeError, match="serializer class or factory"): - fory.register(Person, serializer=serializer) - - -def test_rejects_foreign_factory(): - runtime = pyfory.Fory(xlang=False, compatible=False) - serializer = PersonSerializer(runtime.type_resolver, Person) - fory = ThreadSafeFory(xlang=False, compatible=False) - fory.register(Person, serializer=lambda _resolver: serializer) - - with pytest.raises(TypeError, match="another resolver"): - fory._get_fory() - - -@pytest.mark.parametrize( - "serializer_factory, error", - [ - (lambda _resolver: object(), "must return a Fory serializer"), - (lambda resolver: PersonSerializer(resolver, Address), "another type"), - ], -) -def test_validates_serializer_factory(serializer_factory, error): - fory = ThreadSafeFory(xlang=False, compatible=False) - fory.register(Person, serializer=serializer_factory) - - with pytest.raises(TypeError, match=error): - fory._get_fory() diff --git a/scala/fory-scala/src/main/java/org/apache/fory/serializer/scala/ScalaSerializers.java b/scala/fory-scala/src/main/java/org/apache/fory/serializer/scala/ScalaSerializers.java index e000419fe3..1330e825b6 100644 --- a/scala/fory-scala/src/main/java/org/apache/fory/serializer/scala/ScalaSerializers.java +++ b/scala/fory-scala/src/main/java/org/apache/fory/serializer/scala/ScalaSerializers.java @@ -19,9 +19,6 @@ package org.apache.fory.serializer.scala; -import static org.apache.fory.serializer.scala.ToFactorySerializers.IterableToFactoryClass; -import static org.apache.fory.serializer.scala.ToFactorySerializers.MapToFactoryClass; - import java.util.Objects; import org.apache.fory.Fory; import org.apache.fory.ThreadSafeFory; @@ -33,6 +30,9 @@ import scala.collection.immutable.NumericRange; import scala.collection.immutable.Range; +import static org.apache.fory.serializer.scala.ToFactorySerializers.IterableToFactoryClass; +import static org.apache.fory.serializer.scala.ToFactorySerializers.MapToFactoryClass; + public class ScalaSerializers { public static void registerSerializers(ThreadSafeFory fory) { fory.register(ForyScala$.MODULE$); diff --git a/scala/fory-scala/src/main/scala-3/org/apache/fory/scala/ForySerializer.scala b/scala/fory-scala/src/main/scala-3/org/apache/fory/scala/ForySerializer.scala index 63ce1405cf..87a1d34c8b 100644 --- a/scala/fory-scala/src/main/scala-3/org/apache/fory/scala/ForySerializer.scala +++ b/scala/fory-scala/src/main/scala-3/org/apache/fory/scala/ForySerializer.scala @@ -107,6 +107,7 @@ object ForySerializer { @Internal def registerSerializer[T](fory: Fory, cls: Class[T])(using serializer: ForySerializer[T]): Unit = { val resolver = fory.getTypeResolver + resolver.checkRegistrationOpen() if serializer.isUnion then { throw new IllegalArgumentException("Use ForySerializer.register for Scala union serializers") } @@ -129,7 +130,9 @@ object ForySerializer { resolver.checkRegistrationOpen() serializer match { case _ if serializer.isUnion => + resolver.checkRegistrationOpen() val runtimeSerializer = serializer.createSerializer(resolver) + resolver.checkRegistrationOpen() val runtimeClasses = serializer.handledRuntimeClasses(cls) if typeId != null then { resolver.registerUnion(cls, typeId.longValue(), runtimeSerializer) @@ -147,6 +150,8 @@ object ForySerializer { ScalaSerializers.registerRuntimeTypeAlias(fory, runtimeClass, cls) } case _ => + // Generated TypeDef construction resolves this registered STRUCT identity. Publish the + // identity first, then recheck freeze after construction before installing the serializer. registerType(fory, cls, typeId, namespace, typeName) val runtimeSerializer = serializer.createSerializer(resolver) resolver.checkRegistrationOpen() diff --git a/swift/Sources/Fory/ReadContext.swift b/swift/Sources/Fory/ReadContext.swift index 99f2bc14fb..82debbe5cd 100644 --- a/swift/Sources/Fory/ReadContext.swift +++ b/swift/Sources/Fory/ReadContext.swift @@ -318,10 +318,6 @@ public final class ReadContext { for localTypeInfo: TypeInfo, wireTypeID: TypeId ) throws -> TypeInfo? { - // Generic type lookup must not prepare metadata; this wire owner does so only on a miss. - if localTypeInfo.typeDefBytes == nil { - try localTypeInfo.ensureTypeMeta(resolver: typeResolver) - } let buffer = self.buffer let compatibleTypeDefTypeInfos = self.compatibleTypeDefTypeInfos if !checkClassVersion, @@ -459,6 +455,17 @@ public final class ReadContext { return try requireCompatibleOwner(cached, for: localTypeInfo) } + // Ref and checked header-cache hits must not complete local metadata. Build it only after + // both miss, then compare the received protocol identity before parsing the remote body. + if localTypeInfo.typeDefBytes == nil { + try localTypeInfo.ensureTypeMeta(resolver: typeResolver) + } + if headerHash == localTypeInfo.typeDefHeaderHash { + try buffer.skip(bodySize) + compatibleTypeDefTypeInfos.push(localTypeInfo) + return localTypeInfo + } + let cachedTypeInfo = try readTypeInfoBody( start: typeMetaStart, headerHash: headerHash, diff --git a/swift/Tests/ForyTests/CollectionSerializerTests.swift b/swift/Tests/ForyTests/CollectionSerializerTests.swift index d0be7f3aca..cb8eb9fea4 100644 --- a/swift/Tests/ForyTests/CollectionSerializerTests.swift +++ b/swift/Tests/ForyTests/CollectionSerializerTests.swift @@ -792,7 +792,6 @@ func generatedReadProgress() throws { let fory = Fory(config: Config(trackRef: false, compatible: true)) try fory.register(AdvancingReadStruct.self, id: 9705) - fory.typeResolver.freezeRegistration() let local = try fory.typeResolver.requireTypeInfo(for: AdvancingReadStruct.self) let emptyMeta = try TypeMeta( typeID: TypeId.compatibleStruct.rawValue, diff --git a/swift/Tests/ForyTests/DecoderStateTests.swift b/swift/Tests/ForyTests/DecoderStateTests.swift index 211014f4c7..74cc278bf1 100644 --- a/swift/Tests/ForyTests/DecoderStateTests.swift +++ b/swift/Tests/ForyTests/DecoderStateTests.swift @@ -143,7 +143,6 @@ func remoteSchemaLogicalKeyLimitPersists() throws { let resolver = TypeResolver(config: config) try resolver.register(Person.self, id: 901) try resolver.register(Address.self, id: 902) - resolver.freezeRegistration() let localTypeInfo = try resolver.requireTypeInfo(for: Person.self) func remoteTypeMeta( diff --git a/swift/Tests/ForyTests/ForySwiftTests.swift b/swift/Tests/ForyTests/ForySwiftTests.swift index 40cccd8ef7..9d41361131 100644 --- a/swift/Tests/ForyTests/ForySwiftTests.swift +++ b/swift/Tests/ForyTests/ForySwiftTests.swift @@ -641,7 +641,6 @@ func schemaLimitTracksStructTypesSeparately() throws { let resolver = TypeResolver(config: config) try resolver.register(Person.self, id: 901) try resolver.register(Address.self, id: 902) - resolver.freezeRegistration() func remoteTypeMeta(userTypeID: UInt32, fieldName: String) throws -> TypeMeta { try TypeMeta( @@ -687,7 +686,6 @@ func nonStructTypeMetaUsesSchemaLimit() throws { let config = Config(maxSchemaVersionsPerType: 1) let resolver = TypeResolver(config: config) try resolver.register(SparseStatus.self, name: "example.SharedEnum") - resolver.freezeRegistration() let namespace = try MetaStringEncoder.namespace.encode("example") let typeName = try MetaStringEncoder.typeName.encode("SharedEnum") @@ -728,7 +726,6 @@ func localNonStructMetaBypassesLimit() throws { let config = Config(compatible: true, maxSchemaVersionsPerType: 1) let resolver = TypeResolver(config: config) try resolver.register(SparseStatus.self, name: "example.SharedEnum") - resolver.freezeRegistration() let localTypeInfo = try resolver.requireTypeInfo(for: SparseStatus.self) try localTypeInfo.ensureTypeMeta(resolver: resolver) let namespace = try MetaStringEncoder.namespace.encode("example") @@ -795,7 +792,6 @@ func failedSchemaDoesNotConsumeLimit() throws { let resolver = TypeResolver(config: config) try resolver.register(Person.self, id: 901) try resolver.register(Address.self, id: 902) - resolver.freezeRegistration() func remoteTypeMeta(fieldName: String, fieldType: TypeMeta.FieldType) throws -> TypeMeta { try TypeMeta( @@ -856,7 +852,6 @@ func staticTypeRejectsWrongMetaOwner() throws { let resolver = TypeResolver(config: config) try resolver.register(Person.self, id: 901) try resolver.register(Address.self, id: 902) - resolver.freezeRegistration() let wrongTypeMeta = try TypeMeta( typeID: TypeId.compatibleStruct.rawValue, userTypeID: 901, @@ -896,7 +891,6 @@ func cachedMetaChecksConcreteOwner() throws { let resolver = TypeResolver(config: config) try resolver.register(Person.self, id: 901) try resolver.register(Address.self, id: 902) - resolver.freezeRegistration() let remote = try TypeMeta( typeID: TypeId.compatibleStruct.rawValue, userTypeID: 901, @@ -944,7 +938,6 @@ func failedStaticMetaDoesNotCount() throws { let resolver = TypeResolver(config: config) try resolver.register(Person.self, id: 901) try resolver.register(Address.self, id: 902) - resolver.freezeRegistration() func typeMeta(userTypeID: UInt32, fieldName: String) throws -> TypeMeta { try TypeMeta( diff --git a/swift/Tests/ForyTests/GraphMemoryBudgetTests.swift b/swift/Tests/ForyTests/GraphMemoryBudgetTests.swift index b0f0add009..65939f6fbf 100644 --- a/swift/Tests/ForyTests/GraphMemoryBudgetTests.swift +++ b/swift/Tests/ForyTests/GraphMemoryBudgetTests.swift @@ -716,7 +716,6 @@ func unknownCaseChargesDynamicHeapBox() throws { let config = Config(compatible: false) let resolver = TypeResolver(config: config) try resolver.register(DynamicBoxBudgetV1.self, id: 9821) - resolver.freezeRegistration() let value = DynamicBoxBudgetV1(first: 1, second: 2, third: 3, fourth: 4) let buffer = ByteBuffer() let writeContext = WriteContext( diff --git a/swift/Tests/ForyTests/TypeMetaDepthTests.swift b/swift/Tests/ForyTests/TypeMetaDepthTests.swift index 7d4d55b896..f389a03080 100644 --- a/swift/Tests/ForyTests/TypeMetaDepthTests.swift +++ b/swift/Tests/ForyTests/TypeMetaDepthTests.swift @@ -61,7 +61,6 @@ func remoteTypeMetaUsesFixedDepth() throws { let config = Config(compatible: true, maxDepth: 2) let resolver = TypeResolver(config: config) try resolver.register(Address.self, id: 902) - resolver.freezeRegistration() func context(_ encoded: [UInt8]) -> ReadContext { let buffer = ByteBuffer() @@ -129,7 +128,6 @@ func cachedMetaUsesHeaderHash() throws { let resolver = TypeResolver(config: config) try resolver.register(Person.self, id: 901) try resolver.register(Address.self, id: 902) - resolver.freezeRegistration() let remote = try TypeMeta( typeID: TypeId.compatibleStruct.rawValue, userTypeID: 901, @@ -189,7 +187,6 @@ func localMetaUsesHeaderHash() throws { let resolver = TypeResolver(config: config) try resolver.register(Person.self, id: 901) try resolver.register(Address.self, id: 902) - resolver.freezeRegistration() let firstTypeInfo = try resolver.requireTypeInfo(for: Person.self) try firstTypeInfo.ensureTypeMeta(resolver: resolver) let firstBytes = try #require(firstTypeInfo.typeDefBytes) From 9372a9537069e8df2cca82e98eda2765729e7344 Mon Sep 17 00:00:00 2001 From: chaokunyang Date: Sun, 30 Aug 2026 19:39:09 +0800 Subject: [PATCH 158/168] test(python): retain stateful reproduction coverage --- .../tests/test_stateful_reproduction.py | 152 ++++++++++++++++++ 1 file changed, 152 insertions(+) create mode 100644 python/pyfory/tests/test_stateful_reproduction.py diff --git a/python/pyfory/tests/test_stateful_reproduction.py b/python/pyfory/tests/test_stateful_reproduction.py new file mode 100644 index 0000000000..2e82bf1efd --- /dev/null +++ b/python/pyfory/tests/test_stateful_reproduction.py @@ -0,0 +1,152 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +from pyfory import Fory + + +# Test class with __getstate__ and __setstate__ +class StatefulObject: + def __init__(self, value, secret=None): + self.value = value + self.secret = secret or "default_secret" + self.computed = self.value * 2 + + def __getstate__(self): + # Only serialize value, not secret or computed + return {"value": self.value} + + def __setstate__(self, state): + self.value = state["value"] + self.secret = "restored_secret" + self.computed = self.value * 2 + + def __eq__(self, other): + return isinstance(other, StatefulObject) and self.value == other.value and self.computed == other.computed + # Note: secret is expected to be different after deserialization + + def __repr__(self): + return f"StatefulObject(value={self.value}, secret={self.secret}, computed={self.computed})" + + +# Test class with getnewargs_ex +class ImmutableWithArgs: + def __init__(self, x, y, name="default"): + self._x = x + self._y = y + self._name = name + + def __getnewargs_ex__(self): + return (self._x, self._y), {"name": self._name} + + def __getstate__(self): + return {"extra_data": "some_extra"} + + def __setstate__(self, state): + self._extra = state.get("extra_data", "none") + + def __eq__(self, other): + return ( + isinstance(other, ImmutableWithArgs) + and self._x == other._x + and self._y == other._y + and self._name == other._name + and getattr(self, "_extra", None) == getattr(other, "_extra", None) + ) + + def __repr__(self): + return f"ImmutableWithArgs(x={self._x}, y={self._y}, name={self._name}, extra={getattr(self, '_extra', None)})" + + +# Test class with getnewargs (older style) +class ImmutableOldStyle: + def __init__(self, a, b): + self._a = a + self._b = b + + def __getnewargs__(self): + return self._a, self._b + + def __getstate__(self): + return {"metadata": "old_style"} + + def __setstate__(self, state): + self._metadata = state.get("metadata", "none") + + def __eq__(self, other): + return ( + isinstance(other, ImmutableOldStyle) + and self._a == other._a + and self._b == other._b + and getattr(self, "_metadata", None) == getattr(other, "_metadata", None) + ) + + def __repr__(self): + return f"ImmutableOldStyle(a={self._a}, b={self._b}, metadata={getattr(self, '_metadata', None)})" + + +def test_current_behavior(): + print("Testing current behavior with stateful objects...") + + fory = Fory(xlang=False, ref=True, strict=False, compatible=False) + + # Test basic stateful object + obj1 = StatefulObject(42, "original_secret") + print(f"Original: {obj1}") + + serialized = fory.serialize(obj1) + deserialized = fory.deserialize(serialized) + print(f"Deserialized: {deserialized}") + print(f"Equal: {obj1 == deserialized}") + print() + + # Test with getnewargs_ex + obj2 = ImmutableWithArgs(10, 20, "test") + print(f"Original: {obj2}") + + serialized2 = fory.serialize(obj2) + deserialized2 = fory.deserialize(serialized2) + print(f"Deserialized attributes: {dir(deserialized2)}") + print(f"Deserialized vars: {vars(deserialized2)}") + try: + print(f"Deserialized: {deserialized2}") + print(f"Equal: {obj2 == deserialized2}") + except Exception as e: + print(f"Error in repr/comparison: {e}") + print() + + # Test with getnewargs (old style) + obj3 = ImmutableOldStyle(100, 200) + print(f"Original: {obj3}") + + serialized3 = fory.serialize(obj3) + deserialized3 = fory.deserialize(serialized3) + print(f"Deserialized: {deserialized3}") + print(f"Equal: {obj3 == deserialized3}") + print() + + # Check what serializer is being used + serializer1 = fory.type_resolver.get_serializer(StatefulObject) + serializer2 = fory.type_resolver.get_serializer(ImmutableWithArgs) + serializer3 = fory.type_resolver.get_serializer(ImmutableOldStyle) + + print(f"StatefulObject serializer: {type(serializer1)}") + print(f"ImmutableWithArgs serializer: {type(serializer2)}") + print(f"ImmutableOldStyle serializer: {type(serializer3)}") + + +if __name__ == "__main__": + test_current_behavior() From df4591596b2858adcebfa3389fc8f500a4c0cb13 Mon Sep 17 00:00:00 2001 From: chaokunyang Date: Sun, 30 Aug 2026 21:02:09 +0800 Subject: [PATCH 159/168] perf(swift): avoid repeated registry freeze stores --- swift/Sources/Fory/TypeResolver.swift | 3 +++ 1 file changed, 3 insertions(+) diff --git a/swift/Sources/Fory/TypeResolver.swift b/swift/Sources/Fory/TypeResolver.swift index 1d32eac7dc..41477acba0 100644 --- a/swift/Sources/Fory/TypeResolver.swift +++ b/swift/Sources/Fory/TypeResolver.swift @@ -841,6 +841,9 @@ final class TypeResolver { @inline(__always) func freezeRegistration() { + if registryFrozen { + return + } registryFrozen = true } From 89c413e2fb97b7a2bd5cb2d67118123c1c587506 Mon Sep 17 00:00:00 2001 From: chaokunyang Date: Sun, 30 Aug 2026 21:28:54 +0800 Subject: [PATCH 160/168] fix(javascript): guard nested registration publication --- javascript/packages/core/lib/fory.ts | 43 ++++++++++++++--------- javascript/packages/core/lib/gen/index.ts | 2 ++ javascript/test/fory.test.ts | 5 ++- 3 files changed, 33 insertions(+), 17 deletions(-) diff --git a/javascript/packages/core/lib/fory.ts b/javascript/packages/core/lib/fory.ts index 8ca9f46a85..73a3a384aa 100644 --- a/javascript/packages/core/lib/fory.ts +++ b/javascript/packages/core/lib/fory.ts @@ -149,31 +149,36 @@ export default class Fory { deserialize(bytes: Uint8Array): InstanceType | null; }; register(constructor: any, customSerializer?: CustomSerializer) { - if (this.registrationFrozen) { - throw new Error("types and serializers must be registered before the first root operation"); - } + this.ensureRegistrationOpen(); + // Codegen hooks can start a root operation. Recheck the facade-owned flag before every + // serializer publication performed by this explicit registration graph. + const ensureRegistrationOpen = () => this.ensureRegistrationOpen(); let serializer: Serializer; if (constructor.prototype?.[ForyTypeInfoSymbol]) { const typeInfo: TypeInfo = (constructor.prototype[ForyTypeInfoSymbol] as WithForyClsInfo) .structTypeInfo; typeInfo.freeze(); - serializer = new Gen(this.typeResolver, { - creator: constructor, - customSerializer, - }).generateSerializer(typeInfo); - if (this.registrationFrozen) { - throw new Error("types and serializers must be registered before the first root operation"); - } + serializer = new Gen( + this.typeResolver, + { + creator: constructor, + customSerializer, + }, + ensureRegistrationOpen, + ).generateSerializer(typeInfo); + this.ensureRegistrationOpen(); this.typeResolver.registerSerializer(typeInfo, serializer); } else { const typeInfo = constructor; typeInfo.freeze(); - serializer = new Gen(this.typeResolver, { - customSerializer, - }).generateSerializer(typeInfo); - if (this.registrationFrozen) { - throw new Error("types and serializers must be registered before the first root operation"); - } + serializer = new Gen( + this.typeResolver, + { + customSerializer, + }, + ensureRegistrationOpen, + ).generateSerializer(typeInfo); + this.ensureRegistrationOpen(); this.typeResolver.registerSerializer(typeInfo, serializer); } return { @@ -183,6 +188,12 @@ export default class Fory { }; } + private ensureRegistrationOpen() { + if (this.registrationFrozen) { + throw new Error("types and serializers must be registered before the first root operation"); + } + } + deserialize(bytes: Uint8Array, serializer: Serializer = this.anySerializer): T | null { this.registrationFrozen = true; try { diff --git a/javascript/packages/core/lib/gen/index.ts b/javascript/packages/core/lib/gen/index.ts index 9e62446722..f6ceefba33 100644 --- a/javascript/packages/core/lib/gen/index.ts +++ b/javascript/packages/core/lib/gen/index.ts @@ -65,6 +65,7 @@ export class Gen { constructor( private typeResolver: TypeResolver, private regOptions: { [key: string]: any } = {}, + private ensureRegistrationOpen?: () => void, ) {} private generate(typeInfo: TypeInfo): Serializer { @@ -104,6 +105,7 @@ export class Gen { } private register(typeInfo: TypeInfo, serializer?: Serializer) { + this.ensureRegistrationOpen?.(); this.typeResolver.registerSerializer(typeInfo, serializer); } diff --git a/javascript/test/fory.test.ts b/javascript/test/fory.test.ts index c3e5073a18..37d9ae55b2 100644 --- a/javascript/test/fory.test.ts +++ b/javascript/test/fory.test.ts @@ -106,6 +106,7 @@ describe("fory", () => { test("freezes during serializer generation", () => { let armed = false; let fory: Fory; + const typeInfo = Type.struct(8105, { value: Type.int32() }); fory = new Fory({ compatible: false, hooks: { @@ -119,7 +120,9 @@ describe("fory", () => { }); armed = true; - expect(() => fory.register(Type.struct(8105, {}))).toThrow(); + expect(() => fory.register(typeInfo)).toThrow(); + const serializer = (fory as any).typeResolver.getSerializerByTypeInfo(typeInfo); + expect(serializer._initialized).toBe(false); }); test.each(["serialize", "deserialize"] as const)("freezes after %s", (operation) => { From 67b819af5c2edca9a63ad907ed859aab533c518d Mon Sep 17 00:00:00 2001 From: chaokunyang Date: Sun, 30 Aug 2026 21:31:02 +0800 Subject: [PATCH 161/168] fix(csharp): publish custom serializers before root only --- csharp/src/Fory/Fory.cs | 16 ++++++++--- csharp/src/Fory/TypeResolver.cs | 13 --------- .../tests/Fory.Tests/RuntimeEdgeCaseTests.cs | 27 +++++++++++++++++++ 3 files changed, 40 insertions(+), 16 deletions(-) diff --git a/csharp/src/Fory/Fory.cs b/csharp/src/Fory/Fory.cs index 169f8fdc53..e31e4300b6 100644 --- a/csharp/src/Fory/Fory.cs +++ b/csharp/src/Fory/Fory.cs @@ -122,7 +122,7 @@ public Fory Register(uint typeId) where TSerializer : Serializer, new() { EnsureRegistrationOpen(); - TypeInfo typeInfo = _typeResolver.RegisterSerializer(); + TypeInfo typeInfo = CreateCustomTypeInfo(); _typeResolver.Register(typeof(T), typeId, typeInfo); return this; } @@ -141,7 +141,7 @@ public Fory Register(string name) { EnsureRegistrationOpen(); (string namespaceName, string typeName) = TypeResolver.SplitTypeName(name); - TypeInfo typeInfo = _typeResolver.RegisterSerializer(); + TypeInfo typeInfo = CreateCustomTypeInfo(); _typeResolver.Register(typeof(T), namespaceName, typeName, typeInfo); return this; } @@ -161,7 +161,7 @@ public Fory Register(string typeNamespace, string typeName) { EnsureRegistrationOpen(); TypeResolver.ValidateSplitTypeName(typeNamespace, typeName); - TypeInfo typeInfo = _typeResolver.RegisterSerializer(); + TypeInfo typeInfo = CreateCustomTypeInfo(); _typeResolver.Register(typeof(T), typeNamespace, typeName, typeInfo); return this; } @@ -340,6 +340,16 @@ private void EnsureRegistrationOpen() } } + private TypeInfo CreateCustomTypeInfo() + where TSerializer : Serializer, new() + { + TypeInfo typeInfo = TypeInfo.Create(typeof(T), new TSerializer()); + // Serializer construction and TypeInfo creation can execute application code. Recheck + // after both so a reentrant root cannot be followed by resolver publication. + EnsureRegistrationOpen(); + return typeInfo; + } + [MethodImpl(MethodImplOptions.NoInlining)] private static void ThrowRegistryFrozen() => throw new InvalidOperationException( diff --git a/csharp/src/Fory/TypeResolver.cs b/csharp/src/Fory/TypeResolver.cs index af6c8dee30..b11d60b8f8 100644 --- a/csharp/src/Fory/TypeResolver.cs +++ b/csharp/src/Fory/TypeResolver.cs @@ -485,19 +485,6 @@ private TypeInfo GetOrCreateTypeInfo(Type type, TypeInfo? explicitTypeInfo) return typeInfo; } - internal TypeInfo RegisterSerializer() - where TSerializer : Serializer, new() - { - TypeInfo typeInfo = TypeInfo.Create(typeof(T), new TSerializer()); - RegisterSerializer(typeof(T), typeInfo); - return typeInfo; - } - - internal void RegisterSerializer(Type type, TypeInfo typeInfo) - { - GetOrCreateTypeInfo(type, typeInfo); - } - internal void Register(Type type, uint id, TypeInfo? explicitTypeInfo = null) { TypeInfo typeInfo = GetOrCreateTypeInfo(type, explicitTypeInfo).WithTypeIdRegistration(id); diff --git a/csharp/tests/Fory.Tests/RuntimeEdgeCaseTests.cs b/csharp/tests/Fory.Tests/RuntimeEdgeCaseTests.cs index b959f3889c..aa2df61ce8 100644 --- a/csharp/tests/Fory.Tests/RuntimeEdgeCaseTests.cs +++ b/csharp/tests/Fory.Tests/RuntimeEdgeCaseTests.cs @@ -60,6 +60,13 @@ public sealed class DecimalEnvelope public sealed class CustomPayloadSerializer : Serializer { + public static Action? ConstructionAction; + + public CustomPayloadSerializer() + { + ConstructionAction?.Invoke(); + } + public override CustomPayload DefaultValue => null!; public override void WriteData(WriteContext context, in CustomPayload value, bool hasGenerics) @@ -858,6 +865,26 @@ public void FrozenRegistryRejectsBeforeMutation() Assert.Equal(0, FrozenPayloadSerializer.Constructions); } + [Fact] + public void CustomSerializerRechecksFreeze() + { + ForyRuntime fory = ForyRuntime.Builder().Build(); + CustomPayloadSerializer.ConstructionAction = () => _ = fory.Serialize(1); + try + { + Assert.Throws( + () => fory.Register(721)); + } + finally + { + CustomPayloadSerializer.ConstructionAction = null; + } + + TypeInfo typeInfo = ReadContextFor(fory).TypeResolver.GetTypeInfo(typeof(CustomPayload)); + Assert.False(typeInfo.IsRegistered); + Assert.NotEqual(typeof(CustomPayloadSerializer), typeInfo.SerializerType); + } + [Fact] public void FailedRootFreezesRegistry() { From 2a9818d218b0ce4ce740639982264368161ebc0a Mon Sep 17 00:00:00 2001 From: chaokunyang Date: Sun, 30 Aug 2026 21:32:46 +0800 Subject: [PATCH 162/168] fix(swift): recheck registry before publication --- swift/Sources/Fory/TypeResolver.swift | 8 ++++ swift/Tests/ForyTests/ForySwiftTests.swift | 53 ++++++++++++++++++++++ 2 files changed, 61 insertions(+) diff --git a/swift/Sources/Fory/TypeResolver.swift b/swift/Sources/Fory/TypeResolver.swift index 41477acba0..cd2f4edb24 100644 --- a/swift/Sources/Fory/TypeResolver.swift +++ b/swift/Sources/Fory/TypeResolver.swift @@ -900,6 +900,8 @@ final class TypeResolver { bodyReader: registeredBodyReader(for: T.self) ) + // Static serializer witnesses above are application code and may start a root. Recheck + // after the last witness and before either returning or publishing the TypeInfo. if let existing = bySerializerType.value( for: UInt64(UInt(bitPattern: serializerTypeID))), existing.matches( @@ -910,9 +912,11 @@ final class TypeResolver { typeName: (namespace: "", name: "") ) { + try ensureRegistrationAllowed() return } + try ensureRegistrationAllowed() store(typeInfo, userTypeID: id) } @@ -972,6 +976,8 @@ final class TypeResolver { bodyReader: registeredBodyReader(for: T.self) ) + // Static serializer witnesses above are application code and may start a root. Recheck + // after the last witness and before either returning or publishing the TypeInfo. if let existing = bySerializerType.value( for: UInt64(UInt(bitPattern: serializerTypeID))), existing.matches( @@ -982,9 +988,11 @@ final class TypeResolver { typeName: (namespace: namespace, name: typeName) ) { + try ensureRegistrationAllowed() return } + try ensureRegistrationAllowed() store(typeInfo, typeNameKey: TypeNameKey(namespace: namespace, typeName: typeName)) } diff --git a/swift/Tests/ForyTests/ForySwiftTests.swift b/swift/Tests/ForyTests/ForySwiftTests.swift index 9d41361131..439f7965f6 100644 --- a/swift/Tests/ForyTests/ForySwiftTests.swift +++ b/swift/Tests/ForyTests/ForySwiftTests.swift @@ -26,6 +26,40 @@ struct Address: Equatable { var zip: Int32 } +private enum IDRegistrationProbe: Serializer { + case value + + nonisolated(unsafe) static let resolver = TypeResolver(config: Config()) + + static var staticTypeId: TypeId { + resolver.freezeRegistration() + return .ext + } + + static func defaultValue(_: ReadContext) throws -> IDRegistrationProbe { .value } + + static func writeData(_: IDRegistrationProbe, _: WriteContext) throws {} + + static func readData(_: ReadContext) throws -> IDRegistrationProbe { .value } +} + +private enum NameRegistrationProbe: Serializer { + case value + + nonisolated(unsafe) static let resolver = TypeResolver(config: Config()) + + static var staticTypeId: TypeId { + resolver.freezeRegistration() + return .ext + } + + static func defaultValue(_: ReadContext) throws -> NameRegistrationProbe { .value } + + static func writeData(_: NameRegistrationProbe, _: WriteContext) throws {} + + static func readData(_: ReadContext) throws -> NameRegistrationProbe { .value } +} + @ForyStruct struct Person: Equatable { var id: Int64 @@ -1176,6 +1210,25 @@ func registrationIsRejectedAfterFirstTopLevelUse() throws { } } +@Test +func registrationRechecksBeforeStore() throws { + let idResolver = IDRegistrationProbe.resolver + #expect(throws: ForyError.self) { + try idResolver.register(IDRegistrationProbe.self, id: 901) + } + #expect(throws: ForyError.self) { + _ = try idResolver.requireTypeInfo(for: IDRegistrationProbe.self) + } + + let nameResolver = NameRegistrationProbe.resolver + #expect(throws: ForyError.self) { + try nameResolver.register(NameRegistrationProbe.self, name: "probe.name") + } + #expect(throws: ForyError.self) { + _ = try nameResolver.requireTypeInfo(for: NameRegistrationProbe.self) + } +} + @Test func serializeToAppendsRoots() throws { let fory = Fory() From 54acd63090df071337922264e120c4f484572ffd Mon Sep 17 00:00:00 2001 From: chaokunyang Date: Sun, 30 Aug 2026 21:46:45 +0800 Subject: [PATCH 163/168] docs: define callback-safe registry publication --- .agents/languages/csharp.md | 3 +++ .agents/languages/javascript.md | 3 +++ .agents/languages/swift.md | 3 +++ AGENTS.md | 12 +++++++----- docs/specification/xlang_implementation_guide.md | 15 ++++++++------- 5 files changed, 24 insertions(+), 12 deletions(-) diff --git a/.agents/languages/csharp.md b/.agents/languages/csharp.md index 13e2bbce73..c959d41d1d 100644 --- a/.agents/languages/csharp.md +++ b/.agents/languages/csharp.md @@ -14,6 +14,9 @@ Load this file when changing `csharp/` or C# xlang behavior. set after failure. Explicit registration checks that flag before mutation. `ThreadSafeFory` keeps its existing registration callbacks only to configure newly created child runtimes; do not turn that list into another registry lifecycle state. +- Custom serializer construction and `TypeInfo` creation can execute application code. Complete + both before resolver mutation, then recheck the same `Fory` flag immediately before publishing + the custom binding. Do not publish an intermediate unregistered `TypeInfo`. - Generated C# gRPC service companions are compiler-owned files that depend on application-provided gRPC packages, not `csharp/src/Fory`. Keep gRPC package references out of the Fory runtime package. - C# generated schema modules are source-file owners. Service companions must use that module's `ThreadSafeFory` and must not introduce namespace-owned aliases or duplicate serializer registration paths. - C# external-type serialization is target-keyed. A local diff --git a/.agents/languages/javascript.md b/.agents/languages/javascript.md index 834cc1cb3a..32f3978959 100644 --- a/.agents/languages/javascript.md +++ b/.agents/languages/javascript.md @@ -14,6 +14,9 @@ Load this file when changing `javascript/`. - JavaScript `Fory` owns the one authoritative registration-frozen flag. The first root serialization or deserialization sets it before codec work and leaves it set after failure. `TypeResolver` owns registration maps, not a second lifecycle flag. +- Codegen hooks can start a root while an explicit registration graph is being generated. Recheck + the `Fory`-owned flag before every later serializer publication in that graph; do not add a + lifecycle flag to `TypeResolver`, stage the graph, or roll registry entries back. - A failed root releases its operation-local reference and metadata state before the exception escapes. The next root entry releases state retained by the previous successful operation before reusing the context. Keep the successful root exit allocation-free and do not copy Java diff --git a/.agents/languages/swift.md b/.agents/languages/swift.md index f06a4750f9..6e9a1fdfe2 100644 --- a/.agents/languages/swift.md +++ b/.agents/languages/swift.md @@ -41,6 +41,9 @@ Load this file when changing `swift/` or Swift xlang behavior. deserialization. Do not add another lifecycle state or cache freeze failure separately. Registered TypeInfo owns lazy TypeMeta completion after freeze; do not add an eager whole-registry metadata pass. +- Serializer static properties are application code and can start a root during registration. + Recheck the same resolver flag after the last static property access and immediately before + returning from an idempotent registration or publishing `TypeInfo` by ID or name. - Direct `Any` and `AnyObject` root overloads remain disfavored forwarding facades over `DynamicSerializer` and `DynamicSerializer`, including their Data-buffer forms. Arbitrary protocol roots explicitly select `DynamicSerializer`. Do not add an unconstrained diff --git a/AGENTS.md b/AGENTS.md index 55adb80721..87acebde54 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -185,11 +185,13 @@ This is the entry point for AI guidance in Apache Fory. Read this file first, th maps, generated descriptors, metadata, serializers, or caches. Do not support post-use registration through cache invalidation, descriptor refresh, serializer rebinding, metadata rebuilding, or other late-registration - machinery. Keep one authoritative lifecycle fact for each natural registration - owner or public facade boundary. That fact may be an owner-native flag or the - presence of an immutable registry snapshot. A thread-safe facade with its own - public registration surface may own that boundary fact, but must not mirror a - child registry's lifecycle. Do not add another lifecycle state, a registration + machinery. Keep one authoritative frozen flag for each natural registration + owner or public facade boundary. A thread-safe facade with its own public + registration surface may own that boundary flag, but must not mirror a child + registry's lifecycle. Registration preparation can execute application code + through serializer construction, static serializer metadata, or codegen hooks. + Recheck the same owner flag after the last such callback and immediately before + explicit registry publication. Do not add another lifecycle state, a registration commit or rollback path, or eager whole-registry preparation solely to implement freeze. Copy operations and facade execution callbacks do not freeze registration unless they start a diff --git a/docs/specification/xlang_implementation_guide.md b/docs/specification/xlang_implementation_guide.md index d312a6d9bf..d2412382cc 100644 --- a/docs/specification/xlang_implementation_guide.md +++ b/docs/specification/xlang_implementation_guide.md @@ -83,13 +83,14 @@ not the place where nested serializers do their work. - resetting operation-local context state at the top-level root boundary Explicit type and serializer registration is open only before the first root serialization or -deserialization. Starting either root establishes exactly one authoritative frozen lifecycle fact -for the -natural registration owner before codec work and never clears it, including when the root fails. -Every explicit registration path checks that fact before mutation. A thread-safe facade with its own -public registration surface may own its boundary fact, but must not mirror a child registry's -lifecycle. That fact may be an owner-native flag or the presence of an immutable registry snapshot. -Do not add another lifecycle state, a registration commit or rollback path, or eager whole-registry +deserialization. Starting either root sets exactly one authoritative frozen flag for the natural +registration owner before codec work and never clears it, including when the root fails. +Every explicit registration path checks that flag before mutation. A thread-safe facade with its own +public registration surface may own its boundary flag, but must not mirror a child registry's +lifecycle. Registration preparation can execute application code through custom serializer +construction, static serializer properties, or code-generation hooks. Recheck the same owner flag +after the last such callback and immediately before explicit registry publication. Do not add +another lifecycle state, a registration commit or rollback path, or eager whole-registry preparation solely to implement freeze. In JavaScript, `Fory` owns this flag. `TypeResolver` owns the registration maps but must not carry a From bbd984639b1ad36b8d558f52b478f0bb7da1700f Mon Sep 17 00:00:00 2001 From: chaokunyang Date: Sun, 30 Aug 2026 23:05:15 +0800 Subject: [PATCH 164/168] fix: guard reentrant registry publication --- .agents/languages/csharp.md | 7 +- .agents/languages/java.md | 10 ++ AGENTS.md | 4 +- csharp/src/Fory/Fory.cs | 19 ++- csharp/src/Fory/TypeResolver.cs | 57 +++++++-- .../tests/Fory.Tests/RuntimeEdgeCaseTests.cs | 66 +++++++++++ docs/compiler/generated-code/java.md | 2 + .../kotlin/configuration.md | 2 + .../scala/configuration.md | 2 + .../xlang_implementation_guide.md | 11 ++ .../apache/fory/AbstractThreadSafeFory.java | 57 +++++---- .../main/java/org/apache/fory/BaseFory.java | 2 + .../src/main/java/org/apache/fory/Fory.java | 34 +++++- .../main/java/org/apache/fory/ForyModule.java | 7 +- .../java/org/apache/fory/ThreadLocalFory.java | 18 ++- .../java/org/apache/fory/ThreadSafeFory.java | 5 +- .../org/apache/fory/pool/ThreadPoolFory.java | 8 +- .../apache/fory/resolver/ClassResolver.java | 20 +++- .../apache/fory/resolver/TypeResolver.java | 20 +++- .../apache/fory/resolver/XtypeResolver.java | 19 ++- .../CompressedArraySerializers.java | 12 +- .../org/apache/fory/ThreadSafeForyTest.java | 108 +++++++++++++++++- 22 files changed, 426 insertions(+), 64 deletions(-) diff --git a/.agents/languages/csharp.md b/.agents/languages/csharp.md index c959d41d1d..a3f9e0d4b6 100644 --- a/.agents/languages/csharp.md +++ b/.agents/languages/csharp.md @@ -14,9 +14,10 @@ Load this file when changing `csharp/` or C# xlang behavior. set after failure. Explicit registration checks that flag before mutation. `ThreadSafeFory` keeps its existing registration callbacks only to configure newly created child runtimes; do not turn that list into another registry lifecycle state. -- Custom serializer construction and `TypeInfo` creation can execute application code. Complete - both before resolver mutation, then recheck the same `Fory` flag immediately before publishing - the custom binding. Do not publish an intermediate unregistered `TypeInfo`. +- Generated or custom serializer construction and `TypeInfo` creation can execute application + code. Complete them before explicit ID or name publication, then recheck the same `Fory` flag + immediately before publishing that registration. Do not publish an intermediate unregistered + `TypeInfo` for a custom binding. - Generated C# gRPC service companions are compiler-owned files that depend on application-provided gRPC packages, not `csharp/src/Fory`. Keep gRPC package references out of the Fory runtime package. - C# generated schema modules are source-file owners. Service companions must use that module's `ThreadSafeFory` and must not introduce namespace-owned aliases or duplicate serializer registration paths. - C# external-type serialization is target-keyed. A local diff --git a/.agents/languages/java.md b/.agents/languages/java.md index c1e4d19c85..4512f0675f 100644 --- a/.agents/languages/java.md +++ b/.agents/languages/java.md @@ -98,6 +98,16 @@ Load this file when changing anything under `java/` or when Java drives a cross- mutate another child. Non-root `execute` and copy operations remain concurrent and do not freeze registration. Complete facade registration before concurrent serialization, deserialization, copy, or `execute` calls begin; do not serialize those operations behind a registration lock. +- `ForyModule.install` is registration-only setup. It may install nested modules and construct + child-specific serializers, but must not start a root through the supplied child or a captured + direct or thread-safe facade. Do not propagate the facade lifecycle owner into raw child + registration to support this invalid reentrancy; frozen-facade replay needs the unexposed child + to remain governed by its local resolver until it adopts the shared snapshot. +- Live thread-safe facade registration carries the existing `SharedRegistry` check through + application-controlled serializer preparation and runs it immediately before child publication. + Replay into a new, unexposed thread-local child instead uses that child's local resolver check, + then freezes the child onto the shared snapshot before exposure. Do not make every child resolver + consult the shared frozen flag: that would reject the required frozen-facade setup replay. - `ThreadSafeFory.execute` exposes one borrowed child only for the callback. Do not retain that child or register through it; use the facade registration methods so every current and future child receives the same setup. diff --git a/AGENTS.md b/AGENTS.md index 87acebde54..c3cccbe796 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -210,7 +210,9 @@ This is the entry point for AI guidance in Apache Fory. Read this file first, th Java `Fory.register(ForyModule)` and the corresponding `BaseFory` operation remain available on direct and thread-safe facades before the first root. Kotlin and Scala registration extensions target `BaseFory` so the same API works with `Fory`, `ThreadLocalFory`, and pooled - `ThreadSafeFory` implementations. + `ThreadSafeFory` implementations. Module installation is registration-only setup: it may install + nested modules and child-specific serializers, but must not start a root through the supplied + child or the direct or thread-safe facade installing the module. - Use semantic naming only. Name things after protocol or domain concepts, not history, runtime origin, or workaround style; avoid vague names such as `Internal`, `java_style_*`, `Runtime`, `Session`, `Plan`, `Payload`, or `Binding` when they do not name the real concept. Keep class, method, function, and variable names concise; do not encode the whole scenario or implementation history into one identifier. Never name a class or method with a `Plan` suffix; use the real domain concept instead. For Fory codec/read APIs, do not use generic `payload` naming; name the exact owner and data shape, such as bytes, body, frame, field, string, list, map, compressed bytes, or primitive-array encoding. - Keep one implementation path. Do not keep parallel helpers, serializers, harnesses, wrappers, or registration flows for the same concept; extend the existing owner path instead of inventing another one. - Follow current scope exactly. The latest explicit user instruction overrides earlier plans, and when scope narrows, remove leaked out-of-scope edits immediately. diff --git a/csharp/src/Fory/Fory.cs b/csharp/src/Fory/Fory.cs index e31e4300b6..0b1bb1c319 100644 --- a/csharp/src/Fory/Fory.cs +++ b/csharp/src/Fory/Fory.cs @@ -73,7 +73,8 @@ public static ForyBuilder Builder() public Fory Register(uint typeId) { EnsureRegistrationOpen(); - _typeResolver.Register(typeof(T), typeId); + TypeInfo typeInfo = ResolveRegistrationTypeInfo(); + _typeResolver.Register(typeof(T), typeId, typeInfo); return this; } @@ -89,7 +90,8 @@ public Fory Register(string name) { EnsureRegistrationOpen(); (string namespaceName, string typeName) = TypeResolver.SplitTypeName(name); - _typeResolver.Register(typeof(T), namespaceName, typeName); + TypeInfo typeInfo = ResolveRegistrationTypeInfo(); + _typeResolver.Register(typeof(T), namespaceName, typeName, typeInfo); return this; } @@ -105,7 +107,9 @@ public Fory Register(string name) public Fory Register(string typeNamespace, string typeName) { EnsureRegistrationOpen(); - _typeResolver.Register(typeof(T), typeNamespace, typeName); + TypeResolver.ValidateSplitTypeName(typeNamespace, typeName); + TypeInfo typeInfo = ResolveRegistrationTypeInfo(); + _typeResolver.Register(typeof(T), typeNamespace, typeName, typeInfo); return this; } @@ -350,6 +354,15 @@ private TypeInfo CreateCustomTypeInfo() return typeInfo; } + private TypeInfo ResolveRegistrationTypeInfo() + { + TypeInfo typeInfo = _typeResolver.ResolveRegistrationTypeInfo(typeof(T)); + // Construction may start a root. Keep the candidate unpublished until the Fory-owned + // check succeeds; Register then publishes the binding and explicit ID or name together. + EnsureRegistrationOpen(); + return typeInfo; + } + [MethodImpl(MethodImplOptions.NoInlining)] private static void ThrowRegistryFrozen() => throw new InvalidOperationException( diff --git a/csharp/src/Fory/TypeResolver.cs b/csharp/src/Fory/TypeResolver.cs index b11d60b8f8..dc0fe861af 100644 --- a/csharp/src/Fory/TypeResolver.cs +++ b/csharp/src/Fory/TypeResolver.cs @@ -453,6 +453,25 @@ internal IReadOnlyList TypeMetaFields(TypeInfo typeInfo, bool return typeInfo.TypeMetaFields(trackRef); } + private TypeInfo ResolveTypeInfo( + Type type, + TypeInfo? explicitTypeInfo, + ulong typeKey) + { + TypeInfo typeInfo = explicitTypeInfo ?? CreateBindingCore(type); + if (typeInfo.Type != type) + { + throw new InvalidDataException($"serializer type mismatch for {type}, got {typeInfo.Type}"); + } + + if (_typeInfos.TryGetValue(typeKey, out TypeInfo? previous)) + { + typeInfo = typeInfo.WithRegistrationFrom(previous); + } + + return typeInfo; + } + private TypeInfo GetOrCreateTypeInfo(Type type, TypeInfo? explicitTypeInfo) { ulong typeKey = TypeMapKey.Get(type); @@ -469,25 +488,39 @@ private TypeInfo GetOrCreateTypeInfo(Type type, TypeInfo? explicitTypeInfo) } } - TypeInfo typeInfo = explicitTypeInfo ?? CreateBindingCore(type); - if (typeInfo.Type != type) - { - throw new InvalidDataException($"serializer type mismatch for {type}, got {typeInfo.Type}"); - } + TypeInfo typeInfo = ResolveTypeInfo(type, explicitTypeInfo, typeKey); + _typeInfos.Set(typeKey, typeInfo); + InvalidateVersionHash(); + return typeInfo; + } - if (_typeInfos.TryGetValue(typeKey, out TypeInfo? previous)) + internal TypeInfo ResolveRegistrationTypeInfo(Type type) + { + return ResolveRegistrationTypeInfo(type, null); + } + + private TypeInfo ResolveRegistrationTypeInfo(Type type, TypeInfo? explicitTypeInfo) + { + ulong typeKey = TypeMapKey.Get(type); + if (_typeInfos.TryGetValue(typeKey, out TypeInfo? existing)) { - typeInfo = typeInfo.WithRegistrationFrom(previous); + if (explicitTypeInfo is null || ReferenceEquals(existing, explicitTypeInfo)) + { + return existing; + } + + if (existing.IsRegistered) + { + throw new InvalidDataException($"cannot override serializer for registered type {type}"); + } } - _typeInfos.Set(typeKey, typeInfo); - InvalidateVersionHash(); - return typeInfo; + return ResolveTypeInfo(type, explicitTypeInfo, typeKey); } internal void Register(Type type, uint id, TypeInfo? explicitTypeInfo = null) { - TypeInfo typeInfo = GetOrCreateTypeInfo(type, explicitTypeInfo).WithTypeIdRegistration(id); + TypeInfo typeInfo = ResolveRegistrationTypeInfo(type, explicitTypeInfo).WithTypeIdRegistration(id); _typeInfos.Set(TypeMapKey.Get(type), typeInfo); _byUserTypeId[id] = typeInfo; InvalidateVersionHash(); @@ -530,7 +563,7 @@ internal static void ValidateSplitTypeName(string namespaceName, string typeName internal void Register(Type type, string namespaceName, string typeName, TypeInfo? explicitTypeInfo = null) { ValidateSplitTypeName(namespaceName, typeName); - TypeInfo typeInfo = GetOrCreateTypeInfo(type, explicitTypeInfo); + TypeInfo typeInfo = ResolveRegistrationTypeInfo(type, explicitTypeInfo); MetaString namespaceMeta = MetaStringEncoder.Namespace.Encode(namespaceName, TypeMetaEncodings.NamespaceMetaStringEncodings); MetaString typeNameMeta = MetaStringEncoder.TypeName.Encode(typeName, TypeMetaEncodings.TypeNameMetaStringEncodings); typeInfo = typeInfo.WithTypeNameRegistration(namespaceMeta, typeNameMeta); diff --git a/csharp/tests/Fory.Tests/RuntimeEdgeCaseTests.cs b/csharp/tests/Fory.Tests/RuntimeEdgeCaseTests.cs index aa2df61ce8..26f50d0c60 100644 --- a/csharp/tests/Fory.Tests/RuntimeEdgeCaseTests.cs +++ b/csharp/tests/Fory.Tests/RuntimeEdgeCaseTests.cs @@ -144,6 +144,38 @@ public override LookupFailureValue ReadData(ReadContext context) } } +public enum RegistrationValue +{ + Zero, +} + +public sealed class RegistrationValueSerializer : Serializer +{ + public static Action? ConstructionAction; + public static int ConstructionCount; + + public RegistrationValueSerializer() + { + ConstructionCount++; + ConstructionAction?.Invoke(); + } + + public override RegistrationValue DefaultValue => RegistrationValue.Zero; + + public override void WriteData(WriteContext context, in RegistrationValue value, bool hasGenerics) + { + _ = context; + _ = value; + _ = hasGenerics; + } + + public override RegistrationValue ReadData(ReadContext context) + { + _ = context; + return RegistrationValue.Zero; + } +} + public sealed class FailingWritePayload { public int Value { get; set; } @@ -885,6 +917,40 @@ public void CustomSerializerRechecksFreeze() Assert.NotEqual(typeof(CustomPayloadSerializer), typeInfo.SerializerType); } + [Fact] + public void GeneratedRegistrationRechecksFreeze() + { + TypeResolver.RegisterGenerated(); + foreach (bool registerByName in new[] { false, true }) + { + ForyRuntime fory = ForyRuntime.Builder().Build(); + RegistrationValueSerializer.ConstructionCount = 0; + RegistrationValueSerializer.ConstructionAction = () => _ = fory.Serialize(1); + try + { + Assert.Throws(() => + { + if (registerByName) + { + fory.Register("test.registration_value"); + } + else + { + fory.Register(722); + } + }); + } + finally + { + RegistrationValueSerializer.ConstructionAction = null; + } + + TypeInfo typeInfo = ReadContextFor(fory).TypeResolver.GetTypeInfo(typeof(RegistrationValue)); + Assert.False(typeInfo.IsRegistered); + Assert.Equal(2, RegistrationValueSerializer.ConstructionCount); + } + } + [Fact] public void FailedRootFreezesRegistry() { diff --git a/docs/compiler/generated-code/java.md b/docs/compiler/generated-code/java.md index 43f5326927..25dd187f7b 100644 --- a/docs/compiler/generated-code/java.md +++ b/docs/compiler/generated-code/java.md @@ -92,6 +92,8 @@ public final class Animal extends Union { Each JVM schema generates a `ForyModule`. Imported schema modules are installed through `fory.register(...)`, so shared imports are deduplicated by the Fory instance. +Module installation is registration-only setup and must not start a root serialization or +deserialization operation on the runtime being configured. ```java public final class AddressbookForyModule implements org.apache.fory.ForyModule { diff --git a/docs/object-serialization/kotlin/configuration.md b/docs/object-serialization/kotlin/configuration.md index 384fcb95ca..4abe134dfc 100644 --- a/docs/object-serialization/kotlin/configuration.md +++ b/docs/object-serialization/kotlin/configuration.md @@ -86,6 +86,8 @@ object ForyHolder { `ForyModule` registration and Kotlin reified registration extensions target `BaseFory`, so they are available on both direct and thread-safe facades. Complete registration before the facade's first root serialization or deserialization, and before concurrent use of a thread-safe facade begins. +Module installation is registration-only setup and must not start root serialization or +deserialization through the runtime or facade being configured. ### Using Builder Methods diff --git a/docs/object-serialization/scala/configuration.md b/docs/object-serialization/scala/configuration.md index 90f284eb4b..923f56f8c2 100644 --- a/docs/object-serialization/scala/configuration.md +++ b/docs/object-serialization/scala/configuration.md @@ -124,6 +124,8 @@ object ForyHolder { `BaseFory`, so they are available on both direct and thread-safe facades. Complete registration before the facade's first root serialization or deserialization, and before concurrent use of a thread-safe facade begins. +Module installation is registration-only setup and must not start root serialization or +deserialization through the runtime or facade being configured. ## Configuration diff --git a/docs/specification/xlang_implementation_guide.md b/docs/specification/xlang_implementation_guide.md index d2412382cc..e007969e79 100644 --- a/docs/specification/xlang_implementation_guide.md +++ b/docs/specification/xlang_implementation_guide.md @@ -93,6 +93,13 @@ after the last such callback and immediately before explicit registry publicatio another lifecycle state, a registration commit or rollback path, or eager whole-registry preparation solely to implement freeze. +For a Java thread-safe facade, live child registration carries the facade's existing shared-registry +check through application-controlled preparation and runs it immediately before publishing into the +child. Replaying already accepted setup into a new, unexposed thread-local child uses the child's +local resolver check; the child then adopts the shared frozen snapshot before exposure. The child +resolver must not mirror or directly enforce the facade flag because these owners govern different +boundaries. + In JavaScript, `Fory` owns this flag. `TypeResolver` owns the registration maps but must not carry a second lifecycle flag. @@ -112,6 +119,10 @@ Scala registration extensions target `BaseFory`, so direct and thread-safe facad pre-root API. Copy operations and facade execution callbacks do not freeze registration unless they start a root serialization or deserialization. +`ForyModule.install` is registration-only setup. It may register nested modules and construct +serializers for the supplied child, but it must not start root serialization or deserialization +through that child or the direct or thread-safe facade installing the module. + Nested serializers must not call back into root `serialize(...)` or `deserialize(...)` entry points. diff --git a/java/fory-core/src/main/java/org/apache/fory/AbstractThreadSafeFory.java b/java/fory-core/src/main/java/org/apache/fory/AbstractThreadSafeFory.java index 0098854119..1333a3e9df 100644 --- a/java/fory-core/src/main/java/org/apache/fory/AbstractThreadSafeFory.java +++ b/java/fory-core/src/main/java/org/apache/fory/AbstractThreadSafeFory.java @@ -19,6 +19,7 @@ package org.apache.fory; +import java.util.function.Consumer; import java.util.function.Function; import org.apache.fory.resolver.TypeChecker; import org.apache.fory.resolver.TypeResolver; @@ -26,60 +27,68 @@ import org.apache.fory.serializer.SerializerFactory; public abstract class AbstractThreadSafeFory implements ThreadSafeFory { + private void applyRegistration(Consumer registration) { + registerCallback( + (fory, checkBeforePublication) -> { + checkBeforePublication.run(); + registration.accept(fory); + }); + } + @Override public void register(Class clz) { - registerCallback(fory -> fory.register(clz)); + applyRegistration(fory -> fory.register(clz)); } @Override public void register(Class cls, int id) { - registerCallback(fory -> fory.register(cls, id)); + applyRegistration(fory -> fory.register(cls, id)); } @Override public void register(Class cls, String name) { - registerCallback(fory -> fory.register(cls, name)); + applyRegistration(fory -> fory.register(cls, name)); } @Override public void register(Class cls, String namespace, String typeName) { - registerCallback(fory -> fory.register(cls, namespace, typeName)); + applyRegistration(fory -> fory.register(cls, namespace, typeName)); } @Override public void register(String className) { - registerCallback(fory -> fory.register(className)); + applyRegistration(fory -> fory.register(className)); } @Override public void register(String className, int id) { - registerCallback(fory -> fory.register(className, id)); + applyRegistration(fory -> fory.register(className, id)); } @Override public void register(String className, String name) { - registerCallback(fory -> fory.register(className, name)); + applyRegistration(fory -> fory.register(className, name)); } @Override public void register(String className, String namespace, String typeName) { - registerCallback(fory -> fory.register(className, namespace, typeName)); + applyRegistration(fory -> fory.register(className, namespace, typeName)); } @Override public void register(ForyModule module) { - registerCallback(fory -> fory.register(module)); + applyRegistration(fory -> fory.register(module)); } public void registerUnion( Class cls, int id, org.apache.fory.serializer.Serializer serializer) { - registerCallback(fory -> fory.registerUnion(cls, id, serializer)); + applyRegistration(fory -> fory.registerUnion(cls, id, serializer)); } @Override public void registerUnion( Class cls, String name, org.apache.fory.serializer.Serializer serializer) { - registerCallback(fory -> fory.registerUnion(cls, name, serializer)); + applyRegistration(fory -> fory.registerUnion(cls, name, serializer)); } public void registerUnion( @@ -87,50 +96,58 @@ public void registerUnion( String namespace, String typeName, org.apache.fory.serializer.Serializer serializer) { - registerCallback(fory -> fory.registerUnion(cls, namespace, typeName, serializer)); + applyRegistration(fory -> fory.registerUnion(cls, namespace, typeName, serializer)); } @Override public void registerSerializer(Class type, Class serializerClass) { - registerCallback(fory -> fory.registerSerializer(type, serializerClass)); + registerCallback( + (fory, checkBeforePublication) -> + fory.registerSerializer(type, serializerClass, checkBeforePublication)); } @Override public void registerSerializer(Class type, Serializer serializer) { - registerCallback(fory -> fory.registerSerializer(type, serializer)); + applyRegistration(fory -> fory.registerSerializer(type, serializer)); } @Override public void registerSerializer( Class type, Function> serializerCreator) { - registerCallback(fory -> fory.registerSerializer(type, serializerCreator)); + registerCallback( + (fory, checkBeforePublication) -> + fory.registerSerializer(type, serializerCreator, checkBeforePublication)); } @Override public void registerSerializerAndType( Class type, Class serializerClass) { - registerCallback(fory -> fory.registerSerializerAndType(type, serializerClass)); + registerCallback( + (fory, checkBeforePublication) -> + fory.registerSerializerAndType(type, serializerClass, checkBeforePublication)); } @Override public void registerSerializerAndType(Class type, Serializer serializer) { - registerCallback(fory -> fory.registerSerializerAndType(type, serializer)); + applyRegistration(fory -> fory.registerSerializerAndType(type, serializer)); } @Override public void registerSerializerAndType( Class type, Function> serializerCreator) { - registerCallback(fory -> fory.registerSerializerAndType(type, serializerCreator)); + registerCallback( + (fory, checkBeforePublication) -> + fory.registerSerializerAndType(type, serializerCreator, checkBeforePublication)); } @Override public void registerSerializerFactory(SerializerFactory serializerFactory) { - registerCallback(fory -> fory.registerSerializerFactory(serializerFactory)); + applyRegistration(fory -> fory.registerSerializerFactory(serializerFactory)); } @Override public void setTypeChecker(TypeChecker typeChecker) { - registerCallback(fory -> fory.getTypeResolver().setTypeChecker(typeChecker)); + applyRegistration(fory -> fory.getTypeResolver().setTypeChecker(typeChecker)); } @Override diff --git a/java/fory-core/src/main/java/org/apache/fory/BaseFory.java b/java/fory-core/src/main/java/org/apache/fory/BaseFory.java index a792d37263..d1ab9295ef 100644 --- a/java/fory-core/src/main/java/org/apache/fory/BaseFory.java +++ b/java/fory-core/src/main/java/org/apache/fory/BaseFory.java @@ -92,6 +92,8 @@ public interface BaseFory { /** * Register a runtime module. Direct {@link Fory} instances install the module immediately; * thread-safe runtimes install it into every current and future underlying runtime instance. + * Module installation is registration-only setup and must not start root serialization or + * deserialization through the runtime or facade being configured. */ void register(ForyModule module); diff --git a/java/fory-core/src/main/java/org/apache/fory/Fory.java b/java/fory-core/src/main/java/org/apache/fory/Fory.java index 601f39fe28..48771c6fa4 100644 --- a/java/fory-core/src/main/java/org/apache/fory/Fory.java +++ b/java/fory-core/src/main/java/org/apache/fory/Fory.java @@ -251,6 +251,11 @@ public void registerSerializer(Class type, Class se getTypeResolver().registerSerializer(type, serializerClass); } + void registerSerializer( + Class type, Class serializerClass, Runnable checkBeforePublication) { + getTypeResolver().registerSerializer(type, serializerClass, checkBeforePublication); + } + @Override public void registerSerializer(Class type, Serializer serializer) { getTypeResolver().registerSerializer(type, serializer); @@ -259,8 +264,19 @@ public void registerSerializer(Class type, Serializer serializer) { @Override public void registerSerializer( Class type, Function> serializerCreator) { + registerSerializer(type, serializerCreator, typeResolver::checkRegistrationOpen); + } + + void registerSerializer( + Class type, + Function> serializerCreator, + Runnable checkBeforePublication) { typeResolver.checkRegistrationOpen(); - getTypeResolver().registerSerializer(type, serializerCreator.apply(typeResolver)); + Serializer serializer = serializerCreator.apply(typeResolver); + // A facade root may freeze a different child while application construction is running. The + // facade owner must be rechecked before this child publishes the prepared serializer. + checkBeforePublication.run(); + getTypeResolver().registerSerializer(type, serializer); } @Override @@ -269,6 +285,11 @@ public void registerSerializerAndType( getTypeResolver().registerSerializerAndType(type, serializerClass); } + void registerSerializerAndType( + Class type, Class serializerClass, Runnable checkBeforePublication) { + getTypeResolver().registerSerializerAndType(type, serializerClass, checkBeforePublication); + } + @Override public void registerSerializerAndType(Class type, Serializer serializer) { getTypeResolver().registerSerializerAndType(type, serializer); @@ -277,8 +298,17 @@ public void registerSerializerAndType(Class type, Serializer serializer) { @Override public void registerSerializerAndType( Class type, Function> serializerCreator) { + registerSerializerAndType(type, serializerCreator, typeResolver::checkRegistrationOpen); + } + + void registerSerializerAndType( + Class type, + Function> serializerCreator, + Runnable checkBeforePublication) { typeResolver.checkRegistrationOpen(); - getTypeResolver().registerSerializerAndType(type, serializerCreator.apply(typeResolver)); + Serializer serializer = serializerCreator.apply(typeResolver); + checkBeforePublication.run(); + getTypeResolver().registerSerializerAndType(type, serializer); } @Override diff --git a/java/fory-core/src/main/java/org/apache/fory/ForyModule.java b/java/fory-core/src/main/java/org/apache/fory/ForyModule.java index 0ccae89e78..eedf620f84 100644 --- a/java/fory-core/src/main/java/org/apache/fory/ForyModule.java +++ b/java/fory-core/src/main/java/org/apache/fory/ForyModule.java @@ -22,6 +22,11 @@ /** A reusable Fory runtime module installed during or after runtime construction. */ @FunctionalInterface public interface ForyModule { - /** Install this module into the concrete runtime. */ + /** + * Installs registration setup into the concrete runtime. + * + *

An installation may register nested modules and child-specific serializers, but it must not + * start a root through {@code fory} or the direct or thread-safe facade installing the module. + */ void install(Fory fory); } diff --git a/java/fory-core/src/main/java/org/apache/fory/ThreadLocalFory.java b/java/fory-core/src/main/java/org/apache/fory/ThreadLocalFory.java index 60f99131fe..5aab882ba3 100644 --- a/java/fory-core/src/main/java/org/apache/fory/ThreadLocalFory.java +++ b/java/fory-core/src/main/java/org/apache/fory/ThreadLocalFory.java @@ -24,7 +24,7 @@ import java.util.Collections; import java.util.Map; import java.util.WeakHashMap; -import java.util.function.Consumer; +import java.util.function.BiConsumer; import java.util.function.Function; import java.util.function.Supplier; import javax.annotation.concurrent.ThreadSafe; @@ -45,14 +45,14 @@ public class ThreadLocalFory extends AbstractThreadSafeFory { private final Supplier foryFactory; private final ThreadLocal foryThreadLocal; - private Consumer factoryCallback; + private BiConsumer factoryCallback; private final Map allFory; private final SharedRegistry sharedRegistry; public ThreadLocalFory(Function factory) { sharedRegistry = new SharedRegistry(); foryFactory = () -> factory.apply(Fory.builder().withSharedRegistry(sharedRegistry)); - factoryCallback = f -> {}; + factoryCallback = (fory, checkBeforePublication) -> {}; allFory = Collections.synchronizedMap(new WeakHashMap<>()); foryThreadLocal = ThreadLocal.withInitial(this::newFory); // 1. init and warm for current thread. @@ -65,7 +65,9 @@ public ThreadLocalFory(Function factory) { private Fory newFory() { synchronized (sharedRegistry) { Fory fory = foryFactory.get(); - factoryCallback.accept(fory); + // The facade may already be frozen, but this child is not exposed yet. Replay uses its local + // owner, then freezeRegistration adopts the facade's published snapshot before exposure. + factoryCallback.accept(fory, fory.getTypeResolver()::checkRegistrationOpen); if (sharedRegistry.isRegistrationFrozen()) { fory.getTypeResolver().freezeRegistration(); } @@ -81,11 +83,15 @@ private Fory currentFory() { @Internal @Override - public void registerCallback(Consumer callback) { + public void registerCallback(BiConsumer callback) { synchronized (sharedRegistry) { sharedRegistry.checkRegistrationOpen(); + Runnable publicationCheck = sharedRegistry::checkRegistrationOpen; synchronized (allFory) { - allFory.keySet().forEach(callback); + for (Fory fory : allFory.keySet()) { + callback.accept(fory, publicationCheck); + sharedRegistry.checkRegistrationOpen(); + } } factoryCallback = factoryCallback.andThen(callback); } diff --git a/java/fory-core/src/main/java/org/apache/fory/ThreadSafeFory.java b/java/fory-core/src/main/java/org/apache/fory/ThreadSafeFory.java index 45dc8ae769..407e2ba723 100644 --- a/java/fory-core/src/main/java/org/apache/fory/ThreadSafeFory.java +++ b/java/fory-core/src/main/java/org/apache/fory/ThreadSafeFory.java @@ -19,7 +19,7 @@ package org.apache.fory; -import java.util.function.Consumer; +import java.util.function.BiConsumer; import java.util.function.Function; import org.apache.fory.annotation.Internal; import org.apache.fory.resolver.TypeChecker; @@ -50,6 +50,7 @@ public interface ThreadSafeFory extends BaseFory { */ void setTypeChecker(TypeChecker typeChecker); + /** Applies registration to current and future children using the supplied owner check. */ @Internal - void registerCallback(Consumer callback); + void registerCallback(BiConsumer callback); } diff --git a/java/fory-core/src/main/java/org/apache/fory/pool/ThreadPoolFory.java b/java/fory-core/src/main/java/org/apache/fory/pool/ThreadPoolFory.java index 198c299117..9c3d2c15fd 100644 --- a/java/fory-core/src/main/java/org/apache/fory/pool/ThreadPoolFory.java +++ b/java/fory-core/src/main/java/org/apache/fory/pool/ThreadPoolFory.java @@ -24,7 +24,7 @@ import java.util.concurrent.Semaphore; import java.util.concurrent.atomic.AtomicInteger; import java.util.concurrent.atomic.AtomicReferenceArray; -import java.util.function.Consumer; +import java.util.function.BiConsumer; import java.util.function.Function; import java.util.function.Supplier; import javax.annotation.concurrent.ThreadSafe; @@ -151,11 +151,13 @@ private static int spread(int hash) { @Internal @Override - public void registerCallback(Consumer callback) { + public void registerCallback(BiConsumer callback) { synchronized (sharedRegistry) { sharedRegistry.checkRegistrationOpen(); + Runnable publicationCheck = sharedRegistry::checkRegistrationOpen; for (Fory fory : pooledFory) { - callback.accept(fory); + callback.accept(fory, publicationCheck); + sharedRegistry.checkRegistrationOpen(); } } } diff --git a/java/fory-core/src/main/java/org/apache/fory/resolver/ClassResolver.java b/java/fory-core/src/main/java/org/apache/fory/resolver/ClassResolver.java index 72842d6c60..72e4b274d5 100644 --- a/java/fory-core/src/main/java/org/apache/fory/resolver/ClassResolver.java +++ b/java/fory-core/src/main/java/org/apache/fory/resolver/ClassResolver.java @@ -1224,9 +1224,17 @@ public static boolean requireJavaSerialization(Class clz) { * @param type of class */ public void registerSerializer(Class type, Class serializerClass) { + registerSerializer(type, serializerClass, this::checkRegistrationOpen); + } + + @Override + public void registerSerializer( + Class type, Class serializerClass, Runnable checkBeforePublication) { checkRegistrationOpen(); checkSerializerRegistration(type, serializerClass); - registerSerializerImpl(type, Serializers.newSerializer(this, type, serializerClass)); + Serializer serializer = Serializers.newSerializer(this, type, serializerClass); + checkBeforePublication.run(); + registerSerializerImpl(type, serializer); } @Override @@ -1236,6 +1244,16 @@ public void registerSerializer(Class type, Serializer serializer) { registerSerializerImpl(type, serializer); } + @Override + public void registerSerializerAndType( + Class type, Class serializerClass, Runnable checkBeforePublication) { + checkRegistrationOpen(); + checkSerializerRegistration(type, serializerClass); + Serializer serializer = Serializers.newSerializer(this, type, serializerClass); + checkBeforePublication.run(); + registerSerializerAndType(type, serializer); + } + /** * If a serializer exists before, it will be replaced by new serializer. * diff --git a/java/fory-core/src/main/java/org/apache/fory/resolver/TypeResolver.java b/java/fory-core/src/main/java/org/apache/fory/resolver/TypeResolver.java index b42cb2e75e..5b6e9d8a5c 100644 --- a/java/fory-core/src/main/java/org/apache/fory/resolver/TypeResolver.java +++ b/java/fory-core/src/main/java/org/apache/fory/resolver/TypeResolver.java @@ -378,6 +378,16 @@ public final ObjectInstantiator getObjectInstantiator(Class type) { public abstract void registerSerializer( Class type, Class serializerClass); + /** + * Registers a serializer class after the registration owner confirms publication is still open. + * + *

The check runs after serializer construction because constructors may execute application + * code which starts a root operation. + */ + @Internal + public abstract void registerSerializer( + Class type, Class serializerClass, Runnable checkBeforePublication); + /** * Registers a serializer for internal types (those with fixed IDs in the type system). This * method is used for built-in types like ArrayList, HashMap, etc. @@ -423,13 +433,13 @@ private void publishRegistrationSnapshot() { */ public void registerSerializerAndType( Class type, Class serializerClass) { - checkRegistrationOpen(); - if (!isRegistered(type)) { - register(type); - } - registerSerializer(type, serializerClass); + registerSerializerAndType(type, serializerClass, this::checkRegistrationOpen); } + @Internal + public abstract void registerSerializerAndType( + Class type, Class serializerClass, Runnable checkBeforePublication); + /** * Registers a type (if not already registered) and then registers the serializer instance. * diff --git a/java/fory-core/src/main/java/org/apache/fory/resolver/XtypeResolver.java b/java/fory-core/src/main/java/org/apache/fory/resolver/XtypeResolver.java index 1546adc968..c1c8b1baf4 100644 --- a/java/fory-core/src/main/java/org/apache/fory/resolver/XtypeResolver.java +++ b/java/fory-core/src/main/java/org/apache/fory/resolver/XtypeResolver.java @@ -517,8 +517,16 @@ private TypeInfo newTypeInfo( } public void registerSerializer(Class type, Class serializerClass) { + registerSerializer(type, serializerClass, this::checkRegistrationOpen); + } + + @Override + public void registerSerializer( + Class type, Class serializerClass, Runnable checkBeforePublication) { checkRegistrationOpen(); - registerSerializer(type, newSerializer(type, serializerClass)); + Serializer serializer = newSerializer(type, serializerClass); + checkBeforePublication.run(); + registerSerializer(type, serializer); } public void registerSerializer(Class type, Serializer serializer) { @@ -550,6 +558,15 @@ public void registerSerializer(Class type, Serializer serializer) { } } + @Override + public void registerSerializerAndType( + Class type, Class serializerClass, Runnable checkBeforePublication) { + checkRegistrationOpen(); + Serializer serializer = newSerializer(type, serializerClass); + checkBeforePublication.run(); + registerSerializerAndType(type, serializer); + } + private void checkSerializerRegistration(Class type, Class serializerClass) { if (isCollection(type) || Collection.class.isAssignableFrom(type)) { if (!CollectionLikeSerializer.class.isAssignableFrom(serializerClass)) { diff --git a/java/fory-core/src/main/java/org/apache/fory/serializer/CompressedArraySerializers.java b/java/fory-core/src/main/java/org/apache/fory/serializer/CompressedArraySerializers.java index 7187c67f71..bafe6c46fe 100644 --- a/java/fory-core/src/main/java/org/apache/fory/serializer/CompressedArraySerializers.java +++ b/java/fory-core/src/main/java/org/apache/fory/serializer/CompressedArraySerializers.java @@ -121,7 +121,11 @@ static void registerIfEnabled(Fory fory) { * @param fory the ThreadSafeFory instance to register serializers with */ public static void registerIfEnabled(ThreadSafeFory fory) { - fory.registerCallback(CompressedArraySerializers::registerIfEnabled); + fory.registerCallback( + (child, checkBeforePublication) -> { + checkBeforePublication.run(); + registerIfEnabled(child); + }); } /** @@ -141,7 +145,11 @@ public static void register(Fory fory) { /** Register compressed array serializers with the given Fory instance. */ public static void register(ThreadSafeFory fory) { - fory.registerCallback(CompressedArraySerializers::register); + fory.registerCallback( + (child, checkBeforePublication) -> { + checkBeforePublication.run(); + register(child); + }); } public static final class CompressedIntArraySerializer extends PrimitiveArraySerializer { diff --git a/java/fory-core/src/test/java/org/apache/fory/ThreadSafeForyTest.java b/java/fory-core/src/test/java/org/apache/fory/ThreadSafeForyTest.java index f4506e0eab..981f063a98 100644 --- a/java/fory-core/src/test/java/org/apache/fory/ThreadSafeForyTest.java +++ b/java/fory-core/src/test/java/org/apache/fory/ThreadSafeForyTest.java @@ -33,6 +33,7 @@ import java.util.concurrent.TimeUnit; import java.util.concurrent.atomic.AtomicInteger; import java.util.concurrent.atomic.AtomicReference; +import java.util.function.Consumer; import lombok.Data; import org.apache.fory.context.CopyContext; import org.apache.fory.context.MetaReadContext; @@ -521,6 +522,17 @@ public Foo read(ReadContext readContext) { } } + public static class ReentrantFooSerializer extends FooSerializer { + private static Consumer constructionAction; + + public ReentrantFooSerializer(TypeResolver typeResolver, Class type) { + super(typeResolver, type); + if (constructionAction != null) { + constructionAction.accept(typeResolver); + } + } + } + private static final class BlockingCopyValue {} private static final class BlockingCopySerializer extends Serializer { @@ -762,7 +774,10 @@ public void testExecuteRootFreezesFacade() { AtomicInteger callbacks = new AtomicInteger(); Assert.assertThrows( - ForyException.class, () -> fory.registerCallback(child -> callbacks.incrementAndGet())); + ForyException.class, + () -> + fory.registerCallback( + (child, checkBeforePublication) -> callbacks.incrementAndGet())); assertEquals(callbacks.get(), 0); } } @@ -796,7 +811,8 @@ public void testExecuteRootRegistrationRace() throws InterruptedException { () -> { try { registrationStarted.countDown(); - fory.registerCallback(child -> callbacks.incrementAndGet()); + fory.registerCallback( + (child, checkBeforePublication) -> callbacks.incrementAndGet()); } catch (Throwable t) { registrationError.set(t); } @@ -816,6 +832,94 @@ public void testExecuteRootRegistrationRace() throws InterruptedException { } } + @Test + public void testReentrantRootStopsRegistration() throws InterruptedException { + for (int registrationKind = 0; registrationKind < 3; registrationKind++) { + for (ThreadSafeFory fory : newThreadSafeRuntimes()) { + AtomicReference otherChild = new AtomicReference<>(); + Thread childThread = + new Thread( + () -> + fory.execute( + child -> { + otherChild.set(child); + return null; + })); + childThread.start(); + childThread.join(); + + TypeResolver rootResolver = fory.execute(Fory::getTypeResolver); + if (fory instanceof ThreadLocalFory) { + Assert.assertNotSame(otherChild.get().getTypeResolver(), rootResolver); + } + AtomicReference lateResolver = new AtomicReference<>(); + AtomicInteger roots = new AtomicInteger(); + Consumer startRoot = + resolver -> { + if (resolver != rootResolver && roots.compareAndSet(0, 1)) { + lateResolver.set(resolver); + fory.serialize("freeze"); + } + }; + Class registeredSerializer; + + if (registrationKind == 1) { + registeredSerializer = ReentrantFooSerializer.class; + ReentrantFooSerializer.constructionAction = startRoot; + try { + Assert.assertThrows( + ForyException.class, + () -> fory.registerSerializer(Foo.class, ReentrantFooSerializer.class)); + } finally { + ReentrantFooSerializer.constructionAction = null; + } + } else if (registrationKind == 2) { + registeredSerializer = ReentrantFooSerializer.class; + ReentrantFooSerializer.constructionAction = startRoot; + try { + Assert.assertThrows( + ForyException.class, + () -> fory.registerSerializerAndType(Foo.class, ReentrantFooSerializer.class)); + } finally { + ReentrantFooSerializer.constructionAction = null; + } + } else { + registeredSerializer = FooSerializer.class; + Assert.assertThrows( + ForyException.class, + () -> + fory.registerSerializer( + Foo.class, + resolver -> { + startRoot.accept(resolver); + return new FooSerializer(resolver, Foo.class); + })); + } + + assertEquals(roots.get(), 1); + Assert.assertFalse(lateResolver.get().isRegistered(Foo.class)); + Assert.assertNotEquals( + lateResolver.get().getSerializer(Foo.class).getClass(), registeredSerializer); + Assert.assertThrows(ForyException.class, () -> fory.register(BeanB.class)); + + if (fory instanceof ThreadLocalFory) { + AtomicReference> serializerType = new AtomicReference<>(); + Thread futureChild = + new Thread( + () -> + fory.execute( + child -> { + serializerType.set(child.getSerializer(Foo.class).getClass()); + return null; + })); + futureChild.start(); + futureChild.join(); + Assert.assertNotEquals(serializerType.get(), registeredSerializer); + } + } + } + } + @Test public void testChildFreezeWaitsForFacade() throws InterruptedException { SharedRegistry sharedRegistry = new SharedRegistry(); From a62acd707b76e2c6a50c158219cf985261e83437 Mon Sep 17 00:00:00 2001 From: chaokunyang Date: Sun, 30 Aug 2026 23:51:42 +0800 Subject: [PATCH 165/168] docs: preserve failed-root cleanup invariants --- cpp/fory/serialization/context.cc | 2 ++ .../src/main/java/org/apache/fory/util/ExceptionUtils.java | 2 ++ 2 files changed, 4 insertions(+) diff --git a/cpp/fory/serialization/context.cc b/cpp/fory/serialization/context.cc index 0bf36d586e..2d1b7fd5c1 100644 --- a/cpp/fory/serialization/context.cc +++ b/cpp/fory/serialization/context.cc @@ -754,6 +754,8 @@ ReadContext::read_type_meta_owner(const TypeInfo *expected_type_info) { cached->concrete_owner = local_type_info; if (local_type_info) { // Have local type - assign dispatch IDs by comparing schemas. + // Extension types have no local TypeMeta; only structs can provide local + // field metadata. if (local_type_info->type_meta) { FORY_RETURN_NOT_OK(TypeMeta::assign_local_dispatch_ids( local_type_info->type_meta.get(), parsed_meta->field_infos)); diff --git a/java/fory-core/src/main/java/org/apache/fory/util/ExceptionUtils.java b/java/fory-core/src/main/java/org/apache/fory/util/ExceptionUtils.java index 7d9589ff53..88a745afe1 100644 --- a/java/fory-core/src/main/java/org/apache/fory/util/ExceptionUtils.java +++ b/java/fory-core/src/main/java/org/apache/fory/util/ExceptionUtils.java @@ -51,6 +51,8 @@ public static StackOverflowError trySetStackOverflowErrorMessage( } } + // Do not attach read-reference tables to the exception. Root cleanup must release the failed + // object graph even when application code retains the exception for later inspection. public static RuntimeException handleReadFailed(Throwable t) { if (t instanceof ForyException) { throw (ForyException) t; From fe97d41d3a4744a52568198dd1d028936dc5e4de Mon Sep 17 00:00:00 2001 From: chaokunyang Date: Sun, 30 Aug 2026 23:59:37 +0800 Subject: [PATCH 166/168] refactor: remove noncausal lifecycle drift --- .../python/configuration.md | 9 ++-- .../python/troubleshooting.md | 2 + .../main/java/org/apache/fory/BaseFory.java | 1 - .../src/main/java/org/apache/fory/Fory.java | 4 +- .../apache/fory/io/BlockedStreamUtils.java | 4 +- .../fory/serializer/ObjectSerializer.java | 6 +-- .../org/apache/fory/ThreadSafeForyTest.java | 50 ------------------- python/README.md | 2 +- python/pyfory/serialization.pyx | 1 + 9 files changed, 15 insertions(+), 64 deletions(-) diff --git a/docs/object-serialization/python/configuration.md b/docs/object-serialization/python/configuration.md index e984e0f4a8..287207fc95 100644 --- a/docs/object-serialization/python/configuration.md +++ b/docs/object-serialization/python/configuration.md @@ -54,12 +54,11 @@ Thread-safe serialization interface using a pooled wrapper: ```python class ThreadSafeFory: - def __init__(self, fory_factory=None, **kwargs) + def __init__( + self, fory_factory=None, **kwargs + ) ``` -When supplied, `fory_factory` creates each pooled `Fory` instance. Otherwise, `**kwargs` are passed -to the normal `Fory` constructor. - ## Parameters | Parameter | Type | Default | Description | @@ -83,7 +82,7 @@ to the normal `Fory` constructor. ## Key Methods ```python -fory = pyfory.Fory(xlang=True) +fory = pyfory.ThreadSafeFory(xlang=True) # Serialization (serialize/deserialize are identical to dumps/loads) data: bytes = fory.serialize(obj) diff --git a/docs/object-serialization/python/troubleshooting.md b/docs/object-serialization/python/troubleshooting.md index 0e7eb59ec2..542d3ff2d5 100644 --- a/docs/object-serialization/python/troubleshooting.md +++ b/docs/object-serialization/python/troubleshooting.md @@ -86,6 +86,8 @@ assert result.next.next is result # Circular reference preserved ### Schema Evolution Not Working ```python +# Keep compatible mode enabled. This is the default. + # Version 1: Writer schema @dataclass class UserV1: diff --git a/java/fory-core/src/main/java/org/apache/fory/BaseFory.java b/java/fory-core/src/main/java/org/apache/fory/BaseFory.java index d1ab9295ef..f369903121 100644 --- a/java/fory-core/src/main/java/org/apache/fory/BaseFory.java +++ b/java/fory-core/src/main/java/org/apache/fory/BaseFory.java @@ -97,7 +97,6 @@ public interface BaseFory { */ void register(ForyModule module); - /** Register a union by ID. */ void registerUnion(Class cls, int id, Serializer serializer); /** diff --git a/java/fory-core/src/main/java/org/apache/fory/Fory.java b/java/fory-core/src/main/java/org/apache/fory/Fory.java index 48771c6fa4..e43c07c333 100644 --- a/java/fory-core/src/main/java/org/apache/fory/Fory.java +++ b/java/fory-core/src/main/java/org/apache/fory/Fory.java @@ -646,9 +646,9 @@ public T copy(T obj) { } private void serializeToStream(OutputStream outputStream, Consumer function) { + MemoryBuffer buf = getBuffer(); + buf.writerIndex(0); try { - MemoryBuffer buf = getBuffer(); - buf.writerIndex(0); function.accept(buf); byte[] bytes = buf.getHeapMemory(); if (bytes != null) { diff --git a/java/fory-core/src/main/java/org/apache/fory/io/BlockedStreamUtils.java b/java/fory-core/src/main/java/org/apache/fory/io/BlockedStreamUtils.java index e6d7db15b6..a8ecfa53ae 100644 --- a/java/fory-core/src/main/java/org/apache/fory/io/BlockedStreamUtils.java +++ b/java/fory-core/src/main/java/org/apache/fory/io/BlockedStreamUtils.java @@ -134,9 +134,9 @@ private static void readByteBuffer(ReadableByteChannel channel, ByteBuffer buffe private static void serializeToStream( Fory fory, OutputStream outputStream, Consumer function) { + MemoryBuffer buf = fory.getBuffer(); + buf.writerIndex(0); try { - MemoryBuffer buf = fory.getBuffer(); - buf.writerIndex(0); buf.writeInt32(-1); function.accept(buf); buf.putInt32(0, buf.writerIndex() - 4); diff --git a/java/fory-core/src/main/java/org/apache/fory/serializer/ObjectSerializer.java b/java/fory-core/src/main/java/org/apache/fory/serializer/ObjectSerializer.java index 333366172d..718d0e45e0 100644 --- a/java/fory-core/src/main/java/org/apache/fory/serializer/ObjectSerializer.java +++ b/java/fory-core/src/main/java/org/apache/fory/serializer/ObjectSerializer.java @@ -84,10 +84,10 @@ public ObjectSerializer( super(typeResolver, cls, objectInstantiator); trackingRef = config.trackingRef(); checkClassVersion = typeResolver.checkClassVersion(); + // avoid recursive building serializers. + // Use `setSerializerIfAbsent` to avoid overwriting existing serializer for class when used + // as data serializer. if (resolveParent) { - // avoid recursive building serializers. - // Use `setSerializerIfAbsent` to avoid overwriting existing serializer for class when used - // as data serializer. typeResolver.setSerializerIfAbsent(cls, this); } Collection descriptors; diff --git a/java/fory-core/src/test/java/org/apache/fory/ThreadSafeForyTest.java b/java/fory-core/src/test/java/org/apache/fory/ThreadSafeForyTest.java index 981f063a98..ebc8b1080b 100644 --- a/java/fory-core/src/test/java/org/apache/fory/ThreadSafeForyTest.java +++ b/java/fory-core/src/test/java/org/apache/fory/ThreadSafeForyTest.java @@ -782,56 +782,6 @@ public void testExecuteRootFreezesFacade() { } } - @Test - public void testExecuteRootRegistrationRace() throws InterruptedException { - for (ThreadSafeFory fory : newThreadSafeRuntimes()) { - CountDownLatch rootStarted = new CountDownLatch(1); - CountDownLatch registrationStarted = new CountDownLatch(1); - CountDownLatch finishRoot = new CountDownLatch(1); - AtomicInteger callbacks = new AtomicInteger(); - AtomicReference rootError = new AtomicReference<>(); - AtomicReference registrationError = new AtomicReference<>(); - Thread rootThread = - new Thread( - () -> { - try { - fory.execute( - child -> { - child.serialize("value"); - rootStarted.countDown(); - awaitUnchecked(finishRoot); - return null; - }); - } catch (Throwable t) { - rootError.set(t); - } - }); - Thread registrationThread = - new Thread( - () -> { - try { - registrationStarted.countDown(); - fory.registerCallback( - (child, checkBeforePublication) -> callbacks.incrementAndGet()); - } catch (Throwable t) { - registrationError.set(t); - } - }); - - rootThread.start(); - assertTrue(rootStarted.await(30, TimeUnit.SECONDS)); - registrationThread.start(); - assertTrue(registrationStarted.await(30, TimeUnit.SECONDS)); - finishRoot.countDown(); - rootThread.join(); - registrationThread.join(); - - assertNull(rootError.get()); - assertTrue(registrationError.get() instanceof ForyException); - assertEquals(callbacks.get(), 0); - } - } - @Test public void testReentrantRootStopsRegistration() throws InterruptedException { for (int registrationKind = 0; registrationKind < 3; registrationKind++) { diff --git a/python/README.md b/python/README.md index 86d175d750..b08fa7d836 100644 --- a/python/README.md +++ b/python/README.md @@ -744,7 +744,7 @@ for t in threads: t.join() **Key Methods:** ```python -fory = pyfory.Fory(xlang=True) +fory = pyfory.ThreadSafeFory(xlang=True) # Serialization (serialize/deserialize are identical to dumps/loads) data: bytes = fory.serialize(obj) diff --git a/python/pyfory/serialization.pyx b/python/pyfory/serialization.pyx index 07efc5511d..43b873caff 100644 --- a/python/pyfory/serialization.pyx +++ b/python/pyfory/serialization.pyx @@ -270,6 +270,7 @@ cdef class TypeResolver: cdef flat_hash_map[uint32_t, PyObject *] _c_user_type_id_to_type_info cdef flat_hash_map[uint64_t, PyObject *] _c_types_info cdef flat_hash_map[pair[int64_t, int64_t], PyObject *] _c_meta_hash_to_type_info + def __init__(self, Config config, *, shared_registry): """ Build the Cython resolver and its hot caches. From ca67563149accc9f8788bdf2d174263680badf53 Mon Sep 17 00:00:00 2001 From: chaokunyang Date: Mon, 31 Aug 2026 00:29:50 +0800 Subject: [PATCH 167/168] docs: register thread-safe types before root use --- docs/object-serialization/python/configuration.md | 14 ++++++-------- python/README.md | 14 ++++++-------- 2 files changed, 12 insertions(+), 16 deletions(-) diff --git a/docs/object-serialization/python/configuration.md b/docs/object-serialization/python/configuration.md index 287207fc95..798e1ee9b5 100644 --- a/docs/object-serialization/python/configuration.md +++ b/docs/object-serialization/python/configuration.md @@ -84,6 +84,12 @@ class ThreadSafeFory: ```python fory = pyfory.ThreadSafeFory(xlang=True) +# Complete registration before the first root operation. Choose one form: +fory.register(MyClass, type_id=123) +# fory.register(MyClass, name="my.package.MyClass") +# fory.register(MyClass, type_id=123, serializer=MySerializer) +# fory.register(MyClass, name="my.package.MyClass", serializer=MySerializer) + # Serialization (serialize/deserialize are identical to dumps/loads) data: bytes = fory.serialize(obj) obj = fory.deserialize(data) @@ -91,14 +97,6 @@ obj = fory.deserialize(data) # Alternative API (aliases) data: bytes = fory.dumps(obj) obj = fory.loads(data) - -# Type registration by id -fory.register(MyClass, type_id=123) -fory.register(MyClass, type_id=123, serializer=custom_serializer) - -# Type registration by name -fory.register(MyClass, name="my.package.MyClass") -fory.register(MyClass, name="my.package.MyClass", serializer=custom_serializer) ``` ## Xlang And Native Mode Comparison diff --git a/python/README.md b/python/README.md index b08fa7d836..afa4c46f39 100644 --- a/python/README.md +++ b/python/README.md @@ -746,6 +746,12 @@ for t in threads: t.join() ```python fory = pyfory.ThreadSafeFory(xlang=True) +# Complete registration before the first root operation. Choose one form: +fory.register(MyClass, type_id=123) +# fory.register(MyClass, name="my.package.MyClass") +# fory.register(MyClass, type_id=123, serializer=MySerializer) +# fory.register(MyClass, name="my.package.MyClass", serializer=MySerializer) + # Serialization (serialize/deserialize are identical to dumps/loads) data: bytes = fory.serialize(obj) obj = fory.deserialize(data) @@ -753,14 +759,6 @@ obj = fory.deserialize(data) # Alternative API (aliases) data: bytes = fory.dumps(obj) obj = fory.loads(data) - -# Type registration by id -fory.register(MyClass, type_id=123) -fory.register(MyClass, type_id=123, serializer=custom_serializer) - -# Type registration by name -fory.register(MyClass, name="my.package.MyClass") -fory.register(MyClass, name="my.package.MyClass", serializer=custom_serializer) ``` ### Xlang And Native Mode Comparison From c08c38354b3fed40f81064f7dab72b09b1fc3060 Mon Sep 17 00:00:00 2001 From: chaokunyang Date: Mon, 31 Aug 2026 00:50:29 +0800 Subject: [PATCH 168/168] docs: keep registry snapshots separate from lifecycle --- docs/security/deserialization.md | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/docs/security/deserialization.md b/docs/security/deserialization.md index ccc78fb0d3..3ca9beb3d2 100644 --- a/docs/security/deserialization.md +++ b/docs/security/deserialization.md @@ -619,12 +619,11 @@ that case, classify the behavior by concrete impact: The first root serialization or deserialization permanently closes explicit type and serializer registration, including when that operation fails. Each natural registration owner keeps exactly -one -authoritative lifecycle fact, and every later explicit registration attempt fails before changing +one authoritative frozen flag, and every later explicit registration attempt fails before changing type, serializer, ID, name, metadata, or policy bindings. A thread-safe facade with its own public -registration surface may own its boundary fact, but must not mirror a child registry's lifecycle. -That fact may be an owner-native flag or the presence of an immutable registry snapshot. Do not add -another lifecycle state, a registration commit or rollback path, or eager whole-registry +registration surface may own its boundary flag, but must not mirror a child registry's lifecycle. +An immutable registry snapshot remains registry data and must not replace the frozen flag. Do not +add another lifecycle state, a registration commit or rollback path, or eager whole-registry preparation solely to implement freeze. Registry freeze does not disable native runtime type resolution. When a mode supports unregistered