From afc1b3efa3b262e59995daf4865f112bc54644fb Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?R=C3=A9mi=20Vezy?= Date: Mon, 31 Aug 2026 10:06:25 +0200 Subject: [PATCH 1/8] Optimize dynamic organ lifecycle refresh --- src/composite_model/compilation.jl | 612 +++++++++++++++-- src/composite_model/runtime_outputs.jl | 645 +++++++++++++++--- test/test-model-api-stabilization.jl | 318 ++++++++- test/test-model-distributed-output-runtime.jl | 138 ++++ test/test-model-hard-calls.jl | 411 +++++++++++ test/test-model-previous-timestep-views.jl | 102 +++ test/test-model-status-type-lifecycle.jl | 92 +++ test/test-unified-model-object-api.jl | 67 ++ 8 files changed, 2255 insertions(+), 130 deletions(-) diff --git a/src/composite_model/compilation.jl b/src/composite_model/compilation.jl index 454c96291..bbabc6fa3 100644 --- a/src/composite_model/compilation.jl +++ b/src/composite_model/compilation.jl @@ -278,7 +278,10 @@ function _selector_candidate_index() Dict{Symbol,Vector{Int}}(), Dict{Symbol,Vector{Int}}(), Dict{Symbol,Vector{Int}}(), - Dict{ObjectId,Vector{Int}}(), + Dict{ + Union{ObjectId,Tuple{ObjectId,Symbol,Symbol}}, + Vector{Int}, + }(), ) end @@ -302,7 +305,13 @@ function _selector_scope_anchor( context=nothing, default_scope=nothing, ) - scope = isnothing(matcher.scope) ? default_scope : matcher.scope + scope = if !isnothing(matcher.scope) + matcher.scope + elseif isnothing(matcher.relation) + default_scope + else + nothing + end scope isa CompiledNamedScope && return scope.root_id isnothing(context) && return nothing context_id = _object_id_from_context(context) @@ -350,8 +359,24 @@ function _selector_candidate_destination( context=context, default_scope=default_scope, ) - !isnothing(anchor) && - return (index.by_scope_anchor, (ObjectId(anchor),)) + if !isnothing(anchor) + anchor_id = ObjectId(anchor) + for (label, value) in ( + (:name, matcher.name), + (:scale, matcher.scale), + (:kind, matcher.kind), + (:species, matcher.species), + ) + isnothing(value) || return ( + index.by_scope_anchor, + Tuple( + (anchor_id, label, candidate) + for candidate in _selector_candidate_values(value) + ), + ) + end + return (index.by_scope_anchor, (anchor_id,)) + end for (groups, value) in ( (index.by_name, matcher.name), (index.by_scale, matcher.scale), @@ -409,6 +434,21 @@ function _union_selector_candidates!( end for anchor in _object_ancestor_ids(model.registry, object_id) union!(candidates, get(index.by_scope_anchor, anchor, ())) + for (label, value) in ( + (:name, object.name), + (:scale, object.scale), + (:kind, object.kind), + (:species, object.species), + ) + isnothing(value) || union!( + candidates, + get( + index.by_scope_anchor, + (anchor, label, value), + (), + ), + ) + end end return candidates end @@ -1630,9 +1670,27 @@ function _compile_added_consumer_bindings!( applications_by_object, applications_by_id, distributed_outputs=NoCompiledDistributedOutputs(), + many_binding_cache=nothing, + performance=nothing, ) for plan in input_plans plan.origin == :inferred_same_object && continue + if !isnothing(many_binding_cache) + shared_binding = _shared_many_input_binding( + many_binding_cache, + model, + plan, + consumer_id, + ) + if !isnothing(shared_binding) + push!(bindings, shared_binding) + _runtime_performance_count!( + performance, + :many_input_binding_precompile_reuses, + ) + continue + end + end _push_model_input_binding!( bindings, model, @@ -1718,17 +1776,21 @@ function _final_inferred_output_plans( return isempty(final_matches) ? matches : final_matches end -function _many_binding_scope_anchor(model::CompositeModel, binding::CompiledModelInputBinding) - selector_criteria = criteria(binding.selector) +function _many_binding_scope_anchor( + model::CompositeModel, + selector, + consumer_id::ObjectId, +) + selector_criteria = criteria(selector) !isnothing(_criteria_get(selector_criteria, :relation, nothing)) && - return (:consumer, binding.consumer_id) + return (:consumer, consumer_id) explicit_scope = _criteria_scope(selector_criteria) scope = isnothing(explicit_scope) ? - _default_dependency_scope(model, binding.consumer_id) : explicit_scope + _default_dependency_scope(model, consumer_id) : explicit_scope if isnothing(scope) || scope isa SceneScope return (:scene,) elseif scope isa SelfPlant - return (:plant, _ancestor_id(model, binding.consumer_id; scale=:Plant)) + return (:plant, _ancestor_id(model, consumer_id; scale=:Plant)) elseif scope isa Scope return (:scope, scope.name) elseif scope isa Ancestor @@ -1736,13 +1798,41 @@ function _many_binding_scope_anchor(model::CompositeModel, binding::CompiledMode :ancestor, _ancestor_id( model, - binding.consumer_id; + consumer_id; scale=scope.scale, include_self=false, ), ) end - return (:consumer, binding.consumer_id) + return (:consumer, consumer_id) +end + +_many_binding_scope_anchor( + model::CompositeModel, + binding::CompiledModelInputBinding, +) = _many_binding_scope_anchor(model, binding.selector, binding.consumer_id) + +function _many_binding_plan_share_key( + model::CompositeModel, + plan::CompiledModelInputPlan, + consumer_id::ObjectId, +) + plan.multiplicity == :many || return nothing + anchor = _many_binding_scope_anchor(model, plan.selector, consumer_id) + first(anchor) === :consumer && return nothing + any(isnothing, anchor) && return nothing + return (:compiled_many_input_plan, plan.slot, anchor) +end + +function _many_binding_plan_share_key( + model::CompositeModel, + binding::CompiledModelInputBinding, +) + return _many_binding_plan_share_key( + model, + binding.plan, + binding.consumer_id, + ) end function _many_binding_share_key(model::CompositeModel, binding::CompiledModelInputBinding) @@ -1774,20 +1864,52 @@ function _binding_with_shared_many_sources( ) end +function _shared_many_input_binding( + cache, + model::CompositeModel, + plan::CompiledModelInputPlan, + consumer_id::ObjectId, +) + key = _many_binding_plan_share_key(model, plan, consumer_id) + isnothing(key) && return nothing + canonical = get(cache, key, nothing) + isnothing(canonical) && return nothing + canonical.plan.slot == plan.slot || return nothing + canonical.multiplicity == :many || return nothing + isnothing(canonical.carrier) && return nothing + return CompiledModelInputBinding( + plan, + consumer_id, + canonical.source_ids, + canonical.source_application_ids, + canonical.policy, + canonical.carrier_hint, + canonical.carrier, + ) +end + +function _cache_many_binding_plan!(cache, model::CompositeModel, binding) + key = _many_binding_plan_share_key(model, binding) + isnothing(key) || (cache[key] = binding) + return binding +end + function _share_many_input_binding!(cache, model::CompositeModel, binding) key = _many_binding_share_key(model, binding) isnothing(key) && return binding canonical = get(cache, key, nothing) if isnothing(canonical) cache[key] = binding - return binding + return _cache_many_binding_plan!(cache, model, binding) end if canonical.source_ids != binding.source_ids || canonical.source_application_ids != binding.source_application_ids cache[(key, binding.consumer_id)] = binding return binding end - return _binding_with_shared_many_sources(binding, canonical) + shared = _binding_with_shared_many_sources(binding, canonical) + _cache_many_binding_plan!(cache, model, canonical) + return shared end function _share_many_input_bindings!(model::CompositeModel, bindings; cache=Dict{Any,Any}()) @@ -1801,7 +1923,10 @@ function _many_input_binding_cache(model::CompositeModel, bindings) cache = Dict{Any,Any}() for binding in bindings key = _many_binding_share_key(model, binding) - isnothing(key) || haskey(cache, key) || (cache[key] = binding) + if !isnothing(key) + haskey(cache, key) || (cache[key] = binding) + _cache_many_binding_plan!(cache, model, cache[key]) + end end return cache end @@ -1825,10 +1950,22 @@ function _append_added_many_sources!( context=binding.consumer_id, default_to_context=true, default_scope=default_scope, - ) && !(object_id in binding.source_ids) + ) && !first( + _sorted_object_id_position(binding.source_ids, object_id), + ) ] isempty(new_source_ids) && return true _sort_object_ids!(new_source_ids) + _filter_many_input_sources_by_writer!( + new_source_ids, + binding.selector, + binding.source_var, + binding.process, + binding.application, + applications_by_id, + distributed_outputs, + ) + isempty(new_source_ids) && return true # The growth path allocates monotonically increasing IDs. Appending preserves the # selector's stable order and, critically, keeps the carrier already installed in @@ -1843,13 +1980,11 @@ function _append_added_many_sources!( existing_refs = parent(binding.carrier) new_refs = parent(new_carrier) eltype(new_refs) <: eltype(existing_refs) || return false - append!(existing_refs, new_refs) - append!(binding.source_ids, new_source_ids) - new_application_ids = if _selector_from_status(binding.selector) + source_application_ids = if _selector_from_status(binding.selector) Symbol[] else - _matching_input_source_applications( + new_application_ids = _matching_input_source_applications( applications_by_object, new_source_ids, binding.source_var, @@ -1857,13 +1992,46 @@ function _append_added_many_sources!( binding.application, distributed_outputs; applications_by_id=applications_by_id, - allow_empty=binding.selector isa OptionalOne, + allow_empty=true, ) + if !isempty(binding.source_application_ids) && all( + application_id -> + application_id in binding.source_application_ids, + new_application_ids, + ) + copy(binding.source_application_ids) + else + combined_source_ids = [binding.source_ids; new_source_ids] + resolved = _matching_input_source_applications( + applications_by_object, + combined_source_ids, + binding.source_var, + binding.process, + binding.application, + distributed_outputs; + applications_by_id=applications_by_id, + allow_empty=false, + ) + if length(resolved) > 1 + _final_many_source_applications( + combined_source_ids, + resolved, + binding.source_var, + applications_by_id, + distributed_outputs, + ) + else + resolved + end + end end - for application_id in new_application_ids - application_id in binding.source_application_ids || - push!(binding.source_application_ids, application_id) - end + + # Resolve every fallible piece before mutating the shared carrier/binding. + # Several consumers may share these vectors through `many_binding_cache`. + append!(existing_refs, new_refs) + append!(binding.source_ids, new_source_ids) + empty!(binding.source_application_ids) + append!(binding.source_application_ids, source_application_ids) return true end @@ -2052,6 +2220,113 @@ function _preserve_recompiled_model_status_views!( return current end +function _stage_temporal_many_extension( + model::CompositeModel, + temporal_input::CompiledTemporalInput, + previous_source_ids, + applications_by_id, + application_positions, + distributed_outputs, +) + binding = temporal_input.binding + binding.multiplicity == :many || return nothing + binding.policy isa PreviousTimeStep || return nothing + previous_count = length(previous_source_ids) + previous_count > 0 || return nothing + source_ids = binding.source_ids + length(source_ids) > previous_count || return nothing + @views source_ids[1:previous_count] == previous_source_ids || return nothing + + initial = temporal_input.initial + storage = temporal_input.reference[] + source_applications = temporal_input.source_applications + initial isa Vector || return nothing + storage isa Union{RefVector,ObjectRefVector} || return nothing + source_applications isa Vector || return nothing + length(initial) == previous_count || return nothing + length(storage) == previous_count || return nothing + length(source_applications) == previous_count || return nothing + + source_values = _input_value(binding.carrier) + source_values isa AbstractVector || return nothing + length(source_values) == length(source_ids) || return nothing + new_indices = (previous_count + 1):length(source_ids) + new_initial = Any[ + _private_temporal_value(source_values[index]) + for index in new_indices + ] + new_source_applications = Union{Nothing,Symbol}[ + _temporal_source_application( + binding, + source_ids[index], + applications_by_id, + application_positions, + distributed_outputs, + ) + for index in new_indices + ] + storage_references = parent(storage) + new_storage_references = Base.RefValue[ + Ref(_private_temporal_value(value)) for value in new_initial + ] + all(reference -> reference isa eltype(storage_references), new_storage_references) || + return nothing + all(application -> application isa eltype(source_applications), new_source_applications) || + return nothing + return ( + initial, + storage_references, + source_applications, + new_initial, + new_storage_references, + new_source_applications, + ) +end + +function _extend_temporal_many_inputs_in_place!( + model::CompositeModel, + view::CompiledModelStatusView, + key, + previous_temporal_sources, + applications_by_id, + application_positions, + distributed_outputs, +) + staged = Any[] + for temporal_input in view.temporal_inputs + previous_source_ids = get( + previous_temporal_sources, + (key..., temporal_input.binding.input), + nothing, + ) + isnothing(previous_source_ids) && continue + extension = _stage_temporal_many_extension( + model, + temporal_input, + previous_source_ids, + applications_by_id, + application_positions, + distributed_outputs, + ) + isnothing(extension) && return false + push!(staged, extension) + end + isempty(staged) && return false + for ( + initial, + storage_references, + source_applications, + new_initial, + new_storage_references, + new_source_applications, + ) in staged + append!(initial, new_initial) + append!(storage_references, new_storage_references) + append!(source_applications, new_source_applications) + end + return true +end + function _extend_model_status_views( model::CompositeModel, compiled::CompiledCompositeModel, @@ -2084,6 +2359,21 @@ function _extend_model_status_views( application_id => index for (index, application_id) in pairs(application_order) ) + for key in affected_temporal_keys + view = get(views, key, nothing) + isnothing(view) && continue + if _extend_temporal_many_inputs_in_place!( + model, + view, + key, + previous_temporal_sources, + applications_by_id, + positions, + distributed_outputs, + ) + delete!(affected_keys, key) + end + end for key in affected_keys application_id, object_id = key application = applications_by_id[application_id] @@ -2136,7 +2426,12 @@ function _append_added_many_call_targets!( context=binding.consumer_id, default_to_context=true, default_scope=default_scope, - ) && !(object_id in binding.callee_object_ids) + ) && !first( + _sorted_object_id_position( + binding.callee_object_ids, + object_id, + ), + ) ] isempty(new_target_ids) && return true _sort_object_ids!(new_target_ids) @@ -2166,6 +2461,84 @@ function _append_added_many_call_targets!( return true end +function _propagate_changed_manual_call_owners!( + changed_target_ids, + changed_callee_target_ids, + call_bindings, + call_owners, +) + frontier = Dict{Symbol,Vector{ObjectId}}() + for (application_id, object_id) in changed_callee_target_ids + push!(get!(frontier, application_id, ObjectId[]), object_id) + end + seen_owner_keys = Set{Tuple{Symbol,ObjectId}}(changed_callee_target_ids) + while !isempty(frontier) + owner_application_ids = Set{Symbol}() + for application_id in keys(frontier) + hasproperty(call_owners, application_id) || continue + union!( + owner_application_ids, + getproperty(call_owners, application_id), + ) + end + isempty(owner_application_ids) && break + next_frontier = Dict{Symbol,Vector{ObjectId}}() + for binding in call_bindings + binding.application_id in owner_application_ids || continue + _compiled_call_mode(binding) === :manual || continue + affected = false + for callee_application_id in binding.callee_application_ids + object_ids = get(frontier, callee_application_id, nothing) + isnothing(object_ids) && continue + if any(object_ids) do object_id + first( + _sorted_object_id_position( + binding.callee_object_ids, + object_id, + ), + ) + end + affected = true + break + end + end + affected || continue + owner_key = (binding.application_id, binding.consumer_id) + owner_key in seen_owner_keys && continue + push!(seen_owner_keys, owner_key) + push!(changed_target_ids, owner_key) + push!( + get!( + next_frontier, + binding.application_id, + ObjectId[], + ), + binding.consumer_id, + ) + end + frontier = next_frontier + end + return changed_target_ids +end + +function _changed_previous_time_step_targets( + previous_temporal_sources, + status_views_by_target, +) + changed_target_ids = Set{Tuple{Symbol,ObjectId}}() + for (application_id, object_id, input) in keys(previous_temporal_sources) + target_key = (application_id, object_id) + view = get(status_views_by_target, target_key, nothing) + isnothing(view) && continue + any(view.temporal_inputs) do temporal_input + temporal_input.binding.input == input && + temporal_input.binding.policy isa PreviousTimeStep + end || continue + push!(changed_target_ids, target_key) + end + return changed_target_ids +end + function _prepare_structural_compiled_delta( model::CompositeModel, compiled::CompiledCompositeModel, @@ -2303,7 +2676,12 @@ function _remove_stale_status_views!( for key in candidate_keys application_id, object_id = key application = compiled.applications_by_id[application_id] - object_id in application.target_ids && continue + first( + _sorted_object_id_position( + application.target_ids, + object_id, + ), + ) && continue delete!(compiled.status_views_by_target, key) end return compiled @@ -2747,6 +3125,8 @@ function _extend_compiled_scene( applications_by_object, applications_by_id, distributed_outputs, + many_binding_cache, + performance, ) last_new_binding = length(input_bindings) if first_new_binding <= last_new_binding @@ -2819,8 +3199,20 @@ function _extend_compiled_scene( end (added_applications..., rewired_applications...) end - _prepare_model_input_defaults!(model, affected_input_applications) - _wire_model_input_carriers!(model, changed_bindings) + final_input_references = pure_addition ? + _model_input_status_references(changed_bindings) : + nothing + _prepare_model_input_defaults!( + model, + affected_input_applications; + final_references=final_input_references, + performance=performance, + ) + _wire_model_input_carriers!( + model, + changed_bindings; + final_references=final_input_references, + ) _validate_model_required_inputs!( model, affected_input_applications, @@ -2884,6 +3276,23 @@ function _extend_compiled_scene( ), ) end + changed_manual_target_ids = Set{Tuple{Symbol,ObjectId}}( + key for key in changed_execution_target_ids + if first(key) in manual_application_ids + ) + union!( + changed_manual_target_ids, + _changed_previous_time_step_targets( + previous_temporal_sources, + status_views_by_target, + ), + ) + _propagate_changed_manual_call_owners!( + changed_execution_target_ids, + changed_manual_target_ids, + call_bindings, + call_owners, + ) changed_execution_application_ids = Set( first(key) for key in changed_execution_target_ids ) @@ -3712,7 +4121,12 @@ function _compile_model_writer_ownership( for destination_id in resolved.destination_ids for variable_ in keys(plan.declarations) variable = Symbol(variable_) - if destination_id in application.target_ids && + if first( + _sorted_object_id_position( + application.target_ids, + destination_id, + ), + ) && variable in keys(outputs_(application.spec)) && _publish_mode_for_output(application.spec, variable) == :stream_only @@ -4480,7 +4894,12 @@ function _incremental_distributed_output_addition!( context=binding.execution_object_id, default_to_context=true, default_scope=default_scope, - ) && !(object_id in binding.destination_ids) + ) && !first( + _sorted_object_id_position( + binding.destination_ids, + object_id, + ), + ) ] isempty(destination_ids) && continue _sort_object_ids!(destination_ids) @@ -4632,7 +5051,30 @@ function _refresh_model_distributed_outputs( return current, _changed_distributed_output_target_ids(previous, current) end -function _prepare_model_input_defaults!(model::CompositeModel, applications) +_model_input_status_reference(binding) = + binding.carrier isa Base.RefValue ? + binding.carrier : Ref(binding.carrier) + +function _model_input_status_references(bindings) + references = Dict{Tuple{Symbol,ObjectId,Symbol},Any}() + for binding in bindings + binding.carrier_hint == :temporal_stream && continue + isnothing(binding.carrier) && continue + references[( + binding.application_id, + binding.consumer_id, + binding.input, + )] = _model_input_status_reference(binding) + end + return references +end + +function _prepare_model_input_defaults!( + model::CompositeModel, + applications; + final_references=nothing, + performance=nothing, +) for application in applications schema = _input_schema(application.spec) defaults = _input_default_values(schema) @@ -4641,6 +5083,38 @@ function _prepare_model_input_defaults!(model::CompositeModel, applications) for (variable, value) in pairs(defaults) variable = Symbol(variable) variable in propertynames(status) && continue + reference = isnothing(final_references) ? + nothing : + get( + final_references, + (application.id, object_id, variable), + nothing, + ) + if !isnothing(reference) + # Preserve conversion callbacks, validation, and diagnostic + # records while avoiding a default Status that the resolved + # carrier would replace immediately below. + _no_status_conversion(model.status_conversion) || + _materialize_status_value( + model, + variable, + value; + object_id=object_id, + application_id=application.id, + origin=:model_input_default, + private_copy=true, + ) + status = _status_with_reference( + status, + variable, + reference, + ) + _runtime_performance_count!( + performance, + :input_default_status_rewrites_avoided, + ) + continue + end status = _status_with_default( model, status, @@ -4665,14 +5139,44 @@ function _prepare_model_input_defaults!(model::CompositeModel, applications) return model end -function _wire_model_input_carriers!(model::CompositeModel, bindings) +function _wire_model_input_carriers!( + model::CompositeModel, + bindings; + final_references=nothing, +) for binding in bindings binding.carrier_hint == :temporal_stream && continue isnothing(binding.carrier) && continue object = _model_object(model, binding.consumer_id) status = object.status status isa Status || continue - reference = binding.carrier isa Base.RefValue ? binding.carrier : Ref(binding.carrier) + reference = if isnothing(final_references) + _model_input_status_reference(binding) + else + get( + final_references, + ( + binding.application_id, + binding.consumer_id, + binding.input, + ), + nothing, + ) + end + isnothing(reference) && + (reference = _model_input_status_reference(binding)) + if binding.input in propertynames(status) && + refvalue(status, binding.input) === reference + delete!( + get!( + model.input_default_status_variables, + binding.consumer_id, + Set{Symbol}(), + ), + binding.input, + ) + continue + end _replace_model_object_status!( model, object, @@ -4847,6 +5351,21 @@ function _compile_model_status_view( if binding.carrier_hint == :temporal_stream ) _validate_temporal_input_output_overlap!(application, temporal_bindings) + output_defaults = outputs_(application.spec) + private_output_names = Tuple( + Symbol(variable) for variable in keys(output_defaults) + if _publish_mode_for_output(application.spec, variable) == + :stream_only + ) + if isempty(temporal_bindings) && isempty(private_output_names) + return CompiledModelStatusView( + canonical_status, + canonical_status, + (), + NamedTuple(), + _compiled_bound_many_inputs(input_bindings, canonical_status), + ) + end temporal_inputs = Tuple(begin initial = _temporal_input_initial(binding, canonical_status) CompiledTemporalInput( @@ -4869,12 +5388,6 @@ function _compile_model_status_view( temporal_input.binding.input => temporal_input for temporal_input in temporal_inputs ) - output_defaults = outputs_(application.spec) - private_output_names = Tuple( - Symbol(variable) for variable in keys(output_defaults) - if _publish_mode_for_output(application.spec, variable) == - :stream_only - ) private_outputs = NamedTuple{private_output_names}(Tuple(begin initial, _, _ = _materialize_status_value( model, @@ -5040,7 +5553,12 @@ function _application_writes_object_variable( object_id::ObjectId, variable::Symbol, ) - return object_id in application.target_ids && + return first( + _sorted_object_id_position( + application.target_ids, + object_id, + ), + ) && variable in _model_output_names(application) end @@ -5050,7 +5568,12 @@ function _application_writes_object_variable( object_id::ObjectId, variable::Symbol, ) - object_id in application.target_ids && + first( + _sorted_object_id_position( + application.target_ids, + object_id, + ), + ) && variable in _model_output_names(application) && return true return any( owner -> owner.application_id == application.id, @@ -5702,7 +6225,12 @@ function _final_many_source_applications( if isnothing(final_owner) for application_id in source_application_ids application = applications_by_id[application_id] - source_id in application.target_ids || continue + first( + _sorted_object_id_position( + application.target_ids, + source_id, + ), + ) || continue source_var in _model_output_names(application) || continue application_id in canonical_ids || push!(canonical_ids, application_id) diff --git a/src/composite_model/runtime_outputs.jl b/src/composite_model/runtime_outputs.jl index 6fd6a058b..1f1f3b461 100644 --- a/src/composite_model/runtime_outputs.jl +++ b/src/composite_model/runtime_outputs.jl @@ -429,6 +429,17 @@ struct CallTarget{CS,EB,A,M,S,VS,TI,OB,CT,BI,OT,ENV,TS,OR,C,E} environment::E end +abstract type AbstractExecutionBatch end + +mutable struct LazyCallExecutionBatches <: AbstractVector{AbstractExecutionBatch} + owner::Any + batches::Any +end + +LazyCallExecutionBatches() = LazyCallExecutionBatches(nothing, nothing) + +Base.IndexStyle(::Type{LazyCallExecutionBatches}) = IndexLinear() + """ CallTargets <: AbstractVector{CallTarget} @@ -464,17 +475,26 @@ function CallTargets( publication_allowed, environment, ) - execution_batches = _compiled_call_mode(binding) === :initializer ? - () : - _compiled_call_execution_batches( - compiled, - environment_bindings, - binding, - temporal_streams, - output_retention, - constants, - ) - return CallTargets( + execution_batches = if _compiled_call_mode(binding) === :initializer + () + elseif binding.multiplicity === :many + # Large `Many` bindings are commonly executed with an object filter + # while organs are emitted. Defer their complete batch construction + # until an unfiltered view or execution actually needs it. + LazyCallExecutionBatches() + else + # Preserve the concrete, allocation-free lookup path promised by + # `call_model` for `One` and `OptionalOne` bindings. + _compiled_call_execution_batches( + compiled, + environment_bindings, + binding, + temporal_streams, + output_retention, + constants, + ) + end + targets = CallTargets( compiled, environment_bindings, binding, @@ -486,8 +506,46 @@ function CallTargets( environment, execution_batches, ) + execution_batches isa LazyCallExecutionBatches && + (execution_batches.owner = targets) + return targets +end + +function _materialize_call_execution_batches!(targets::CallTargets) + cached = targets.execution_batches + cached isa LazyCallExecutionBatches || return cached + isnothing(cached.batches) || return cached.batches + cached.batches = _compiled_call_execution_batches( + targets.compiled, + targets.environment_bindings, + targets.binding, + targets.temporal_streams, + targets.output_retention, + targets.constants, + ) + return cached.batches end +function _cached_call_execution_batches(targets::CallTargets) + cached = targets.execution_batches + cached isa LazyCallExecutionBatches || return cached + return cached.batches +end + +_call_execution_batches_materialized(targets::CallTargets) = + !isnothing(_cached_call_execution_batches(targets)) + +Base.length(batches::LazyCallExecutionBatches) = + length(_materialize_call_execution_batches!(batches.owner)) + +Base.size(batches::LazyCallExecutionBatches) = (length(batches),) + +Base.getindex(batches::LazyCallExecutionBatches, index::Int) = + getindex(_materialize_call_execution_batches!(batches.owner), index) + +Base.iterate(batches::LazyCallExecutionBatches, state...) = + iterate(_materialize_call_execution_batches!(batches.owner), state...) + Base.IndexStyle(::Type{<:CallTargets}) = IndexLinear() Base.eltype(::Type{<:CallTargets}) = CallTarget @@ -577,7 +635,6 @@ end return nothing end -abstract type AbstractExecutionBatch end struct UnspecifiedModelEnvironment end const _UNSPECIFIED_SCENE_ENVIRONMENT = UnspecifiedModelEnvironment() struct RawGlobalModelEnvironment{M} @@ -1192,6 +1249,109 @@ function _model_output_reference( ) end +function _initialize_model_output_stream!( + streams, + compiled::CompiledCompositeModel, + retention::OutputRetentionPlan, + application, + object_id::ObjectId, + variable::Symbol, + sizehint_steps::Integer, +) + key = _model_stream_key(application.id, object_id, variable) + haskey(streams, key) && return streams + reference = _model_output_reference( + compiled, + application, + object_id, + variable, + ) + isnothing(reference) && error( + "Application `$(application.id)` declares retained output ", + "`$(variable)`, but object `$(object_id.value)` status has no ", + "such variable.", + ) + stream = _model_new_output_stream( + reference[], + retention, + application.id, + variable, + ) + if stream isa Vector && sizehint_steps > 0 + sizehint!( + stream, + max( + 1, + ceil( + Int, + float(sizehint_steps) / + float(application.clock.dt), + ), + ), + ) + end + streams[key] = stream + return streams +end + +function _initialize_changed_model_output_streams!( + streams, + compiled::CompiledCompositeModel, + retention::OutputRetentionPlan, + sizehint_steps::Integer, + target_keys, +) + for (application_id, object_id) in target_keys + variables = get( + retention.retained_outputs_by_application, + application_id, + (), + ) + isempty(variables) && continue + application = _compiled_application_by_id(compiled, application_id) + model_outputs = keys(outputs_(application.spec)) + if haskey(compiled.status_views_by_target, (application_id, object_id)) + for variable in variables + variable in model_outputs || continue + _initialize_model_output_stream!( + streams, + compiled, + retention, + application, + object_id, + variable, + sizehint_steps, + ) + end + end + compiled.distributed_outputs isa CompiledDistributedOutputs || continue + groups = get( + compiled.distributed_outputs.by_execution_target, + (application_id, object_id), + nothing, + ) + isnothing(groups) && continue + for binding in values(groups) + for variable_ in keys(binding.declarations) + variable = Symbol(variable_) + variable in variables || continue + for destination_id in binding.destination_ids + _initialize_model_output_stream!( + streams, + compiled, + retention, + application, + destination_id, + variable, + sizehint_steps, + ) + end + end + end + end + return streams +end + function _initialize_model_output_streams!( streams, compiled::CompiledCompositeModel, @@ -1199,6 +1359,13 @@ function _initialize_model_output_streams!( sizehint_steps::Integer=0, target_keys=nothing, ) + !isnothing(target_keys) && return _initialize_changed_model_output_streams!( + streams, + compiled, + retention, + sizehint_steps, + target_keys, + ) for (application_id, variables) in retention.retained_outputs_by_application application = _compiled_application_by_id(compiled, application_id) for variable in variables @@ -1207,43 +1374,15 @@ function _initialize_model_output_streams!( application, variable, ) - !isnothing(target_keys) && - !((application_id, object_id) in target_keys) && - compiled.distributed_outputs isa NoCompiledDistributedOutputs && - continue - key = _model_stream_key(application_id, object_id, variable) - haskey(streams, key) && continue - reference = _model_output_reference( + _initialize_model_output_stream!( + streams, compiled, + retention, application, object_id, variable, + sizehint_steps, ) - isnothing(reference) && error( - "Application `$(application_id)` declares retained output ", - "`$(variable)`, but object `$(object_id.value)` status has no ", - "such variable.", - ) - stream = _model_new_output_stream( - reference[], - retention, - application_id, - variable, - ) - if stream isa Vector && sizehint_steps > 0 - sizehint!( - stream, - max( - 1, - ceil( - Int, - float(sizehint_steps) / - float(application.clock.dt), - ), - ), - ) - end - streams[key] = stream end end end @@ -3205,17 +3344,51 @@ end function _model_execution_inputs_match( runtime_inputs::Tuple, compiled_inputs::Tuple, + temporal_streams=nothing, ) length(runtime_inputs) == length(compiled_inputs) || return false + additions = Tuple{Any,Vector{Any}}[] for index in eachindex(runtime_inputs) runtime_input = runtime_inputs[index] compiled_input = compiled_inputs[index] if runtime_input isa RuntimeTemporalInput runtime_input.compiled === compiled_input || return false + binding = compiled_input.binding + binding.multiplicity == :many || continue + source_streams = runtime_input.source_streams + if !(source_streams isa Vector) + length(source_streams) == length(binding.source_ids) || + return false + continue + end + current_count = length(source_streams) + source_count = length(binding.source_ids) + current_count <= source_count || return false + current_count == source_count && continue + binding.policy isa PreviousTimeStep || return false + isnothing(temporal_streams) && return false + length(compiled_input.initial) == source_count || return false + length(compiled_input.reference[]) == source_count || return false + length(compiled_input.source_applications) == source_count || + return false + new_streams = Any[] + for source_index in (current_count + 1):source_count + stream = _runtime_temporal_source_stream( + temporal_streams, + compiled_input, + source_index, + ) + stream isa eltype(source_streams) || return false + push!(new_streams, stream) + end + push!(additions, (source_streams, new_streams)) else runtime_input === compiled_input || return false end end + for (source_streams, new_streams) in additions + append!(source_streams, new_streams) + end return true end @@ -3225,6 +3398,7 @@ function _model_execution_outputs_match( application::CompiledModelApplication, object_id::ObjectId, output_retention, + temporal_streams=nothing, ) variables = output_retention isa OutputRetentionPlan ? get( @@ -3232,29 +3406,102 @@ function _model_execution_outputs_match( application.id, (), ) : () + output_index = 0 + dependency_horizons = output_retention isa OutputRetentionPlan ? + output_retention.dependency_horizons : + nothing + status_view = get( + compiled.status_views_by_target, + (application.id, object_id), + nothing, + ) + for variable in variables + variable in keys(outputs_(application.spec)) || continue + output_index += 1 + output_index <= length(runtime_outputs) || return false + output = runtime_outputs[output_index] + output isa RuntimeOutputStream || return false + _runtime_output_variable(output) == variable || return false + expected_horizon = isnothing(dependency_horizons) ? + 0.0 : + get( + dependency_horizons, + (application.id, variable), + 0.0, + ) + output.dependency_horizon == expected_horizon || return false + isnothing(temporal_streams) && continue + key = _model_stream_key(application.id, object_id, variable) + output.stream === get(temporal_streams, key, nothing) || return false + isnothing(status_view) && return false + output.reference === refvalue(status_view.status, variable) || return false + end + + additions = Tuple{Any,Vector{Any}}[] if compiled.distributed_outputs isa CompiledDistributedOutputs groups = get( compiled.distributed_outputs.by_execution_target, (application.id, object_id), nothing, ) - if !isnothing(groups) && any( - variable -> haskey( - compiled.distributed_outputs.destination_ids_by_application_variable, - (application.id, variable), - ), - variables, - ) - # Distributed columns carry lifecycle-specific destination - # references. Rebuild this execution target at a lifecycle barrier - # instead of trying to compare one runtime entry per variable. - return false + if !isnothing(groups) + isnothing(temporal_streams) && return false + for binding in values(groups) + for variable_ in keys(binding.declarations) + variable = Symbol(variable_) + variable in variables || continue + output_index += 1 + output_index <= length(runtime_outputs) || return false + output = runtime_outputs[output_index] + output isa RuntimeDistributedOutputStream || return false + _runtime_output_variable(output) == variable || return false + output.binding === binding || return false + output.references === + getproperty(binding.columns, variable) || return false + expected_horizon = isnothing(dependency_horizons) ? + 0.0 : + get( + dependency_horizons, + (application.id, variable), + 0.0, + ) + output.dependency_horizon == expected_horizon || return false + destination_ids = binding.destination_ids + runtime_streams = output.streams + length(runtime_streams) <= length(destination_ids) || + return false + for index in eachindex(runtime_streams) + key = _model_stream_key( + application.id, + destination_ids[index], + variable, + ) + runtime_streams[index] === + get(temporal_streams, key, nothing) || return false + end + length(runtime_streams) == length(destination_ids) && + continue + runtime_streams isa Vector || return false + new_streams = Any[] + for index in (length(runtime_streams) + 1):length(destination_ids) + key = _model_stream_key( + application.id, + destination_ids[index], + variable, + ) + stream = get(temporal_streams, key, nothing) + isnothing(stream) && return false + stream isa eltype(runtime_streams) || return false + push!(new_streams, stream) + end + push!(additions, (runtime_streams, new_streams)) + end + end end end - length(runtime_outputs) == length(variables) || return false - for index in eachindex(runtime_outputs) - _runtime_output_variable(runtime_outputs[index]) == variables[index] || - return false + output_index == length(runtime_outputs) || return false + for (runtime_streams, new_streams) in additions + append!(runtime_streams, new_streams) end return true end @@ -3305,12 +3552,39 @@ function _model_execution_output_targets_match( ) end +function _manual_call_binding_has_changed_target( + binding, + compiled::CompiledCompositeModel, +) + _compiled_call_mode(binding) === :manual || return false + for (application_id, object_id) in compiled.changed_execution_target_ids + application_id in binding.callee_application_ids || continue + first( + _sorted_object_id_position( + binding.callee_object_ids, + object_id, + ), + ) || continue + application = get(compiled.applications_by_id, application_id, nothing) + isnothing(application) && continue + first( + _sorted_object_id_position( + application.target_ids, + object_id, + ), + ) || continue + return true + end + return false +end + function _model_execution_target_change_reason( target::CompiledExecutionTarget, compiled::CompiledCompositeModel, env_bindings::CompiledEnvironmentBindings, application::CompiledModelApplication, output_retention=nothing, + temporal_streams=nothing, ) object_id = target.object_id key = (application.id, object_id) @@ -3332,6 +3606,7 @@ function _model_execution_target_change_reason( _model_execution_inputs_match( target.input_bindings, status_view.temporal_inputs, + temporal_streams, ) || return :temporal_inputs _model_execution_outputs_match( @@ -3340,18 +3615,26 @@ function _model_execution_target_change_reason( application, object_id, output_retention, + temporal_streams, ) || return :output_bindings + target.environment_binding === _environment_binding_for( + env_bindings, + application.id, + object_id, + ) || return :environment_binding target.call_bindings === get(compiled.call_bindings_by_target, key, ()) || return :call_bindings target.call_bindings_signature == _call_bindings_signature(target.call_bindings) || return :call_bindings - target.environment_binding === _environment_binding_for( - env_bindings, - application.id, - object_id, - ) || return :environment_binding + any( + binding -> _manual_call_binding_has_changed_target( + binding, + compiled, + ), + target.call_bindings, + ) && return :call_bindings return nothing end @@ -3399,6 +3682,7 @@ function _model_execution_group_reusable( env_bindings::CompiledEnvironmentBindings, application::CompiledModelApplication, output_retention=nothing, + temporal_streams=nothing, ) group.application === application || return false target_index = 0 @@ -3414,6 +3698,7 @@ function _model_execution_group_reusable( env_bindings, application, output_retention, + temporal_streams, )) || return false end end @@ -3489,7 +3774,7 @@ function _model_execution_batch_accepts_target( return isequal(provider, batch.environment_provider) end -function _extend_execution_target_call_batches!( +function _extend_execution_target_call_batches_by_prefix!( target::CompiledExecutionTarget, compiled::CompiledCompositeModel, env_bindings::CompiledEnvironmentBindings, @@ -3512,37 +3797,45 @@ function _extend_execution_target_call_batches!( _compiled_call_name(binding) || return false push!(staged_bindings, (call_targets, binding)) _compiled_call_mode(binding) === :initializer && continue - current_application_ids = Set(binding.callee_application_ids) + execution_batches = _cached_call_execution_batches(call_targets) + isnothing(execution_batches) && continue all( - batch -> batch.application.id in current_application_ids, - call_targets.execution_batches, + batch -> batch.application.id in binding.callee_application_ids, + execution_batches, ) || return false for application_id in binding.callee_application_ids application = _compiled_application_by_id(compiled, application_id) batches = AbstractExecutionBatch[ - batch for batch in call_targets.execution_batches + batch for batch in execution_batches if batch.application.id == application_id ] isempty(batches) && return false - existing_ids = Set( - execution_target.object_id - for batch in batches - for execution_target in batch.targets - ) current_ids = ObjectId[ object_id for object_id in binding.callee_object_ids if _call_binding_target_matches(binding, application, object_id) ] - all(object_id -> object_id in current_ids, existing_ids) || - return false - added_ids = ObjectId[ - object_id for object_id in current_ids - if !(object_id in existing_ids) - ] - isempty(added_ids) && continue - _sort_object_ids!(added_ids) + # Call batches are compiled by walking the sorted callee IDs and + # only split when their concrete target type changes. A monotonic + # lifecycle addition therefore preserves the flattened old target + # sequence as an exact prefix. Any removal, reordering, or target + # replacement falls back to the general rebuild path. + existing_count = 0 + for batch in batches + for execution_target in batch.targets + existing_count += 1 + existing_count <= length(current_ids) || return false + execution_target.object_id == current_ids[existing_count] || + return false + ( + application.id, + execution_target.object_id, + ) in compiled.changed_execution_target_ids && return false + end + end + existing_count == length(current_ids) && continue destination_batch = last(batches) - for object_id in added_ids + for index in (existing_count + 1):lastindex(current_ids) + object_id = current_ids[index] execution_target = _compiled_model_execution_target( compiled, env_bindings, @@ -3574,6 +3867,175 @@ function _extend_execution_target_call_batches!( return true end +function _pure_addition_changed_call_target_ids( + binding, + compiled::CompiledCompositeModel, +) + application_ids = binding.callee_application_ids + changed_ids = ObjectId[] + affected_application_ids = Symbol[] + for (application_id, object_id) in compiled.changed_execution_target_ids + application_id in application_ids || continue + first( + _sorted_object_id_position( + binding.callee_object_ids, + object_id, + ), + ) || continue + application = _compiled_application_by_id(compiled, application_id) + first( + _sorted_object_id_position( + application.target_ids, + object_id, + ), + ) || continue + push!(changed_ids, object_id) + application_id in affected_application_ids || + push!(affected_application_ids, application_id) + end + _sort_object_ids!(changed_ids) + return changed_ids, affected_application_ids +end + +function _extend_execution_target_call_batches_for_pure_addition!( + target::CompiledExecutionTarget, + compiled::CompiledCompositeModel, + env_bindings::CompiledEnvironmentBindings, + temporal_streams, + output_retention, + constants, + performance=nothing, +) + compiled.status_view_refresh_is_pure_addition || return nothing + context = target.context + context isa RunContext || return nothing + current_call_bindings = get( + compiled.call_bindings_by_target, + (context.application.id, target.object_id), + (), + ) + length(context.calls) == length(current_call_bindings) || return nothing + staged = Tuple{Any,Any}[] + staged_bindings = Tuple{Any,Any}[] + for (call_targets, binding) in zip(context.calls, current_call_bindings) + _compiled_call_name(call_targets.binding) == + _compiled_call_name(binding) || return nothing + push!(staged_bindings, (call_targets, binding)) + _compiled_call_mode(binding) === :initializer && continue + execution_batches = _cached_call_execution_batches(call_targets) + isnothing(execution_batches) && continue + all( + batch -> batch.application.id in binding.callee_application_ids, + execution_batches, + ) || return nothing + changed_ids, affected_application_ids = + _pure_addition_changed_call_target_ids(binding, compiled) + isempty(changed_ids) && continue + length(affected_application_ids) == 1 || return nothing + length(binding.callee_application_ids) == 1 || return nothing + application_id = only(affected_application_ids) + application = _compiled_application_by_id(compiled, application_id) + batches = AbstractExecutionBatch[ + batch for batch in execution_batches + if batch.application.id == application_id + ] + isempty(batches) && return false + last_old_id = last(last(batches).targets).object_id + all( + object_id -> _object_id_isless(last_old_id, object_id), + changed_ids, + ) || return false + found_boundary, boundary = _sorted_object_id_position( + binding.callee_object_ids, + last_old_id, + ) + found_boundary || return false + for index in (boundary + 1):lastindex(binding.callee_object_ids) + object_id = binding.callee_object_ids[index] + first( + _sorted_object_id_position( + application.target_ids, + object_id, + ), + ) || continue + first( + _sorted_object_id_position( + changed_ids, + object_id, + ), + ) || return false + end + destination_batch = last(batches) + for object_id in changed_ids + execution_target = _compiled_model_execution_target( + compiled, + env_bindings, + application, + object_id, + temporal_streams, + output_retention, + constants, + ) + _model_execution_batch_accepts_target( + destination_batch, + execution_target, + env_bindings, + application, + ) || return false + push!(staged, (destination_batch.targets, execution_target)) + end + end + for (targets, execution_target) in staged + push!(targets, execution_target) + end + for (call_targets, binding) in staged_bindings + call_targets.binding = binding + end + target.call_bindings = current_call_bindings + target.call_bindings_signature = + _call_bindings_signature(current_call_bindings) + _runtime_performance_count!( + performance, + :execution_target_call_delta_extensions, + ) + _runtime_performance_count!( + performance, + :execution_target_call_delta_targets_added, + length(staged), + ) + return true +end + +function _extend_execution_target_call_batches!( + target::CompiledExecutionTarget, + compiled::CompiledCompositeModel, + env_bindings::CompiledEnvironmentBindings, + temporal_streams, + output_retention, + constants, + performance=nothing, +) + pure_addition_result = + _extend_execution_target_call_batches_for_pure_addition!( + target, + compiled, + env_bindings, + temporal_streams, + output_retention, + constants, + performance, + ) + isnothing(pure_addition_result) || return pure_addition_result + return _extend_execution_target_call_batches_by_prefix!( + target, + compiled, + env_bindings, + temporal_streams, + output_retention, + constants, + ) +end + function _refresh_model_execution_group_delta!( previous_group::CompiledApplicationExecutionGroup, compiled::CompiledCompositeModel, @@ -3629,6 +4091,7 @@ function _refresh_model_execution_group_delta!( env_bindings, application, output_retention, + temporal_streams, ) end isnothing(change_reason) && continue @@ -3640,6 +4103,7 @@ function _refresh_model_execution_group_delta!( temporal_streams, output_retention, constants, + performance, ) _runtime_performance_count!( performance, @@ -3791,6 +4255,7 @@ function _refresh_model_execution_plan( env_bindings, application, output_retention, + temporal_streams, ) push!(groups, previous_group) append!(batches, previous_group.batches) @@ -3838,6 +4303,7 @@ function _refresh_model_execution_plan( env_bindings, application, output_retention, + temporal_streams, ) end if isnothing(change_reason) @@ -4312,7 +4778,12 @@ end function _call_binding_target_matches(binding, application, object_id::ObjectId) return (binding.multiplicity != :many && length(binding.callee_application_ids) == 1) || - object_id in application.target_ids + first( + _sorted_object_id_position( + application.target_ids, + object_id, + ), + ) end _call_target_matches(targets::CallTargets, application, object_id::ObjectId) = @@ -4491,7 +4962,9 @@ end _compiled_call_mode(targets.binding) === :manual || _initializer_requires_dedicated_api(context, name) length(targets) == 1 || _call_model_target_count_error(name, targets) - return _single_call_model(targets.execution_batches) + return _single_call_model( + _materialize_call_execution_batches!(targets), + ) end @@ -5340,7 +5813,7 @@ function _run_call_targets!( ) _run_compiled_call_batches!( targets, - targets.execution_batches, + _materialize_call_execution_batches!(targets), publish, sampled_environment, environment, diff --git a/test/test-model-api-stabilization.jl b/test/test-model-api-stabilization.jl index be68751b9..7e7f7137e 100644 --- a/test/test-model-api-stabilization.jl +++ b/test/test-model-api-stabilization.jl @@ -94,6 +94,24 @@ function PlantSimEngine.run!( return nothing end +struct StabilizationSafeLaggedSumModel <: AbstractStabilization_Lagged_SumModel end + +PlantSimEngine.inputs_(::StabilizationSafeLaggedSumModel) = + (previous_signals=Default([0.0]),) +PlantSimEngine.outputs_(::StabilizationSafeLaggedSumModel) = + (lagged_total=0.0,) + +function PlantSimEngine.run!( + ::StabilizationSafeLaggedSumModel, + status, + environment, + constants, + context, +) + status.lagged_total = sum(status.previous_signals; init=0.0) + return nothing +end + @testset "one-object lowering and initialization report" begin model = CompositeModel( StabilizationSourceModel(), @@ -1005,6 +1023,80 @@ function stabilization_selector_candidate_scene(nplants) ) end +function stabilization_scoped_label_candidate_scene() + return CompositeModel( + Object(:scene; scale=:Scene, kind=:scene), + Object(:plant_1; scale=:Plant, parent=:scene), + Object( + :plant_1_leaf; + scale=:Leaf, + parent=:plant_1, + status=Status(supplied=0.0), + ), + Object(:plant_2; scale=:Plant, parent=:scene), + Object( + :plant_2_leaf; + scale=:Leaf, + parent=:plant_2, + status=Status(supplied=0.0), + ); + applications=( + ModelSpec( + StabilizationSourceModel(); + name=:plant_source, + on=Many(scale=:Plant), + ), + ModelSpec( + StabilizationConsumerModel(); + name=:leaf_consumer, + on=Many(scale=:Leaf), + inputs=( + :signal => One( + scale=(:Plant, :Axis), + within=SelfPlant(), + application=:plant_source, + var=:signal, + ), + ), + ), + ), + environment=(duration=Hour(1),), + ) +end + +function stabilization_relation_candidate_scene() + return CompositeModel( + Object(:plant; scale=:Plant), + Object( + :leaf_a; + scale=:Leaf, + name=:leaf_a, + parent=:plant, + ); + applications=( + ModelSpec( + StabilizationSourceModel(); + name=:leaf_source, + on=Many(scale=:Leaf), + ), + ModelSpec( + StabilizationSafeLaggedSumModel(); + name=:sibling_sum, + on=One(name=:leaf_a), + inputs=( + :previous_signals => Many( + Relation(:siblings); + scale=:Leaf, + application=:leaf_source, + var=:signal, + ), + ), + ), + ), + environment=(duration=Hour(1),), + ) +end + function stabilization_selector_candidate_refresh_allocations(nplants) model = stabilization_selector_candidate_scene(nplants) simulation = run!(model; outputs=:none) @@ -1053,6 +1145,54 @@ end @test isempty(plant_bindings[:plant_1]) @test plant_bindings[:plant_2] == [:plant_1_leaf, :plant_2_leaf] + scoped_label_model = stabilization_scoped_label_candidate_scene() + scoped_label_simulation = run!( + scoped_label_model; + outputs=:none, + performance=true, + ) + register_object!( + scoped_label_model, + Object( + :zz_new_leaf; + scale=:Leaf, + parent=:plant_1, + status=Status(supplied=0.0), + ), + ) + continue!(scoped_label_simulation) + scoped_label_counts = + Advanced.runtime_performance(scoped_label_simulation).counts + @test scoped_label_counts[:selector_input_binding_candidates] == 0 + new_leaf_binding = only( + row for row in explain_bindings(scoped_label_model) + if row.application_id == :leaf_consumer && + row.consumer_id == :zz_new_leaf && + row.input == :signal + ) + @test new_leaf_binding.source_ids == [:plant_1] + + relation_model = stabilization_relation_candidate_scene() + relation_simulation = run!( + relation_model; + outputs=:none, + performance=true, + ) + register_object!( + relation_model, + Object(:leaf_b; scale=:Leaf, parent=:plant), + ) + continue!(relation_simulation) + relation_counts = Advanced.runtime_performance(relation_simulation).counts + @test relation_counts[:selector_input_binding_candidates] == 1 + sibling_binding = only( + row for row in explain_bindings(relation_model) + if row.application_id == :sibling_sum && + row.consumer_id == :leaf_a && + row.input == :previous_signals + ) + @test sibling_binding.source_ids == [:leaf_b] + stabilization_selector_candidate_refresh_allocations(8) small_allocations = minimum( stabilization_selector_candidate_refresh_allocations(8) @@ -1476,6 +1616,11 @@ end original_view = simulation.compiled.status_views_by_target[(:source, ObjectId(:leaf_1))] + original_status = only(model_objects(model; scale=:Leaf)).status + @test original_view.status === original_status + @test original_view.canonical_status === original_status + @test isempty(original_view.temporal_inputs) + @test isempty(original_view.private_outputs) previous_runtime_revision = model.runtime_revision register_object!(model, Object(:leaf_2; scale=:Leaf)) @test model.runtime_revision == previous_runtime_revision + 1 @@ -1553,6 +1698,18 @@ end original_plant_view = simulation.compiled.status_views_by_target[(:lagged_sum, ObjectId(:plant))] original_temporal_input = only(original_plant_view.temporal_inputs) + original_temporal_storage = original_temporal_input.reference[] + original_temporal_reference = only(parent(original_temporal_storage)) + original_lagged_target = only( + target + for batch in simulation.execution_plan.batches + if batch.application.id == :lagged_sum + for target in batch.targets + ) + original_runtime_temporal_input = only(original_lagged_target.input_bindings) + original_source_stream = only( + original_runtime_temporal_input.source_streams, + ) @test collect(original_temporal_input.reference[]) == [0.0] register_object!( @@ -1572,15 +1729,33 @@ end simulation.compiled.status_views_by_target[(:lagged_sum, ObjectId(:plant))] refreshed_temporal_input = only(refreshed_plant_view.temporal_inputs) @test refreshed_leaf_view === original_leaf_view - @test refreshed_plant_view !== original_plant_view + @test refreshed_plant_view === original_plant_view + @test refreshed_temporal_input === original_temporal_input + @test refreshed_temporal_input.reference[] === original_temporal_storage + @test first(parent(refreshed_temporal_input.reference[])) === + original_temporal_reference @test refreshed_temporal_input.binding.source_ids == ObjectId.([:leaf_1, :leaf_2]) + refreshed_lagged_target = only( + target + for batch in simulation.execution_plan.batches + if batch.application.id == :lagged_sum + for target in batch.targets + ) + refreshed_runtime_temporal_input = + only(refreshed_lagged_target.input_bindings) + @test refreshed_lagged_target === original_lagged_target + @test refreshed_runtime_temporal_input === + original_runtime_temporal_input + @test first(refreshed_runtime_temporal_input.source_streams) === + original_source_stream + @test length(refreshed_runtime_temporal_input.source_streams) == 2 @test plant_status.lagged_total == 1.0 continue!(simulation) @test plant_status.lagged_total == 3.0 performance = Advanced.runtime_performance(simulation) - @test performance.counts[:status_views_constructed] == 2 + @test performance.counts[:status_views_constructed] == 1 @test performance.counts[:lifecycle_barriers] == 1 @test performance.counts[:lifecycle_added_objects] == 1 @test performance.counts[:lifecycle_environment_dirty_objects] == 1 @@ -1588,6 +1763,145 @@ end @test !haskey(performance.counts, :output_retention_compiles) end +@testset "incremental Many source applications use the complete source set" begin + model = CompositeModel( + Object(:scene; scale=:Scene), + Object(:plant; scale=:Plant, parent=:scene), + Object( + :leaf_1; + scale=:Leaf, + kind=:produced, + parent=:plant, + ); + applications=( + ModelSpec( + StabilizationSourceModel(); + name=:source, + on=Many(scale=:Leaf, kind=:produced), + ), + ModelSpec( + StabilizationSafeLaggedSumModel(); + name=:sum, + on=One(scale=:Plant), + inputs=( + :previous_signals => Many( + scale=:Leaf, + within=Subtree(), + application=:source, + var=:signal, + ), + ), + ), + ), + environment=(duration=Hour(1),), + ) + simulation = run!(model; outputs=:none) + plant = only(model_objects(model; scale=:Plant)).status + binding = only( + simulation.compiled.input_bindings_by_target[ + (:sum, ObjectId(:plant)) + ], + ) + carrier = binding.carrier + @test plant.lagged_total == 1.0 + @test binding.source_application_ids == [:source] + + register_object!( + model, + Object( + :leaf_2; + scale=:Leaf, + kind=:supplied, + parent=:plant, + status=Status(signal=10.0), + ), + ) + continue!(simulation) + + refreshed_binding = only( + simulation.compiled.input_bindings_by_target[ + (:sum, ObjectId(:plant)) + ], + ) + @test refreshed_binding === binding + @test refreshed_binding.carrier === carrier + @test refreshed_binding.source_ids == ObjectId.([:leaf_1, :leaf_2]) + @test refreshed_binding.source_application_ids == [:source] + @test plant.lagged_total == 12.0 +end + +@testset "new consumers reuse an updated plant-wide Many carrier" begin + model = CompositeModel( + Object(:scene; scale=:Scene), + Object(:plant; scale=:Plant, parent=:scene), + Object(:leaf_1; scale=:Leaf, parent=:plant), + Object(:axis_1; scale=:Axis, parent=:plant); + applications=( + ModelSpec( + StabilizationSourceModel(); + name=:source, + on=Many(scale=:Leaf), + ), + ModelSpec( + StabilizationSafeLaggedSumModel(); + name=:plant_wide_sum, + on=Many(scale=:Axis), + inputs=( + :previous_signals => Many( + scale=:Leaf, + within=SelfPlant(), + application=:source, + var=:signal, + ), + ), + ), + ), + environment=(duration=Hour(1),), + ) + simulation = run!(model; outputs=:none, performance=true) + original_binding = only( + simulation.compiled.input_bindings_by_target[ + (:plant_wide_sum, ObjectId(:axis_1)) + ], + ) + + register_object!(model, Object(:leaf_2; scale=:Leaf, parent=:plant)) + register_object!(model, Object(:axis_2; scale=:Axis, parent=:plant)) + continue!(simulation) + + refreshed_binding = only( + simulation.compiled.input_bindings_by_target[ + (:plant_wide_sum, ObjectId(:axis_1)) + ], + ) + new_binding = only( + simulation.compiled.input_bindings_by_target[ + (:plant_wide_sum, ObjectId(:axis_2)) + ], + ) + @test refreshed_binding === original_binding + @test new_binding.source_ids === original_binding.source_ids + @test new_binding.source_application_ids === + original_binding.source_application_ids + @test new_binding.carrier === original_binding.carrier + @test new_binding.source_ids == ObjectId.([:leaf_1, :leaf_2]) + axis_2_status = + PlantSimEngine._model_object(model, ObjectId(:axis_2)).status + @test propertynames(axis_2_status) == + (:lagged_total, :previous_signals) + @test axis_2_status.previous_signals === new_binding.carrier + @test :previous_signals ∉ get( + model.input_default_status_variables, + ObjectId(:axis_2), + Set{Symbol}(), + ) + @test PlantSimEngine._model_object(model, ObjectId(:axis_1)).status !== + axis_2_status + performance = Advanced.runtime_performance(simulation) + @test performance.counts[:many_input_binding_precompile_reuses] == 1 + @test performance.counts[:input_default_status_rewrites_avoided] == 1 +end + @testset "incremental execution plan reuses unaffected groups" begin model = CompositeModel( Object(:scene; scale=:Scene), diff --git a/test/test-model-distributed-output-runtime.jl b/test/test-model-distributed-output-runtime.jl index 19990c0a7..407e665c3 100644 --- a/test/test-model-distributed-output-runtime.jl +++ b/test/test-model-distributed-output-runtime.jl @@ -86,6 +86,21 @@ function _distributed_runtime_value(object_id::ObjectId, time::Real) return coefficient * float(time) end +function _distributed_runtime_execution_target( + simulation, + application_id::Symbol, + object_id::ObjectId, +) + return only( + target + for group in simulation.execution_plan.groups + if group.application.id == application_id + for batch in group.batches + for target in batch.targets + if target.object_id == object_id + ) +end + function PlantSimEngine.run!( model::DistributedRuntimeSceneWriterModel, status, @@ -319,12 +334,32 @@ end application=:distributed_runtime_writer, ) retained_simulation = run!(retained_model; outputs=request) + empty_writer_target = _distributed_runtime_execution_target( + retained_simulation, + :distributed_runtime_writer, + ObjectId(:scene), + ) + empty_distributed_output = only( + output for output in empty_writer_target.output_bindings + if output isa PlantSimEngine.RuntimeDistributedOutputStream + ) + @test isempty(empty_distributed_output.streams) register_object!( retained_model, Object(:late_leaf; scale=:Leaf); parent=:plant, ) continue!(retained_simulation) + populated_writer_target = _distributed_runtime_execution_target( + retained_simulation, + :distributed_runtime_writer, + ObjectId(:scene), + ) + @test populated_writer_target !== empty_writer_target + @test length(only( + output for output in populated_writer_target.output_bindings + if output isa PlantSimEngine.RuntimeDistributedOutputStream + ).streams) == 1 @test outputs(retained_simulation)[ ( :distributed_runtime_writer, @@ -341,6 +376,81 @@ end @test getproperty.(retained_rows, :object_id) == [:late_leaf] end +@testset "monotonic distributed additions extend retained streams in place" begin + model = _distributed_runtime_two_plant_scene() + simulation = run!(model; outputs=:all) + writer_target = _distributed_runtime_execution_target( + simulation, + :distributed_runtime_writer, + ObjectId(:scene), + ) + distributed_output = only( + output for output in writer_target.output_bindings + if output isa PlantSimEngine.RuntimeDistributedOutputStream + ) + runtime_streams = distributed_output.streams + leaf_a_stream = outputs(simulation)[ + (:distributed_runtime_writer, ObjectId(:leaf_a), :incident_par) + ] + @test length(runtime_streams) == 2 + + register_object!( + model, + Object(:z_late_leaf; scale=:Leaf); + parent=:plant_a, + ) + continue!(simulation) + + refreshed_writer_target = _distributed_runtime_execution_target( + simulation, + :distributed_runtime_writer, + ObjectId(:scene), + ) + refreshed_distributed_output = only( + output for output in refreshed_writer_target.output_bindings + if output isa PlantSimEngine.RuntimeDistributedOutputStream + ) + @test refreshed_writer_target === writer_target + @test refreshed_distributed_output === distributed_output + @test refreshed_distributed_output.streams === runtime_streams + @test length(runtime_streams) == 3 + @test runtime_streams[1] === leaf_a_stream + @test outputs(simulation)[ + (:distributed_runtime_writer, ObjectId(:z_late_leaf), :incident_par) + ] == [(2.0, 2.0)] +end + +@testset "targeted stream initialization follows affected distributed writers" begin + model = _distributed_runtime_two_plant_scene() + compiled = Advanced.refresh_bindings!(model) + retention = PlantSimEngine.compile_model_output_retention( + compiled, + (); + retain_all=true, + ) + streams = Dict{Tuple{Symbol,ObjectId,Symbol},Any}() + PlantSimEngine._initialize_model_output_streams!( + streams, + compiled, + retention, + 0, + Set([(:distributed_runtime_writer, ObjectId(:scene))]), + ) + + @test Set(keys(streams)) == Set([ + ( + :distributed_runtime_writer, + ObjectId(:leaf_a), + :incident_par, + ), + ( + :distributed_runtime_writer, + ObjectId(:leaf_b), + :incident_par, + ), + ]) +end + @testset "Updates defines the final distributed-output producer" begin events = DistributedRuntimeEvent[] model = CompositeModel( @@ -779,6 +889,16 @@ end ) retained_stream = outputs(simulation)[stream_key] @test retained_stream == [(1.0, 1.0)] + plant_a_target = _distributed_runtime_execution_target( + simulation, + :distributed_runtime_plant_writer, + ObjectId(:plant_a), + ) + plant_b_target = _distributed_runtime_execution_target( + simulation, + :distributed_runtime_plant_writer, + ObjectId(:plant_b), + ) initial_targets = simulation.compiled.distributed_outputs.by_execution_target @test initial_targets[ @@ -795,6 +915,18 @@ end reparented_targets = simulation.compiled.distributed_outputs.by_execution_target + reparented_plant_a_target = _distributed_runtime_execution_target( + simulation, + :distributed_runtime_plant_writer, + ObjectId(:plant_a), + ) + reparented_plant_b_target = _distributed_runtime_execution_target( + simulation, + :distributed_runtime_plant_writer, + ObjectId(:plant_b), + ) + @test reparented_plant_a_target !== plant_a_target + @test reparented_plant_b_target !== plant_b_target @test isempty( reparented_targets[ (:distributed_runtime_plant_writer, ObjectId(:plant_a)) @@ -822,6 +954,12 @@ end removed_targets = simulation.compiled.distributed_outputs.by_execution_target + removed_plant_b_target = _distributed_runtime_execution_target( + simulation, + :distributed_runtime_plant_writer, + ObjectId(:plant_b), + ) + @test removed_plant_b_target !== reparented_plant_b_target @test all( isempty( removed_targets[ diff --git a/test/test-model-hard-calls.jl b/test/test-model-hard-calls.jl index 521e7ff29..81889b27c 100644 --- a/test/test-model-hard-calls.jl +++ b/test/test-model-hard-calls.jl @@ -5,17 +5,29 @@ using Test PlantSimEngine.@process "nested_call_leaf" verbose = false PlantSimEngine.@process "nested_call_middle" verbose = false PlantSimEngine.@process "nested_call_root" verbose = false +PlantSimEngine.@process "nested_many_middle" verbose = false +PlantSimEngine.@process "nested_many_root" verbose = false PlantSimEngine.@process "many_call_controller" verbose = false +PlantSimEngine.@process "selective_many_call_controller" verbose = false PlantSimEngine.@process "call_return_shape" verbose = false struct NestedCallLeafModel <: AbstractNested_Call_LeafModel end struct NestedCallMiddleModel <: AbstractNested_Call_MiddleModel end struct NestedCallRootModel <: AbstractNested_Call_RootModel end +struct NestedManyMiddleModel <: AbstractNested_Many_MiddleModel end +struct NestedManyRootModel <: AbstractNested_Many_RootModel end struct ManyCallControllerModel <: AbstractMany_Call_ControllerModel end +struct SelectiveManyCallControllerModel{O} <: + AbstractSelective_Many_Call_ControllerModel + objects::O +end struct CallReturnShapeModel <: AbstractCall_Return_ShapeModel end const CALL_RETURN_CONTEXT = Ref{Any}() const NESTED_ROOT_CONTEXT = Ref{Any}() +const NESTED_MANY_MIDDLE_CONTEXT = Ref{Any}() +const MANY_CALL_CONTEXT = Ref{Any}() +const LAZY_MANY_CALL_CONTEXT = Ref{Any}() function call_lookup_allocations(context) call_targets(context, :one) @@ -83,9 +95,46 @@ function PlantSimEngine.run!( return nothing end +PlantSimEngine.inputs_(::NestedManyMiddleModel) = NamedTuple() +PlantSimEngine.outputs_(::NestedManyMiddleModel) = (total=0.0, ncalls=0) + +function PlantSimEngine.run!( + ::NestedManyMiddleModel, + status, + environment, + constants, + context, +) + NESTED_MANY_MIDDLE_CONTEXT[] = context + leaves = run_call!(context, :leaves; publish=true) + status.ncalls = length(leaves) + status.total = sum((leaf.status.value for leaf in leaves); init=0.0) + return nothing +end + +PlantSimEngine.inputs_(::NestedManyRootModel) = NamedTuple() +PlantSimEngine.outputs_(::NestedManyRootModel) = (total=0.0, ncalls=0) + +function PlantSimEngine.run!( + ::NestedManyRootModel, + status, + environment, + constants, + context, +) + middle = only(run_call!(context, :middle; publish=true)) + status.ncalls = middle.status.ncalls + status.total = middle.status.total + return nothing +end + PlantSimEngine.inputs_(::ManyCallControllerModel) = NamedTuple() PlantSimEngine.outputs_(::ManyCallControllerModel) = (total=0.0, ncalls=0) +PlantSimEngine.inputs_(::SelectiveManyCallControllerModel) = NamedTuple() +PlantSimEngine.outputs_(::SelectiveManyCallControllerModel) = + (selected_total=0.0, selected_count=0) + function PlantSimEngine.run!( ::ManyCallControllerModel, status, @@ -93,12 +142,35 @@ function PlantSimEngine.run!( constants, context, ) + MANY_CALL_CONTEXT[] = context targets = run_call!(context, :children; publish=true) status.ncalls = length(targets) status.total = sum((target.status.value for target in targets); init=0.0) return nothing end +function PlantSimEngine.run!( + model::SelectiveManyCallControllerModel, + status, + environment, + constants, + context, +) + LAZY_MANY_CALL_CONTEXT[] = context + selected = run_call!( + context, + :children; + objects=model.objects, + publish=true, + ) + status.selected_count = length(selected) + status.selected_total = sum( + (target.status.value for target in selected); + init=0.0, + ) + return nothing +end + PlantSimEngine.inputs_(::CallReturnShapeModel) = NamedTuple() PlantSimEngine.outputs_(::CallReturnShapeModel) = ( one_count=0, @@ -175,6 +247,64 @@ end @test statuses[:middle].calls == 3 end +@testset "nested manual Many owners refresh transitively" begin + model = CompositeModel( + Object(:scene; scale=:Scene, name=:scene), + Object(:middle; scale=:Plant, name=:middle, parent=:scene), + Object(:leaf_a; scale=:Leaf, parent=:middle); + applications=( + ModelSpec( + NestedManyRootModel(); + name=:root, + on=One(name=:scene), + calls=( + :middle => One( + name=:middle, + within=Subtree(), + application=:middle, + ), + ), + ), + ModelSpec( + NestedManyMiddleModel(); + name=:middle, + on=One(name=:middle), + calls=( + :leaves => Many( + scale=:Leaf, + within=Subtree(), + application=:leaf, + ), + ), + ), + ModelSpec( + NestedCallLeafModel(); + name=:leaf, + on=Many(scale=:Leaf), + ), + ), + environment=(duration=Hour(1),), + ) + + simulation = run!(model; outputs=:none) + root = only(model_objects(model; name=:scene)).status + scenario_plan = simulation.compiled.scenario_plan + @test root.ncalls == 1 + @test root.total == 1.0 + @test length(call_targets(NESTED_MANY_MIDDLE_CONTEXT[], :leaves)) == 1 + + register_object!( + model, + Object(:leaf_b; scale=:Leaf, parent=:middle), + ) + continue!(simulation) + + @test simulation.compiled.scenario_plan === scenario_plan + @test root.ncalls == 2 + @test root.total == 3.0 + @test length(call_targets(NESTED_MANY_MIDDLE_CONTEXT[], :leaves)) == 2 +end + @testset "hard-call ownership cycles fail before execution" begin model = CompositeModel( Object(:scene; scale=:Scene, name=:scene), @@ -389,6 +519,14 @@ end ) @test getproperty.(rows, :object_id) == [:leaf_a, :leaf_b] @test getproperty.(rows, :value) == [1.0, 1.0] + initial_call_view = call_targets(MANY_CALL_CONTEXT[], :children) + initial_execution_targets = [ + target + for batch in initial_call_view.execution_batches + for target in batch.targets + ] + @test getproperty.(initial_execution_targets, :object_id) == + ObjectId[ObjectId(:leaf_a), ObjectId(:leaf_b)] register_object!( model, @@ -397,6 +535,17 @@ end continue!(simulation; steps=1) performance = Advanced.runtime_performance(simulation) @test performance.counts[:execution_target_call_batches_extended] == 1 + refreshed_call_view = call_targets(MANY_CALL_CONTEXT[], :children) + refreshed_execution_targets = [ + target + for batch in refreshed_call_view.execution_batches + for target in batch.targets + ] + @test refreshed_call_view === initial_call_view + @test refreshed_execution_targets[1] === initial_execution_targets[1] + @test refreshed_execution_targets[2] === initial_execution_targets[2] + @test getproperty.(refreshed_execution_targets, :object_id) == + ObjectId[ObjectId(:leaf_a), ObjectId(:leaf_b), ObjectId(:leaf_c)] @test controller.ncalls == 3 @test controller.total == 5.0 @@ -406,6 +555,268 @@ end @test controller.total == 5.0 end +@testset "large monotonic Many call extension preserves existing targets" begin + initial_count = 128 + initial_leaf_ids = Symbol[ + Symbol("leaf_", lpad(string(index), 3, '0')) + for index in 1:initial_count + ] + model = CompositeModel( + Object(:scene; scale=:Scene, name=:scene), + ( + Object(leaf_id; scale=:Leaf, parent=:scene) + for leaf_id in initial_leaf_ids + )...; + applications=( + ModelSpec( + ManyCallControllerModel(); + name=:controller, + on=One(name=:scene), + calls=( + :children => Many( + scale=:Leaf, + within=SceneScope(), + application=:leaf_calls, + ), + ), + ), + ModelSpec( + NestedCallLeafModel(); + name=:leaf_calls, + on=Many(scale=:Leaf), + ), + ), + environment=(duration=Hour(1),), + ) + + simulation = run!(model; outputs=:none, performance=true) + controller = only(model_objects(model; scale=:Scene)).status + initial_call_view = call_targets(MANY_CALL_CONTEXT[], :children) + initial_execution_targets = [ + target + for batch in initial_call_view.execution_batches + for target in batch.targets + ] + @test length(initial_execution_targets) == initial_count + @test getproperty.(initial_execution_targets, :object_id) == + ObjectId.(initial_leaf_ids) + @test controller.ncalls == initial_count + @test controller.total == initial_count + + added_leaf_id = :leaf_129 + register_object!( + model, + Object(added_leaf_id; scale=:Leaf, parent=:scene), + ) + continue!(simulation) + + refreshed_call_view = call_targets(MANY_CALL_CONTEXT[], :children) + refreshed_execution_targets = [ + target + for batch in refreshed_call_view.execution_batches + for target in batch.targets + ] + @test refreshed_call_view === initial_call_view + @test length(refreshed_execution_targets) == initial_count + 1 + @test all( + refreshed_execution_targets[index] === initial_execution_targets[index] + for index in eachindex(initial_execution_targets) + ) + @test refreshed_execution_targets[end].object_id == ObjectId(added_leaf_id) + @test all( + refreshed_execution_targets[end] !== initial_target + for initial_target in initial_execution_targets + ) + @test Advanced.runtime_performance(simulation).counts[ + :execution_target_call_batches_extended + ] == 1 + @test Advanced.runtime_performance(simulation).counts[ + :execution_target_call_delta_extensions + ] == 1 + @test Advanced.runtime_performance(simulation).counts[ + :execution_target_call_delta_targets_added + ] == 1 + @test controller.ncalls == initial_count + 1 + @test controller.total == 2 * initial_count + 1 +end + +@testset "large selective Many call remains lazy until full inspection" begin + initial_count = 128 + initial_leaf_ids = Symbol[ + Symbol("lazy_leaf_", lpad(string(index), 3, '0')) + for index in 1:initial_count + ] + selected_leaf_ids = ( + initial_leaf_ids[1], + initial_leaf_ids[div(initial_count, 2)], + initial_leaf_ids[end], + ) + model = CompositeModel( + Object(:scene; scale=:Scene, name=:scene), + ( + Object(leaf_id; scale=:Leaf, parent=:scene) + for leaf_id in initial_leaf_ids + )...; + applications=( + ModelSpec( + SelectiveManyCallControllerModel(selected_leaf_ids); + name=:selective_controller, + on=One(name=:scene), + calls=( + :children => Many( + scale=:Leaf, + within=SceneScope(), + application=:selective_leaf_calls, + ), + ), + ), + ModelSpec( + NestedCallLeafModel(); + name=:selective_leaf_calls, + on=Many(scale=:Leaf), + ), + ), + environment=(duration=Hour(1),), + ) + + simulation = run!(model; outputs=:none) + controller = only(model_objects(model; scale=:Scene)).status + context = LAZY_MANY_CALL_CONTEXT[] + full_call_view = call_targets(context, :children) + @test !PlantSimEngine._call_execution_batches_materialized( + full_call_view, + ) + @test controller.selected_count == length(selected_leaf_ids) + @test controller.selected_total == length(selected_leaf_ids) + @test all( + model_status(model, leaf_id).calls == 1 + for leaf_id in selected_leaf_ids + ) + @test all( + model_status(model, leaf_id).calls == 0 + for leaf_id in setdiff(initial_leaf_ids, selected_leaf_ids) + ) + + @test call_targets(context, :children) === full_call_view + @test !PlantSimEngine._call_execution_batches_materialized( + full_call_view, + ) + @test length(full_call_view) == initial_count + @test PlantSimEngine._call_execution_batches_materialized(full_call_view) + materialized_targets = collect(full_call_view) + @test getproperty.(materialized_targets, :object_id) == + ObjectId.(initial_leaf_ids) + @test all( + model_status(model, leaf_id).calls == + (leaf_id in selected_leaf_ids ? 1 : 0) + for leaf_id in initial_leaf_ids + ) + + executed_targets = run_call!(context, :children; publish=true) + @test executed_targets === full_call_view + @test PlantSimEngine._call_execution_batches_materialized( + executed_targets, + ) + @test sum(target.status.calls for target in executed_targets) == + initial_count + length(selected_leaf_ids) + @test sum(target.status.value for target in executed_targets) == + initial_count + length(selected_leaf_ids) +end + +@testset "non-monotonic Many call additions use the rebuild path" begin + model = CompositeModel( + Object(:scene; scale=:Scene, name=:scene), + Object(:leaf_b; scale=:Leaf, parent=:scene), + Object(:leaf_c; scale=:Leaf, parent=:scene); + applications=( + ModelSpec( + ManyCallControllerModel(); + name=:controller, + on=One(name=:scene), + calls=( + :children => Many( + scale=:Leaf, + within=SceneScope(), + application=:leaf_calls, + ), + ), + ), + ModelSpec( + NestedCallLeafModel(); + name=:leaf_calls, + on=Many(scale=:Leaf), + ), + ), + environment=(duration=Hour(1),), + ) + + simulation = run!(model; outputs=:none, performance=true) + initial_call_view = call_targets(MANY_CALL_CONTEXT[], :children) + register_object!( + model, + Object(:leaf_a; scale=:Leaf, parent=:scene), + ) + continue!(simulation) + + refreshed_call_view = call_targets(MANY_CALL_CONTEXT[], :children) + refreshed_object_ids = ObjectId[ + target.object_id + for batch in refreshed_call_view.execution_batches + for target in batch.targets + ] + @test refreshed_object_ids == + ObjectId[ObjectId(:leaf_a), ObjectId(:leaf_b), ObjectId(:leaf_c)] + @test refreshed_call_view !== initial_call_view + @test get( + Advanced.runtime_performance(simulation).counts, + :execution_target_call_batches_extended, + 0, + ) == 0 +end + +@testset "reparented Many call targets rebuild within a stable scope" begin + model = CompositeModel( + Object(:scene; scale=:Scene, name=:scene), + Object(:plant_a; scale=:Plant, parent=:scene), + Object(:plant_b; scale=:Plant, parent=:scene), + Object(:leaf; scale=:Leaf, parent=:plant_a); + applications=( + ModelSpec( + ManyCallControllerModel(); + name=:controller, + on=One(name=:scene), + calls=( + :children => Many( + scale=:Leaf, + within=SceneScope(), + application=:leaf_calls, + ), + ), + ), + ModelSpec( + NestedCallLeafModel(); + name=:leaf_calls, + on=Many(scale=:Leaf), + ), + ), + environment=(duration=Hour(1),), + ) + + simulation = run!(model; outputs=:none, performance=true) + reparent_object!(model, :leaf, :plant_b) + continue!(simulation) + + refreshed_call_view = call_targets(MANY_CALL_CONTEXT[], :children) + refreshed_execution_target = + only(only(refreshed_call_view.execution_batches).targets) + @test refreshed_execution_target.object_id == ObjectId(:leaf) + @test get( + Advanced.runtime_performance(simulation).counts, + :execution_target_call_batches_extended, + 0, + ) == 0 +end + @testset "call targets refresh after reparenting" begin model = CompositeModel( Object(:scene; scale=:Scene), diff --git a/test/test-model-previous-timestep-views.jl b/test/test-model-previous-timestep-views.jl index 25140041e..67ca36b88 100644 --- a/test/test-model-previous-timestep-views.jl +++ b/test/test-model-previous-timestep-views.jl @@ -59,12 +59,22 @@ function PlantSimEngine.run!( end PlantSimEngine.@process "temporal_view_many_consumer" verbose = false +PlantSimEngine.@process "temporal_view_manual_controller" verbose = false struct TemporalViewManyConsumer <: AbstractTemporal_View_Many_ConsumerModel end +struct TemporalViewManualController <: + AbstractTemporal_View_Manual_ControllerModel end + +const TEMPORAL_VIEW_MANUAL_CONTEXT = Ref{Any}() + PlantSimEngine.inputs_(::TemporalViewManyConsumer) = (signals=Required(Vector{Float64}),) PlantSimEngine.outputs_(::TemporalViewManyConsumer) = (signal_total=0.0,) +PlantSimEngine.inputs_(::TemporalViewManualController) = NamedTuple() +PlantSimEngine.outputs_(::TemporalViewManualController) = + (signal_total=0.0, source_count=0) + function PlantSimEngine.run!( ::TemporalViewManyConsumer, status, @@ -77,6 +87,20 @@ function PlantSimEngine.run!( return nothing end +function PlantSimEngine.run!( + ::TemporalViewManualController, + status, + environment, + constants, + context, +) + TEMPORAL_VIEW_MANUAL_CONTEXT[] = context + consumer = only(run_call!(context, :consumer; publish=true)) + status.signal_total = consumer.status.signal_total + status.source_count = length(consumer.status.signals) + return nothing +end + PlantSimEngine.@process "temporal_view_mixed_consumer" verbose = false struct TemporalViewMixedConsumer <: AbstractTemporal_View_Mixed_ConsumerModel end @@ -696,6 +720,84 @@ end ) end +@testset "manual callee extends PreviousTimeStep Many sources in place" begin + model = CompositeModel( + Object(:scene; scale=:Scene, status=Status(signals=[0.0])), + Object( + :leaf_1; + scale=:Leaf, + parent=:scene, + status=Status(signal=1.0), + ); + applications=( + _temporal_view_source_spec(target=Many(scale=:Leaf)), + ModelSpec( + TemporalViewManualController(); + name=:manual_controller, + on=One(scale=:Scene), + calls=( + :consumer => One( + scale=:Scene, + application=:manual_many_consumer, + ), + ), + ), + ModelSpec( + TemporalViewManyConsumer(); + name=:manual_many_consumer, + on=One(scale=:Scene), + inputs=( + PreviousTimeStep(:signals) => Many( + scale=:Leaf, + within=SceneScope(), + application=:signal_source, + var=:signal, + ), + ), + ), + ), + ) + + simulation = run!(model; outputs=:all) + schedule = Dict( + row.application_id => row + for row in Diagnostics.explain_schedule(simulation.compiled) + ) + @test schedule[:manual_many_consumer].manual_call_only + initial_call_view = call_targets( + TEMPORAL_VIEW_MANUAL_CONTEXT[], + :consumer, + ) + initial_target = only(only(initial_call_view.execution_batches).targets) + initial_temporal_inputs = initial_target.input_bindings + @test final_state(simulation, :scene).signal_total == 1.0 + @test final_state(simulation, :scene).source_count == 1 + + register_object!( + model, + Object(:leaf_2; scale=:Leaf, status=Status(signal=10.0)); + parent=:scene, + ) + @test_nowarn continue!(simulation; steps=2) + + refreshed_call_view = call_targets( + TEMPORAL_VIEW_MANUAL_CONTEXT[], + :consumer, + ) + refreshed_target = + only(only(refreshed_call_view.execution_batches).targets) + @test refreshed_call_view !== initial_call_view + @test refreshed_target !== initial_target + @test refreshed_target.input_bindings !== initial_temporal_inputs + @test length(refreshed_target.status.signals) == 2 + @test outputs(simulation)[ + (:manual_controller, ObjectId(:scene), :signal_total) + ] == [(1.0, 1.0), (2.0, 12.0), (3.0, 14.0)] + @test outputs(simulation)[ + (:manual_controller, ObjectId(:scene), :source_count) + ] == [(1.0, 1), (2.0, 2), (3.0, 2)] +end + @testset "Heterogeneous PreviousTimeStep materialization is allocation-free" begin mixed_object = CompositeModel( Object(:scene; scale=:Scene, status=Status(signal=10.0)), diff --git a/test/test-model-status-type-lifecycle.jl b/test/test-model-status-type-lifecycle.jl index 03c5d4783..520a946a9 100644 --- a/test/test-model-status-type-lifecycle.jl +++ b/test/test-model-status-type-lifecycle.jl @@ -6,6 +6,7 @@ using Test PlantSimEngine.@process "status_type_lifecycle_increment" verbose = false PlantSimEngine.@process "status_type_lifecycle_stream_writer" verbose = false PlantSimEngine.@process "status_type_lifecycle_stream_consumer" verbose = false +PlantSimEngine.@process "status_type_lifecycle_bound_consumer" verbose = false struct StatusTypeLifecycleIncrementModel <: AbstractStatus_Type_Lifecycle_IncrementModel end @@ -58,6 +59,25 @@ function PlantSimEngine.run!( return nothing end +struct StatusTypeLifecycleBoundConsumerModel <: + AbstractStatus_Type_Lifecycle_Bound_ConsumerModel end + +PlantSimEngine.inputs_(::StatusTypeLifecycleBoundConsumerModel) = + (bound_value=Default(-1.0),) +PlantSimEngine.outputs_(::StatusTypeLifecycleBoundConsumerModel) = + (seen=0.0,) + +function PlantSimEngine.run!( + ::StatusTypeLifecycleBoundConsumerModel, + status, + environment, + constants, + context, +) + status.seen = status.bound_value + return nothing +end + struct StatusTypeLifecycleTransformCounter calls::Base.RefValue{Int} end @@ -67,6 +87,78 @@ function (counter::StatusTypeLifecycleTransformCounter)(variable, value) return value end +@testset "pure additions preserve bound-default conversion diagnostics" begin + calls = Ref(0) + model = CompositeModel( + Object(:scene; scale=:Scene), + Object(:plant; scale=:Plant, parent=:scene), + Object(:leaf_1; scale=:Leaf, parent=:plant); + applications=( + ModelSpec( + StatusTypeLifecycleIncrementModel(); + name=:lifecycle_increment, + on=Many(scale=:Leaf), + ), + ModelSpec( + StatusTypeLifecycleBoundConsumerModel(); + name=:bound_consumer, + on=Many(scale=:Leaf), + inputs=( + bound_value=One( + within=Self(), + application=:lifecycle_increment, + var=:value, + ), + ), + ), + ), + environment=(duration=Hour(1),), + type_promotion=Dict(Float64 => Float32), + status_transform=StatusTypeLifecycleTransformCounter(calls), + ) + simulation = run!(model; outputs=:none, performance=true) + calls_before_addition = calls[] + + register_object!( + model, + Object(:leaf_2; scale=:Leaf, parent=:plant), + ) + continue!(simulation) + + status = model_status(model, :leaf_2) + @test propertynames(status) == (:value, :seen, :bound_value) + @test PlantSimEngine.refvalue(status, :bound_value) === + PlantSimEngine.refvalue(status, :value) + @test status.bound_value === Float32(1) + @test status.seen === Float32(1) + @test calls[] == calls_before_addition + 3 + @test haskey( + model.status_conversion_records, + ( + :model_input_default, + :bound_consumer, + ObjectId(:leaf_2), + :bound_value, + ), + ) + @test :bound_value ∉ get( + model.input_default_status_variables, + ObjectId(:leaf_2), + Set{Symbol}(), + ) + initialization = only( + row for row in Diagnostics.explain_initialization(simulation) + if row.application_id == :bound_consumer && + row.object_id == :leaf_2 && + row.variable == :bound_value + ) + @test initialization.disposition == :producer_bound + @test initialization.status_transform_applied + @test initialization.type_mapping_applied + performance = Advanced.runtime_performance(simulation) + @test performance.counts[:input_default_status_rewrites_avoided] == 1 +end + @testset "register_object! converts status after compilation" begin model = CompositeModel( Object(:scene; scale=:Scene), diff --git a/test/test-unified-model-object-api.jl b/test/test-unified-model-object-api.jl index 98e5b2e81..717e63b21 100644 --- a/test/test-unified-model-object-api.jl +++ b/test/test-unified-model-object-api.jl @@ -4500,6 +4500,73 @@ end if row.application_id == :moving_probe ).handle.cell == :cell_b + moving_hard_call_backend = ModelObjectMutableEnvironmentBackend( + :cell_a => 20.0, + :cell_b => 30.0, + ) + moving_hard_call_scene = CompositeModel( + Object( + :scene; + scale=:Scene, + kind=:scene, + geometry=(cell=:cell_a,), + status=Status(move_count=0), + ), + Object( + :leaf_1; + scale=:Leaf, + kind=:plant, + parent=:scene, + geometry=(cell=:cell_a,), + status=Status(temperature_seen=0.0, called_temperature=0.0), + ); + applications=( + ModelSpec( + ModelObjectGeometryMoverModel(); + name=:geometry_mover, + on=One(scale=:Scene), + ), + ModelSpec( + ModelObjectEnvironmentCallSourceModel(); + name=:environment_source, + on=One(scale=:Leaf), + environment=Environment(provider=:grid), + ), + ModelSpec( + ModelObjectEnvironmentCallControllerModel(32.5, true); + name=:environment_controller, + on=One(scale=:Leaf), + environment=Environment(provider=:grid, sink=:grid), + calls=( + :source => One( + scale=:Leaf, + application=:environment_source, + ), + ), + ), + ), + environment=moving_hard_call_backend, + ) + moving_hard_call_simulation = run!( + moving_hard_call_scene; + outputs=:none, + performance=true, + ) + moving_hard_call_status = + only(model_objects(moving_hard_call_scene; scale=:Leaf)).status + @test moving_hard_call_status.temperature_seen == 32.5 + @test moving_hard_call_status.called_temperature == 32.5 + @test moving_hard_call_backend.values == + Dict(:cell_a => 20.0, :cell_b => 32.5) + @test only(moving_hard_call_backend.writes).cell == :cell_b + moving_controller_group = only( + group for group in moving_hard_call_simulation.execution_plan.groups + if group.application.id == :environment_controller + ) + moving_controller_target = + only(only(moving_controller_group.batches).targets) + @test moving_controller_target.environment_binding.handle.cell == :cell_b + multirate_scene = CompositeModel( Object(:scene; scale=:Scene, kind=:scene), Object(:leaf_1; scale=:Leaf, kind=:plant, parent=:scene, status=Status(signal=0.0)); From f521b8dbf8ecf25e7a3e41bb64caa74d1ff41f33 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?R=C3=A9mi=20Vezy?= Date: Mon, 31 Aug 2026 19:40:24 +0200 Subject: [PATCH 2/8] perf: accelerate dynamic organ lifecycle --- benchmark/benchmarks.jl | 27 + benchmark/test-hard-call-path-benchmark.jl | 11 + benchmark/test-organ-lifecycle-benchmark.jl | 72 ++ .../test-selector-resolution-benchmark.jl | 52 ++ benchmark/test/runtests.jl | 24 + src/composite_model/compilation.jl | 829 +++++++++++++++--- src/composite_model/runtime_outputs.jl | 617 ++++++++++++- src/composite_model/selectors.jl | 92 +- src/composite_model/status_conversion.jl | 5 +- src/visualization/model_graph_view.jl | 101 ++- test/test-model-api-stabilization.jl | 591 ++++++++++++- test/test-model-hard-calls.jl | 515 +++++++++++ test/test-model-status-type-lifecycle.jl | 194 ++++ 13 files changed, 2944 insertions(+), 186 deletions(-) create mode 100644 benchmark/test-selector-resolution-benchmark.jl diff --git a/benchmark/benchmarks.jl b/benchmark/benchmarks.jl index 6858e890e..cf53869d1 100644 --- a/benchmark/benchmarks.jl +++ b/benchmark/benchmarks.jl @@ -56,6 +56,23 @@ if SUPPORTS_COMPOSITE_OBJECT_BENCHMARKS ) setup = (data = setup_status_registry_benchmark($nobjects)) end + include(joinpath(@__DIR__, "test-selector-resolution-benchmark.jl")) + const SELECTOR_RESOLUTION_BENCHMARK_PHYTOMERS = 2_048 + SUITE[suite_name]["PSE_selector_subtree_tip_2048"] = + @benchmarkable benchmark_subtree_selector_resolution( + data, + data.tip_context, + ) setup = (data = setup_subtree_selector_resolution_benchmark( + $SELECTOR_RESOLUTION_BENCHMARK_PHYTOMERS, + )) + SUITE[suite_name]["PSE_selector_subtree_root_2048"] = + @benchmarkable benchmark_subtree_selector_resolution( + data, + data.root_context, + ) setup = (data = setup_subtree_selector_resolution_benchmark( + $SELECTOR_RESOLUTION_BENCHMARK_PHYTOMERS, + )) + include(joinpath(@__DIR__, "test-organ-lifecycle-benchmark.jl")) for nobjects in (32, 256, 1_024) SUITE[suite_name]["PSE_organ_adaptation_$(nobjects)"] = @@ -76,6 +93,12 @@ if SUPPORTS_COMPOSITE_OBJECT_BENCHMARKS ) setup = (data = setup_organ_refresh_benchmark( $nobjects, )) evals = 1 + SUITE[suite_name]["PSE_organ_status_recipe_refresh_$(nobjects)"] = + @benchmarkable benchmark_status_recipe_refresh_after_add!( + data, + ) setup = (data = setup_organ_status_recipe_refresh_benchmark( + $nobjects, + )) evals = 1 SUITE[suite_name]["PSE_organ_add_refresh_$(nobjects)"] = @benchmarkable benchmark_add_and_refresh!( data, @@ -334,6 +357,10 @@ if SUPPORTS_COMPOSITE_OBJECT_BENCHMARKS target_count=$target_count, )) end + SUITE[suite_name]["PSE_call_binding_signature_4096"] = + @benchmarkable benchmark_call_binding_signature( + binding, + ) setup = (binding = setup_call_binding_signature_benchmark(4_096)) SUITE[suite_name]["PSE_lifecycle_small"] = @benchmarkable benchmark_lifecycle_event( simulation, diff --git a/benchmark/test-hard-call-path-benchmark.jl b/benchmark/test-hard-call-path-benchmark.jl index da335d45a..b5b94c8b1 100644 --- a/benchmark/test-hard-call-path-benchmark.jl +++ b/benchmark/test-hard-call-path-benchmark.jl @@ -397,6 +397,17 @@ end benchmark_compiled_hard_call_step(simulation) = step!(simulation) +function setup_call_binding_signature_benchmark(target_count::Int=4_096) + simulation = setup_compiled_hard_call_step( + kind=:many, + target_count=target_count, + ) + return only(simulation.compiled.call_bindings) +end + +Base.@noinline benchmark_call_binding_signature(binding) = + PlantSimEngine._call_bindings_signature((binding,)) + function benchmark_compiled_hard_call_invocation( context::T; repeats=1, diff --git a/benchmark/test-organ-lifecycle-benchmark.jl b/benchmark/test-organ-lifecycle-benchmark.jl index f0312907e..6ca3ba68e 100644 --- a/benchmark/test-organ-lifecycle-benchmark.jl +++ b/benchmark/test-organ-lifecycle-benchmark.jl @@ -19,6 +19,39 @@ function PlantSimEngine.run!( return nothing end +PlantSimEngine.@process "organ_lifecycle_status_recipe" verbose = false + +struct OrganLifecycleStatusRecipeModel <: + AbstractOrgan_Lifecycle_Status_RecipeModel end + +const ORGAN_LIFECYCLE_RECIPE_OUTPUT_NAMES = + ntuple(index -> Symbol(:recipe_output_, index), 32) +const ORGAN_LIFECYCLE_RECIPE_INPUT_NAMES = + ntuple(index -> Symbol(:recipe_input_, index), 32) + +PlantSimEngine.outputs_(::OrganLifecycleStatusRecipeModel) = + NamedTuple{ORGAN_LIFECYCLE_RECIPE_OUTPUT_NAMES}( + ntuple(_ -> 0.0, length(ORGAN_LIFECYCLE_RECIPE_OUTPUT_NAMES)), + ) + +PlantSimEngine.inputs_(::OrganLifecycleStatusRecipeModel) = + NamedTuple{ORGAN_LIFECYCLE_RECIPE_INPUT_NAMES}( + ntuple( + _ -> PlantSimEngine.Default(1.0), + length(ORGAN_LIFECYCLE_RECIPE_INPUT_NAMES), + ), + ) + +function PlantSimEngine.run!( + ::OrganLifecycleStatusRecipeModel, + status, + environment, + constants=nothing, + context=nothing, +) + return nothing +end + _organ_lifecycle_status(node) = PlantSimEngine.Status(node=node, signal=0.0) @@ -61,6 +94,20 @@ Base.@noinline function benchmark_adapt_organ_model(topology) ) end +Base.@noinline function benchmark_adapt_organ_status_recipe_model(topology) + return PlantSimEngine.CompositeModel( + topology.root; + applications=( + PlantSimEngine.ModelSpec( + OrganLifecycleStatusRecipeModel(); + name=:organ_lifecycle_status_recipe, + on=PlantSimEngine.Many(scale=:Leaf), + ), + ), + status=node -> PlantSimEngine.Status(node=node), + ) +end + function setup_organ_lifecycle_benchmark( nobjects::Int; start_simulation::Bool=false, @@ -86,6 +133,15 @@ function setup_organ_refresh_benchmark(nobjects::Int) return merge(data, (; status)) end +function setup_organ_status_recipe_refresh_benchmark(nobjects::Int) + topology = setup_organ_topology_benchmark(nobjects) + model = benchmark_adapt_organ_status_recipe_model(topology) + PlantSimEngine.Advanced.refresh_bindings!(model) + data = merge(topology, (; model)) + benchmark_add_status_recipe_organ!(data) + return data +end + Base.@noinline function benchmark_add_organ!(data) return PlantSimEngine.add_organ!( data.plant, @@ -98,11 +154,27 @@ Base.@noinline function benchmark_add_organ!(data) ) end +Base.@noinline function benchmark_add_status_recipe_organ!(data) + return PlantSimEngine.add_organ!( + data.plant, + data.model, + "+", + :Leaf, + 2; + index=data.nobjects + 1, + ) +end + Base.@noinline function benchmark_refresh_after_add!(data) PlantSimEngine.Advanced.refresh_bindings!(data.model) return data.model end +Base.@noinline function benchmark_status_recipe_refresh_after_add!(data) + PlantSimEngine.Advanced.refresh_bindings!(data.model) + return data.model +end + Base.@noinline function benchmark_add_and_refresh!(data) status = benchmark_add_organ!(data) PlantSimEngine.Advanced.refresh_bindings!(data.model) diff --git a/benchmark/test-selector-resolution-benchmark.jl b/benchmark/test-selector-resolution-benchmark.jl new file mode 100644 index 000000000..61575806b --- /dev/null +++ b/benchmark/test-selector-resolution-benchmark.jl @@ -0,0 +1,52 @@ +using PlantSimEngine + +""" +Build an XPalm-shaped chain where every phytomer owns one reproductive organ. +The newest phytomer has a two-object subtree, while the oldest phytomer spans +the complete chain, so the same selector benchmarks both the bounded +scope-first path and its registry-index fallback. +""" +function setup_subtree_selector_resolution_benchmark(nphytomers::Int=2_048) + nphytomers > 0 || throw(ArgumentError("`nphytomers` must be positive.")) + objects = Object[ + Object(:scene; scale=:Scene), + Object(:plant; scale=:Plant, parent=:scene), + ] + previous_phytomer_id = :plant + for index in 1:nphytomers + phytomer_id = Symbol(:phytomer_, index) + male_id = Symbol(:male_, index) + push!( + objects, + Object( + phytomer_id; + scale=:Phytomer, + parent=previous_phytomer_id, + ), + ) + push!( + objects, + Object(male_id; scale=:Male, parent=phytomer_id), + ) + previous_phytomer_id = phytomer_id + end + + model = CompositeModel(objects...) + selector = Many(scale=:Male, within=Subtree()) + return ( + model=model, + selector=selector, + matcher=PlantSimEngine._compile_selector_matcher(model, selector), + tip_context=ObjectId(Symbol(:phytomer_, nphytomers)), + root_context=ObjectId(:phytomer_1), + ) +end + +Base.@noinline function benchmark_subtree_selector_resolution(data, context) + return PlantSimEngine._resolve_object_ids( + data.model, + data.selector, + data.matcher; + context=context, + ) +end diff --git a/benchmark/test/runtests.jl b/benchmark/test/runtests.jl index ee97d5e7b..2604c22e0 100644 --- a/benchmark/test/runtests.jl +++ b/benchmark/test/runtests.jl @@ -185,6 +185,26 @@ if benchmark_test_enabled("organ lifecycle benchmark API smoke") :organ_lifecycle_leaf ].target_ids + recipe = setup_organ_status_recipe_refresh_benchmark(32) + benchmark_status_recipe_refresh_after_add!(recipe) + recipe_status = PlantSimEngine.model_status( + recipe.model, + PlantSimEngine.ObjectId(recipe.next_node_id), + ) + @test propertynames(recipe_status) == ( + :node, + ORGAN_LIFECYCLE_RECIPE_OUTPUT_NAMES..., + ORGAN_LIFECYCLE_RECIPE_INPUT_NAMES..., + ) + @test all( + variable -> recipe_status[variable] == 0.0, + ORGAN_LIFECYCLE_RECIPE_OUTPUT_NAMES, + ) + @test all( + variable -> recipe_status[variable] == 1.0, + ORGAN_LIFECYCLE_RECIPE_INPUT_NAMES, + ) + combined = setup_organ_lifecycle_benchmark(32) combined_status = benchmark_add_and_refresh!(combined) @test PlantSimEngine.object_id(combined.model, combined_status) == @@ -858,13 +878,17 @@ if benchmark_test_enabled("internal-only benchmark suite assembly smoke") @test haskey(suite, "PSE_status_read_write") @test haskey(suite, "PSE_status_registry_lookup_32") @test haskey(suite, "PSE_status_registry_sweep_1024") + @test haskey(suite, "PSE_selector_subtree_tip_2048") + @test haskey(suite, "PSE_selector_subtree_root_2048") @test haskey(suite, "PSE_organ_adaptation_32") @test haskey(suite, "PSE_organ_add_1024") @test haskey(suite, "PSE_organ_refresh_1024") + @test haskey(suite, "PSE_organ_status_recipe_refresh_1024") @test haskey(suite, "PSE_organ_add_refresh_1024") @test haskey(suite, "PSE_organ_add_continue_1024") @test haskey(suite, "PSE") @test haskey(suite, "PSE_hard_calls_zero") + @test haskey(suite, "PSE_call_binding_signature_4096") @test haskey(suite, "PSE_lifecycle_large") @test haskey(suite, "PSE_immutable_scenario_none") @test haskey(suite, "PSE_immutable_scenario_requests") diff --git a/src/composite_model/compilation.jl b/src/composite_model/compilation.jl index bbabc6fa3..4c4de3275 100644 --- a/src/composite_model/compilation.jl +++ b/src/composite_model/compilation.jl @@ -1183,6 +1183,35 @@ struct CompiledModelCallBinding{P} consumer_id::ObjectId callee_object_ids::Vector{ObjectId} callee_application_ids::Vector{Symbol} + # The binding stays immutable while lifecycle refreshes extend or rebuild + # its shared target vectors in place. Keep the corresponding generation in + # a reference so execution-target signatures can observe those mutations + # without hashing every callee. This cache detail is intentionally omitted + # from `propertynames`. + membership_generation::Base.RefValue{UInt64} + # Manual `Many` memberships stay cold until their complete target view is + # materialized. The flag is reference-backed for the same reason as the + # generation: incremental compiler shells retain binding identity. + membership_observed::Base.RefValue{Bool} +end + +function CompiledModelCallBinding( + plan::P, + consumer_id::ObjectId, + callee_object_ids::Vector{ObjectId}, + callee_application_ids::Vector{Symbol}, +) where {P} + return CompiledModelCallBinding{P}( + plan, + consumer_id, + callee_object_ids, + callee_application_ids, + Ref(UInt64(0)), + Ref( + _compiled_call_mode(plan) !== :manual || + plan.multiplicity !== :many, + ), + ) end @inline function Base.getproperty( @@ -1226,6 +1255,29 @@ Base.propertynames(binding::CompiledModelCallBinding) = ( @inline _compiled_call_mode(binding::CompiledModelCallBinding) = _compiled_call_mode(binding.plan) +@inline _compiled_call_membership_generation( + binding::CompiledModelCallBinding, +) = getfield(binding, :membership_generation)[] + +@inline function _mark_compiled_call_membership_changed!( + binding::CompiledModelCallBinding, + revision::Integer, +) + getfield(binding, :membership_generation)[] = UInt64(revision) + return binding +end + +@inline _compiled_call_membership_is_observed( + binding::CompiledModelCallBinding, +) = getfield(binding, :membership_observed)[] + +@inline function _mark_compiled_call_membership_observed!( + binding::CompiledModelCallBinding, +) + getfield(binding, :membership_observed)[] = true + return binding +end + struct CompiledEnvironmentSamplingRule{T,S} end CompiledEnvironmentSamplingRule(target::Symbol, source::Symbol) = @@ -1335,43 +1387,84 @@ mutable struct CompiledCompositeModel{SC,SP,AP,AI,OA,ABO,IB,CB,IBI,CBI,DBI,DCBI, revision::Int end -function _index_dynamic_bindings(model::CompositeModel, bindings) +function _index_dynamic_input_binding!( + index::SelectorCandidateIndex, + model::CompositeModel, + binding::CompiledModelInputBinding, + binding_index::Int, + many_binding_cache, +) + binding.origin == :inferred_same_object && return index + if _dynamic_input_binding_shares_canonical_sources( + model, + binding, + many_binding_cache, + ) + return index + end + _index_selector_candidate!( + index, + model, + binding.matcher, + binding_index; + context=binding.consumer_id, + default_scope=_default_dependency_scope( + model, + binding.consumer_id, + ), + ) + return index +end + +function _index_dynamic_input_bindings( + model::CompositeModel, + bindings, + many_binding_cache, +) index = _selector_candidate_index() for (binding_index, binding) in pairs(bindings) - binding.origin == :inferred_same_object && continue - _index_selector_candidate!( + _index_dynamic_input_binding!( index, model, - binding.matcher, - binding_index; - context=binding.consumer_id, - default_scope=_default_dependency_scope( - model, - binding.consumer_id, - ), + binding, + binding_index, + many_binding_cache, ) end return index end -_index_dynamic_input_bindings(model::CompositeModel, bindings) = - _index_dynamic_bindings(model, bindings) +function _index_dynamic_call_binding!( + index::SelectorCandidateIndex, + model::CompositeModel, + binding::CompiledModelCallBinding, + binding_index::Int, +) + _compiled_call_mode(binding) === :manual || return index + _compiled_call_membership_is_observed(binding) || return index + binding.origin == :inferred_same_object && return index + _index_selector_candidate!( + index, + model, + binding.matcher, + binding_index; + context=binding.consumer_id, + default_scope=_default_dependency_scope( + model, + binding.consumer_id, + ), + ) + return index +end function _index_dynamic_call_bindings(model::CompositeModel, bindings) index = _selector_candidate_index() for (binding_index, binding) in pairs(bindings) - _compiled_call_mode(binding) === :manual || continue - binding.origin == :inferred_same_object && continue - _index_selector_candidate!( + _index_dynamic_call_binding!( index, model, - binding.matcher, - binding_index; - context=binding.consumer_id, - default_scope=_default_dependency_scope( - model, - binding.consumer_id, - ), + binding, + binding_index, ) end return index @@ -1522,10 +1615,12 @@ function _compile_scene( :consumer_id, ), ) - call_bindings_by_target = _index_model_bindings( - call_bindings, - :application_id, - :consumer_id, + call_bindings_by_target = Dict{Any,Any}( + _index_model_bindings( + call_bindings, + :application_id, + :consumer_id, + ), ) call_owners = scenario_plan.call_owners application_children = scenario_plan.application_children @@ -1560,7 +1655,11 @@ function _compile_scene( call_bindings, input_bindings_by_target, call_bindings_by_target, - _index_dynamic_input_bindings(model, input_bindings), + _index_dynamic_input_bindings( + model, + input_bindings, + many_input_binding_cache, + ), _index_dynamic_call_bindings(model, call_bindings), many_input_binding_cache, distributed_outputs, @@ -1931,6 +2030,45 @@ function _many_input_binding_cache(model::CompositeModel, bindings) return cache end +function _shared_same_rate_many_storage( + binding::CompiledModelInputBinding, + canonical::CompiledModelInputBinding, +) + binding.multiplicity == :many || return false + binding.carrier_hint === :ref_vector || return false + canonical.multiplicity == :many || return false + canonical.carrier_hint === :ref_vector || return false + return binding.source_ids === canonical.source_ids && + binding.source_application_ids === + canonical.source_application_ids && + binding.carrier === canonical.carrier +end + +function _dynamic_input_binding_shares_canonical_sources( + model::CompositeModel, + binding::CompiledModelInputBinding, + many_binding_cache, +) + binding.multiplicity == :many || return false + binding.carrier_hint === :ref_vector || return false + key = _many_binding_share_key(model, binding) + isnothing(key) && return false + canonical = get(many_binding_cache, key, nothing) + canonical isa CompiledModelInputBinding || return false + _shared_same_rate_many_storage(binding, canonical) || return false + return binding.slot != canonical.slot || + binding.consumer_id != canonical.consumer_id +end + +function _shared_same_rate_many_binding_indices(bindings, binding) + binding.multiplicity == :many || return Int[] + binding.carrier_hint === :ref_vector || return Int[] + return Int[ + index for (index, candidate) in pairs(bindings) + if _shared_same_rate_many_storage(candidate, binding) + ] +end + function _append_added_many_sources!( model::CompositeModel, binding::CompiledModelInputBinding, @@ -2458,6 +2596,7 @@ function _append_added_many_call_targets!( push!(binding.callee_application_ids, application_id) end end + _mark_compiled_call_membership_changed!(binding, model.revision) return true end @@ -2486,6 +2625,7 @@ function _propagate_changed_manual_call_owners!( for binding in call_bindings binding.application_id in owner_application_ids || continue _compiled_call_mode(binding) === :manual || continue + _compiled_call_membership_is_observed(binding) || continue affected = false for callee_application_id in binding.callee_application_ids object_ids = get(frontier, callee_application_id, nothing) @@ -2608,6 +2748,7 @@ function _prepare_structural_compiled_delta( end forced_call_target_keys = Set{Tuple{Symbol,ObjectId}}() for binding in call_bindings + _compiled_call_membership_is_observed(binding) || continue any( dirty_id -> first( _sorted_object_id_position( @@ -2623,11 +2764,15 @@ function _prepare_structural_compiled_delta( (binding.application_id, binding.consumer_id), ) end - call_bindings_by_target = _index_model_bindings( - call_bindings, - :application_id, - :consumer_id, + call_bindings_by_target = Dict{Any,Any}( + _index_model_bindings( + call_bindings, + :application_id, + :consumer_id, + ), ) + many_input_binding_cache = + _many_input_binding_cache(model, input_bindings) stripped = CompiledCompositeModel( model, @@ -2640,9 +2785,13 @@ function _prepare_structural_compiled_delta( call_bindings, input_bindings_by_target, call_bindings_by_target, - _index_dynamic_input_bindings(model, input_bindings), + _index_dynamic_input_bindings( + model, + input_bindings, + many_input_binding_cache, + ), _index_dynamic_call_bindings(model, call_bindings), - _many_input_binding_cache(model, input_bindings), + many_input_binding_cache, compiled.distributed_outputs, compiled.call_owners, compiled.application_children, @@ -2908,32 +3057,35 @@ function _extend_compiled_scene( "`$(first(key))` on object `$(last(key).value)`.", ) replacement = replacements[replacement_index] - empty!(binding.callee_object_ids) - append!(binding.callee_object_ids, replacement.callee_object_ids) - empty!(binding.callee_application_ids) - append!( - binding.callee_application_ids, - replacement.callee_application_ids, - ) + if binding.callee_object_ids != replacement.callee_object_ids || + binding.callee_application_ids != + replacement.callee_application_ids + empty!(binding.callee_object_ids) + append!( + binding.callee_object_ids, + replacement.callee_object_ids, + ) + empty!(binding.callee_application_ids) + append!( + binding.callee_application_ids, + replacement.callee_application_ids, + ) + _mark_compiled_call_membership_changed!( + binding, + model.revision, + ) + end end end append!(call_bindings, new_call_bindings) dynamic_call_binding_indices = compiled.dynamic_call_binding_indices first_new_call_binding = length(call_bindings) - length(new_call_bindings) + 1 for binding_index in first_new_call_binding:length(call_bindings) - binding = call_bindings[binding_index] - _compiled_call_mode(binding) === :manual || continue - binding.origin == :inferred_same_object && continue - _index_selector_candidate!( + _index_dynamic_call_binding!( dynamic_call_binding_indices, model, - binding.matcher, - binding_index; - context=binding.consumer_id, - default_scope=_default_dependency_scope( - model, - binding.consumer_id, - ), + call_bindings[binding_index], + binding_index, ) end _validate_model_writers_for_objects!( @@ -2942,7 +3094,15 @@ function _extend_compiled_scene( added_ids, compiled.scenario_plan.manual_application_ids, ) - _prepare_model_output_statuses!(model, added_applications) + if pure_addition + _prepare_model_output_statuses_batched!( + model, + added_applications; + performance=performance, + ) + else + _prepare_model_output_statuses!(model, added_applications) + end _runtime_performance_finish!( performance, :call_binding_refresh, @@ -2974,6 +3134,7 @@ function _extend_compiled_scene( previous_temporal_sources = copy(previous_temporal_sources_seed) processed_many_sources = IdDict{Any,Nothing}() previous_shared_many_sources = IdDict{Any,Vector{ObjectId}}() + rebuild_dynamic_input_index = !pure_addition candidate_binding_indices = Set{Int}() if !isempty(forced_input_binding_keys) for (binding_index, binding) in pairs(input_bindings) @@ -2995,7 +3156,12 @@ function _extend_compiled_scene( :selector_input_binding_candidates, length(candidate_binding_indices), ) - for index in candidate_binding_indices + queued_binding_indices = copy(candidate_binding_indices) + pending_binding_indices = sort!(collect(candidate_binding_indices)) + pending_binding_position = 1 + while pending_binding_position <= length(pending_binding_indices) + index = pending_binding_indices[pending_binding_position] + pending_binding_position += 1 binding = input_bindings[index] force_rebuild = !isempty(forced_input_binding_keys) && @@ -3004,7 +3170,9 @@ function _extend_compiled_scene( binding.consumer_id, binding.input, ) in forced_input_binding_keys - if binding.multiplicity == :many && haskey(processed_many_sources, binding.source_ids) + if !force_rebuild && + binding.multiplicity == :many && + haskey(processed_many_sources, binding.source_ids) if binding.carrier_hint == :temporal_stream key = (binding.application_id, binding.consumer_id) push!(affected_temporal_keys, key) @@ -3068,6 +3236,20 @@ function _extend_compiled_scene( processed_many_sources[binding.source_ids] = nothing continue end + rebuild_dynamic_input_index = true + # The reverse selector index keeps only one representative for a + # same-rate `Many(...)` group whose mutable source storage is shared. + # Appending updates every consumer at once. A rebuild replaces the + # carrier, however, so enqueue every member of that exact identity + # group and rewire them through the existing general path. + for shared_index in _shared_same_rate_many_binding_indices( + input_bindings, + binding, + ) + shared_index in queued_binding_indices && continue + push!(queued_binding_indices, shared_index) + push!(pending_binding_indices, shared_index) + end application = applications_by_id[binding.application_id] replacement = CompiledModelInputBinding[] _push_model_input_binding!( @@ -3143,21 +3325,26 @@ function _extend_compiled_scene( end end end - dynamic_input_binding_indices = compiled.dynamic_input_binding_indices - for binding_index in (previous_binding_count + 1):length(input_bindings) - binding = input_bindings[binding_index] - binding.origin == :inferred_same_object && continue - _index_selector_candidate!( - dynamic_input_binding_indices, + dynamic_input_binding_indices = if rebuild_dynamic_input_index + many_binding_cache = + _many_input_binding_cache(model, input_bindings) + _index_dynamic_input_bindings( model, - binding.matcher, - binding_index; - context=binding.consumer_id, - default_scope=_default_dependency_scope( - model, - binding.consumer_id, - ), + input_bindings, + many_binding_cache, ) + else + index = compiled.dynamic_input_binding_indices + for binding_index in (previous_binding_count + 1):length(input_bindings) + _index_dynamic_input_binding!( + index, + model, + input_bindings[binding_index], + binding_index, + many_binding_cache, + ) + end + index end affected_input_applications = if pure_addition || @@ -3202,17 +3389,27 @@ function _extend_compiled_scene( final_input_references = pure_addition ? _model_input_status_references(changed_bindings) : nothing - _prepare_model_input_defaults!( - model, - affected_input_applications; - final_references=final_input_references, - performance=performance, - ) - _wire_model_input_carriers!( - model, - changed_bindings; - final_references=final_input_references, - ) + if pure_addition + _prepare_model_input_statuses_batched!( + model, + affected_input_applications, + changed_bindings; + final_references=final_input_references, + performance=performance, + ) + else + _prepare_model_input_defaults!( + model, + affected_input_applications; + final_references=final_input_references, + performance=performance, + ) + _wire_model_input_carriers!( + model, + changed_bindings; + final_references=final_input_references, + ) + end _validate_model_required_inputs!( model, affected_input_applications, @@ -4476,6 +4673,163 @@ function _ref_vector_carrier(refs) return RefVector(typed_refs) end +""" +Mutable, storage-neutral recipe used to assemble one canonical object status. + +The recipe keeps existing field order and reference identity, stages additions +or replacements in place, then materializes at most one new `Status`. Keeping +the logical names separate from the current `Ref` backing also leaves a +small seam for a future columnar status-storage experiment. +""" +mutable struct _CanonicalStatusRecipe + original::Union{Nothing,Status} + names::Vector{Symbol} + references::Union{Vector{Base.RefValue},Vector{Ref}} + positions::Dict{Symbol,Int} + changed::Bool + field_changes::Int +end + +_status_recipe_references(references::Tuple{Vararg{Base.RefValue}}) = + Base.RefValue[references...] +_status_recipe_references(references) = Ref[references...] + +function _CanonicalStatusRecipe(status::Union{Nothing,Status}) + names = isnothing(status) ? Symbol[] : collect(propertynames(status)) + references = if isnothing(status) + Base.RefValue[] + else + _status_recipe_references(refvalues(status)) + end + positions = Dict{Symbol,Int}() + sizehint!(positions, length(names)) + for (index, name) in pairs(names) + positions[name] = index + end + return _CanonicalStatusRecipe( + status, + names, + references, + positions, + isnothing(status), + 0, + ) +end + +@inline _status_recipe_has_variable(recipe::_CanonicalStatusRecipe, variable::Symbol) = + haskey(recipe.positions, variable) + +function _status_recipe_reference( + recipe::_CanonicalStatusRecipe, + variable::Symbol, +) + return recipe.references[recipe.positions[variable]] +end + +function _status_recipe_set_reference!( + recipe::_CanonicalStatusRecipe, + variable::Symbol, + reference::Ref, +) + position = get(recipe.positions, variable, 0) + if iszero(position) + push!(recipe.names, variable) + push!(recipe.references, reference) + recipe.positions[variable] = length(recipe.names) + elseif recipe.references[position] === reference + return false + else + recipe.references[position] = reference + end + recipe.changed = true + recipe.field_changes += 1 + return true +end + +function _status_recipe_add_default!( + model::CompositeModel, + recipe::_CanonicalStatusRecipe, + object_id::ObjectId, + variable::Symbol, + value; + application_id=nothing, + origin=:model_default, + conversion_records=model.status_conversion_records, +) + _status_recipe_has_variable(recipe, variable) && return false + initial, _, _ = _materialize_status_value( + model, + variable, + value; + object_id=object_id, + application_id=application_id, + origin=origin, + private_copy=true, + conversion_records=conversion_records, + ) + return _status_recipe_set_reference!(recipe, variable, Ref(initial)) +end + +function _finish_status_recipe(recipe::_CanonicalStatusRecipe) + recipe.changed || return recipe.original + names = Tuple(recipe.names) + references = Tuple(recipe.references) + return Status(NamedTuple{names}(references)) +end + +function _status_recipe_for_object!( + recipes::Dict{ObjectId,_CanonicalStatusRecipe}, + recipe_order::Vector{ObjectId}, + model::CompositeModel, + object_id::ObjectId, +) + return get!(recipes, object_id) do + object = _model_object(model, object_id) + status = object.status + (isnothing(status) || status isa Status) || error( + "Model object `$(object_id.value)` uses model applications but its status has type " * + "`$(typeof(status))`. Use `Status(...)` or leave status as `nothing`." + ) + push!(recipe_order, object_id) + _CanonicalStatusRecipe(status) + end +end + +function _apply_status_recipes!( + model::CompositeModel, + recipes::Dict{ObjectId,_CanonicalStatusRecipe}, + recipe_order::Vector{ObjectId}; + performance=nothing, +) + for object_id in recipe_order + recipe = recipes[object_id] + recipe.changed || continue + _replace_model_object_status!( + model, + object_id, + _finish_status_recipe(recipe), + ) + _runtime_performance_count!( + performance, + :lifecycle_binding_refresh_status_recipe_materializations, + ) + _runtime_performance_count!( + performance, + :lifecycle_binding_refresh_status_recipe_fields_applied, + recipe.field_changes, + ) + _runtime_performance_count!( + performance, + :lifecycle_binding_refresh_status_recipe_materializations_avoided, + max( + recipe.field_changes - (isnothing(recipe.original) ? 0 : 1), + 0, + ), + ) + end + return model +end + function _status_with_reference(status::Status, variable::Symbol, reference::Base.RefValue) names = propertynames(status) if variable in names @@ -4546,6 +4900,52 @@ function _prepare_model_output_statuses!(model::CompositeModel, applications) return model end +function _prepare_model_output_statuses_batched!( + model::CompositeModel, + applications; + performance=nothing, +) + recipes = Dict{ObjectId,_CanonicalStatusRecipe}() + recipe_order = ObjectId[] + conversion_records = _no_status_conversion(model.status_conversion) ? + model.status_conversion_records : + Dict{Any,Any}() + for application in applications + defaults = outputs_(application.spec) + for object_id in application.target_ids + recipe = _status_recipe_for_object!( + recipes, + recipe_order, + model, + object_id, + ) + for (variable, value) in pairs(defaults) + _publish_mode_for_output(application.spec, variable) == + :canonical || continue + _status_recipe_add_default!( + model, + recipe, + object_id, + variable, + value; + application_id=application.id, + origin=:model_output_default, + conversion_records=conversion_records, + ) + end + end + end + result = _apply_status_recipes!( + model, + recipes, + recipe_order; + performance=performance, + ) + conversion_records === model.status_conversion_records || + merge!(model.status_conversion_records, conversion_records) + return result +end + function _validate_required_model_output_destinations!( model::CompositeModel, resolved_destinations, @@ -4609,28 +5009,24 @@ function _prepare_model_output_destination_statuses!( model::CompositeModel, resolved_destinations, ) - staged = Dict{ObjectId,Status}() - staged_order = ObjectId[] + recipes = Dict{ObjectId,_CanonicalStatusRecipe}() + recipe_order = ObjectId[] records_before = copy(model.status_conversion_records) try for resolved in resolved_destinations for destination_id in resolved.destination_ids - status = get(staged, destination_id, nothing) - if isnothing(status) - current = _model_object(model, destination_id).status - status = isnothing(current) ? Status() : current - status isa Status || error( - "Output destination `$(destination_id.value)` status must be " * - "a `Status` or `nothing`, got `$(typeof(status))`.", - ) - push!(staged_order, destination_id) - end + recipe = _status_recipe_for_object!( + recipes, + recipe_order, + model, + destination_id, + ) for (variable_, declaration) in pairs(resolved.plan.declarations) declaration isa Default || continue variable = Symbol(variable_) - status = _status_with_default( + _status_recipe_add_default!( model, - status, + recipe, destination_id, variable, _input_default(declaration); @@ -4638,7 +5034,6 @@ function _prepare_model_output_destination_statuses!( origin=:distributed_output_default, ) end - staged[destination_id] = status end end catch @@ -4646,10 +5041,7 @@ function _prepare_model_output_destination_statuses!( merge!(model.status_conversion_records, records_before) rethrow() end - for destination_id in staged_order - _replace_model_object_status!(model, destination_id, staged[destination_id]) - end - return model + return _apply_status_recipes!(model, recipes, recipe_order) end function _model_output_destination_columns( @@ -5194,6 +5586,148 @@ function _wire_model_input_carriers!( return model end +function _prepare_model_input_statuses_batched!( + model::CompositeModel, + applications, + bindings; + final_references=nothing, + performance=nothing, +) + recipes = Dict{ObjectId,_CanonicalStatusRecipe}() + recipe_order = ObjectId[] + conversion_records = _no_status_conversion(model.status_conversion) ? + model.status_conversion_records : + Dict{Any,Any}() + default_status_updates = Dict{Tuple{ObjectId,Symbol},Bool}() + input_default_rewrites_avoided = 0 + for application in applications + schema = _input_schema(application.spec) + defaults = _input_default_values(schema) + for object_id in application.target_ids + recipe = _status_recipe_for_object!( + recipes, + recipe_order, + model, + object_id, + ) + for (variable_, value) in pairs(defaults) + variable = Symbol(variable_) + _status_recipe_has_variable(recipe, variable) && continue + reference = isnothing(final_references) ? + nothing : + get( + final_references, + (application.id, object_id, variable), + nothing, + ) + if !isnothing(reference) + # Preserve conversion callbacks, validation, and diagnostic + # records even though the resolved carrier owns storage. + _no_status_conversion(model.status_conversion) || + _materialize_status_value( + model, + variable, + value; + object_id=object_id, + application_id=application.id, + origin=:model_input_default, + private_copy=true, + conversion_records=conversion_records, + ) + _status_recipe_set_reference!( + recipe, + variable, + reference, + ) + input_default_rewrites_avoided += 1 + continue + end + _status_recipe_add_default!( + model, + recipe, + object_id, + variable, + value; + application_id=application.id, + origin=:model_input_default, + conversion_records=conversion_records, + ) + default_status_updates[(object_id, variable)] = true + end + end + end + + for binding in bindings + binding.carrier_hint == :temporal_stream && continue + isnothing(binding.carrier) && continue + recipe = get(recipes, binding.consumer_id, nothing) + if isnothing(recipe) + status = _model_object(model, binding.consumer_id).status + status isa Status || continue + recipe = _status_recipe_for_object!( + recipes, + recipe_order, + model, + binding.consumer_id, + ) + end + reference = if isnothing(final_references) + _model_input_status_reference(binding) + else + get( + final_references, + ( + binding.application_id, + binding.consumer_id, + binding.input, + ), + nothing, + ) + end + isnothing(reference) && + (reference = _model_input_status_reference(binding)) + if _status_recipe_has_variable(recipe, binding.input) && + _status_recipe_reference(recipe, binding.input) === reference + default_status_updates[(binding.consumer_id, binding.input)] = false + continue + end + _status_recipe_set_reference!( + recipe, + binding.input, + reference, + ) + default_status_updates[(binding.consumer_id, binding.input)] = false + end + + result = _apply_status_recipes!( + model, + recipes, + recipe_order; + performance=performance, + ) + conversion_records === model.status_conversion_records || + merge!(model.status_conversion_records, conversion_records) + for ((object_id, variable), is_default) in default_status_updates + variables = get!( + model.input_default_status_variables, + object_id, + Set{Symbol}(), + ) + if is_default + push!(variables, variable) + else + delete!(variables, variable) + end + end + iszero(input_default_rewrites_avoided) || + _runtime_performance_count!( + performance, + :input_default_status_rewrites_avoided, + input_default_rewrites_avoided, + ) + return result +end + function _private_temporal_value(value) value isa AbstractArray && return copy(value) return value @@ -6402,6 +6936,93 @@ function _matching_callee_applications(applications, object_id::ObjectId, proc, return matches end +"""Resolve one call binding against the model and compiled application set. + +This helper is deliberately non-mutating. Cold manual `Many` bindings use it +for diagnostics and graph inspection without enrolling themselves in lifecycle +tracking. +""" +function _resolved_compiled_call_membership( + compiled::CompiledCompositeModel, + binding::CompiledModelCallBinding, +) + model = compiled.model + object_ids = _dependency_object_ids( + model, + binding.selector, + binding.matcher, + binding.consumer_id, + ) + if bindings_dirty(model) && !isempty(lifecycle_delta(model).added) + # A newborn has no compiled application/status/environment target until + # the safe refresh barrier. Keep it out of a first complete view so the + # observed binding's normal addition delta can append it exactly once. + pending_added_ids = Set(snapshot.id for snapshot in lifecycle_delta(model).added) + filter!(object_id -> object_id ∉ pending_added_ids, object_ids) + end + application_ids = Symbol[] + for object_id in object_ids + append!( + application_ids, + _matching_callee_applications( + compiled.applications_by_object, + object_id, + binding.process, + binding.application, + ), + ) + end + unique!(application_ids) + return object_ids, application_ids +end + +function _current_compiled_call_membership( + compiled::CompiledCompositeModel, + binding::CompiledModelCallBinding, +) + _compiled_call_membership_is_observed(binding) && return ( + binding.callee_object_ids, + binding.callee_application_ids, + ) + return _resolved_compiled_call_membership(compiled, binding) +end + +function _observe_compiled_call_membership!( + compiled::CompiledCompositeModel, + binding::CompiledModelCallBinding, + ; + resolve_current::Bool=true, +) + _compiled_call_membership_is_observed(binding) && return binding + if resolve_current + object_ids, application_ids = + _resolved_compiled_call_membership(compiled, binding) + if binding.callee_object_ids != object_ids || + binding.callee_application_ids != application_ids + empty!(binding.callee_object_ids) + append!(binding.callee_object_ids, object_ids) + empty!(binding.callee_application_ids) + append!(binding.callee_application_ids, application_ids) + _mark_compiled_call_membership_changed!( + binding, + compiled.model.revision, + ) + end + end + _mark_compiled_call_membership_observed!(binding) + binding_index = findfirst( + candidate -> candidate === binding, + compiled.call_bindings, + ) + isnothing(binding_index) || _index_dynamic_call_binding!( + compiled.dynamic_call_binding_indices, + compiled.model, + binding, + binding_index, + ) + return binding +end + function _compile_model_call_bindings( model::CompositeModel, applications, @@ -6781,6 +7402,9 @@ explain_bindings(model::CompositeModel) = explain_bindings(refresh_bindings!(mod function explain_calls(compiled::CompiledCompositeModel) return [ + let + callee_object_ids, callee_application_ids = + _current_compiled_call_membership(compiled, binding) ( call_plan_slot=binding.plan.slot, application_slot=binding.plan.application_slot, @@ -6789,10 +7413,10 @@ function explain_calls(compiled::CompiledCompositeModel) call=binding.call, mode=_compiled_call_mode(binding), origin=binding.origin, - callee_object_ids=[id.value for id in binding.callee_object_ids], - callee_application_ids=binding.callee_application_ids, - potential_callee_application_ids= - binding.plan.potential_callee_application_ids, + callee_object_ids=[id.value for id in callee_object_ids], + callee_application_ids=callee_application_ids, + potential_callee_application_ids= + binding.plan.potential_callee_application_ids, process=binding.process, application=binding.application, multiplicity=binding.multiplicity, @@ -6800,9 +7424,10 @@ function explain_calls(compiled::CompiledCompositeModel) :canonical_status_only : :explicit_accept, default_publish=false, accepted_publish=_compiled_call_mode(binding) === :manual, - resolved=!isempty(binding.callee_application_ids), + resolved=!isempty(callee_application_ids), selector=binding.selector, ) + end for binding in compiled.call_bindings ] end diff --git a/src/composite_model/runtime_outputs.jl b/src/composite_model/runtime_outputs.jl index 1f1f3b461..816592e61 100644 --- a/src/composite_model/runtime_outputs.jl +++ b/src/composite_model/runtime_outputs.jl @@ -434,9 +434,27 @@ abstract type AbstractExecutionBatch end mutable struct LazyCallExecutionBatches <: AbstractVector{AbstractExecutionBatch} owner::Any batches::Any + tracks_full_membership::Bool end -LazyCallExecutionBatches() = LazyCallExecutionBatches(nothing, nothing) +LazyCallExecutionBatches(; tracks_full_membership::Bool=true) = + LazyCallExecutionBatches( + nothing, + nothing, + tracks_full_membership, + ) + +struct _TrackedCallExecutionBatchCache + # Keep the already-boxed runtime values behind `Any` fields. This cache is + # itself stored behind an `Any` boundary; parameterizing its fields would + # recover their types only through an existential cache type and re-box + # them on every hot `Many` validity check or execution. + batches::Any + binding::Any + membership_generation::UInt64 + compiled::Any + environment_bindings::Any +end Base.IndexStyle(::Type{LazyCallExecutionBatches}) = IndexLinear() @@ -474,6 +492,8 @@ function CallTargets( constants, publication_allowed, environment, + ; + tracks_full_membership::Bool=true, ) execution_batches = if _compiled_call_mode(binding) === :initializer () @@ -481,7 +501,9 @@ function CallTargets( # Large `Many` bindings are commonly executed with an object filter # while organs are emitted. Defer their complete batch construction # until an unfiltered view or execution actually needs it. - LazyCallExecutionBatches() + LazyCallExecutionBatches( + ; tracks_full_membership=tracks_full_membership, + ) else # Preserve the concrete, allocation-free lookup path promised by # `call_model` for `One` and `OptionalOne` bindings. @@ -511,11 +533,195 @@ function CallTargets( return targets end +function _current_call_binding_for_retained_view( + compiled::CompiledCompositeModel, + binding::CompiledModelCallBinding, +) + bindings = get( + compiled.call_bindings_by_target, + (binding.application_id, binding.consumer_id), + (), + ) + binding_index = findfirst( + candidate -> + _compiled_call_name(candidate) === _compiled_call_name(binding) && + _compiled_call_mode(candidate) === _compiled_call_mode(binding), + bindings, + ) + isnothing(binding_index) && error( + "Retained hard-call view `$(_compiled_call_name(binding))` no longer has " * + "a compiled binding for application `$(binding.application_id)` on " * + "object `$(binding.consumer_id.value)`.", + ) + return bindings[binding_index] +end + +@inline function _environment_bindings_match_compiled( + environment_bindings, + compiled::CompiledCompositeModel, +) + return environment_bindings isa CompiledEnvironmentBindings && + environment_bindings.model_revision == compiled.revision && + environment_bindings.applications_identity == + objectid(compiled.applications) +end + +function _prebarrier_environment_bindings( + model::CompositeModel, + compiled::CompiledCompositeModel, + fallback, +) + cached = compiled_environment_bindings(model) + _environment_bindings_match_compiled(cached, compiled) && return cached + _environment_bindings_match_compiled(fallback, compiled) && return fallback + return nothing +end + +function _synchronize_tracked_call_targets!(targets::CallTargets) + binding = targets.binding + model = targets.compiled.model + topology_dirty = bindings_dirty(model) + current_compiled = if topology_dirty + cached = getfield(model, :binding_cache) + cached isa CompiledCompositeModel ? cached : targets.compiled + else + cached = compiled_bindings(model) + isnothing(cached) ? targets.compiled : cached + end + current_environment_bindings = if topology_dirty + _prebarrier_environment_bindings( + model, + current_compiled, + targets.environment_bindings, + ) + else + refresh_environment_bindings!(model, current_compiled) + end + if isnothing(current_environment_bindings) + # A dirty lifecycle may temporarily expose a newer structural cache + # without a matching pre-barrier environment cache. Keep the last + # internally coherent runtime shell instead of mixing generations. + current_compiled = targets.compiled + current_environment_bindings = targets.environment_bindings + end + if current_compiled !== targets.compiled + current_binding = _current_call_binding_for_retained_view( + current_compiled, + binding, + ) + targets.compiled = current_compiled + targets.binding = current_binding + end + targets.environment_bindings = current_environment_bindings + return targets +end + +@inline function _call_execution_batch_cache_is_current( + cached::LazyCallExecutionBatches, + targets::CallTargets, +) + state = cached.batches + isnothing(state) && return false + cached.tracks_full_membership || return true + state isa _TrackedCallExecutionBatchCache || return false + return state.binding === targets.binding && + state.membership_generation == + _compiled_call_membership_generation(targets.binding) && + state.compiled === targets.compiled && + state.environment_bindings === targets.environment_bindings +end + +@inline function _tracked_call_execution_batch_cache_can_reuse( + cached::LazyCallExecutionBatches, + targets::CallTargets, +) + _call_execution_batch_cache_is_current(cached, targets) || return false + model = targets.compiled.model + if bindings_dirty(model) + prebarrier_compiled = getfield(model, :binding_cache) + if prebarrier_compiled isa CompiledCompositeModel + prebarrier_compiled === targets.compiled || return false + end + prebarrier_environment = _prebarrier_environment_bindings( + model, + targets.compiled, + targets.environment_bindings, + ) + return prebarrier_environment === targets.environment_bindings + end + environment_bindings_dirty(model) && return false + compiled_bindings(model) === targets.compiled || return false + current_environment_bindings = compiled_environment_bindings(model) + # With both dirty flags clear, the model-owned compiled and environment + # caches are a coherent pair. Identity against both retained values is the + # sufficient hot-path check; recomputing their revision/object identity + # here introduces a small allocation before every materialized `Many` call. + return current_environment_bindings === targets.environment_bindings +end + +function _record_call_execution_batch_cache!( + cached::LazyCallExecutionBatches, + targets::CallTargets, + batches, +) + cached.batches = if cached.tracks_full_membership + _TrackedCallExecutionBatchCache( + batches, + targets.binding, + _compiled_call_membership_generation(targets.binding), + targets.compiled, + targets.environment_bindings, + ) + else + batches + end + return batches +end + +function _record_extended_call_execution_batch_cache!( + targets::CallTargets, + compiled::CompiledCompositeModel, + environment_bindings::CompiledEnvironmentBindings, + binding::CompiledModelCallBinding, +) + cached = targets.execution_batches + cached isa LazyCallExecutionBatches || return targets + cached.tracks_full_membership || return targets + batches = _cached_call_execution_batches(targets) + isnothing(batches) && return targets + cached.batches = _TrackedCallExecutionBatchCache( + batches, + binding, + _compiled_call_membership_generation(binding), + compiled, + environment_bindings, + ) + return targets +end + function _materialize_call_execution_batches!(targets::CallTargets) cached = targets.execution_batches cached isa LazyCallExecutionBatches || return cached - isnothing(cached.batches) || return cached.batches - cached.batches = _compiled_call_execution_batches( + if cached.tracks_full_membership + _tracked_call_execution_batch_cache_can_reuse(cached, targets) && + return _cached_call_execution_batches(targets) + _synchronize_tracked_call_targets!(targets) + _call_execution_batch_cache_is_current(cached, targets) && + return _cached_call_execution_batches(targets) + elseif !isnothing(cached.batches) + return cached.batches + end + # A complete manual `Many` view becomes lifecycle-tracked only here. Merely + # retrieving the cached CallTargets wrapper, diagnostics, and targeted + # `objects=` calls leave the full membership cold. + if cached.tracks_full_membership + _observe_compiled_call_membership!( + targets.compiled, + targets.binding; + resolve_current=!bindings_dirty(targets.compiled.model), + ) + end + batches = _compiled_call_execution_batches( targets.compiled, targets.environment_bindings, targets.binding, @@ -523,13 +729,15 @@ function _materialize_call_execution_batches!(targets::CallTargets) targets.output_retention, targets.constants, ) - return cached.batches + return _record_call_execution_batch_cache!(cached, targets, batches) end function _cached_call_execution_batches(targets::CallTargets) cached = targets.execution_batches cached isa LazyCallExecutionBatches || return cached - return cached.batches + state = cached.batches + state isa _TrackedCallExecutionBatchCache && return state.batches + return state end _call_execution_batches_materialized(targets::CallTargets) = @@ -664,8 +872,13 @@ function _call_bindings_signature(call_bindings) signature = hash(length(call_bindings)) for binding in call_bindings signature = hash(_compiled_call_name(binding), signature) - signature = hash(Tuple(binding.callee_application_ids), signature) - signature = hash(Tuple(binding.callee_object_ids), signature) + # Lifecycle code advances this generation whenever either callee + # vector changes, so signature refresh remains independent of the + # number of organs selected by the call. + signature = hash( + _compiled_call_membership_generation(binding), + signature, + ) end return signature end @@ -3557,6 +3770,7 @@ function _manual_call_binding_has_changed_target( compiled::CompiledCompositeModel, ) _compiled_call_mode(binding) === :manual || return false + _compiled_call_membership_is_observed(binding) || return false for (application_id, object_id) in compiled.changed_execution_target_ids application_id in binding.callee_application_ids || continue first( @@ -3860,6 +4074,12 @@ function _extend_execution_target_call_batches_by_prefix!( end for (call_targets, binding) in staged_bindings call_targets.binding = binding + _record_extended_call_execution_batch_cache!( + call_targets, + compiled, + env_bindings, + binding, + ) end target.call_bindings = current_call_bindings target.call_bindings_signature = @@ -3990,6 +4210,12 @@ function _extend_execution_target_call_batches_for_pure_addition!( end for (call_targets, binding) in staged_bindings call_targets.binding = binding + _record_extended_call_execution_batch_cache!( + call_targets, + compiled, + env_bindings, + binding, + ) end target.call_bindings = current_call_bindings target.call_bindings_signature = @@ -4045,16 +4271,10 @@ function _refresh_model_execution_group_delta!( output_retention, constants, performance, - changed_execution_target_ids, + changed_object_ids, ) previous_group.application === application || return nothing - changed_object_ids = ObjectId[ - object_id - for (application_id, object_id) in changed_execution_target_ids - if application_id == application.id - ] isempty(changed_object_ids) && return nothing - _sort_object_ids!(changed_object_ids) actions = NamedTuple[] targets_constructed = 0 for object_id in changed_object_ids @@ -4193,6 +4413,194 @@ function _refresh_model_execution_group_delta!( ) end +function _changed_execution_target_ids_by_application( + changed_execution_target_ids, +) + changed_by_application = Dict{Symbol,Vector{ObjectId}}() + for (application_id, object_id) in changed_execution_target_ids + object_ids = get!(changed_by_application, application_id) do + ObjectId[] + end + push!(object_ids, object_id) + end + for object_ids in values(changed_by_application) + _sort_object_ids!(object_ids) + end + return changed_by_application +end + +function _try_refresh_model_execution_plan_for_pure_addition!( + previous::CompiledExecutionPlan, + compiled::CompiledCompositeModel, + env_bindings::CompiledEnvironmentBindings, + temporal_streams, + output_retention, + constants, + performance, + changed_application_ids, + changed_targets_by_application, + manual_application_ids, + is_pure_addition::Bool, +) + is_pure_addition || return nothing + isnothing(changed_application_ids) && return nothing + + staged_targets = Dict{Tuple{Symbol,ObjectId},Any}() + applications = CompiledModelApplication[] + for application_id in changed_application_ids + application_id in manual_application_ids && continue + application = get(compiled.applications_by_id, application_id, nothing) + isnothing(application) && return ( + plan=nothing, + staged_targets=staged_targets, + ) + push!(applications, application) + end + sort!(applications; by=application -> application.slot) + + # Validate every affected group before constructing a target. Target + # construction is then staged globally, and no batch is mutated unless all + # applications can extend their existing final batch. + candidates = NamedTuple[] + for application in applications + previous_group = previous.groups_by_application_slot[application.slot] + isnothing(previous_group) && return ( + plan=nothing, + staged_targets=staged_targets, + ) + previous_group.application === application || return ( + plan=nothing, + staged_targets=staged_targets, + ) + changed_object_ids = get( + changed_targets_by_application, + application.id, + (), + ) + isempty(changed_object_ids) && return ( + plan=nothing, + staged_targets=staged_targets, + ) + isempty(previous_group.batches) && return ( + plan=nothing, + staged_targets=staged_targets, + ) + last_batch = last(previous_group.batches) + isempty(last_batch.targets) && return ( + plan=nothing, + staged_targets=staged_targets, + ) + last_existing_id = last(last_batch.targets).object_id + _object_id_isless(last_existing_id, first(changed_object_ids)) || + return ( + plan=nothing, + staged_targets=staged_targets, + ) + length(application.target_ids) >= length(changed_object_ids) || return ( + plan=nothing, + staged_targets=staged_targets, + ) + suffix_start = length(application.target_ids) - length(changed_object_ids) + 1 + all( + index -> + application.target_ids[suffix_start + index - 1] == + changed_object_ids[index], + eachindex(changed_object_ids), + ) || return ( + plan=nothing, + staged_targets=staged_targets, + ) + previous_target_count = sum( + batch -> length(batch.targets), + previous_group.batches; + init=0, + ) + ( + previous_target_count + length(changed_object_ids) == + length(application.target_ids) + ) || return ( + plan=nothing, + staged_targets=staged_targets, + ) + for object_id in changed_object_ids + isnothing( + _model_execution_target_location(previous_group, object_id), + ) || return ( + plan=nothing, + staged_targets=staged_targets, + ) + end + push!( + candidates, + ( + application=application, + batch=last_batch, + object_ids=changed_object_ids, + ), + ) + end + + staged_appends = NamedTuple[] + for candidate in candidates + targets = Any[] + for object_id in candidate.object_ids + target = _compiled_model_execution_target( + compiled, + env_bindings, + candidate.application, + object_id, + temporal_streams, + output_retention, + constants, + ) + staged_targets[(candidate.application.id, object_id)] = target + _model_execution_batch_accepts_target( + candidate.batch, + target, + env_bindings, + candidate.application, + ) || return ( + plan=nothing, + staged_targets=staged_targets, + ) + push!(targets, target) + end + push!( + staged_appends, + ( + application=candidate.application, + batch=candidate.batch, + targets=targets, + ), + ) + end + + for staged in staged_appends + for target in staged.targets + push!(staged.batch.targets, target) + _count_model_execution_target_rebuild!(performance, :new_target) + end + isempty(staged.targets) || _runtime_performance_count!( + performance, + :execution_groups_updated_in_place, + ) + end + + return ( + plan=CompiledExecutionPlan( + previous.groups, + previous.batches, + previous.groups_by_application_slot, + previous.schedule, + compiled.revision, + env_bindings.environment_revision, + ), + targets_constructed=length(staged_targets), + batches_constructed=0, + groups_reused=length(previous.groups) - length(staged_appends), + ) +end + function _refresh_model_execution_plan( previous::CompiledExecutionPlan, compiled::CompiledCompositeModel, @@ -4205,6 +4613,44 @@ function _refresh_model_execution_plan( changed_execution_target_ids=compiled.changed_execution_target_ids, ) manual_application_ids = _manual_call_application_ids(compiled) + changed_targets_by_application = isnothing(changed_application_ids) ? + nothing : + _changed_execution_target_ids_by_application( + changed_execution_target_ids, + ) + staged_execution_targets = nothing + force_general_changed_groups = false + if !isnothing(changed_targets_by_application) + pure_addition_refresh = + _try_refresh_model_execution_plan_for_pure_addition!( + previous, + compiled, + env_bindings, + temporal_streams, + output_retention, + constants, + performance, + changed_application_ids, + changed_targets_by_application, + manual_application_ids, + compiled.status_view_refresh_is_pure_addition && + changed_execution_target_ids === + compiled.changed_execution_target_ids, + ) + if !isnothing(pure_addition_refresh) + isnothing(pure_addition_refresh.plan) || + return pure_addition_refresh + staged_execution_targets = + pure_addition_refresh.staged_targets + # The transactional attempt did not touch the old plan. Rebuild + # changed groups directly only when target construction actually + # started, reusing every staged target instead of retrying the + # in-place group delta. A preflight rejection leaves this empty and + # must retain the normal incremental-delta fallback. + force_general_changed_groups = + !isempty(staged_execution_targets) + end + end previous_groups = Dict( group.application.id => group for group in previous.groups ) @@ -4227,27 +4673,33 @@ function _refresh_model_execution_plan( groups_reused += 1 continue end - delta_group = _refresh_model_execution_group_delta!( - previous_group, - compiled, - env_bindings, - application, - temporal_streams, - output_retention, - constants, - performance, - changed_execution_target_ids, - ) - if !isnothing(delta_group) - if !isnothing(delta_group.group) - push!(groups, delta_group.group) - append!(batches, delta_group.group.batches) + if !force_general_changed_groups + delta_group = _refresh_model_execution_group_delta!( + previous_group, + compiled, + env_bindings, + application, + temporal_streams, + output_retention, + constants, + performance, + get( + changed_targets_by_application, + application.id, + (), + ), + ) + if !isnothing(delta_group) + if !isnothing(delta_group.group) + push!(groups, delta_group.group) + append!(batches, delta_group.group.batches) + end + targets_constructed += + delta_group.targets_constructed + batches_constructed += + delta_group.batches_constructed + continue end - targets_constructed += - delta_group.targets_constructed - batches_constructed += - delta_group.batches_constructed - continue end elseif _model_execution_group_reusable( previous_group, @@ -4309,9 +4761,15 @@ function _refresh_model_execution_plan( if isnothing(change_reason) push!(targets, previous_target) else - push!( - targets, - _compiled_model_execution_target( + execution_target = isnothing(staged_execution_targets) ? + nothing : + get( + staged_execution_targets, + target_key, + nothing, + ) + if isnothing(execution_target) + execution_target = _compiled_model_execution_target( compiled, env_bindings, application, @@ -4319,7 +4777,11 @@ function _refresh_model_execution_plan( temporal_streams, output_retention, constants, - ), + ) + end + push!( + targets, + execution_target, ) _count_model_execution_target_rebuild!( performance, @@ -5298,6 +5760,17 @@ function _targeted_new_object_call_targets( "Hard call `$(name)` from application `$(context.application.id)` does not ", "resolve requested object(s) `$(Tuple(id.value for id in unresolved_ids))`." ) + if !initializer && binding.multiplicity != :many + # A targeted newborn does not relax the authored singular selector. + # Resolve its complete live scope before compiling or preparing any + # newborn status so an existing match plus the newborn fails atomically. + _dependency_object_ids( + model, + binding.selector, + binding.matcher, + binding.consumer_id, + ) + end compiled = context.compiled targeted_runtime = _targeted_topology_runtime!(context, model) @@ -5327,9 +5800,20 @@ function _targeted_new_object_call_targets( ) return nothing end + if !initializer && binding.multiplicity != :many + pair_count = sum( + count(object_id -> object_id in requested_ids, application.target_ids) + for application in callee_applications; + init=0, + ) + pair_count == 1 || error( + "Hard call `$(name)` from application `$(context.application.id)` " * + "expected exactly one callee application/object pair, got $(pair_count).", + ) + end if !application_set.outputs_prepared - _prepare_model_output_statuses!(model, output_applications) + _prepare_model_output_statuses_batched!(model, output_applications) application_set.outputs_prepared = true end input_bindings = CompiledModelInputBinding[] @@ -5351,8 +5835,13 @@ function _targeted_new_object_call_targets( ) end end - _prepare_model_input_defaults!(model, callee_applications) - _wire_model_input_carriers!(model, input_bindings) + final_input_references = _model_input_status_references(input_bindings) + _prepare_model_input_statuses_batched!( + model, + callee_applications, + input_bindings; + final_references=final_input_references, + ) _validate_model_required_inputs!( model, callee_applications, @@ -5490,17 +5979,55 @@ function _current_topology_call_targets( ) end resolved_binding = bindings[binding] - unresolved_ids = setdiff(requested_ids, resolved_binding.callee_object_ids) + _compiled_call_mode(resolved_binding) === :manual || + _initializer_requires_dedicated_api(context, name) + resolved_binding.multiplicity != :many && length(requested_ids) > 1 && + error( + "Hard call `$(name)` from application `$(context.application.id)` ", + "accepts at most one requested object.", + ) + default_scope = _default_dependency_scope(model, context.object_id) + unresolved_ids = ObjectId[] + callee_application_ids = Symbol[] + for object_id in requested_ids + if !_selector_matches_object_id( + model, + resolved_binding.matcher, + object_id; + context=context.object_id, + default_to_context=true, + default_scope=default_scope, + ) + push!(unresolved_ids, object_id) + continue + end + matching_applications = _matching_callee_applications( + compiled.applications_by_object, + object_id, + resolved_binding.process, + resolved_binding.application, + ) + if isempty(matching_applications) + push!(unresolved_ids, object_id) + continue + end + append!(callee_application_ids, matching_applications) + end isempty(unresolved_ids) || error( "Hard call `$(name)` from application `$(context.application.id)` does not ", "resolve requested object(s) `$(Tuple(id.value for id in unresolved_ids))`." ) + unique!(callee_application_ids) selected_binding = CompiledModelCallBinding( resolved_binding.plan, resolved_binding.consumer_id, requested_ids, - resolved_binding.callee_application_ids, + callee_application_ids, ) + # This synthetic binding is already the complete explicitly requested + # selection. Mark only it observed so materialization never activates the + # cached full `Many` binding or its lifecycle reverse index. + _mark_compiled_call_membership_observed!(selected_binding) return CallTargets( compiled, environment_bindings, @@ -5511,6 +6038,8 @@ function _current_topology_call_targets( context.constants, context.publication_allowed, context.environment, + ; + tracks_full_membership=false, ) end diff --git a/src/composite_model/selectors.jl b/src/composite_model/selectors.jl index e2b53f197..f7b51ec29 100644 --- a/src/composite_model/selectors.jl +++ b/src/composite_model/selectors.jl @@ -505,6 +505,38 @@ function _descendant_ids(model::CompositeModel, root_id::ObjectId) return _append_descendant_ids!(ObjectId[], model, root_id) end +# Object-relative selectors created with a newborn organ commonly cover only a +# handful of objects. Probe at most 32 objects before falling back to the +# registry-index heuristic so that those tiny scopes avoid copying a scene-wide +# label index without making large plant subtrees pay for a full extra walk. +const _SCOPE_FIRST_DESCENDANT_LIMIT = 32 + +function _append_descendant_ids_up_to!( + ids::Vector{ObjectId}, + model::CompositeModel, + root_id::ObjectId, + limit::Int, +) + length(ids) >= limit && return false + push!(ids, root_id) + object = _model_object(model, root_id) + for child_id in object.children + _append_descendant_ids_up_to!(ids, model, child_id, limit) || + return false + end + return true +end + +function _descendant_ids_up_to( + model::CompositeModel, + root_id::ObjectId, + limit::Int, +) + ids = ObjectId[] + complete = _append_descendant_ids_up_to!(ids, model, root_id, limit) + return complete ? ids : nothing +end + function _ancestor_id( model::CompositeModel, current_id::ObjectId; @@ -1188,14 +1220,43 @@ function _resolve_object_ids( return ObjectId[_object_id_from_context(context)] end - indexed_ids = _indexed_object_ids( - model; - scale=scale, - kind=kind, - species=species, - name=name, - ) - candidate_ids = if isnothing(relation) + has_indexed_criteria = !isnothing(scale) || + !isnothing(kind) || + !isnothing(species) || + !isnothing(name) + scope_candidate_ids = if isnothing(relation) && has_indexed_criteria + if scope isa Self + _scope_object_ids(model, scope, context) + elseif scope isa Subtree + current_id = _object_id_from_context(context) + isnothing(current_id) && error( + "`Subtree()` selectors require a current object context.", + ) + _descendant_ids_up_to( + model, + current_id, + _SCOPE_FIRST_DESCENDANT_LIMIT, + ) + else + nothing + end + else + nothing + end + indexed_ids = if isnothing(scope_candidate_ids) + _indexed_object_ids( + model; + scale=scale, + kind=kind, + species=species, + name=name, + ) + else + nothing + end + candidate_ids = if !isnothing(scope_candidate_ids) + scope_candidate_ids + elseif isnothing(relation) if scope isa Ancestor && !isnothing(scale) && scale == scope.scale current_id = _object_id_from_context(context) isnothing(current_id) && error( @@ -1232,11 +1293,22 @@ function _resolve_object_ids( end ids = ObjectId[ id for id in candidate_ids - if _matches_object_criteria(_model_object(model, id); scale=scale, kind=kind, species=species, name=name) + if _matches_object_criteria( + _model_object(model, id); + scale=scale, + kind=kind, + species=species, + name=name, + ) ] _sort_object_ids!(ids) - diagnostic_candidate_ids = if isempty(ids) && !isnothing(indexed_ids) + diagnostic_candidate_ids = if !isnothing(scope_candidate_ids) + # Indexed candidates already satisfy the requested labels. Preserve + # that diagnostic contract for ambiguous singular matches, while an + # empty match still reports every label available inside the scope. + isempty(ids) ? scope_candidate_ids : ids + elseif isempty(ids) && !isnothing(indexed_ids) if isnothing(relation) _scope_object_ids(model, scope, context) else diff --git a/src/composite_model/status_conversion.jl b/src/composite_model/status_conversion.jl index 2a2bd4e0c..2d30bae39 100644 --- a/src/composite_model/status_conversion.jl +++ b/src/composite_model/status_conversion.jl @@ -265,6 +265,7 @@ function _materialize_status_value( private_copy::Bool=false, reuse::Bool=false, declared_type=typeof(value), + conversion_records=model.status_conversion_records, ) if _no_status_conversion(model.status_conversion) initial = private_copy ? _private_initial_value(value) : value @@ -278,7 +279,7 @@ function _materialize_status_value( ) if reuse cached = _cached_status_materialization( - get(model.status_conversion_records, key, nothing), + get(conversion_records, key, nothing), value, private_copy, ) @@ -295,7 +296,7 @@ function _materialize_status_value( application_id=application_id, origin=origin, ) - model.status_conversion_records[key] = StatusConversionRecord( + conversion_records[key] = StatusConversionRecord( variable, object_id, application_id, diff --git a/src/visualization/model_graph_view.jl b/src/visualization/model_graph_view.jl index 531bd7060..6dfa85c44 100644 --- a/src/visualization/model_graph_view.jl +++ b/src/visualization/model_graph_view.jl @@ -214,6 +214,8 @@ function _model_graph_compiled( scenario_plan.application_order, distributed_outputs, ) + many_input_binding_cache = + _many_input_binding_cache(model, input_bindings) return CompiledCompositeModel( model, scenario_plan, @@ -225,9 +227,13 @@ function _model_graph_compiled( call_bindings, input_by_target, call_by_target, - _index_dynamic_input_bindings(model, input_bindings), + _index_dynamic_input_bindings( + model, + input_bindings, + many_input_binding_cache, + ), _index_dynamic_call_bindings(model, call_bindings), - _many_input_binding_cache(model, input_bindings), + many_input_binding_cache, distributed_outputs, scenario_plan.call_owners, scenario_plan.application_children, @@ -1177,38 +1183,79 @@ function _model_graph_binding_edges(report, level) return collect(values(edges)) end +function _model_graph_resolved_call_pairs( + report, + binding, + callee_object_ids, + callee_application_ids, +) + applications_by_id = isnothing(report.compiled) ? + _applications_by_id(report.applications) : + report.compiled.applications_by_id + pairs = Tuple{Symbol,ObjectId}[] + for callee_application_id in callee_application_ids + application = get( + applications_by_id, + callee_application_id, + nothing, + ) + isnothing(application) && continue + for callee_object_id in callee_object_ids + _call_binding_target_matches( + binding, + application, + callee_object_id, + ) || continue + push!(pairs, (callee_application_id, callee_object_id)) + end + end + return pairs +end + function _model_graph_call_edges(report, level) edges = Dict{String,Dict{String,Any}}() for binding in report.call_bindings + callee_object_ids, resolved_application_ids = if level == :resolved && + !isnothing(report.compiled) + _current_compiled_call_membership(report.compiled, binding) + else + (binding.callee_object_ids, binding.callee_application_ids) + end callee_application_ids = level == :resolved ? - binding.callee_application_ids : + resolved_application_ids : binding.potential_callee_application_ids call_kind = _compiled_call_mode(binding) === :initializer ? "initializer" : "manual_call" - for callee_application_id in callee_application_ids - if level == :resolved - for callee_object_id in binding.callee_object_ids - edge_id = string( - "call:", binding.application_id, ":", binding.consumer_id.value, - ":", binding.call, ":", callee_application_id, ":", callee_object_id.value, - ) - edges[edge_id] = Dict{String,Any}( - "id" => edge_id, - "source" => _model_graph_execution_node_id(binding.application_id, binding.consumer_id), - "target" => _model_graph_execution_node_id(callee_application_id, callee_object_id), - "sourcePort" => nothing, - "targetPort" => nothing, - "kind" => call_kind, - "mode" => string(_compiled_call_mode(binding)), - "projection" => "resolved", - "call" => string(binding.call), - "origin" => string(binding.origin), - "multiplicity" => string(binding.multiplicity), - "selector" => _model_graph_selector_dict(binding.selector), - "cycle" => false, - ) - end - else + if level == :resolved + resolved_pairs = _model_graph_resolved_call_pairs( + report, + binding, + callee_object_ids, + callee_application_ids, + ) + for (callee_application_id, callee_object_id) in resolved_pairs + edge_id = string( + "call:", binding.application_id, ":", binding.consumer_id.value, + ":", binding.call, ":", callee_application_id, ":", callee_object_id.value, + ) + edges[edge_id] = Dict{String,Any}( + "id" => edge_id, + "source" => _model_graph_execution_node_id(binding.application_id, binding.consumer_id), + "target" => _model_graph_execution_node_id(callee_application_id, callee_object_id), + "sourcePort" => nothing, + "targetPort" => nothing, + "kind" => call_kind, + "mode" => string(_compiled_call_mode(binding)), + "projection" => "resolved", + "call" => string(binding.call), + "origin" => string(binding.origin), + "multiplicity" => string(binding.multiplicity), + "selector" => _model_graph_selector_dict(binding.selector), + "cycle" => false, + ) + end + else + for callee_application_id in callee_application_ids edge_id = string("call:", binding.application_id, ":", binding.call, ":", callee_application_id) edges[edge_id] = Dict{String,Any}( "id" => edge_id, diff --git a/test/test-model-api-stabilization.jl b/test/test-model-api-stabilization.jl index 7e7f7137e..1085bf691 100644 --- a/test/test-model-api-stabilization.jl +++ b/test/test-model-api-stabilization.jl @@ -56,6 +56,23 @@ function PlantSimEngine.run!( return nothing end +struct StabilizationAlternativeContextModel <: AbstractStabilization_ContextModel end + +PlantSimEngine.inputs_(::StabilizationAlternativeContextModel) = NamedTuple() +PlantSimEngine.outputs_(::StabilizationAlternativeContextModel) = + (seen_revision=0,) + +function PlantSimEngine.run!( + ::StabilizationAlternativeContextModel, + status, + environment, + constants, + context, +) + status.seen_revision = -Advanced.model_revision(runtime_model(context)) + return nothing +end + PlantSimEngine.@process "stabilization_environment" verbose = false struct StabilizationEnvironmentModel <: AbstractStabilization_EnvironmentModel end @@ -112,6 +129,24 @@ function PlantSimEngine.run!( return nothing end +struct StabilizationMixedManySumModel <: AbstractStabilization_Lagged_SumModel end + +PlantSimEngine.inputs_(::StabilizationMixedManySumModel) = + (previous_signals=Default(Any[]),) +PlantSimEngine.outputs_(::StabilizationMixedManySumModel) = + (lagged_total=0.0,) + +function PlantSimEngine.run!( + ::StabilizationMixedManySumModel, + status, + environment, + constants, + context, +) + status.lagged_total = sum(status.previous_signals; init=0.0) + return nothing +end + @testset "one-object lowering and initialization report" begin model = CompositeModel( StabilizationSourceModel(), @@ -889,6 +924,106 @@ end ObjectId[] @test resolve_object_ids(model, Many(scale=:Leaf, within=Subtree()); context=:plant) == ObjectId[ObjectId(:leaf_1), ObjectId(:leaf_2)] + + default_scope_selector = One(scale=:Leaf) + @test PlantSimEngine._resolve_object_ids( + model, + default_scope_selector, + PlantSimEngine._compile_selector_matcher( + model, + default_scope_selector, + ); + context=ObjectId(:leaf_1), + default_to_context=true, + default_scope=Self(), + ) == ObjectId[ObjectId(:leaf_1)] + + ambiguous_scoped_error = try + resolve_object_ids( + model, + One(scale=:Leaf, within=Subtree()); + context=:plant, + ) + nothing + catch error + sprint(showerror, error) + end + @test contains( + ambiguous_scoped_error, + "available=(scales = [:Leaf],", + ) + + missing_scoped_error = try + resolve_object_ids( + model, + One(scale=:Leef, within=Subtree()); + context=:plant, + ) + nothing + catch error + sprint(showerror, error) + end + @test contains( + missing_scoped_error, + "available=(scales = [:Leaf, :Plant],", + ) + @test contains(missing_scoped_error, "suggestions=(scale = [:Leaf]") + + long_scope_objects = Object[Object(:long_root; scale=:Plant)] + long_scope_leaf_ids = Set{ObjectId}() + parent_id = :long_root + for index in 1:40 + object_id = Symbol(:long_scope_, index) + object_scale = isodd(index) ? :Leaf : :Axis + push!( + long_scope_objects, + Object(object_id; scale=object_scale, parent=parent_id), + ) + object_scale == :Leaf && push!(long_scope_leaf_ids, ObjectId(object_id)) + parent_id = object_id + end + long_scope_model = CompositeModel(long_scope_objects...) + @test length( + PlantSimEngine._descendant_ids_up_to( + long_scope_model, + ObjectId(:long_scope_10), + PlantSimEngine._SCOPE_FIRST_DESCENDANT_LIMIT, + ), + ) == 31 + @test length( + PlantSimEngine._descendant_ids_up_to( + long_scope_model, + ObjectId(:long_scope_9), + PlantSimEngine._SCOPE_FIRST_DESCENDANT_LIMIT, + ), + ) == PlantSimEngine._SCOPE_FIRST_DESCENDANT_LIMIT + @test isnothing( + PlantSimEngine._descendant_ids_up_to( + long_scope_model, + ObjectId(:long_scope_8), + PlantSimEngine._SCOPE_FIRST_DESCENDANT_LIMIT, + ), + ) + @test isnothing( + PlantSimEngine._descendant_ids_up_to( + long_scope_model, + ObjectId(:long_root), + PlantSimEngine._SCOPE_FIRST_DESCENDANT_LIMIT, + ), + ) + @test Set( + resolve_object_ids( + long_scope_model, + Many(scale=:Leaf, within=Subtree()); + context=:long_root, + ), + ) == long_scope_leaf_ids + @test resolve_object_ids( + long_scope_model, + Many(scale=:Leaf, within=Subtree()); + context=:long_scope_39, + ) == ObjectId[ObjectId(:long_scope_39)] + @test_throws "No matching ancestor" resolve_object_ids( model, Many(within=Ancestor(scale=:Plant)); @@ -1902,6 +2037,308 @@ end @test performance.counts[:input_default_status_rewrites_avoided] == 1 end +function stabilization_shared_many_consumer_scene( + axis_count; + initial_leaf_id=:leaf_a, +) + objects = Object[ + Object(:scene; scale=:Scene), + Object(:plant; scale=:Plant, parent=:scene), + Object(initial_leaf_id; scale=:Leaf, parent=:plant), + ] + append!( + objects, + ( + Object(Symbol(:axis_, index); scale=:Axis, parent=:plant) + for index in 1:axis_count + ), + ) + return CompositeModel( + objects...; + applications=( + ModelSpec( + StabilizationSourceModel(); + name=:shared_many_source, + on=Many(scale=:Leaf), + ), + ModelSpec( + StabilizationSafeLaggedSumModel(); + name=:shared_many_consumer, + on=Many(scale=:Axis), + inputs=( + :previous_signals => Many( + scale=:Leaf, + within=SelfPlant(), + application=:shared_many_source, + var=:signal, + ), + ), + ), + ), + environment=(duration=Hour(1),), + ) +end + +function stabilization_shared_many_bindings(simulation, axis_count) + return PlantSimEngine.CompiledModelInputBinding[ + only( + simulation.compiled.input_bindings_by_target[ + (:shared_many_consumer, ObjectId(Symbol(:axis_, index))) + ], + ) for index in 1:axis_count + ] +end + +@testset "shared same-rate Many bindings use one lifecycle candidate" begin + axis_count = 32 + model = stabilization_shared_many_consumer_scene(axis_count) + simulation = run!(model; outputs=:none, performance=true) + original_bindings = + stabilization_shared_many_bindings(simulation, axis_count) + original = first(original_bindings) + @test all( + binding -> + binding.source_ids === original.source_ids && + binding.source_application_ids === + original.source_application_ids && + binding.carrier === original.carrier, + original_bindings, + ) + + register_object!( + model, + Object(:leaf_z; scale=:Leaf, parent=:plant), + ) + continue!(simulation) + + refreshed_bindings = + stabilization_shared_many_bindings(simulation, axis_count) + refreshed = first(refreshed_bindings) + @test Advanced.runtime_performance(simulation).counts[ + :selector_input_binding_candidates + ] == 1 + @test refreshed === original + @test refreshed.source_ids == ObjectId.([:leaf_a, :leaf_z]) + @test refreshed.carrier === original.carrier + @test all( + binding -> + binding.source_ids === refreshed.source_ids && + binding.source_application_ids === + refreshed.source_application_ids && + binding.carrier === refreshed.carrier, + refreshed_bindings, + ) + @test all( + model_object(model, Symbol(:axis_, index)).status.lagged_total == 3.0 + for index in 1:axis_count + ) +end + +@testset "shared Many fallback rewires every consumer and rebuilds its index" begin + axis_count = 2 + model = stabilization_shared_many_consumer_scene( + axis_count; + initial_leaf_id=:leaf_z, + ) + simulation = run!(model; outputs=:none, performance=true) + initial_bindings = + stabilization_shared_many_bindings(simulation, axis_count) + initial_carrier = first(initial_bindings).carrier + + # This ID sorts before the existing source, forcing the general rebuild + # path instead of the monotonic in-place append. + register_object!( + model, + Object(:leaf_a; scale=:Leaf, parent=:plant), + ) + continue!(simulation) + + rebuilt_bindings = + stabilization_shared_many_bindings(simulation, axis_count) + rebuilt = first(rebuilt_bindings) + @test Advanced.runtime_performance(simulation).counts[ + :selector_input_binding_candidates + ] == 1 + @test rebuilt.carrier !== initial_carrier + @test rebuilt.source_ids == ObjectId.([:leaf_a, :leaf_z]) + @test all(binding -> binding.carrier === rebuilt.carrier, rebuilt_bindings) + @test all( + model_object(model, Symbol(:axis_, index)).status.previous_signals === + rebuilt.carrier for index in 1:axis_count + ) + + # Forced structural keys remain exact: both consumer bindings are selected + # for source removal even though their dynamic addition index is compact. + remove_object!(model, :leaf_a) + continue!(simulation) + register_object!( + model, + Object(:leaf_zz; scale=:Leaf, parent=:plant), + ) + continue!(simulation) + + final_bindings = + stabilization_shared_many_bindings(simulation, axis_count) + final = first(final_bindings) + @test Advanced.runtime_performance(simulation).counts[ + :selector_input_binding_candidates + ] == 4 + @test final.source_ids == ObjectId.([:leaf_z, :leaf_zz]) + @test all(binding -> binding.carrier === final.carrier, final_bindings) +end + +@testset "shared ObjectRefVector carriers keep one lifecycle candidate" begin + axis_count = 4 + objects = Object[ + Object(:scene; scale=:Scene), + Object(:plant; scale=:Plant, parent=:scene), + Object( + :leaf_a; + scale=:Leaf, + parent=:plant, + status=Status(signal=1.0), + ), + Object( + :leaf_b; + scale=:Leaf, + parent=:plant, + status=Status(signal=2), + ), + ] + append!( + objects, + ( + Object(Symbol(:axis_, index); scale=:Axis, parent=:plant) + for index in 1:axis_count + ), + ) + model = CompositeModel( + objects...; + applications=( + ModelSpec( + StabilizationMixedManySumModel(); + name=:mixed_many_consumer, + on=Many(scale=:Axis), + inputs=( + :previous_signals => Many( + scale=:Leaf, + within=SelfPlant(), + var=:signal, + from_status=true, + ), + ), + ), + ), + environment=(duration=Hour(1),), + ) + simulation = run!(model; outputs=:none, performance=true) + bindings = PlantSimEngine.CompiledModelInputBinding[ + only( + simulation.compiled.input_bindings_by_target[ + (:mixed_many_consumer, ObjectId(Symbol(:axis_, index))) + ], + ) for index in 1:axis_count + ] + carrier = first(bindings).carrier + @test carrier isa PlantSimEngine.ObjectRefVector + @test all(binding -> binding.carrier === carrier, bindings) + + register_object!( + model, + Object( + :leaf_z; + scale=:Leaf, + parent=:plant, + status=Status(signal=3 // 1), + ), + ) + continue!(simulation) + + refreshed = PlantSimEngine.CompiledModelInputBinding[ + only( + simulation.compiled.input_bindings_by_target[ + (:mixed_many_consumer, ObjectId(Symbol(:axis_, index))) + ], + ) for index in 1:axis_count + ] + @test Advanced.runtime_performance(simulation).counts[ + :selector_input_binding_candidates + ] == 1 + @test all(binding -> binding.carrier === carrier, refreshed) + @test collect(carrier) == [1.0, 2, 3 // 1] +end + +@testset "temporal Many bindings keep per-consumer lifecycle candidates" begin + axis_count = 2 + model = CompositeModel( + Object(:scene; scale=:Scene), + Object(:plant; scale=:Plant, parent=:scene), + Object(:leaf_a; scale=:Leaf, parent=:plant), + Object(:axis_1; scale=:Axis, parent=:plant), + Object(:axis_2; scale=:Axis, parent=:plant); + applications=( + ModelSpec( + StabilizationSourceModel(); + name=:temporal_many_source, + on=Many(scale=:Leaf), + ), + ModelSpec( + StabilizationSafeLaggedSumModel(); + name=:temporal_many_consumer, + on=Many(scale=:Axis), + inputs=( + PreviousTimeStep(:previous_signals) => Many( + scale=:Leaf, + within=SelfPlant(), + application=:temporal_many_source, + var=:signal, + ), + ), + ), + ), + environment=(duration=Hour(1),), + ) + simulation = run!( + model; + steps=2, + outputs=:none, + performance=true, + ) + + register_object!( + model, + Object(:leaf_z; scale=:Leaf, parent=:plant), + ) + continue!(simulation) + + @test Advanced.runtime_performance(simulation).counts[ + :selector_input_binding_candidates + ] == axis_count + first_view = simulation.compiled.status_views_by_target[ + (:temporal_many_consumer, ObjectId(:axis_1)) + ] + second_view = simulation.compiled.status_views_by_target[ + (:temporal_many_consumer, ObjectId(:axis_2)) + ] + @test length(first_view.status.previous_signals) == 2 + @test length(second_view.status.previous_signals) == 2 + @test first_view.status.previous_signals !== + second_view.status.previous_signals + + remove_object!(model, :leaf_z) + continue!(simulation) + first_view_after_removal = simulation.compiled.status_views_by_target[ + (:temporal_many_consumer, ObjectId(:axis_1)) + ] + second_view_after_removal = simulation.compiled.status_views_by_target[ + (:temporal_many_consumer, ObjectId(:axis_2)) + ] + @test length(first_view_after_removal.status.previous_signals) == 1 + @test length(second_view_after_removal.status.previous_signals) == 1 + @test first_view_after_removal.status.previous_signals !== + second_view_after_removal.status.previous_signals +end + @testset "incremental execution plan reuses unaffected groups" begin model = CompositeModel( Object(:scene; scale=:Scene), @@ -1912,8 +2349,19 @@ end ), ) simulation = run!(model; performance=true) + original_groups = simulation.execution_plan.groups + original_batches = simulation.execution_plan.batches + original_groups_by_application_slot = + simulation.execution_plan.groups_by_application_slot + original_schedule = simulation.execution_plan.schedule + original_leaf_group = only( + group for group in original_groups + if group.application.id == :leaf_source + ) + original_leaf_batch = only(original_leaf_group.batches) + original_leaf_target = only(original_leaf_batch.targets) original_scene_group = only( - group for group in simulation.execution_plan.groups + group for group in original_groups if group.application.id == :scene_source ) original_scene_target = only(only(original_scene_group.batches).targets) @@ -1928,8 +2376,22 @@ end group for group in simulation.execution_plan.groups if group.application.id == :scene_source ) + refreshed_leaf_group = only( + group for group in simulation.execution_plan.groups + if group.application.id == :leaf_source + ) refreshed_scene_target = only(only(refreshed_scene_group.batches).targets) performance = Advanced.runtime_performance(simulation) + @test simulation.execution_plan.groups === original_groups + @test simulation.execution_plan.batches === original_batches + @test simulation.execution_plan.groups_by_application_slot === + original_groups_by_application_slot + @test simulation.execution_plan.schedule === original_schedule + @test refreshed_leaf_group === original_leaf_group + @test only(refreshed_leaf_group.batches) === original_leaf_batch + @test first(original_leaf_batch.targets) === original_leaf_target + @test [target.object_id for target in original_leaf_batch.targets] == + ObjectId.([:leaf_1, :leaf_2]) @test refreshed_scene_group === original_scene_group @test refreshed_scene_target === original_scene_target @test performance.counts[:execution_groups_reused] == 1 @@ -1938,6 +2400,133 @@ end @test performance.counts[:execution_groups_updated_in_place] == 1 end +@testset "execution plan addition falls back when a group is created" begin + model = CompositeModel( + Object(:scene; scale=:Scene); + applications=( + ModelSpec( + StabilizationSourceModel(); + name=:leaf_source, + on=Many(scale=:Leaf), + ), + ModelSpec( + StabilizationSourceModel(); + name=:scene_source, + on=One(scale=:Scene), + ), + ), + ) + simulation = run!(model; outputs=:none) + original_groups = simulation.execution_plan.groups + original_batches = simulation.execution_plan.batches + + register_object!( + model, + Object(:leaf_1; scale=:Leaf, parent=:scene), + ) + continue!(simulation) + + @test simulation.execution_plan.groups !== original_groups + @test simulation.execution_plan.batches !== original_batches + leaf_group = only( + group for group in simulation.execution_plan.groups + if group.application.id == :leaf_source + ) + @test only(only(leaf_group.batches).targets).object_id == + ObjectId(:leaf_1) + @test model_object(model, :leaf_1).status.signal == 1.0 +end + +@testset "execution plan pure-addition staging is transactional" begin + context_models = PlantSimEngine.ObjectModelOverrides( + StabilizationContextModel(), + Dict( + ObjectId(:leaf_z) => StabilizationAlternativeContextModel(), + ), + ) + model = CompositeModel( + Object(:scene; scale=:Scene), + Object(:leaf_a; scale=:Leaf, parent=:scene), + Object(:leaf_z; scale=:Leaf, parent=:scene); + applications=( + ModelSpec( + StabilizationSourceModel(); + name=:leaf_source, + on=Many(scale=:Leaf), + ), + ModelSpec( + context_models; + name=:leaf_context, + on=Many(scale=:Leaf), + ), + ), + ) + simulation = run!(model; outputs=:none, performance=true) + original_groups = simulation.execution_plan.groups + original_batches = simulation.execution_plan.batches + original_source_group = only( + group for group in original_groups + if group.application.id == :leaf_source + ) + original_source_batch = only(original_source_group.batches) + original_context_group = only( + group for group in original_groups + if group.application.id == :leaf_context + ) + @test [ + [target.object_id for target in batch.targets] + for batch in original_context_group.batches + ] == [ObjectId.([:leaf_a]), ObjectId.([:leaf_z])] + + register_object!( + model, + Object(:leaf_zz; scale=:Leaf, parent=:scene), + ) + continue!(simulation) + + # The first application was append-compatible, but the second needed a + # new concrete batch. Global staging must leave both old groups untouched + # before the general fallback replaces the plan. + @test [target.object_id for target in original_source_batch.targets] == + ObjectId.([:leaf_a, :leaf_z]) + @test [ + [target.object_id for target in batch.targets] + for batch in original_context_group.batches + ] == [ObjectId.([:leaf_a]), ObjectId.([:leaf_z])] + @test simulation.execution_plan.groups !== original_groups + @test simulation.execution_plan.batches !== original_batches + + refreshed_source_group = only( + group for group in simulation.execution_plan.groups + if group.application.id == :leaf_source + ) + refreshed_context_group = only( + group for group in simulation.execution_plan.groups + if group.application.id == :leaf_context + ) + @test [ + target.object_id for batch in refreshed_source_group.batches + for target in batch.targets + ] == ObjectId.([:leaf_a, :leaf_z, :leaf_zz]) + @test [ + [target.object_id for target in batch.targets] + for batch in refreshed_context_group.batches + ] == [ + ObjectId.([:leaf_a]), + ObjectId.([:leaf_z]), + ObjectId.([:leaf_zz]), + ] + performance = Advanced.runtime_performance(simulation) + @test performance.counts[:execution_targets_constructed] == 2 + @test performance.counts[:execution_batches_constructed] == 4 + @test get( + performance.counts, + :execution_groups_updated_in_place, + 0, + ) == 0 + @test model_object(model, :leaf_zz).status.signal == 1.0 +end + function stabilization_lifecycle_scene(nleaves_per_plant) objects = Object[ Object(:scene; scale=:Scene), diff --git a/test/test-model-hard-calls.jl b/test/test-model-hard-calls.jl index 81889b27c..688062c89 100644 --- a/test/test-model-hard-calls.jl +++ b/test/test-model-hard-calls.jl @@ -43,6 +43,12 @@ function call_model_lookup_allocations(context::T) where {T} return @allocated literal_call_model(context) end +function call_binding_signature_allocations(binding) + bindings = (binding,) + PlantSimEngine._call_bindings_signature(bindings) + return @allocated PlantSimEngine._call_bindings_signature(bindings) +end + PlantSimEngine.inputs_(::NestedCallLeafModel) = NamedTuple() PlantSimEngine.outputs_(::NestedCallLeafModel) = (value=0.0, calls=0) @@ -389,10 +395,58 @@ end :consumer_id, :callee_object_ids, :callee_application_ids, + :membership_generation, + :membership_observed, + ) + @test propertynames(one_binding) == ( + :plan, + :consumer_id, + :callee_object_ids, + :callee_application_ids, + :mode, + :slot, + :application_slot, + :application_id, + :call, + :selector, + :matcher, + :origin, + :process, + :application, + :multiplicity, + :potential_callee_application_ids, ) + @test :membership_generation ∉ propertynames(one_binding) + @test :membership_observed ∉ propertynames(one_binding) @test one_binding.plan === one_plan @test one_binding.application_id == :controller @test one_binding.multiplicity == :one + reconstructed_binding = PlantSimEngine.CompiledModelCallBinding( + one_binding.plan, + one_binding.consumer_id, + copy(one_binding.callee_object_ids), + copy(one_binding.callee_application_ids), + ) + @test reconstructed_binding.plan === one_binding.plan + @test reconstructed_binding.consumer_id == one_binding.consumer_id + @test reconstructed_binding.callee_object_ids == + one_binding.callee_object_ids + @test reconstructed_binding.callee_application_ids == + one_binding.callee_application_ids + @test propertynames(reconstructed_binding) == propertynames(one_binding) + @test getfield(reconstructed_binding, :membership_generation)[] == 0 + @test getfield(reconstructed_binding, :membership_observed)[] + + large_binding = PlantSimEngine.CompiledModelCallBinding( + one_binding.plan, + one_binding.consumer_id, + ObjectId[ + ObjectId(Symbol("signature_leaf_", index)) + for index in 1:4096 + ], + copy(one_binding.callee_application_ids), + ) + @test call_binding_signature_allocations(large_binding) == 0 one_call_row = only( row for row in explain_calls(simulation.compiled) if row.call == :one ) @@ -508,6 +562,11 @@ end call = only(explain_calls(compiled)) @test call.callee_object_ids == [:leaf_a, :leaf_b] @test call.callee_application_ids == [:leaf_calls] + call_binding = only(compiled.call_bindings) + initial_membership_generation = + PlantSimEngine._compiled_call_membership_generation(call_binding) + initial_call_signature = + PlantSimEngine._call_bindings_signature((call_binding,)) simulation = run!(model; outputs=:all, performance=true) controller = only(model_objects(model; scale=:Scene)).status @@ -535,6 +594,14 @@ end continue!(simulation; steps=1) performance = Advanced.runtime_performance(simulation) @test performance.counts[:execution_target_call_batches_extended] == 1 + @test only(simulation.compiled.call_bindings) === call_binding + addition_membership_generation = + PlantSimEngine._compiled_call_membership_generation(call_binding) + @test addition_membership_generation == + UInt64(Advanced.model_revision(model)) + @test addition_membership_generation > initial_membership_generation + @test PlantSimEngine._call_bindings_signature((call_binding,)) != + initial_call_signature refreshed_call_view = call_targets(MANY_CALL_CONTEXT[], :children) refreshed_execution_targets = [ target @@ -551,6 +618,11 @@ end remove_object!(model, :leaf_b) continue!(simulation; steps=1) + removal_membership_generation = + PlantSimEngine._compiled_call_membership_generation(call_binding) + @test removal_membership_generation == + UInt64(Advanced.model_revision(model)) + @test removal_membership_generation > addition_membership_generation @test controller.ncalls == 2 @test controller.total == 5.0 end @@ -723,6 +795,449 @@ end initial_count + length(selected_leaf_ids) end +@testset "cold Many membership survives lifecycle changes before inspection" begin + model = CompositeModel( + Object(:scene; scale=:Scene, name=:scene), + Object(:plant_a; scale=:Plant, name=:plant_a, parent=:scene), + Object(:plant_b; scale=:Plant, name=:plant_b, parent=:scene), + Object(:leaf_keep; scale=:Leaf, parent=:plant_a), + Object(:leaf_remove; scale=:Leaf, parent=:plant_a); + applications=( + ModelSpec( + SelectiveManyCallControllerModel((:leaf_keep,)); + name=:selective_controller, + on=One(name=:plant_a), + calls=( + :children => Many( + scale=:Leaf, + within=Subtree(), + application=:selective_leaf_calls, + ), + ), + ), + ModelSpec( + NestedCallLeafModel(); + name=:selective_leaf_calls, + on=Many(scale=:Leaf), + ), + ), + environment=(duration=Hour(1),), + ) + + simulation = run!(model; outputs=:none, performance=true) + context = LAZY_MANY_CALL_CONTEXT[] + full_call_view = call_targets(context, :children) + call_binding = only(simulation.compiled.call_bindings) + @test !PlantSimEngine._compiled_call_membership_is_observed(call_binding) + @test !PlantSimEngine._call_execution_batches_materialized(full_call_view) + + register_object!( + model, + Object(:leaf_move; scale=:Leaf, parent=:plant_a), + ) + continue!(simulation) + reparent_object!(model, :leaf_move, :plant_b) + continue!(simulation) + remove_object!(model, :leaf_remove) + continue!(simulation) + + # Diagnostics and the resolved graph must report current topology without + # enrolling the complete membership in lifecycle tracking. + call = only(explain_calls(simulation.compiled)) + @test call.callee_object_ids == [:leaf_keep] + resolved_view = PlantSimEngine.model_graph_view( + simulation.compiled; + level=:resolved, + ) + call_edges = [ + edge for edge in resolved_view.edges + if edge["kind"] == "manual_call" && + edge["call"] == "children" && + edge["projection"] == "resolved" + ] + @test length(call_edges) == 1 + @test only(call_edges)["target"] == + "execution:selective_leaf_calls:leaf_keep" + @test !PlantSimEngine._compiled_call_membership_is_observed(call_binding) + @test get( + Advanced.runtime_performance(simulation).counts, + :selector_call_binding_candidates, + 0, + ) == 0 + + # The wrapper obtained before all three lifecycle changes stays cached. + # Its first complete inspection catches up once, then activates the normal + # incremental extension path for later objects. + @test call_targets(LAZY_MANY_CALL_CONTEXT[], :children) === full_call_view + @test length(full_call_view) == 1 + @test PlantSimEngine._compiled_call_membership_is_observed(call_binding) + @test getproperty.(collect(full_call_view), :object_id) == + ObjectId[ObjectId(:leaf_keep)] + + register_object!( + model, + Object(:leaf_z_after_observation; scale=:Leaf, parent=:plant_a), + ) + continue!(simulation) + @test call_targets(LAZY_MANY_CALL_CONTEXT[], :children) === full_call_view + @test getproperty.(collect(full_call_view), :object_id) == + ObjectId[ + ObjectId(:leaf_keep), + ObjectId(:leaf_z_after_observation), + ] + @test Advanced.runtime_performance(simulation).counts[ + :selector_call_binding_candidates + ] == 1 +end + +@testset "retained cold Many synchronizes when a multirate owner is not due" begin + model = CompositeModel( + Object(:scene; scale=:Scene, name=:scene), + Object(:plant_a; scale=:Plant, name=:plant_a, parent=:scene), + Object(:plant_b; scale=:Plant, name=:plant_b, parent=:scene), + Object(:leaf_keep; scale=:Leaf, parent=:plant_a), + Object(:leaf_move; scale=:Leaf, parent=:plant_a); + applications=( + ModelSpec( + SelectiveManyCallControllerModel((:leaf_keep,)); + name=:slow_selective_controller, + on=One(name=:plant_a), + calls=( + :children => Many( + scale=:Leaf, + within=Subtree(), + application=:slow_selective_leaf_calls, + ), + ), + every=Hour(2), + ), + ModelSpec( + NestedCallLeafModel(); + name=:slow_selective_leaf_calls, + on=Many(scale=:Leaf), + ), + ), + environment=(duration=Hour(1),), + ) + + simulation = run!(model; steps=1, outputs=:none, performance=true) + retained_view = call_targets(LAZY_MANY_CALL_CONTEXT[], :children) + initial_compiled = retained_view.compiled + @test !PlantSimEngine._compiled_call_membership_is_observed( + retained_view.binding, + ) + + reparent_object!(model, :leaf_move, :plant_b) + # Step two refreshes lifecycle state, but the two-hour owner is not due and + # therefore cannot synchronize its retained RunContext itself. + continue!(simulation; steps=1) + @test simulation.compiled !== initial_compiled + @test retained_view.compiled === initial_compiled + + @test length(retained_view) == 1 + @test retained_view.compiled === simulation.compiled + @test retained_view.binding === only(simulation.compiled.call_bindings) + @test PlantSimEngine._compiled_call_membership_is_observed( + retained_view.binding, + ) + @test getproperty.(collect(retained_view), :object_id) == + ObjectId[ObjectId(:leaf_keep)] + + register_object!( + model, + Object(:leaf_z_after_sync; scale=:Leaf, parent=:plant_a), + ) + continue!(simulation; steps=1) + @test call_targets(LAZY_MANY_CALL_CONTEXT[], :children) === retained_view + @test getproperty.(collect(retained_view), :object_id) == + ObjectId[ObjectId(:leaf_keep), ObjectId(:leaf_z_after_sync)] +end + +@testset "cold Many materialization preserves the pre-barrier shell" begin + model = CompositeModel( + Object(:scene; scale=:Scene, name=:scene), + Object(:plant_a; scale=:Plant, name=:plant_a, parent=:scene), + Object(:plant_b; scale=:Plant, name=:plant_b, parent=:scene), + Object(:leaf_keep; scale=:Leaf, parent=:plant_a), + Object(:leaf_move; scale=:Leaf, parent=:plant_b); + applications=( + ModelSpec( + SelectiveManyCallControllerModel((:leaf_keep,)); + name=:prebarrier_controller, + on=One(name=:plant_a), + calls=( + :children => Many( + scale=:Leaf, + within=Subtree(), + application=:prebarrier_leaf_calls, + ), + ), + ), + ModelSpec( + NestedCallLeafModel(); + name=:prebarrier_leaf_calls, + on=Many(scale=:Leaf), + ), + ), + environment=(duration=Hour(1),), + ) + + simulation = run!(model; outputs=:none) + retained_view = call_targets(LAZY_MANY_CALL_CONTEXT[], :children) + reparent_object!(model, :leaf_move, :plant_a) + @test Advanced.bindings_dirty(model) + + # A first complete view inside the pending lifecycle event stays on the + # previous safe barrier instead of combining live topology with old + # applications and environment handles. + @test getproperty.(collect(retained_view), :object_id) == + ObjectId[ObjectId(:leaf_keep)] + @test Advanced.bindings_dirty(model) + @test PlantSimEngine._compiled_call_membership_is_observed( + retained_view.binding, + ) + + continue!(simulation) + @test getproperty.(collect(retained_view), :object_id) == + ObjectId[ObjectId(:leaf_keep), ObjectId(:leaf_move)] +end + +@testset "detached retained Many rebuilds after later lifecycle changes" begin + model = CompositeModel( + Object(:scene; scale=:Scene, name=:scene), + Object(:plant_a; scale=:Plant, name=:plant_a, parent=:scene), + Object(:plant_b; scale=:Plant, name=:plant_b, parent=:scene), + Object(:leaf_keep; scale=:Leaf, parent=:plant_a); + applications=( + ModelSpec( + SelectiveManyCallControllerModel((:leaf_keep,)); + name=:detached_controller, + on=One(name=:plant_a), + calls=( + :children => Many( + scale=:Leaf, + within=Subtree(), + application=:detached_leaf_calls, + ), + ), + every=Hour(2), + ), + ModelSpec( + NestedCallLeafModel(); + name=:detached_leaf_calls, + on=Many(scale=:Leaf), + ), + ), + environment=(duration=Hour(1),), + ) + + simulation = run!(model; steps=1, outputs=:none) + retained_view = call_targets(LAZY_MANY_CALL_CONTEXT[], :children) + initial_binding = retained_view.binding + + # Reparenting the consumer replaces its execution target and binding while + # the two-hour owner is not due. The retained wrapper is now detached from + # the execution plan but must remain a live full-membership view. + reparent_object!(model, :plant_a, :plant_b) + continue!(simulation; steps=1) + current_binding = only(simulation.compiled.call_bindings) + @test current_binding !== initial_binding + @test retained_view.binding === initial_binding + + @test getproperty.(collect(retained_view), :object_id) == + ObjectId[ObjectId(:leaf_keep)] + @test retained_view.binding === current_binding + first_batches = + PlantSimEngine._cached_call_execution_batches(retained_view) + + register_object!( + model, + Object(:leaf_z_after_detach; scale=:Leaf, parent=:plant_a), + ) + continue!(simulation; steps=1) + @test call_targets(LAZY_MANY_CALL_CONTEXT[], :children) !== retained_view + @test getproperty.(collect(retained_view), :object_id) == + ObjectId[ObjectId(:leaf_keep), ObjectId(:leaf_z_after_detach)] + @test PlantSimEngine._cached_call_execution_batches(retained_view) !== + first_batches +end + +@testset "tracked Many refreshes a dirty environment before rebuilding" begin + model = CompositeModel( + Object(:scene; scale=:Scene, name=:scene), + Object(:plant_a; scale=:Plant, name=:plant_a, parent=:scene), + Object(:leaf_keep; scale=:Leaf, parent=:plant_a); + applications=( + ModelSpec( + SelectiveManyCallControllerModel((:leaf_keep,)); + name=:environment_controller, + on=One(name=:plant_a), + calls=( + :children => Many( + scale=:Leaf, + within=Subtree(), + application=:environment_leaf_calls, + ), + ), + ), + ModelSpec( + NestedCallLeafModel(); + name=:environment_leaf_calls, + on=Many(scale=:Leaf), + ), + ), + environment=(duration=Hour(1),), + ) + + run!(model; outputs=:none) + retained_view = call_targets(LAZY_MANY_CALL_CONTEXT[], :children) + @test length(retained_view) == 1 + initial_environment_bindings = retained_view.environment_bindings + initial_batches = + PlantSimEngine._cached_call_execution_batches(retained_view) + + mark_environment_binding_dirty!(model) + @test !Advanced.bindings_dirty(model) + @test Advanced.environment_bindings_dirty(model) + @test length(retained_view) == 1 + @test !Advanced.environment_bindings_dirty(model) + @test retained_view.environment_bindings !== initial_environment_bindings + @test PlantSimEngine._cached_call_execution_batches(retained_view) !== + initial_batches +end + +@testset "targeted objects validate current callee applications without warming Many" begin + model = CompositeModel( + Object(:scene; scale=:Scene, name=:scene), + Object( + :callable_leaf; + scale=:Leaf, + kind=:CallableLeaf, + parent=:scene, + ), + Object( + :uncallable_leaf; + scale=:Leaf, + kind=:OtherLeaf, + parent=:scene, + ); + applications=( + ModelSpec( + SelectiveManyCallControllerModel((:callable_leaf,)); + name=:selective_controller, + on=One(name=:scene), + calls=( + :children => Many( + scale=:Leaf, + within=SceneScope(), + application=:selective_leaf_calls, + ), + ), + ), + ModelSpec( + NestedCallLeafModel(); + name=:selective_leaf_calls, + on=Many(kind=:CallableLeaf), + ), + ), + environment=(duration=Hour(1),), + ) + + simulation = run!(model; outputs=:none) + context = LAZY_MANY_CALL_CONTEXT[] + binding = only(simulation.compiled.call_bindings) + @test !PlantSimEngine._compiled_call_membership_is_observed(binding) + call = only(explain_calls(simulation.compiled)) + @test call.callee_object_ids == [:callable_leaf, :uncallable_leaf] + resolved_view = PlantSimEngine.model_graph_view( + simulation.compiled; + level=:resolved, + ) + call_edges = [ + edge for edge in resolved_view.edges + if edge["kind"] == "manual_call" && + edge["call"] == "children" && + edge["projection"] == "resolved" + ] + @test length(call_edges) == 1 + @test only(call_edges)["target"] == + "execution:selective_leaf_calls:callable_leaf" + @test !any( + edge -> occursin("uncallable_leaf", edge["target"]), + call_edges, + ) + @test !PlantSimEngine._compiled_call_membership_is_observed(binding) + @test_throws "does not resolve requested object(s)" call_targets( + context, + :children; + objects=:uncallable_leaf, + ) + @test !PlantSimEngine._compiled_call_membership_is_observed(binding) + selected = call_targets( + context, + :children; + objects=:callable_leaf, + ) + @test length(selected) == 1 + @test only(selected).object_id == ObjectId(:callable_leaf) + @test selected.execution_batches isa + PlantSimEngine.LazyCallExecutionBatches + @test !selected.execution_batches.tracks_full_membership + @test !PlantSimEngine._compiled_call_membership_is_observed(binding) +end + +@testset "targeted singular calls reject ambiguous newborns before preparation" begin + model = CompositeModel( + Object(:scene; scale=:Scene, name=:scene), + Object(:existing_leaf; scale=:Leaf, parent=:scene); + applications=( + ModelSpec( + SelectiveManyCallControllerModel((:existing_leaf,)); + name=:singular_controller, + on=One(name=:scene), + calls=( + :children => One( + scale=:Leaf, + within=SceneScope(), + application=:singular_leaf_call, + ), + ), + ), + ModelSpec( + NestedCallLeafModel(); + name=:singular_leaf_call, + on=Many(scale=:Leaf), + ), + ), + environment=(duration=Hour(1),), + ) + + simulation = run!(model; outputs=:none) + context = LAZY_MANY_CALL_CONTEXT[] + existing_calls = model_status(model, :existing_leaf).calls + newborn = register_object!( + model, + Object( + :newborn_leaf; + scale=:Leaf, + parent=:scene, + status=Status(marker=1.0), + ), + ) + newborn_status = model_status(model, newborn) + @test propertynames(newborn_status) == (:marker,) + @test_throws "Expected exactly one object" run_call!( + context, + :children; + objects=newborn, + publish=true, + ) + @test propertynames(model_status(model, newborn)) == (:marker,) + @test model_status(model, :existing_leaf).calls == existing_calls + @test Advanced.bindings_dirty(model) + @test simulation.compiled === context.compiled +end + @testset "non-monotonic Many call additions use the rebuild path" begin model = CompositeModel( Object(:scene; scale=:Scene, name=:scene), diff --git a/test/test-model-status-type-lifecycle.jl b/test/test-model-status-type-lifecycle.jl index 520a946a9..eda2cda6a 100644 --- a/test/test-model-status-type-lifecycle.jl +++ b/test/test-model-status-type-lifecycle.jl @@ -7,6 +7,8 @@ PlantSimEngine.@process "status_type_lifecycle_increment" verbose = false PlantSimEngine.@process "status_type_lifecycle_stream_writer" verbose = false PlantSimEngine.@process "status_type_lifecycle_stream_consumer" verbose = false PlantSimEngine.@process "status_type_lifecycle_bound_consumer" verbose = false +PlantSimEngine.@process "status_type_lifecycle_output_transaction" verbose = false +PlantSimEngine.@process "status_type_lifecycle_input_transaction" verbose = false struct StatusTypeLifecycleIncrementModel <: AbstractStatus_Type_Lifecycle_IncrementModel end @@ -87,6 +89,49 @@ function (counter::StatusTypeLifecycleTransformCounter)(variable, value) return value end +struct StatusTypeLifecycleFailOn + variable::Symbol +end + +function (transform::StatusTypeLifecycleFailOn)(variable, value) + variable == transform.variable && error("intentional conversion failure") + return value +end + +struct StatusTypeLifecycleOutputTransactionModel <: + AbstractStatus_Type_Lifecycle_Output_TransactionModel end + +PlantSimEngine.outputs_(::StatusTypeLifecycleOutputTransactionModel) = + (first_output=0.0, failing_output=0.0) + +function PlantSimEngine.run!( + ::StatusTypeLifecycleOutputTransactionModel, + status, + environment, + constants, + context, +) + return nothing +end + +struct StatusTypeLifecycleInputTransactionModel <: + AbstractStatus_Type_Lifecycle_Input_TransactionModel end + +PlantSimEngine.inputs_(::StatusTypeLifecycleInputTransactionModel) = + (first_default=Default(1.0), failing_default=Default(2.0)) +PlantSimEngine.outputs_(::StatusTypeLifecycleInputTransactionModel) = + (seen=0.0,) + +function PlantSimEngine.run!( + ::StatusTypeLifecycleInputTransactionModel, + status, + environment, + constants, + context, +) + return nothing +end + @testset "pure additions preserve bound-default conversion diagnostics" begin calls = Ref(0) model = CompositeModel( @@ -127,6 +172,9 @@ end status = model_status(model, :leaf_2) @test propertynames(status) == (:value, :seen, :bound_value) + references = PlantSimEngine.refvalues(status) + @test isconcretetype(typeof(references)) + @test all(isconcretetype, fieldtypes(typeof(references))) @test PlantSimEngine.refvalue(status, :bound_value) === PlantSimEngine.refvalue(status, :value) @test status.bound_value === Float32(1) @@ -157,6 +205,152 @@ end @test initialization.type_mapping_applied performance = Advanced.runtime_performance(simulation) @test performance.counts[:input_default_status_rewrites_avoided] == 1 + @test performance.counts[ + :lifecycle_binding_refresh_status_recipe_materializations + ] == 2 + @test performance.counts[ + :lifecycle_binding_refresh_status_recipe_fields_applied + ] == 3 + @test performance.counts[ + :lifecycle_binding_refresh_status_recipe_materializations_avoided + ] == 2 + @test only( + row for row in Diagnostics.explain_runtime_performance(simulation) + if row.metric == + :lifecycle_binding_refresh_status_recipe_materializations + ).phase == :lifecycle_buffer_update +end + +@testset "batched recipes preserve arbitrary Ref aliases" begin + external_storage = [7.5] + external_reference = Ref(external_storage, 1) + initial_status = Status((external_value=external_reference,)) + @test PlantSimEngine.refvalue(initial_status, :external_value) === + external_reference + + model = CompositeModel( + Object(:scene; scale=:Scene), + Object(:plant; scale=:Plant, parent=:scene); + applications=( + ModelSpec( + StatusTypeLifecycleIncrementModel(); + name=:lifecycle_increment, + on=Many(scale=:Leaf), + ), + ModelSpec( + StatusTypeLifecycleBoundConsumerModel(); + name=:bound_consumer, + on=Many(scale=:Leaf), + inputs=( + bound_value=One( + within=Self(), + application=:lifecycle_increment, + var=:value, + ), + ), + ), + ), + environment=(duration=Hour(1),), + ) + Advanced.refresh_bindings!(model) + register_object!( + model, + Object( + :late_leaf; + scale=:Leaf, + parent=:plant, + status=initial_status, + ), + ) + + Advanced.refresh_bindings!(model) + refreshed = model_status(model, :late_leaf) + @test propertynames(refreshed) == + (:external_value, :value, :seen, :bound_value) + @test PlantSimEngine.refvalue(refreshed, :external_value) === + external_reference + @test PlantSimEngine.refvalue(refreshed, :bound_value) === + PlantSimEngine.refvalue(refreshed, :value) + + external_storage[1] = 9.25 + @test refreshed.external_value == 9.25 +end + +@testset "batched output recipes commit conversion metadata atomically" begin + model = CompositeModel( + Object(:scene; scale=:Scene), + Object(:plant; scale=:Plant, parent=:scene); + applications=( + ModelSpec( + StatusTypeLifecycleOutputTransactionModel(); + name=:output_transaction, + on=Many(scale=:Leaf), + ), + ), + environment=(duration=Hour(1),), + status_transform=StatusTypeLifecycleFailOn(:failing_output), + ) + Advanced.refresh_bindings!(model) + register_object!( + model, + Object(:late_leaf; scale=:Leaf, parent=:plant), + ) + + @test_throws ErrorException Advanced.refresh_bindings!(model) + @test isnothing(model_status(model, :late_leaf)) + @test !any( + key -> key[3] == ObjectId(:late_leaf), + keys(model.status_conversion_records), + ) +end + +@testset "batched input recipes commit metadata atomically" begin + model = CompositeModel( + Object(:scene; scale=:Scene), + Object(:plant; scale=:Plant, parent=:scene); + applications=( + ModelSpec( + StatusTypeLifecycleInputTransactionModel(); + name=:input_transaction, + on=Many(scale=:Leaf), + ), + ), + environment=(duration=Hour(1),), + status_transform=StatusTypeLifecycleFailOn(:failing_default), + ) + Advanced.refresh_bindings!(model) + register_object!( + model, + Object(:late_leaf; scale=:Leaf, parent=:plant), + ) + + performance = Advanced.RuntimePerformanceCounters() + @test_throws ErrorException Advanced.refresh_bindings!( + model; + performance=performance, + ) + @test propertynames(model_status(model, :late_leaf)) == (:seen,) + @test !haskey( + model.status_conversion_records, + ( + :model_input_default, + :input_transaction, + ObjectId(:late_leaf), + :first_default, + ), + ) + @test isempty( + get( + model.input_default_status_variables, + ObjectId(:late_leaf), + Set{Symbol}(), + ), + ) + @test get( + performance.counts, + :input_default_status_rewrites_avoided, + 0, + ) == 0 end @testset "register_object! converts status after compilation" begin From 492c521f3ecd111f44226f44d75014942bdb6220 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?R=C3=A9mi=20Vezy?= Date: Mon, 31 Aug 2026 22:18:07 +0200 Subject: [PATCH 3/8] perf: append stable many input sources in place --- src/composite_model/compilation.jl | 62 ++++++++++++++++++- test/test-model-api-stabilization.jl | 30 ++++++++- test/test-model-distributed-output-runtime.jl | 35 ++++++++++- 3 files changed, 123 insertions(+), 4 deletions(-) diff --git a/src/composite_model/compilation.jl b/src/composite_model/compilation.jl index 4c4de3275..44aad4476 100644 --- a/src/composite_model/compilation.jl +++ b/src/composite_model/compilation.jl @@ -2069,6 +2069,55 @@ function _shared_same_rate_many_binding_indices(bindings, binding) ] end +function _can_append_stable_many_sources(binding::CompiledModelInputBinding) + binding.multiplicity == :many || return false + binding.carrier_hint === :ref_vector || return false + binding.carrier isa RefVector || return false + potential_application_ids = binding.potential_source_application_ids + source_application_ids = binding.source_application_ids + length(source_application_ids) == length(potential_application_ids) || + return false + return all( + application_id -> application_id in source_application_ids, + potential_application_ids, + ) +end + +function _append_stable_many_source_references!( + model::CompositeModel, + binding::CompiledModelInputBinding, + new_source_ids, + performance, +) + _can_append_stable_many_sources(binding) || return false + existing_references = parent(binding.carrier) + added_references = eltype(existing_references)[] + sizehint!(added_references, length(new_source_ids)) + for object_id in new_source_ids + reference = _status_ref_or_nothing( + _model_object(model, object_id).status, + binding.source_var, + ) + reference isa eltype(existing_references) || return false + push!(added_references, reference) + end + + # Resolve every fallible reference before mutating the vectors shared by + # all consumers in this same-rate `Many(...)` group. + append!(existing_references, added_references) + append!(binding.source_ids, new_source_ids) + _runtime_performance_count!( + performance, + :lifecycle_many_input_binding_direct_appends, + ) + _runtime_performance_count!( + performance, + :lifecycle_many_input_binding_direct_sources_appended, + length(new_source_ids), + ) + return true +end + function _append_added_many_sources!( model::CompositeModel, binding::CompiledModelInputBinding, @@ -2076,6 +2125,8 @@ function _append_added_many_sources!( applications_by_object, applications_by_id, distributed_outputs, + ; + performance=nothing, ) binding.multiplicity == :many || return false default_scope = _default_dependency_scope(model, binding.consumer_id) @@ -2113,6 +2164,13 @@ function _append_added_many_sources!( return false end + _append_stable_many_source_references!( + model, + binding, + new_source_ids, + performance, + ) && return true + new_carrier = _input_carrier(model, binding.selector, new_source_ids, binding.source_var) isnothing(new_carrier) && return false existing_refs = parent(binding.carrier) @@ -3201,7 +3259,7 @@ function _extend_compiled_scene( continue end end - if !force_rebuild + if !force_rebuild && binding.multiplicity != :many _selector_matches_any_object_id( model, binding.matcher, @@ -3221,6 +3279,8 @@ function _extend_compiled_scene( applications_by_object, applications_by_id, distributed_outputs, + ; + performance=performance, ) end if appended_sources diff --git a/test/test-model-api-stabilization.jl b/test/test-model-api-stabilization.jl index 1085bf691..6adb72fe9 100644 --- a/test/test-model-api-stabilization.jl +++ b/test/test-model-api-stabilization.jl @@ -2096,6 +2096,10 @@ end original_bindings = stabilization_shared_many_bindings(simulation, axis_count) original = first(original_bindings) + original_source_ids = original.source_ids + original_source_application_ids = original.source_application_ids + original_carrier = original.carrier + original_references = parent(original.carrier) @test all( binding -> binding.source_ids === original.source_ids && @@ -2117,9 +2121,26 @@ end @test Advanced.runtime_performance(simulation).counts[ :selector_input_binding_candidates ] == 1 + @test Advanced.runtime_performance(simulation).counts[ + :lifecycle_many_input_binding_direct_appends + ] == 1 + @test Advanced.runtime_performance(simulation).counts[ + :lifecycle_many_input_binding_direct_sources_appended + ] == 1 + direct_append_row = only( + row for row in Diagnostics.explain_runtime_performance(simulation) + if row.metric == :lifecycle_many_input_binding_direct_appends + ) + @test direct_append_row.phase == :lifecycle_buffer_update @test refreshed === original + @test refreshed.source_ids === original_source_ids + @test refreshed.source_application_ids === + original_source_application_ids @test refreshed.source_ids == ObjectId.([:leaf_a, :leaf_z]) - @test refreshed.carrier === original.carrier + @test refreshed.carrier === original_carrier + @test parent(refreshed.carrier) === original_references + @test parent(refreshed.carrier)[end] === + PlantSimEngine.refvalue(model_status(model, :leaf_z), :signal) @test all( binding -> binding.source_ids === refreshed.source_ids && @@ -2132,6 +2153,8 @@ end model_object(model, Symbol(:axis_, index)).status.lagged_total == 3.0 for index in 1:axis_count ) + model_status(model, :leaf_z).signal = 11.0 + @test refreshed.carrier[end] == 11.0 end @testset "shared Many fallback rewires every consumer and rebuilds its index" begin @@ -2159,6 +2182,11 @@ end @test Advanced.runtime_performance(simulation).counts[ :selector_input_binding_candidates ] == 1 + @test get( + Advanced.runtime_performance(simulation).counts, + :lifecycle_many_input_binding_direct_appends, + 0, + ) == 0 @test rebuilt.carrier !== initial_carrier @test rebuilt.source_ids == ObjectId.([:leaf_a, :leaf_z]) @test all(binding -> binding.carrier === rebuilt.carrier, rebuilt_bindings) diff --git a/test/test-model-distributed-output-runtime.jl b/test/test-model-distributed-output-runtime.jl index 407e665c3..043846a6b 100644 --- a/test/test-model-distributed-output-runtime.jl +++ b/test/test-model-distributed-output-runtime.jl @@ -78,7 +78,7 @@ function _distributed_runtime_value(object_id::ObjectId, time::Real) 10.0 elseif object_id == ObjectId(:leaf_b) 20.0 - elseif object_id == ObjectId(:late_leaf) + elseif object_id in (ObjectId(:late_leaf), ObjectId(:leaf_z)) 30.0 else 1.0 @@ -1044,9 +1044,40 @@ end @test binding.source_application_ids == [:distributed_runtime_sun_only_writer] - simulation = run!(model; outputs=:none) + original_carrier = binding.carrier + original_references = parent(original_carrier) + simulation = run!(model; outputs=:none, performance=true) @test final_state(simulation, :plant).integrated_total == 10.0 @test final_state(simulation, :leaf_b).incident_par == 20.0 + + register_object!( + model, + Object( + :leaf_z; + scale=:Leaf, + kind=:sun, + parent=:plant, + ), + ) + continue!(simulation) + + refreshed = only( + candidate for candidate in simulation.compiled.input_bindings + if candidate.application_id == + :distributed_runtime_sun_only_consumer + ) + @test refreshed === binding + @test refreshed.carrier === original_carrier + @test parent(refreshed.carrier) === original_references + @test refreshed.source_ids == ObjectId.([:leaf_a, :leaf_z]) + @test parent(refreshed.carrier)[end] === PlantSimEngine.refvalue( + model_status(model, :leaf_z), + :incident_par, + ) + @test final_state(simulation, :plant).integrated_total == 80.0 + counts = Advanced.runtime_performance(simulation).counts + @test counts[:lifecycle_many_input_binding_direct_appends] == 1 + @test counts[:lifecycle_many_input_binding_direct_sources_appended] == 1 end @testset "Many default policy follows final distributed Updates writer" begin From ce06cf26af0f43db7db0719a71ede9346d4d0e02 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?R=C3=A9mi=20Vezy?= Date: Tue, 1 Sep 2026 01:51:47 +0200 Subject: [PATCH 4/8] perf: cache lifecycle targets and runtime contexts --- src/composite_model/compilation.jl | 150 +++++++++++-- src/composite_model/runtime_outputs.jl | 177 ++++++++++----- test/test-model-api-stabilization.jl | 262 ++++++++++++++++++++++- test/test-model-multirate-integration.jl | 58 +++++ 4 files changed, 569 insertions(+), 78 deletions(-) diff --git a/src/composite_model/compilation.jl b/src/composite_model/compilation.jl index 44aad4476..a8352f2d7 100644 --- a/src/composite_model/compilation.jl +++ b/src/composite_model/compilation.jl @@ -262,16 +262,19 @@ struct CompiledApplicationSchedule{E,A,P,G} end """Reverse candidate index for compiled selector matchers.""" -struct SelectorCandidateIndex{W,S,K,SP,N,A} +struct SelectorCandidateIndex{W,S,K,SP,N,A,R,L,C} wildcard::W by_scale::S by_kind::K by_species::SP by_name::N by_scope_anchor::A + scope_roots::R + template_label_values::L + application_target_templates::C end -function _selector_candidate_index() +function _selector_candidate_index(; application_targets::Bool=false) return SelectorCandidateIndex( Int[], Dict{Symbol,Vector{Int}}(), @@ -282,6 +285,9 @@ function _selector_candidate_index() Union{ObjectId,Tuple{ObjectId,Symbol,Symbol}}, Vector{Int}, }(), + Set{ObjectId}(), + Dict{Symbol,Set{Symbol}}(), + application_targets ? Dict{Any,Any}() : nothing, ) end @@ -294,6 +300,12 @@ function _freeze_selector_candidate_index(index::SelectorCandidateIndex) freeze(index.by_species), freeze(index.by_name), freeze(index.by_scope_anchor), + Set(index.scope_roots), + Dict( + label => Set(values) + for (label, values) in index.template_label_values + ), + Dict{Any,Any}(), ) end @@ -353,6 +365,16 @@ function _selector_candidate_destination( if scope isa Self && isnothing(matcher.relation) return nothing end + # `Ancestor(scale=S)` combined with a source at `scale=S` selects the + # nearest matching ancestor. Adding a descendant cannot change that source + # for an existing consumer, so this binding only needs the structural + # removal/reparenting refresh path. + if isnothing(matcher.relation) && + scope isa Ancestor && + !isnothing(matcher.scale) && + matcher.scale == scope.scale + return nothing + end anchor = _selector_scope_anchor( model, matcher; @@ -397,6 +419,18 @@ function _index_selector_candidate!( context=nothing, default_scope=nothing, ) + tracks_application_targets = + !isnothing(index.application_target_templates) + if tracks_application_targets + for label in (:scale, :kind, :species, :name) + value = getproperty(matcher, label) + isnothing(value) && continue + union!( + get!(index.template_label_values, label, Set{Symbol}()), + _selector_candidate_values(value), + ) + end + end destination = _selector_candidate_destination( index, model, @@ -411,6 +445,11 @@ function _index_selector_candidate!( else for value in values push!(get!(groups, value, Int[]), candidate_index) + tracks_application_targets && + groups === index.by_scope_anchor && push!( + index.scope_roots, + value isa Tuple ? first(value) : value, + ) end end return index @@ -454,7 +493,7 @@ function _union_selector_candidates!( end function _application_target_candidate_index(model, application_plans) - index = _selector_candidate_index() + index = _selector_candidate_index(; application_targets=true) for plan in application_plans _index_selector_candidate!( index, @@ -1674,6 +1713,84 @@ function _compile_scene( ) end +struct _ApplicationTargetTemplate{C,M} + candidate_slots::C + matched_slots::M +end + +function _application_target_template_label( + index::SelectorCandidateIndex, + label::Symbol, + value, +) + selected_values = get(index.template_label_values, label, nothing) + isnothing(selected_values) && return nothing + return value in selected_values ? value : nothing +end + +function _application_target_template_key( + index::SelectorCandidateIndex, + model::CompositeModel, + object_id::ObjectId, +) + object = _model_object(model, object_id) + ancestors = _object_ancestor_ids(model.registry, object_id) + scope_roots = Tuple( + ancestor_id for ancestor_id in ancestors + if ancestor_id in index.scope_roots + ) + return ( + _application_target_template_label(index, :scale, object.scale), + _application_target_template_label(index, :kind, object.kind), + _application_target_template_label(index, :species, object.species), + _application_target_template_label(index, :name, object.name), + scope_roots, + ) +end + +function _application_target_template( + model::CompositeModel, + compiled::CompiledCompositeModel, + object_id::ObjectId, + performance=nothing, +) + index = compiled.scenario_plan.application_target_candidates + key = _application_target_template_key(index, model, object_id) + cached = get(index.application_target_templates, key, nothing) + if !isnothing(cached) + _runtime_performance_count!( + performance, + :lifecycle_application_target_template_cache_hits, + ) + return cached + end + candidate_slots = Set{Int}() + _union_selector_candidates!( + candidate_slots, + index, + model, + object_id, + ) + sorted_candidates = Tuple(sort!(collect(candidate_slots))) + matched_slots = Tuple( + slot for slot in sorted_candidates if _selector_matches_object_id( + model, + compiled.applications[slot].target_matcher, + object_id, + ) + ) + template = _ApplicationTargetTemplate( + sorted_candidates, + matched_slots, + ) + index.application_target_templates[key] = template + _runtime_performance_count!( + performance, + :lifecycle_application_target_template_cache_misses, + ) + return template +end + function _new_application_targets( model::CompositeModel, compiled::CompiledCompositeModel, @@ -1681,37 +1798,32 @@ function _new_application_targets( performance=nothing, ) candidate_slots = Set{Int}() + targets = Dict{Symbol,Vector{ObjectId}}() for object_id in added_ids - _union_selector_candidates!( - candidate_slots, - compiled.scenario_plan.application_target_candidates, + template = _application_target_template( model, + compiled, object_id, + performance, ) + union!(candidate_slots, template.candidate_slots) + for slot in template.matched_slots + application = compiled.applications[slot] + push!(get!(targets, application.id, ObjectId[]), object_id) + end end _runtime_performance_count!( performance, :selector_application_candidates, length(candidate_slots), ) - targets = Dict{Symbol,Vector{ObjectId}}() - for slot in sort!(collect(candidate_slots)) - application = compiled.applications[slot] - matched = ObjectId[ - object_id for object_id in added_ids - if _selector_matches_object_id( - model, - application.target_matcher, - object_id, - ) - ] - isempty(matched) && continue + for (application_id, matched) in targets + application = compiled.applications_by_id[application_id] new_count = length(application.target_ids) + length(matched) if (application.applies_to isa One && new_count != 1) || (application.applies_to isa OptionalOne && new_count > 1) return nothing end - targets[application.id] = matched end return targets end diff --git a/src/composite_model/runtime_outputs.jl b/src/composite_model/runtime_outputs.jl index 816592e61..50bd9143c 100644 --- a/src/composite_model/runtime_outputs.jl +++ b/src/composite_model/runtime_outputs.jl @@ -883,13 +883,34 @@ function _call_bindings_signature(call_bindings) return signature end +mutable struct ExecutionBatchContextState + # Scheduled root batches always use the default publication/environment + # state. Hard-call batches deliberately bypass this synchronization cache. + compiled::Any + environment_bindings::Any +end + struct CompiledExecutionBatch{A,T<:AbstractVector,MP,OP} <: AbstractExecutionBatch application::A targets::T environment_provider::MP output_publication::OP + context_state::ExecutionBatchContextState end +CompiledExecutionBatch( + application, + targets, + environment_provider, + output_publication, +) = CompiledExecutionBatch( + application, + targets, + environment_provider, + output_publication, + ExecutionBatchContextState(nothing, nothing), +) + struct CompiledApplicationExecutionGroup{A,B} application::A batches::B @@ -2528,6 +2549,43 @@ end ) end +function _synchronize_model_execution_batch_contexts!( + batch::CompiledExecutionBatch, + compiled, + environment_bindings, + temporal_streams, + output_retention, + time::Real, + constants, +) + state = batch.context_state + state.compiled === compiled && + state.environment_bindings === environment_bindings && return nothing + for target in batch.targets + context = target.context + context isa RunContext || continue + _prepare_model_execution_context!( + context, + compiled, + environment_bindings, + batch.application, + target.object_id, + target.bound_inputs, + target.output_targets, + temporal_streams, + output_retention, + time, + constants, + ) + end + # Mark the batch current only after every retained context has been + # synchronized. If preparation throws, the next execution retries the + # complete slow path. + state.compiled = compiled + state.environment_bindings = environment_bindings + return nothing +end + @inline function _run_model_execution_target_without_calls!( compiled::CompiledCompositeModel, env_bindings::CompiledEnvironmentBindings, @@ -2558,19 +2616,8 @@ end time, ) : sampled_environment context = if target.context isa RunContext - _prepare_model_execution_context!( - target.context, - compiled, - env_bindings, - application, - target.object_id, - target.bound_inputs, - target.output_targets, - temporal_streams, - output_retention, - time, - constants, - ) + target.context.time = time + target.context else RunContext( compiled, @@ -2582,7 +2629,7 @@ end target.output_targets, temporal_streams, output_retention, - float(time), + time, constants, true, _NO_ENVIRONMENT_OVERRIDE, @@ -2637,19 +2684,24 @@ end target.environment_binding, time, ) : sampled_environment - context = _prepare_model_execution_context!( - target.context, - compiled, - env_bindings, - application, - target.object_id, - target.bound_inputs, - target.output_targets, - temporal_streams, - output_retention, - time, - constants, - ) + context = if target.context isa RunContext + target.context.time = time + target.context + else + _prepare_model_execution_context!( + target.context, + compiled, + env_bindings, + application, + target.object_id, + target.bound_inputs, + target.output_targets, + temporal_streams, + output_retention, + time, + constants, + ) + end run!(target.model, status, environment_value, constants, context) if publish_outputs isempty(target.output_bindings) || @@ -2715,12 +2767,22 @@ end temporal_streams, output_retention, ) + batch_time = float(time) shared_environment = _model_execution_batch_environment( batch.environment_provider, env_bindings, batch.application, time, ) + _synchronize_model_execution_batch_contexts!( + batch, + compiled, + env_bindings, + temporal_streams, + output_retention, + batch_time, + constants, + ) publish_outputs = batch.output_publication.enabled if isempty(first(batch.targets).call_bindings) if isnothing(temporal_streams) @@ -2730,7 +2792,7 @@ end env_bindings, batch.application, target, - time, + batch_time, constants, temporal_streams, output_retention, @@ -2760,19 +2822,7 @@ end ) : shared_environment context = target.context if context isa RunContext - _prepare_model_execution_context!( - context, - compiled, - env_bindings, - batch.application, - target.object_id, - target.bound_inputs, - target.output_targets, - temporal_streams, - output_retention, - time, - constants, - ) + context.time = batch_time else context = _prepare_model_execution_context!( context, @@ -2784,7 +2834,7 @@ end target.output_targets, temporal_streams, output_retention, - time, + batch_time, constants, ) end @@ -2811,7 +2861,7 @@ end env_bindings, batch.application, target, - time, + batch_time, constants, temporal_streams, output_retention, @@ -2832,6 +2882,7 @@ function _run_model_execution_batch_profiled!( output_retention, performance::RuntimePerformanceCounters, ) + batch_time = float(time) started_at = _runtime_performance_start(performance) shared_environment = _model_execution_batch_environment( batch.environment_provider, @@ -2844,6 +2895,15 @@ function _run_model_execution_batch_profiled!( :environment_sampling, started_at, ) + _synchronize_model_execution_batch_contexts!( + batch, + compiled, + env_bindings, + temporal_streams, + output_retention, + batch_time, + constants, + ) publish_outputs = batch.output_publication.enabled for target in batch.targets started_at = _runtime_performance_start(performance) @@ -2878,19 +2938,24 @@ function _run_model_execution_batch_profiled!( started_at, ) - context = _prepare_model_execution_context!( - target.context, - compiled, - env_bindings, - batch.application, - target.object_id, - target.bound_inputs, - target.output_targets, - temporal_streams, - output_retention, - time, - constants, - ) + context = if target.context isa RunContext + target.context.time = batch_time + target.context + else + _prepare_model_execution_context!( + target.context, + compiled, + env_bindings, + batch.application, + target.object_id, + target.bound_inputs, + target.output_targets, + temporal_streams, + output_retention, + batch_time, + constants, + ) + end started_at = _runtime_performance_start(performance) run!( target.model, diff --git a/test/test-model-api-stabilization.jl b/test/test-model-api-stabilization.jl index 6adb72fe9..0be25fbee 100644 --- a/test/test-model-api-stabilization.jl +++ b/test/test-model-api-stabilization.jl @@ -1232,12 +1232,52 @@ function stabilization_relation_candidate_scene() ) end +function stabilization_nearest_ancestor_candidate_scene() + return CompositeModel( + Object(:scene; scale=:Scene), + Object(:plant; scale=:Plant, parent=:scene), + Object(:phytomer_a; scale=:Phytomer, parent=:plant), + Object( + :female_a; + scale=:Female, + parent=:phytomer_a, + status=Status(supplied=0.0), + ); + applications=( + ModelSpec( + StabilizationSourceModel(); + name=:ancestor_source, + on=Many(scale=:Phytomer), + ), + ModelSpec( + StabilizationConsumerModel(); + name=:ancestor_consumer, + on=Many(scale=:Female), + inputs=( + :signal => One( + scale=:Phytomer, + within=Ancestor(scale=:Phytomer), + application=:ancestor_source, + var=:signal, + ), + ), + ), + ), + environment=(duration=Hour(1),), + ) +end + function stabilization_selector_candidate_refresh_allocations(nplants) model = stabilization_selector_candidate_scene(nplants) simulation = run!(model; outputs=:none) register_object!( model, - Object(:zz_new_leaf; scale=:Leaf, parent=:plant_1), + Object( + :zz_new_leaf; + scale=:Leaf, + name=:unique_new_leaf_1, + parent=:plant_1, + ), ) return @allocated PlantSimEngine._refresh_simulation_runtime!(simulation) end @@ -1245,15 +1285,46 @@ end @testset "lifecycle reverse selector candidates remain local" begin model = stabilization_selector_candidate_scene(64) simulation = run!(model; outputs=:none, performance=true) + for index in ( + simulation.compiled.dynamic_input_binding_indices, + simulation.compiled.dynamic_call_binding_indices, + ) + @test isnothing(index.application_target_templates) + @test isempty(index.template_label_values) + @test isempty(index.scope_roots) + end register_object!( model, - Object(:zz_new_leaf; scale=:Leaf, parent=:plant_1), + Object( + :zz_new_leaf; + scale=:Leaf, + name=:unique_new_leaf_1, + parent=:plant_1, + ), ) continue!(simulation) counts = Advanced.runtime_performance(simulation).counts @test counts[:selector_application_candidates] == 1 @test counts[:selector_input_binding_candidates] == 1 @test get(counts, :selector_call_binding_candidates, 0) == 0 + @test counts[:lifecycle_application_target_template_cache_misses] == 1 + + register_object!( + model, + Object( + :zz_new_leaf_2; + scale=:Leaf, + name=:unique_new_leaf_2, + parent=:plant_1, + ), + ) + continue!(simulation) + @test Advanced.runtime_performance(simulation).counts[ + :lifecycle_application_target_template_cache_hits + ] == 1 + @test Advanced.runtime_performance(simulation).counts[ + :lifecycle_application_target_template_cache_misses + ] == 1 reparented_model = stabilization_selector_candidate_scene(64) reparented_simulation = run!( @@ -1340,6 +1411,143 @@ end @test large_allocations <= small_allocations + 32_768 end +@testset "application target templates preserve instance scope" begin + template = CompositeModelTemplate(( + ModelSpec( + StabilizationSourceModel(); + name=:source, + on=Many(scale=:Leaf), + ), + )) + palm_1 = ObjectInstance( + :palm_1, + template; + root=Object(:template_plant_1; scale=:Plant, parent=:scene), + objects=( + Object( + :template_leaf_1a; + scale=:Leaf, + name=:unique_leaf_1a, + parent=:template_plant_1, + ), + Object( + :template_leaf_1b; + scale=:Leaf, + name=:unique_leaf_1b, + parent=:template_plant_1, + ), + ), + ) + palm_2 = ObjectInstance( + :palm_2, + template; + root=Object(:template_plant_2; scale=:Plant, parent=:scene), + objects=( + Object( + :template_leaf_2; + scale=:Leaf, + name=:unique_leaf_2, + parent=:template_plant_2, + ), + ), + ) + model = CompositeModel( + Object(:scene; scale=:Scene), + palm_1, + palm_2, + ) + compiled = Advanced.refresh_bindings!(model) + index = compiled.scenario_plan.application_target_candidates + key_1a = PlantSimEngine._application_target_template_key( + index, + model, + ObjectId(:template_leaf_1a), + ) + key_1b = PlantSimEngine._application_target_template_key( + index, + model, + ObjectId(:template_leaf_1b), + ) + key_2 = PlantSimEngine._application_target_template_key( + index, + model, + ObjectId(:template_leaf_2), + ) + @test key_1a == key_1b + @test key_1a != key_2 + @test index.template_label_values == Dict(:scale => Set([:Leaf])) + + target_template_1 = PlantSimEngine._application_target_template( + model, + compiled, + ObjectId(:template_leaf_1a), + ) + target_template_2 = PlantSimEngine._application_target_template( + model, + compiled, + ObjectId(:template_leaf_2), + ) + @test Tuple( + compiled.applications[slot].id + for slot in target_template_1.matched_slots + ) == (:palm_1__source,) + @test Tuple( + compiled.applications[slot].id + for slot in target_template_2.matched_slots + ) == (:palm_2__source,) +end + +@testset "nearest Ancestor bindings ignore descendant additions" begin + model = stabilization_nearest_ancestor_candidate_scene() + simulation = run!(model; outputs=:none, performance=true) + + register_object!( + model, + Object(:phytomer_z; scale=:Phytomer, parent=:phytomer_a), + ) + continue!(simulation) + + @test Advanced.runtime_performance(simulation).counts[ + :selector_input_binding_candidates + ] == 0 + existing_binding = only( + row for row in explain_bindings(model) + if row.application_id == :ancestor_consumer && + row.consumer_id == :female_a && + row.input == :signal + ) + @test existing_binding.source_ids == [:phytomer_a] + @test model_status(model, :female_a).observed == 2.0 + + register_object!( + model, + Object( + :female_z; + scale=:Female, + parent=:phytomer_z, + status=Status(supplied=0.0), + ), + ) + continue!(simulation) + new_binding = only( + row for row in explain_bindings(model) + if row.application_id == :ancestor_consumer && + row.consumer_id == :female_z && + row.input == :signal + ) + @test new_binding.source_ids == [:phytomer_z] + + reparent_object!(model, :female_z, :phytomer_a) + continue!(simulation) + reparented_binding = only( + row for row in explain_bindings(model) + if row.application_id == :ancestor_consumer && + row.consumer_id == :female_z && + row.input == :signal + ) + @test reparented_binding.source_ids == [:phytomer_a] +end + @testset "repeated applications require explicit identity" begin model = CompositeModel( Object(:leaf; scale=:Leaf); @@ -2376,7 +2584,7 @@ end ModelSpec(StabilizationSourceModel(); name=:scene_source, on=One(scale=:Scene)), ), ) - simulation = run!(model; performance=true) + simulation = run!(model; outputs=:all, performance=true) original_groups = simulation.execution_plan.groups original_batches = simulation.execution_plan.batches original_groups_by_application_slot = @@ -2388,6 +2596,8 @@ end ) original_leaf_batch = only(original_leaf_group.batches) original_leaf_target = only(original_leaf_batch.targets) + original_leaf_context = original_leaf_target.context + @test original_leaf_context isa PlantSimEngine.RunContext original_scene_group = only( group for group in original_groups if group.application.id == :scene_source @@ -2418,14 +2628,60 @@ end @test refreshed_leaf_group === original_leaf_group @test only(refreshed_leaf_group.batches) === original_leaf_batch @test first(original_leaf_batch.targets) === original_leaf_target + @test first(original_leaf_batch.targets).context === original_leaf_context @test [target.object_id for target in original_leaf_batch.targets] == ObjectId.([:leaf_1, :leaf_2]) + @test original_leaf_batch.context_state.compiled === simulation.compiled + @test original_leaf_batch.context_state.environment_bindings === + simulation.environment_bindings + @test all(original_leaf_batch.targets) do target + target.context.compiled === simulation.compiled && + target.context.environment_bindings === + simulation.environment_bindings && + target.context.time == current_step(simulation) + end @test refreshed_scene_group === original_scene_group @test refreshed_scene_target === original_scene_target @test performance.counts[:execution_groups_reused] == 1 @test performance.counts[:execution_targets_constructed] == 1 @test performance.counts[:execution_batches_constructed] == 0 @test performance.counts[:execution_groups_updated_in_place] == 1 + + current_compiled = simulation.compiled + current_environment_bindings = simulation.environment_bindings + continue!(simulation) + @test original_leaf_batch.context_state.compiled === current_compiled + @test original_leaf_batch.context_state.environment_bindings === + current_environment_bindings + @test all( + target -> target.context.time == current_step(simulation), + original_leaf_batch.targets, + ) + + float32_time = Float32(current_step(simulation) + 1) + @test isnothing(PlantSimEngine._synchronize_model_execution_batch_contexts!( + original_leaf_batch, + simulation.compiled, + simulation.environment_bindings, + simulation.temporal_streams, + simulation.output_retention, + float32_time, + simulation.constants, + )) + float32_target = first(original_leaf_batch.targets) + @test PlantSimEngine._run_model_execution_target!( + simulation.compiled, + simulation.environment_bindings, + original_leaf_batch.application, + float32_target, + float32_time, + simulation.constants, + simulation.temporal_streams, + simulation.output_retention, + nothing, + false, + ) === float32_target.status + @test float32_target.context.time == Float64(float32_time) end @testset "execution plan addition falls back when a group is created" begin diff --git a/test/test-model-multirate-integration.jl b/test/test-model-multirate-integration.jl index 9e41f2457..714d5b6ec 100644 --- a/test/test-model-multirate-integration.jl +++ b/test/test-model-multirate-integration.jl @@ -212,6 +212,64 @@ end @test all(row.event_driven for row in values(schedule)) end +@testset "non-due batches synchronize retained contexts when next due" begin + log = Tuple{Int,Symbol}[] + model = CompositeModel( + Object(:scene; scale=:Scene), + Object(:leaf_1; scale=:Leaf, parent=:scene), + Object(:leaf_2; scale=:Leaf, parent=:scene); + applications=( + ModelSpec( + EventScheduleProbeModel(:leaf_probe, log); + name=:leaf_probe, + on=Many(scale=:Leaf), + every=Hour(2), + ), + ), + environment=(duration=Hour(1),), + ) + simulation = run!(model; steps=1, outputs=:all) + batch = only(simulation.execution_plan.batches) + contexts = getproperty.(batch.targets, :context) + initial_compiled = simulation.compiled + initial_environment_bindings = simulation.environment_bindings + + register_object!( + model, + Object(:unmatched_axis; scale=:Axis, parent=:scene), + ) + continue!(simulation) + + @test only(simulation.execution_plan.batches) === batch + @test simulation.compiled !== initial_compiled + @test batch.context_state.compiled === initial_compiled + @test batch.context_state.environment_bindings === + initial_environment_bindings + @test all(context -> context.time == 1.0, contexts) + + continue!(simulation) + + @test batch.context_state.compiled === simulation.compiled + @test batch.context_state.environment_bindings === + simulation.environment_bindings + @test all(contexts) do context + context.compiled === simulation.compiled && + context.environment_bindings === + simulation.environment_bindings && + context.time == 3.0 + end + @test all( + object -> object.status.runs == 2, + model_objects(model; scale=:Leaf), + ) + @test log == [ + (1, :leaf_probe), + (1, :leaf_probe), + (3, :leaf_probe), + (3, :leaf_probe), + ] +end + @testset "generic clocks retain phase semantics" begin model = CompositeModel( Object(:generic_slot; scale=:ScheduleSlot); From d9ac84c28676b36afcd6d86967ffa0f74d579cc5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?R=C3=A9mi=20Vezy?= Date: Tue, 1 Sep 2026 04:30:44 +0200 Subject: [PATCH 5/8] perf: reuse planned singleton input producers --- src/composite_model/compilation.jl | 74 +++++++++++++------ test/test-model-api-stabilization.jl | 8 ++ test/test-model-distributed-output-runtime.jl | 2 + test/test-model-initializers.jl | 6 ++ 4 files changed, 69 insertions(+), 21 deletions(-) diff --git a/src/composite_model/compilation.jl b/src/composite_model/compilation.jl index a8352f2d7..577e9acae 100644 --- a/src/composite_model/compilation.jl +++ b/src/composite_model/compilation.jl @@ -2298,7 +2298,8 @@ function _append_added_many_sources!( binding.source_var, binding.process, binding.application, - distributed_outputs; + distributed_outputs, + binding.potential_source_application_ids; applications_by_id=applications_by_id, allow_empty=true, ) @@ -2316,7 +2317,8 @@ function _append_added_many_sources!( binding.source_var, binding.process, binding.application, - distributed_outputs; + distributed_outputs, + binding.potential_source_application_ids; applications_by_id=applications_by_id, allow_empty=false, ) @@ -6181,29 +6183,58 @@ function _matching_input_source_applications( source_var::Symbol, process_filter, application_filter, - distributed_outputs=NoCompiledDistributedOutputs(); + distributed_outputs=NoCompiledDistributedOutputs(), + potential_source_application_ids=(); applications_by_id=nothing, allow_empty::Bool=false, ) - matches = Symbol[] - for source_id in source_ids - for application in get(applications_by_object, source_id, Any[]) - source_var in _model_output_names(application) || continue - isnothing(process_filter) || application.process == process_filter || continue - isnothing(application_filter) || application.id == application_filter || continue - push!(matches, application.id) + potential_application_id = + length(potential_source_application_ids) == 1 ? + only(potential_source_application_ids) : nothing + use_planned_singleton = !isnothing(potential_application_id) && + !isnothing(applications_by_id) && + haskey(applications_by_id, potential_application_id) + matches = if use_planned_singleton + application_id = potential_application_id + application = applications_by_id[application_id] + matched = any(source_ids) do source_id + _application_writes_object_variable( + distributed_outputs, + application, + source_id, + source_var, + ) && return true + # Targeted newborn initializers use a partial application overlay + # until the lifecycle barrier extends the compiled application. + # Preserve that path without rescanning unrelated applications. + return any(get(applications_by_object, source_id, ())) do candidate + candidate.id == application_id || return false + return source_var in _model_output_names(candidate) + end end - _append_distributed_input_source_applications!( - matches, - distributed_outputs, - source_id, - source_var, - process_filter, - application_filter, - applications_by_id, - ) + matched ? Symbol[application_id] : Symbol[] + else + resolved = Symbol[] + for source_id in source_ids + for application in get(applications_by_object, source_id, Any[]) + source_var in _model_output_names(application) || continue + isnothing(process_filter) || application.process == process_filter || continue + isnothing(application_filter) || application.id == application_filter || continue + push!(resolved, application.id) + end + _append_distributed_input_source_applications!( + resolved, + distributed_outputs, + source_id, + source_var, + process_filter, + application_filter, + applications_by_id, + ) + end + unique!(resolved) + resolved end - unique!(matches) if !allow_empty && (!isnothing(process_filter) || !isnothing(application_filter)) && isempty(matches) @@ -6779,7 +6810,8 @@ function _push_model_input_binding!( source_var, process_filter, application_filter, - distributed_outputs; + distributed_outputs, + plan.potential_source_application_ids; applications_by_id=applications_by_id, allow_empty=selector isa OptionalOne || (selector isa Many && isempty(source_ids)), diff --git a/test/test-model-api-stabilization.jl b/test/test-model-api-stabilization.jl index 0be25fbee..cb5f8ae5e 100644 --- a/test/test-model-api-stabilization.jl +++ b/test/test-model-api-stabilization.jl @@ -1597,6 +1597,14 @@ end row for row in explain_bindings(ordered_writer_scene) if row.application_id == :consumer && row.input == :signal ) + ordered_input_plans = + Advanced.refresh_bindings!(ordered_writer_scene).scenario_plan.input_plans + ordered_plan = only( + plan for plan in ordered_input_plans + if plan.application_id == :consumer && plan.input == :signal + ) + @test ordered_plan.potential_source_application_ids == + (:source_a, :source_b) @test ordered_binding.source_application_ids == [:source_b] end diff --git a/test/test-model-distributed-output-runtime.jl b/test/test-model-distributed-output-runtime.jl index 043846a6b..8df3e9227 100644 --- a/test/test-model-distributed-output-runtime.jl +++ b/test/test-model-distributed-output-runtime.jl @@ -1041,6 +1041,8 @@ end :distributed_runtime_sun_only_consumer ) @test binding.source_ids == ObjectId[ObjectId(:leaf_a)] + @test binding.plan.potential_source_application_ids == + (:distributed_runtime_sun_only_writer,) @test binding.source_application_ids == [:distributed_runtime_sun_only_writer] diff --git a/test/test-model-initializers.jl b/test/test-model-initializers.jl index c0c440812..041ab90f0 100644 --- a/test/test-model-initializers.jl +++ b/test/test-model-initializers.jl @@ -1255,6 +1255,12 @@ end compiled = Advanced.refresh_bindings!(model) @test compiled.applications_by_id[:split_creator].target_ids == [ObjectId(:chain_creator_a), ObjectId(:chain_creator_b)] + @test isempty(compiled.applications_by_id[:chain_source].target_ids) + sink_plan = only( + plan for plan in compiled.scenario_plan.input_plans + if plan.application_id == :chain_sink + ) + @test sink_plan.potential_source_application_ids == (:chain_source,) run!(model; steps=1, outputs=:none) @test model_status(model, :split_chain_source).signal == 7.0 @test model_status(model, :split_chain_sink).seen == 7.0 From 503f7a7da24a57fb7189f750a3e7981489bda051 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?R=C3=A9mi=20Vezy?= Date: Tue, 1 Sep 2026 05:24:18 +0200 Subject: [PATCH 6/8] perf: specialize uninstrumented execution groups --- src/composite_model/runtime_outputs.jl | 123 ++++++++++++++++--------- test/test-model-api-stabilization.jl | 2 + 2 files changed, 80 insertions(+), 45 deletions(-) diff --git a/src/composite_model/runtime_outputs.jl b/src/composite_model/runtime_outputs.jl index 50bd9143c..93b58bbfb 100644 --- a/src/composite_model/runtime_outputs.jl +++ b/src/composite_model/runtime_outputs.jl @@ -6953,11 +6953,18 @@ function _refresh_simulation_runtime!(simulation::Simulation) return simulation end -function _simulation_runtime_dirty(simulation::Simulation) - _runtime_performance_count!( - simulation.performance, - :runtime_dirty_checks, - ) +@inline function _simulation_runtime_dirty( + simulation::Simulation, + ::Nothing, +) + return simulation.runtime_revision != simulation.model.runtime_revision +end + +@inline function _simulation_runtime_dirty( + simulation::Simulation, + performance::RuntimePerformanceCounters, +) + _runtime_performance_count!(performance, :runtime_dirty_checks) return simulation.runtime_revision != simulation.model.runtime_revision end @@ -6976,6 +6983,64 @@ function _mark_schedule_prefix_completed!( return completed_applications end +@inline function _run_model_execution_group!( + group::CompiledApplicationExecutionGroup, + simulation::Simulation, + step::Integer, + ::Nothing, +) + for batch in group.batches + _run_model_execution_batch!( + batch, + simulation.compiled, + simulation.environment_bindings, + step, + simulation.constants, + simulation.temporal_streams, + simulation.output_retention, + ) + end + return nothing +end + +@inline function _run_model_execution_group!( + group::CompiledApplicationExecutionGroup, + simulation::Simulation, + step::Integer, + performance::RuntimePerformanceCounters, +) + _runtime_performance_count!( + performance, + :application_groups_considered, + ) + for batch in group.batches + _run_model_execution_batch_profiled!( + batch, + simulation.compiled, + simulation.environment_bindings, + step, + simulation.constants, + simulation.temporal_streams, + simulation.output_retention, + performance, + ) + _runtime_performance_count!( + performance, + :execution_batches_visited, + ) + _runtime_performance_count!( + performance, + :execution_targets_visited, + length(batch.targets), + ) + end + _runtime_performance_count!( + performance, + :application_groups_visited, + ) + return nothing +end + function _run_model_execution_step!(simulation::Simulation, step::Integer) started_at = _runtime_performance_start(simulation.performance) empty!(simulation.environment_bindings.sample_cache) @@ -7007,46 +7072,11 @@ function _run_model_execution_step!(simulation::Simulation, step::Integer) schedule_entry.application_slot ] isnothing(group) && continue - _runtime_performance_count!( - simulation.performance, - :application_groups_considered, - ) - for batch in group.batches - if isnothing(simulation.performance) - _run_model_execution_batch!( - batch, - simulation.compiled, - simulation.environment_bindings, - step, - simulation.constants, - simulation.temporal_streams, - simulation.output_retention, - ) - else - _run_model_execution_batch_profiled!( - batch, - simulation.compiled, - simulation.environment_bindings, - step, - simulation.constants, - simulation.temporal_streams, - simulation.output_retention, - simulation.performance, - ) - end - _runtime_performance_count!( - simulation.performance, - :execution_batches_visited, - ) - _runtime_performance_count!( - simulation.performance, - :execution_targets_visited, - length(batch.targets), - ) - end - _runtime_performance_count!( + _run_model_execution_group!( + group, + simulation, + step, simulation.performance, - :application_groups_visited, ) if !isnothing(completed_applications) @@ -7058,7 +7088,10 @@ function _run_model_execution_step!(simulation::Simulation, step::Integer) ) completed_schedule_entry = entry_index end - _simulation_runtime_dirty(simulation) || continue + _simulation_runtime_dirty( + simulation, + simulation.performance, + ) || continue if isnothing(completed_applications) completed_applications = Set{Symbol}() _mark_schedule_prefix_completed!( diff --git a/test/test-model-api-stabilization.jl b/test/test-model-api-stabilization.jl index cb5f8ae5e..18671736b 100644 --- a/test/test-model-api-stabilization.jl +++ b/test/test-model-api-stabilization.jl @@ -1912,9 +1912,11 @@ end performance = Advanced.runtime_performance(simulation) @test performance.counts[:steps_executed] == 3 + @test performance.counts[:application_groups_considered] == 3 @test performance.counts[:application_groups_visited] == 3 @test performance.counts[:execution_batches_visited] == 3 @test performance.counts[:execution_targets_visited] == 3 + @test performance.counts[:runtime_dirty_checks] == 3 @test performance.counts[:initial_application_plans_compiled] == 1 @test performance.counts[:initial_input_plans_compiled] == 0 @test performance.counts[:initial_call_plans_compiled] == 0 From 2dcedda40f895d1805e20716d3e2d10581189b07 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?R=C3=A9mi=20Vezy?= Date: Tue, 1 Sep 2026 12:23:13 +0200 Subject: [PATCH 7/8] perf: index observed manual call owners --- src/composite_model/compilation.jl | 144 ++++++++++++++++------ src/visualization/model_graph_view.jl | 1 + test/test-model-hard-calls.jl | 168 +++++++++++++++++++++++++- 3 files changed, 273 insertions(+), 40 deletions(-) diff --git a/src/composite_model/compilation.jl b/src/composite_model/compilation.jl index 577e9acae..25bc08c64 100644 --- a/src/composite_model/compilation.jl +++ b/src/composite_model/compilation.jl @@ -1401,7 +1401,7 @@ end # scenario plan and application plans remain immutable values; only their # runtime shell is reference-backed so passing it across a batch boundary does # not box and copy the complete typed scenario-plan tuple. -mutable struct CompiledCompositeModel{SC,SP,AP,AI,OA,ABO,IB,CB,IBI,CBI,DBI,DCBI,MBC,DO,CO,AC,SVI,CE,CET,PA,AO} +mutable struct CompiledCompositeModel{SC,SP,AP,AI,OA,ABO,IB,CB,IBI,CBI,DBI,DCBI,MCBI,MBC,DO,CO,AC,SVI,CE,CET,PA,AO} model::SC scenario_plan::SP applications::AP @@ -1414,6 +1414,7 @@ mutable struct CompiledCompositeModel{SC,SP,AP,AI,OA,ABO,IB,CB,IBI,CBI,DBI,DCBI, call_bindings_by_target::CBI dynamic_input_binding_indices::DBI dynamic_call_binding_indices::DCBI + manual_call_binding_indices_by_owner_application::MCBI many_input_binding_cache::MBC distributed_outputs::DO call_owners::CO @@ -1509,6 +1510,32 @@ function _index_dynamic_call_bindings(model::CompositeModel, bindings) return index end +function _index_manual_call_binding_by_owner_application!( + index, + binding::CompiledModelCallBinding, + binding_index::Int, +) + _compiled_call_mode(binding) === :manual || return index + _compiled_call_membership_is_observed(binding) || return index + binding_indices = get!(index, binding.application_id) do + Int[] + end + push!(binding_indices, binding_index) + return index +end + +function _index_manual_call_bindings_by_owner_application(bindings) + index = Dict{Symbol,Vector{Int}}() + for (binding_index, binding) in pairs(bindings) + _index_manual_call_binding_by_owner_application!( + index, + binding, + binding_index, + ) + end + return index +end + function _index_model_bindings(bindings, application_field::Symbol, object_field::Symbol) grouped = Dict{Tuple{Symbol,ObjectId},Vector{Any}}() for binding in bindings @@ -1700,6 +1727,7 @@ function _compile_scene( many_input_binding_cache, ), _index_dynamic_call_bindings(model, call_bindings), + _index_manual_call_bindings_by_owner_application(call_bindings), many_input_binding_cache, distributed_outputs, call_owners, @@ -2776,14 +2804,23 @@ function _propagate_changed_manual_call_owners!( changed_target_ids, changed_callee_target_ids, call_bindings, + manual_call_binding_indices_by_owner_application, call_owners, + performance=nothing, ) frontier = Dict{Symbol,Vector{ObjectId}}() for (application_id, object_id) in changed_callee_target_ids - push!(get!(frontier, application_id, ObjectId[]), object_id) + object_ids = get!(frontier, application_id) do + ObjectId[] + end + push!(object_ids, object_id) end seen_owner_keys = Set{Tuple{Symbol,ObjectId}}(changed_callee_target_ids) while !isempty(frontier) + _runtime_performance_count!( + performance, + :lifecycle_manual_call_owner_frontier_waves, + ) owner_application_ids = Set{Symbol}() for application_id in keys(frontier) hasproperty(call_owners, application_id) || continue @@ -2794,39 +2831,52 @@ function _propagate_changed_manual_call_owners!( end isempty(owner_application_ids) && break next_frontier = Dict{Symbol,Vector{ObjectId}}() - for binding in call_bindings - binding.application_id in owner_application_ids || continue - _compiled_call_mode(binding) === :manual || continue - _compiled_call_membership_is_observed(binding) || continue - affected = false - for callee_application_id in binding.callee_application_ids - object_ids = get(frontier, callee_application_id, nothing) - isnothing(object_ids) && continue - if any(object_ids) do object_id - first( - _sorted_object_id_position( - binding.callee_object_ids, - object_id, - ), - ) - end - affected = true - break + for owner_application_id in owner_application_ids + binding_indices = get( + manual_call_binding_indices_by_owner_application, + owner_application_id, + nothing, + ) + isnothing(binding_indices) && continue + for binding_index in binding_indices + _runtime_performance_count!( + performance, + :lifecycle_manual_call_owner_binding_candidates, + ) + binding = call_bindings[binding_index] + owner_key = (binding.application_id, binding.consumer_id) + owner_key in seen_owner_keys && continue + affected = false + for callee_application_id in binding.callee_application_ids + object_ids = get(frontier, callee_application_id, nothing) + isnothing(object_ids) && continue + if any(object_ids) do object_id + first( + _sorted_object_id_position( + binding.callee_object_ids, + object_id, + ), + ) + end + affected = true + break + end end - end - affected || continue - owner_key = (binding.application_id, binding.consumer_id) - owner_key in seen_owner_keys && continue - push!(seen_owner_keys, owner_key) - push!(changed_target_ids, owner_key) - push!( - get!( + affected || continue + push!(seen_owner_keys, owner_key) + push!(changed_target_ids, owner_key) + _runtime_performance_count!( + performance, + :lifecycle_manual_call_owner_targets_propagated, + ) + next_object_ids = get!( next_frontier, binding.application_id, - ObjectId[], - ), - binding.consumer_id, - ) + ) do + ObjectId[] + end + push!(next_object_ids, binding.consumer_id) + end end frontier = next_frontier end @@ -2963,6 +3013,7 @@ function _prepare_structural_compiled_delta( many_input_binding_cache, ), _index_dynamic_call_bindings(model, call_bindings), + _index_manual_call_bindings_by_owner_application(call_bindings), many_input_binding_cache, compiled.distributed_outputs, compiled.call_owners, @@ -3251,6 +3302,8 @@ function _extend_compiled_scene( end append!(call_bindings, new_call_bindings) dynamic_call_binding_indices = compiled.dynamic_call_binding_indices + manual_call_binding_indices_by_owner_application = + compiled.manual_call_binding_indices_by_owner_application first_new_call_binding = length(call_bindings) - length(new_call_bindings) + 1 for binding_index in first_new_call_binding:length(call_bindings) _index_dynamic_call_binding!( @@ -3259,6 +3312,11 @@ function _extend_compiled_scene( call_bindings[binding_index], binding_index, ) + _index_manual_call_binding_by_owner_application!( + manual_call_binding_indices_by_owner_application, + call_bindings[binding_index], + binding_index, + ) end _validate_model_writers_for_objects!( applications, @@ -3662,7 +3720,9 @@ function _extend_compiled_scene( changed_execution_target_ids, changed_manual_target_ids, call_bindings, + manual_call_binding_indices_by_owner_application, call_owners, + performance, ) changed_execution_application_ids = Set( first(key) for key in changed_execution_target_ids @@ -3685,6 +3745,7 @@ function _extend_compiled_scene( call_bindings_by_target, dynamic_input_binding_indices, dynamic_call_binding_indices, + manual_call_binding_indices_by_owner_application, many_binding_cache, distributed_outputs, call_owners, @@ -7218,12 +7279,19 @@ function _observe_compiled_call_membership!( candidate -> candidate === binding, compiled.call_bindings, ) - isnothing(binding_index) || _index_dynamic_call_binding!( - compiled.dynamic_call_binding_indices, - compiled.model, - binding, - binding_index, - ) + if !isnothing(binding_index) + _index_dynamic_call_binding!( + compiled.dynamic_call_binding_indices, + compiled.model, + binding, + binding_index, + ) + _index_manual_call_binding_by_owner_application!( + compiled.manual_call_binding_indices_by_owner_application, + binding, + binding_index, + ) + end return binding end diff --git a/src/visualization/model_graph_view.jl b/src/visualization/model_graph_view.jl index 6dfa85c44..3f84b75da 100644 --- a/src/visualization/model_graph_view.jl +++ b/src/visualization/model_graph_view.jl @@ -233,6 +233,7 @@ function _model_graph_compiled( many_input_binding_cache, ), _index_dynamic_call_bindings(model, call_bindings), + _index_manual_call_bindings_by_owner_application(call_bindings), many_input_binding_cache, distributed_outputs, scenario_plan.call_owners, diff --git a/test/test-model-hard-calls.jl b/test/test-model-hard-calls.jl index 688062c89..815f3221a 100644 --- a/test/test-model-hard-calls.jl +++ b/test/test-model-hard-calls.jl @@ -257,7 +257,19 @@ end model = CompositeModel( Object(:scene; scale=:Scene, name=:scene), Object(:middle; scale=:Plant, name=:middle, parent=:scene), - Object(:leaf_a; scale=:Leaf, parent=:middle); + Object(:leaf_a; scale=:Leaf, parent=:middle), + Object( + :unrelated_middle; + scale=:Plant, + name=:unrelated_middle, + parent=:scene, + ), + Object( + :unrelated_leaf; + scale=:Leaf, + name=:unrelated_leaf, + parent=:unrelated_middle, + ); applications=( ModelSpec( NestedManyRootModel(); @@ -283,6 +295,18 @@ end ), ), ), + ModelSpec( + NestedCallMiddleModel(); + name=:unrelated_middle, + on=One(name=:unrelated_middle), + calls=( + :leaf => One( + name=:unrelated_leaf, + within=Subtree(), + application=:leaf, + ), + ), + ), ModelSpec( NestedCallLeafModel(); name=:leaf, @@ -292,7 +316,7 @@ end environment=(duration=Hour(1),), ) - simulation = run!(model; outputs=:none) + simulation = run!(model; outputs=:none, performance=true) root = only(model_objects(model; name=:scene)).status scenario_plan = simulation.compiled.scenario_plan @test root.ncalls == 1 @@ -309,6 +333,17 @@ end @test root.ncalls == 2 @test root.total == 3.0 @test length(call_targets(NESTED_MANY_MIDDLE_CONTEXT[], :leaves)) == 2 + performance = Advanced.runtime_performance(simulation) + @test length(simulation.compiled.call_bindings) == 3 + @test performance.counts[ + :lifecycle_manual_call_owner_binding_candidates + ] == 3 + @test performance.counts[ + :lifecycle_manual_call_owner_binding_candidates + ] < 2 * length(simulation.compiled.call_bindings) + @test performance.counts[ + :lifecycle_manual_call_owner_targets_propagated + ] == 1 end @testset "hard-call ownership cycles fail before execution" begin @@ -570,6 +605,10 @@ end simulation = run!(model; outputs=:all, performance=true) controller = only(model_objects(model; scale=:Scene)).status + manual_owner_index = + simulation.compiled.manual_call_binding_indices_by_owner_application + call_binding_index = only(eachindex(simulation.compiled.call_bindings)) + @test manual_owner_index[:controller] == [call_binding_index] @test controller.ncalls == 2 @test controller.total == 2.0 rows = filter( @@ -594,6 +633,9 @@ end continue!(simulation; steps=1) performance = Advanced.runtime_performance(simulation) @test performance.counts[:execution_target_call_batches_extended] == 1 + @test simulation.compiled.manual_call_binding_indices_by_owner_application === + manual_owner_index + @test manual_owner_index[:controller] == [call_binding_index] @test only(simulation.compiled.call_bindings) === call_binding addition_membership_generation = PlantSimEngine._compiled_call_membership_generation(call_binding) @@ -618,6 +660,11 @@ end remove_object!(model, :leaf_b) continue!(simulation; steps=1) + rebuilt_manual_owner_index = + simulation.compiled.manual_call_binding_indices_by_owner_application + @test rebuilt_manual_owner_index !== manual_owner_index + @test rebuilt_manual_owner_index[:controller] == + [only(eachindex(simulation.compiled.call_bindings))] removal_membership_generation = PlantSimEngine._compiled_call_membership_generation(call_binding) @test removal_membership_generation == @@ -627,6 +674,74 @@ end @test controller.total == 5.0 end +@testset "new manual call owner extends observed owner index" begin + model = CompositeModel( + Object(:scene; scale=:Scene, name=:scene), + Object(:plant_a; scale=:Plant, parent=:scene), + Object(:leaf_a; scale=:Leaf, parent=:plant_a); + applications=( + ModelSpec( + ManyCallControllerModel(); + name=:controller, + on=Many(scale=:Plant), + calls=( + :children => Many( + scale=:Leaf, + within=Subtree(), + application=:leaf_calls, + ), + ), + ), + ModelSpec( + NestedCallLeafModel(); + name=:leaf_calls, + on=Many(scale=:Leaf), + ), + ), + environment=(duration=Hour(1),), + ) + + simulation = run!(model; outputs=:none) + manual_owner_index = + simulation.compiled.manual_call_binding_indices_by_owner_application + initial_binding_indices = copy(manual_owner_index[:controller]) + @test length(initial_binding_indices) == 1 + @test all( + PlantSimEngine._compiled_call_membership_is_observed( + simulation.compiled.call_bindings[binding_index], + ) + for binding_index in initial_binding_indices + ) + + register_object!( + model, + Object(:plant_b; scale=:Plant, parent=:scene), + ) + register_object!( + model, + Object(:leaf_b; scale=:Leaf, parent=:plant_b), + ) + continue!(simulation) + + @test simulation.compiled.manual_call_binding_indices_by_owner_application === + manual_owner_index + extended_binding_indices = manual_owner_index[:controller] + @test length(extended_binding_indices) == 2 + @test length(unique(extended_binding_indices)) == 2 + @test first(initial_binding_indices) in extended_binding_indices + indexed_bindings = + simulation.compiled.call_bindings[extended_binding_indices] + @test Set(getproperty.(indexed_bindings, :consumer_id)) == Set( + ObjectId[ObjectId(:plant_a), ObjectId(:plant_b)], + ) + @test all( + PlantSimEngine._compiled_call_membership_is_observed, + indexed_bindings, + ) + @test model_status(model, :plant_a).ncalls == 1 + @test model_status(model, :plant_b).ncalls == 1 +end + @testset "large monotonic Many call extension preserves existing targets" begin initial_count = 128 initial_leaf_ids = Symbol[ @@ -830,6 +945,13 @@ end call_binding = only(simulation.compiled.call_bindings) @test !PlantSimEngine._compiled_call_membership_is_observed(call_binding) @test !PlantSimEngine._call_execution_batches_materialized(full_call_view) + @test isempty( + get( + simulation.compiled.manual_call_binding_indices_by_owner_application, + :selective_controller, + Int[], + ), + ) register_object!( model, @@ -864,6 +986,18 @@ end :selector_call_binding_candidates, 0, ) == 0 + @test get( + Advanced.runtime_performance(simulation).counts, + :lifecycle_manual_call_owner_binding_candidates, + 0, + ) == 0 + @test isempty( + get( + simulation.compiled.manual_call_binding_indices_by_owner_application, + :selective_controller, + Int[], + ), + ) # The wrapper obtained before all three lifecycle changes stays cached. # Its first complete inspection catches up once, then activates the normal @@ -873,6 +1007,25 @@ end @test PlantSimEngine._compiled_call_membership_is_observed(call_binding) @test getproperty.(collect(full_call_view), :object_id) == ObjectId[ObjectId(:leaf_keep)] + call_binding_index = findfirst( + candidate -> candidate === call_binding, + simulation.compiled.call_bindings, + ) + @test !isnothing(call_binding_index) + indexed_call_bindings = copy( + get( + simulation.compiled.manual_call_binding_indices_by_owner_application, + :selective_controller, + Int[], + ), + ) + @test indexed_call_bindings == [call_binding_index] + @test length(full_call_view) == 1 + @test get( + simulation.compiled.manual_call_binding_indices_by_owner_application, + :selective_controller, + Int[], + ) == indexed_call_bindings register_object!( model, @@ -888,6 +1041,17 @@ end @test Advanced.runtime_performance(simulation).counts[ :selector_call_binding_candidates ] == 1 + @test Advanced.runtime_performance(simulation).counts[ + :lifecycle_manual_call_owner_binding_candidates + ] == 1 + @test Advanced.runtime_performance(simulation).counts[ + :lifecycle_manual_call_owner_targets_propagated + ] == 1 + @test get( + simulation.compiled.manual_call_binding_indices_by_owner_application, + :selective_controller, + Int[], + ) == indexed_call_bindings end @testset "retained cold Many synchronizes when a multirate owner is not due" begin From b053534e820dc1408defa77bb810a8aba87e58d3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?R=C3=A9mi=20Vezy?= Date: Tue, 1 Sep 2026 16:32:09 +0200 Subject: [PATCH 8/8] perf: index manual calls by exact callee target --- src/composite_model/compilation.jl | 358 ++++++++++++++++++------- src/composite_model/runtime_outputs.jl | 11 - src/visualization/model_graph_view.jl | 5 +- test/test-model-hard-calls.jl | 264 +++++++++++++++--- 4 files changed, 499 insertions(+), 139 deletions(-) diff --git a/src/composite_model/compilation.jl b/src/composite_model/compilation.jl index 25bc08c64..5f80711fe 100644 --- a/src/composite_model/compilation.jl +++ b/src/composite_model/compilation.jl @@ -1414,7 +1414,10 @@ mutable struct CompiledCompositeModel{SC,SP,AP,AI,OA,ABO,IB,CB,IBI,CBI,DBI,DCBI, call_bindings_by_target::CBI dynamic_input_binding_indices::DBI dynamic_call_binding_indices::DCBI - manual_call_binding_indices_by_owner_application::MCBI + # Observed manual bindings are indexed by executable callee + # application/object targets so lifecycle propagation can start from an + # exact changed target rather than scanning every owner binding. + observed_manual_call_binding_indices_by_callee_target::MCBI many_input_binding_cache::MBC distributed_outputs::DO call_owners::CO @@ -1510,27 +1513,182 @@ function _index_dynamic_call_bindings(model::CompositeModel, bindings) return index end -function _index_manual_call_binding_by_owner_application!( +function _push_observed_manual_call_binding_index!( + index, + callee_application_id::Symbol, + callee_object_id::ObjectId, + binding_index::Int, +) + binding_indices = get!( + index, + (callee_application_id, callee_object_id), + ) do + Int[] + end + position = searchsortedfirst(binding_indices, binding_index) + if position > lastindex(binding_indices) || + @inbounds(binding_indices[position] != binding_index) + insert!(binding_indices, position, binding_index) + end + return index +end + +function _call_binding_target_matches( + binding, + application, + object_id::ObjectId, +) + return (binding.multiplicity != :many && + length(binding.callee_application_ids) == 1) || + first( + _sorted_object_id_position( + application.target_ids, + object_id, + ), + ) +end + +function _index_observed_manual_call_binding_application_targets!( + index, + binding::CompiledModelCallBinding, + binding_index::Int, + callee_application_id::Symbol, + application, +) + if binding.multiplicity != :many && + length(binding.callee_application_ids) == 1 + for callee_object_id in binding.callee_object_ids + _push_observed_manual_call_binding_index!( + index, + callee_application_id, + callee_object_id, + binding_index, + ) + end + return index + end + + application_target_ids = application.target_ids + binding_target_ids = binding.callee_object_ids + if length(application_target_ids) <= length(binding_target_ids) + for callee_object_id in application_target_ids + first( + _sorted_object_id_position( + binding_target_ids, + callee_object_id, + ), + ) || continue + _push_observed_manual_call_binding_index!( + index, + callee_application_id, + callee_object_id, + binding_index, + ) + end + else + for callee_object_id in binding_target_ids + first( + _sorted_object_id_position( + application_target_ids, + callee_object_id, + ), + ) || continue + _push_observed_manual_call_binding_index!( + index, + callee_application_id, + callee_object_id, + binding_index, + ) + end + end + return index +end + +function _index_observed_manual_call_binding_by_callee_target!( index, binding::CompiledModelCallBinding, binding_index::Int, + applications_by_id, ) _compiled_call_mode(binding) === :manual || return index _compiled_call_membership_is_observed(binding) || return index - binding_indices = get!(index, binding.application_id) do - Int[] + for callee_application_id in binding.callee_application_ids + application = applications_by_id[callee_application_id] + _index_observed_manual_call_binding_application_targets!( + index, + binding, + binding_index, + callee_application_id, + application, + ) end - push!(binding_indices, binding_index) return index end -function _index_manual_call_bindings_by_owner_application(bindings) - index = Dict{Symbol,Vector{Int}}() +function _index_observed_manual_call_bindings_by_callee_target( + bindings, + applications_by_id, +) + index = Dict{Tuple{Symbol,ObjectId},Vector{Int}}() for (binding_index, binding) in pairs(bindings) - _index_manual_call_binding_by_owner_application!( + _index_observed_manual_call_binding_by_callee_target!( index, binding, binding_index, + applications_by_id, + ) + end + return index +end + +function _extend_observed_manual_call_binding_callee_target_index!( + index, + binding::CompiledModelCallBinding, + binding_index::Int, + previous_object_count::Int, + previous_application_count::Int, + applications_by_id, + applications_by_object, +) + _compiled_call_mode(binding) === :manual || return index + _compiled_call_membership_is_observed(binding) || return index + + # Index new objects through their actual applications rather than scanning + # the application/object product. A newly discovered application is then + # intersected with the complete binding membership to cover any existing + # targets without materializing that product. + new_object_positions = + (previous_object_count + 1):length(binding.callee_object_ids) + for object_position in new_object_positions + callee_object_id = binding.callee_object_ids[object_position] + for application in get( + applications_by_object, + callee_object_id, + (), + ) + callee_application_id = application.id + callee_application_id in binding.callee_application_ids || + continue + _push_observed_manual_call_binding_index!( + index, + callee_application_id, + callee_object_id, + binding_index, + ) + end + end + new_application_positions = + (previous_application_count + 1):length(binding.callee_application_ids) + for application_position in new_application_positions + callee_application_id = + binding.callee_application_ids[application_position] + application = applications_by_id[callee_application_id] + _index_observed_manual_call_binding_application_targets!( + index, + binding, + binding_index, + callee_application_id, + application, ) end return index @@ -1727,7 +1885,10 @@ function _compile_scene( many_input_binding_cache, ), _index_dynamic_call_bindings(model, call_bindings), - _index_manual_call_bindings_by_owner_application(call_bindings), + _index_observed_manual_call_bindings_by_callee_target( + call_bindings, + applications_by_id, + ), many_input_binding_cache, distributed_outputs, call_owners, @@ -2753,7 +2914,9 @@ function _append_added_many_call_targets!( added_ids, applications_by_object, ) - binding.multiplicity == :many || return false + binding.multiplicity == :many || return nothing + previous_object_count = length(binding.callee_object_ids) + previous_application_count = length(binding.callee_application_ids) default_scope = _default_dependency_scope(model, binding.consumer_id) new_target_ids = ObjectId[ object_id for object_id in added_ids @@ -2771,7 +2934,10 @@ function _append_added_many_call_targets!( ), ) ] - isempty(new_target_ids) && return true + isempty(new_target_ids) && return ( + previous_object_count=previous_object_count, + previous_application_count=previous_application_count, + ) _sort_object_ids!(new_target_ids) # Monotonically increasing lifecycle IDs preserve the selector's compiled @@ -2781,7 +2947,7 @@ function _append_added_many_call_targets!( last(binding.callee_object_ids), first(new_target_ids), ) - return false + return nothing end append!(binding.callee_object_ids, new_target_ids) @@ -2797,86 +2963,55 @@ function _append_added_many_call_targets!( end end _mark_compiled_call_membership_changed!(binding, model.revision) - return true + return ( + previous_object_count=previous_object_count, + previous_application_count=previous_application_count, + ) end function _propagate_changed_manual_call_owners!( changed_target_ids, changed_callee_target_ids, call_bindings, - manual_call_binding_indices_by_owner_application, - call_owners, + observed_manual_call_binding_indices_by_callee_target, performance=nothing, ) - frontier = Dict{Symbol,Vector{ObjectId}}() - for (application_id, object_id) in changed_callee_target_ids - object_ids = get!(frontier, application_id) do - ObjectId[] - end - push!(object_ids, object_id) - end + frontier = Tuple{Symbol,ObjectId}[ + key for key in changed_callee_target_ids + ] seen_owner_keys = Set{Tuple{Symbol,ObjectId}}(changed_callee_target_ids) while !isempty(frontier) _runtime_performance_count!( performance, :lifecycle_manual_call_owner_frontier_waves, ) - owner_application_ids = Set{Symbol}() - for application_id in keys(frontier) - hasproperty(call_owners, application_id) || continue - union!( - owner_application_ids, - getproperty(call_owners, application_id), - ) - end - isempty(owner_application_ids) && break - next_frontier = Dict{Symbol,Vector{ObjectId}}() - for owner_application_id in owner_application_ids + candidate_binding_indices = Set{Int}() + for callee_key in frontier binding_indices = get( - manual_call_binding_indices_by_owner_application, - owner_application_id, + observed_manual_call_binding_indices_by_callee_target, + callee_key, nothing, ) - isnothing(binding_indices) && continue - for binding_index in binding_indices - _runtime_performance_count!( - performance, - :lifecycle_manual_call_owner_binding_candidates, - ) - binding = call_bindings[binding_index] - owner_key = (binding.application_id, binding.consumer_id) - owner_key in seen_owner_keys && continue - affected = false - for callee_application_id in binding.callee_application_ids - object_ids = get(frontier, callee_application_id, nothing) - isnothing(object_ids) && continue - if any(object_ids) do object_id - first( - _sorted_object_id_position( - binding.callee_object_ids, - object_id, - ), - ) - end - affected = true - break - end - end - affected || continue - push!(seen_owner_keys, owner_key) - push!(changed_target_ids, owner_key) - _runtime_performance_count!( - performance, - :lifecycle_manual_call_owner_targets_propagated, - ) - next_object_ids = get!( - next_frontier, - binding.application_id, - ) do - ObjectId[] - end - push!(next_object_ids, binding.consumer_id) - end + isnothing(binding_indices) || + union!(candidate_binding_indices, binding_indices) + end + isempty(candidate_binding_indices) && break + next_frontier = Tuple{Symbol,ObjectId}[] + for binding_index in candidate_binding_indices + _runtime_performance_count!( + performance, + :lifecycle_manual_call_owner_binding_candidates, + ) + binding = call_bindings[binding_index] + owner_key = (binding.application_id, binding.consumer_id) + owner_key in seen_owner_keys && continue + push!(seen_owner_keys, owner_key) + push!(changed_target_ids, owner_key) + _runtime_performance_count!( + performance, + :lifecycle_manual_call_owner_targets_propagated, + ) + push!(next_frontier, owner_key) end frontier = next_frontier end @@ -3013,7 +3148,10 @@ function _prepare_structural_compiled_delta( many_input_binding_cache, ), _index_dynamic_call_bindings(model, call_bindings), - _index_manual_call_bindings_by_owner_application(call_bindings), + _index_observed_manual_call_bindings_by_callee_target( + call_bindings, + compiled.applications_by_id, + ), many_input_binding_cache, compiled.distributed_outputs, compiled.call_owners, @@ -3169,6 +3307,8 @@ function _extend_compiled_scene( Set{Tuple{Symbol,ObjectId}}(forced_call_target_keys) changed_call_target_keys = Set{Tuple{Symbol,ObjectId}}(forced_call_target_keys) + observed_manual_call_binding_indices_by_callee_target = + compiled.observed_manual_call_binding_indices_by_callee_target if has_calls candidate_call_binding_indices = Set{Int}() for object_id in added_ids @@ -3196,15 +3336,27 @@ function _extend_compiled_scene( ) || continue key = (binding.application_id, binding.consumer_id) push!(changed_call_target_keys, key) - appended = key in forced_call_target_keys ? - false : - _append_added_many_call_targets!( + extension = key in forced_call_target_keys ? + nothing : + _append_added_many_call_targets!( model, binding, added_ids, applications_by_object, ) - appended || push!(rebuilt_existing_call_targets, key) + if isnothing(extension) + push!(rebuilt_existing_call_targets, key) + elseif pure_addition + _extend_observed_manual_call_binding_callee_target_index!( + observed_manual_call_binding_indices_by_callee_target, + binding, + binding_index, + extension.previous_object_count, + extension.previous_application_count, + applications_by_id, + applications_by_object, + ) + end end end new_call_bindings = has_calls ? @@ -3268,6 +3420,14 @@ function _extend_compiled_scene( "for application `$(first(key))` on object `$(last(key).value)`.", ) for binding in existing + binding_index = findfirst( + candidate -> candidate === binding, + call_bindings, + ) + isnothing(binding_index) && error( + "Incremental hard-call refresh lost the existing compiled binding ", + "for application `$(first(key))` on object `$(last(key).value)`.", + ) replacement_index = findfirst( candidate -> _compiled_call_name(candidate) === @@ -3297,13 +3457,25 @@ function _extend_compiled_scene( binding, model.revision, ) + pure_addition && + _index_observed_manual_call_binding_by_callee_target!( + observed_manual_call_binding_indices_by_callee_target, + binding, + binding_index, + applications_by_id, + ) end end end append!(call_bindings, new_call_bindings) dynamic_call_binding_indices = compiled.dynamic_call_binding_indices - manual_call_binding_indices_by_owner_application = - compiled.manual_call_binding_indices_by_owner_application + if !pure_addition + observed_manual_call_binding_indices_by_callee_target = + _index_observed_manual_call_bindings_by_callee_target( + call_bindings, + applications_by_id, + ) + end first_new_call_binding = length(call_bindings) - length(new_call_bindings) + 1 for binding_index in first_new_call_binding:length(call_bindings) _index_dynamic_call_binding!( @@ -3312,11 +3484,13 @@ function _extend_compiled_scene( call_bindings[binding_index], binding_index, ) - _index_manual_call_binding_by_owner_application!( - manual_call_binding_indices_by_owner_application, - call_bindings[binding_index], - binding_index, - ) + pure_addition && + _index_observed_manual_call_binding_by_callee_target!( + observed_manual_call_binding_indices_by_callee_target, + call_bindings[binding_index], + binding_index, + applications_by_id, + ) end _validate_model_writers_for_objects!( applications, @@ -3720,8 +3894,7 @@ function _extend_compiled_scene( changed_execution_target_ids, changed_manual_target_ids, call_bindings, - manual_call_binding_indices_by_owner_application, - call_owners, + observed_manual_call_binding_indices_by_callee_target, performance, ) changed_execution_application_ids = Set( @@ -3745,7 +3918,7 @@ function _extend_compiled_scene( call_bindings_by_target, dynamic_input_binding_indices, dynamic_call_binding_indices, - manual_call_binding_indices_by_owner_application, + observed_manual_call_binding_indices_by_callee_target, many_binding_cache, distributed_outputs, call_owners, @@ -7286,10 +7459,11 @@ function _observe_compiled_call_membership!( binding, binding_index, ) - _index_manual_call_binding_by_owner_application!( - compiled.manual_call_binding_indices_by_owner_application, + _index_observed_manual_call_binding_by_callee_target!( + compiled.observed_manual_call_binding_indices_by_callee_target, binding, binding_index, + compiled.applications_by_id, ) end return binding diff --git a/src/composite_model/runtime_outputs.jl b/src/composite_model/runtime_outputs.jl index 93b58bbfb..28dfdeb33 100644 --- a/src/composite_model/runtime_outputs.jl +++ b/src/composite_model/runtime_outputs.jl @@ -5302,17 +5302,6 @@ end return found end -function _call_binding_target_matches(binding, application, object_id::ObjectId) - return (binding.multiplicity != :many && - length(binding.callee_application_ids) == 1) || - first( - _sorted_object_id_position( - application.target_ids, - object_id, - ), - ) -end - _call_target_matches(targets::CallTargets, application, object_id::ObjectId) = _call_binding_target_matches(targets.binding, application, object_id) diff --git a/src/visualization/model_graph_view.jl b/src/visualization/model_graph_view.jl index 3f84b75da..bb819ff89 100644 --- a/src/visualization/model_graph_view.jl +++ b/src/visualization/model_graph_view.jl @@ -233,7 +233,10 @@ function _model_graph_compiled( many_input_binding_cache, ), _index_dynamic_call_bindings(model, call_bindings), - _index_manual_call_bindings_by_owner_application(call_bindings), + _index_observed_manual_call_bindings_by_callee_target( + call_bindings, + applications_by_id, + ), many_input_binding_cache, distributed_outputs, scenario_plan.call_owners, diff --git a/test/test-model-hard-calls.jl b/test/test-model-hard-calls.jl index 815f3221a..11ae6de1d 100644 --- a/test/test-model-hard-calls.jl +++ b/test/test-model-hard-calls.jl @@ -337,7 +337,10 @@ end @test length(simulation.compiled.call_bindings) == 3 @test performance.counts[ :lifecycle_manual_call_owner_binding_candidates - ] == 3 + ] == 2 + @test performance.counts[ + :lifecycle_manual_call_owner_frontier_waves + ] == 2 @test performance.counts[ :lifecycle_manual_call_owner_binding_candidates ] < 2 * length(simulation.compiled.call_bindings) @@ -605,10 +608,15 @@ end simulation = run!(model; outputs=:all, performance=true) controller = only(model_objects(model; scale=:Scene)).status - manual_owner_index = - simulation.compiled.manual_call_binding_indices_by_owner_application + observed_callee_target_index = + simulation.compiled.observed_manual_call_binding_indices_by_callee_target call_binding_index = only(eachindex(simulation.compiled.call_bindings)) - @test manual_owner_index[:controller] == [call_binding_index] + @test observed_callee_target_index[ + (:leaf_calls, ObjectId(:leaf_a)) + ] == [call_binding_index] + @test observed_callee_target_index[ + (:leaf_calls, ObjectId(:leaf_b)) + ] == [call_binding_index] @test controller.ncalls == 2 @test controller.total == 2.0 rows = filter( @@ -633,9 +641,11 @@ end continue!(simulation; steps=1) performance = Advanced.runtime_performance(simulation) @test performance.counts[:execution_target_call_batches_extended] == 1 - @test simulation.compiled.manual_call_binding_indices_by_owner_application === - manual_owner_index - @test manual_owner_index[:controller] == [call_binding_index] + @test simulation.compiled.observed_manual_call_binding_indices_by_callee_target === + observed_callee_target_index + @test observed_callee_target_index[ + (:leaf_calls, ObjectId(:leaf_c)) + ] == [call_binding_index] @test only(simulation.compiled.call_bindings) === call_binding addition_membership_generation = PlantSimEngine._compiled_call_membership_generation(call_binding) @@ -660,11 +670,16 @@ end remove_object!(model, :leaf_b) continue!(simulation; steps=1) - rebuilt_manual_owner_index = - simulation.compiled.manual_call_binding_indices_by_owner_application - @test rebuilt_manual_owner_index !== manual_owner_index - @test rebuilt_manual_owner_index[:controller] == - [only(eachindex(simulation.compiled.call_bindings))] + rebuilt_observed_callee_target_index = + simulation.compiled.observed_manual_call_binding_indices_by_callee_target + @test rebuilt_observed_callee_target_index !== observed_callee_target_index + @test !haskey( + rebuilt_observed_callee_target_index, + (:leaf_calls, ObjectId(:leaf_b)), + ) + @test rebuilt_observed_callee_target_index[ + (:leaf_calls, ObjectId(:leaf_a)) + ] == [only(eachindex(simulation.compiled.call_bindings))] removal_membership_generation = PlantSimEngine._compiled_call_membership_generation(call_binding) @test removal_membership_generation == @@ -674,7 +689,7 @@ end @test controller.total == 5.0 end -@testset "new manual call owner extends observed owner index" begin +@testset "new manual call owner extends observed callee target index" begin model = CompositeModel( Object(:scene; scale=:Scene, name=:scene), Object(:plant_a; scale=:Plant, parent=:scene), @@ -702,9 +717,11 @@ end ) simulation = run!(model; outputs=:none) - manual_owner_index = - simulation.compiled.manual_call_binding_indices_by_owner_application - initial_binding_indices = copy(manual_owner_index[:controller]) + observed_callee_target_index = + simulation.compiled.observed_manual_call_binding_indices_by_callee_target + initial_binding_indices = copy( + observed_callee_target_index[(:leaf_calls, ObjectId(:leaf_a))], + ) @test length(initial_binding_indices) == 1 @test all( PlantSimEngine._compiled_call_membership_is_observed( @@ -723,9 +740,12 @@ end ) continue!(simulation) - @test simulation.compiled.manual_call_binding_indices_by_owner_application === - manual_owner_index - extended_binding_indices = manual_owner_index[:controller] + @test simulation.compiled.observed_manual_call_binding_indices_by_callee_target === + observed_callee_target_index + extended_binding_indices = vcat( + observed_callee_target_index[(:leaf_calls, ObjectId(:leaf_a))], + observed_callee_target_index[(:leaf_calls, ObjectId(:leaf_b))], + ) @test length(extended_binding_indices) == 2 @test length(unique(extended_binding_indices)) == 2 @test first(initial_binding_indices) in extended_binding_indices @@ -742,6 +762,154 @@ end @test model_status(model, :plant_b).ncalls == 1 end +@testset "observed callee target index extends exact membership" begin + model = CompositeModel( + Object(:scene; scale=:Scene, name=:scene), + Object(:leaf_a; scale=:Leaf, kind=:a, parent=:scene); + applications=( + ModelSpec( + ManyCallControllerModel(); + name=:controller, + on=One(name=:scene), + calls=( + :children => Many( + scale=:Leaf, + process=:nested_call_leaf, + within=SceneScope(), + ), + ), + ), + ModelSpec( + NestedCallLeafModel(); + name=:leaf_a_calls, + on=Many(scale=:Leaf, kind=:a), + ), + ModelSpec( + NestedCallLeafModel(); + name=:leaf_b_calls, + on=Many(scale=:Leaf, kind=:b), + ), + ), + environment=(duration=Hour(1),), + ) + + simulation = run!(model; outputs=:none, performance=true) + call_binding = only(simulation.compiled.call_bindings) + call_binding_index = only(eachindex(simulation.compiled.call_bindings)) + observed_callee_target_index = + simulation.compiled.observed_manual_call_binding_indices_by_callee_target + @test call_binding.callee_application_ids == [:leaf_a_calls] + @test observed_callee_target_index == Dict( + (:leaf_a_calls, ObjectId(:leaf_a)) => [call_binding_index], + ) + + register_object!( + model, + Object(:leaf_b; scale=:Leaf, kind=:b, parent=:scene), + ) + register_object!( + model, + Object(:leaf_c; scale=:Leaf, kind=:b, parent=:scene), + ) + continue!(simulation) + + @test simulation.compiled.observed_manual_call_binding_indices_by_callee_target === + observed_callee_target_index + @test Set(call_binding.callee_application_ids) == + Set((:leaf_a_calls, :leaf_b_calls)) + @test Set(keys(observed_callee_target_index)) == Set(( + (:leaf_a_calls, ObjectId(:leaf_a)), + (:leaf_b_calls, ObjectId(:leaf_b)), + (:leaf_b_calls, ObjectId(:leaf_c)), + )) + @test all( + binding_indices == [call_binding_index] + for binding_indices in values(observed_callee_target_index) + ) + @test model_status(model, :scene).ncalls == 3 + @test model_status(model, :scene).total == 4.0 + @test Advanced.runtime_performance(simulation).counts[ + :lifecycle_manual_call_owner_binding_candidates + ] == 1 +end + +@testset "shared exact callee target reindexes after owner removal" begin + model = CompositeModel( + Object(:scene; scale=:Scene, name=:scene), + Object(:plant_a; scale=:Plant, parent=:scene), + Object(:plant_b; scale=:Plant, parent=:scene); + applications=( + ModelSpec( + ManyCallControllerModel(); + name=:controller, + on=Many(scale=:Plant), + calls=( + :children => Many( + name=:shared_leaf, + within=SceneScope(), + application=:leaf_calls, + ), + ), + ), + ModelSpec( + NestedCallLeafModel(); + name=:leaf_calls, + on=Many(name=:shared_leaf), + ), + ), + environment=(duration=Hour(1),), + ) + + simulation = run!(model; outputs=:none, performance=true) + @test isempty( + simulation.compiled.observed_manual_call_binding_indices_by_callee_target, + ) + + register_object!( + model, + Object( + :shared_leaf; + scale=:Leaf, + name=:shared_leaf, + parent=:scene, + ), + ) + continue!(simulation) + + shared_key = (:leaf_calls, ObjectId(:shared_leaf)) + observed_callee_target_index = + simulation.compiled.observed_manual_call_binding_indices_by_callee_target + @test observed_callee_target_index[shared_key] == [1, 2] + @test getproperty.(simulation.compiled.call_bindings, :consumer_id) == + ObjectId[ObjectId(:plant_a), ObjectId(:plant_b)] + for binding_index in reverse(eachindex(simulation.compiled.call_bindings)) + PlantSimEngine._index_observed_manual_call_binding_by_callee_target!( + observed_callee_target_index, + simulation.compiled.call_bindings[binding_index], + binding_index, + simulation.compiled.applications_by_id, + ) + end + @test observed_callee_target_index[shared_key] == [1, 2] + @test Advanced.runtime_performance(simulation).counts[ + :lifecycle_manual_call_owner_binding_candidates + ] == 2 + @test Advanced.runtime_performance(simulation).counts[ + :lifecycle_manual_call_owner_targets_propagated + ] == 2 + + remove_object!(model, :plant_a) + continue!(simulation) + + rebuilt_observed_callee_target_index = + simulation.compiled.observed_manual_call_binding_indices_by_callee_target + @test rebuilt_observed_callee_target_index !== + observed_callee_target_index + @test rebuilt_observed_callee_target_index[shared_key] == [1] + @test only(simulation.compiled.call_bindings).consumer_id == + ObjectId(:plant_b) +end + @testset "large monotonic Many call extension preserves existing targets" begin initial_count = 128 initial_leaf_ids = Symbol[ @@ -946,11 +1114,7 @@ end @test !PlantSimEngine._compiled_call_membership_is_observed(call_binding) @test !PlantSimEngine._call_execution_batches_materialized(full_call_view) @test isempty( - get( - simulation.compiled.manual_call_binding_indices_by_owner_application, - :selective_controller, - Int[], - ), + simulation.compiled.observed_manual_call_binding_indices_by_callee_target, ) register_object!( @@ -992,11 +1156,7 @@ end 0, ) == 0 @test isempty( - get( - simulation.compiled.manual_call_binding_indices_by_owner_application, - :selective_controller, - Int[], - ), + simulation.compiled.observed_manual_call_binding_indices_by_callee_target, ) # The wrapper obtained before all three lifecycle changes stays cached. @@ -1012,18 +1172,20 @@ end simulation.compiled.call_bindings, ) @test !isnothing(call_binding_index) + observed_callee_target_index = + simulation.compiled.observed_manual_call_binding_indices_by_callee_target indexed_call_bindings = copy( get( - simulation.compiled.manual_call_binding_indices_by_owner_application, - :selective_controller, + observed_callee_target_index, + (:selective_leaf_calls, ObjectId(:leaf_keep)), Int[], ), ) @test indexed_call_bindings == [call_binding_index] @test length(full_call_view) == 1 @test get( - simulation.compiled.manual_call_binding_indices_by_owner_application, - :selective_controller, + observed_callee_target_index, + (:selective_leaf_calls, ObjectId(:leaf_keep)), Int[], ) == indexed_call_bindings @@ -1047,9 +1209,11 @@ end @test Advanced.runtime_performance(simulation).counts[ :lifecycle_manual_call_owner_targets_propagated ] == 1 + @test simulation.compiled.observed_manual_call_binding_indices_by_callee_target === + observed_callee_target_index @test get( - simulation.compiled.manual_call_binding_indices_by_owner_application, - :selective_controller, + observed_callee_target_index, + (:selective_leaf_calls, ObjectId(:leaf_z_after_observation)), Int[], ) == indexed_call_bindings end @@ -1482,9 +1646,21 @@ end ) simulation = run!(model; outputs=:none, performance=true) + observed_callee_target_index = + simulation.compiled.observed_manual_call_binding_indices_by_callee_target + @test observed_callee_target_index[ + (:leaf_calls, ObjectId(:leaf)) + ] == [only(eachindex(simulation.compiled.call_bindings))] reparent_object!(model, :leaf, :plant_b) continue!(simulation) + rebuilt_observed_callee_target_index = + simulation.compiled.observed_manual_call_binding_indices_by_callee_target + @test rebuilt_observed_callee_target_index !== observed_callee_target_index + @test rebuilt_observed_callee_target_index[ + (:leaf_calls, ObjectId(:leaf)) + ] == [only(eachindex(simulation.compiled.call_bindings))] + refreshed_call_view = call_targets(MANY_CALL_CONTEXT[], :children) refreshed_execution_target = only(only(refreshed_call_view.execution_batches).targets) @@ -1522,6 +1698,9 @@ end @test schedule[:leaf_calls].manual_call_only @test !schedule[:leaf_calls].root_scheduled @test controller.ncalls == 0 + initial_observed_callee_target_index = + simulation.compiled.observed_manual_call_binding_indices_by_callee_target + @test isempty(initial_observed_callee_target_index) reparent_object!(model, :leaf, :plant_a) continue!(simulation; steps=1) @@ -1533,6 +1712,13 @@ end @test performance.counts[:selector_call_binding_candidates] == 1 @test controller.ncalls == 1 @test controller.total == 1.0 + entered_observed_callee_target_index = + simulation.compiled.observed_manual_call_binding_indices_by_callee_target + @test entered_observed_callee_target_index !== + initial_observed_callee_target_index + @test entered_observed_callee_target_index[ + (:leaf_calls, ObjectId(:leaf)) + ] == [only(eachindex(simulation.compiled.call_bindings))] reparent_object!(model, :leaf, :plant_b) continue!(simulation; steps=1) @@ -1546,6 +1732,14 @@ end @test performance.counts[:selector_call_binding_candidates] == 1 @test controller.ncalls == 0 @test controller.total == 0.0 + left_observed_callee_target_index = + simulation.compiled.observed_manual_call_binding_indices_by_callee_target + @test left_observed_callee_target_index !== + entered_observed_callee_target_index + @test !haskey( + left_observed_callee_target_index, + (:leaf_calls, ObjectId(:leaf)), + ) end @testset "manual target cadence contract" begin